1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 23 /* All Rights Reserved */ 24 25 26 /* 27 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 28 * Use is subject to license terms. 29 */ 30 31 #pragma ident "%Z%%M% %I% %E% SMI" 32 33 #include <sys/types.h> 34 #include <sys/sysmacros.h> 35 #include <sys/param.h> 36 #include <sys/errno.h> 37 #include <sys/signal.h> 38 #include <sys/proc.h> 39 #include <sys/conf.h> 40 #include <sys/cred.h> 41 #include <sys/user.h> 42 #include <sys/vnode.h> 43 #include <sys/file.h> 44 #include <sys/session.h> 45 #include <sys/stream.h> 46 #include <sys/strsubr.h> 47 #include <sys/stropts.h> 48 #include <sys/poll.h> 49 #include <sys/systm.h> 50 #include <sys/cpuvar.h> 51 #include <sys/uio.h> 52 #include <sys/cmn_err.h> 53 #include <sys/priocntl.h> 54 #include <sys/procset.h> 55 #include <sys/vmem.h> 56 #include <sys/bitmap.h> 57 #include <sys/kmem.h> 58 #include <sys/siginfo.h> 59 #include <sys/vtrace.h> 60 #include <sys/callb.h> 61 #include <sys/debug.h> 62 #include <sys/modctl.h> 63 #include <sys/vmsystm.h> 64 #include <vm/page.h> 65 #include <sys/atomic.h> 66 #include <sys/suntpi.h> 67 #include <sys/strlog.h> 68 #include <sys/promif.h> 69 #include <sys/project.h> 70 #include <sys/vm.h> 71 #include <sys/taskq.h> 72 #include <sys/sunddi.h> 73 #include <sys/sunldi_impl.h> 74 #include <sys/strsun.h> 75 #include <sys/isa_defs.h> 76 #include <sys/multidata.h> 77 #include <sys/pattr.h> 78 #include <sys/strft.h> 79 #include <sys/zone.h> 80 81 #define O_SAMESTR(q) (((q)->q_next) && \ 82 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR))) 83 84 /* 85 * WARNING: 86 * The variables and routines in this file are private, belonging 87 * to the STREAMS subsystem. These should not be used by modules 88 * or drivers. Compatibility will not be guaranteed. 89 */ 90 91 /* 92 * Id value used to distinguish between different multiplexor links. 93 */ 94 static int32_t lnk_id = 0; 95 96 #define STREAMS_LOPRI MINCLSYSPRI 97 static pri_t streams_lopri = STREAMS_LOPRI; 98 99 #define STRSTAT(x) (str_statistics.x.value.ui64++) 100 typedef struct str_stat { 101 kstat_named_t sqenables; 102 kstat_named_t stenables; 103 kstat_named_t syncqservice; 104 kstat_named_t freebs; 105 kstat_named_t qwr_outer; 106 kstat_named_t rservice; 107 kstat_named_t strwaits; 108 kstat_named_t taskqfails; 109 kstat_named_t bufcalls; 110 kstat_named_t qhelps; 111 kstat_named_t qremoved; 112 kstat_named_t sqremoved; 113 kstat_named_t bcwaits; 114 kstat_named_t sqtoomany; 115 } str_stat_t; 116 117 static str_stat_t str_statistics = { 118 { "sqenables", KSTAT_DATA_UINT64 }, 119 { "stenables", KSTAT_DATA_UINT64 }, 120 { "syncqservice", KSTAT_DATA_UINT64 }, 121 { "freebs", KSTAT_DATA_UINT64 }, 122 { "qwr_outer", KSTAT_DATA_UINT64 }, 123 { "rservice", KSTAT_DATA_UINT64 }, 124 { "strwaits", KSTAT_DATA_UINT64 }, 125 { "taskqfails", KSTAT_DATA_UINT64 }, 126 { "bufcalls", KSTAT_DATA_UINT64 }, 127 { "qhelps", KSTAT_DATA_UINT64 }, 128 { "qremoved", KSTAT_DATA_UINT64 }, 129 { "sqremoved", KSTAT_DATA_UINT64 }, 130 { "bcwaits", KSTAT_DATA_UINT64 }, 131 { "sqtoomany", KSTAT_DATA_UINT64 }, 132 }; 133 134 static kstat_t *str_kstat; 135 136 /* 137 * qrunflag was used previously to control background scheduling of queues. It 138 * is not used anymore, but kept here in case some module still wants to access 139 * it via qready() and setqsched macros. 140 */ 141 char qrunflag; /* Unused */ 142 143 /* 144 * Most of the streams scheduling is done via task queues. Task queues may fail 145 * for non-sleep dispatches, so there are two backup threads servicing failed 146 * requests for queues and syncqs. Both of these threads also service failed 147 * dispatches freebs requests. Queues are put in the list specified by `qhead' 148 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs 149 * requests are put into `freebs_list' which has no tail pointer. All three 150 * lists are protected by a single `service_queue' lock and use 151 * `services_to_run' condition variable for signaling background threads. Use of 152 * a single lock should not be a problem because it is only used under heavy 153 * loads when task queues start to fail and at that time it may be a good idea 154 * to throttle scheduling requests. 155 * 156 * NOTE: queues and syncqs should be scheduled by two separate threads because 157 * queue servicing may be blocked waiting for a syncq which may be also 158 * scheduled for background execution. This may create a deadlock when only one 159 * thread is used for both. 160 */ 161 162 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */ 163 164 static kmutex_t service_queue; /* protects all of servicing vars */ 165 static kcondvar_t services_to_run; /* wake up background service thread */ 166 static kcondvar_t syncqs_to_run; /* wake up background service thread */ 167 168 /* 169 * List of queues scheduled for background processing dueue to lack of resources 170 * in the task queues. Protected by service_queue lock; 171 */ 172 static struct queue *qhead; 173 static struct queue *qtail; 174 175 /* 176 * Same list for syncqs 177 */ 178 static syncq_t *sqhead; 179 static syncq_t *sqtail; 180 181 static mblk_t *freebs_list; /* list of buffers to free */ 182 183 /* 184 * Backup threads for servicing queues and syncqs 185 */ 186 kthread_t *streams_qbkgrnd_thread; 187 kthread_t *streams_sqbkgrnd_thread; 188 189 /* 190 * Bufcalls related variables. 191 */ 192 struct bclist strbcalls; /* list of waiting bufcalls */ 193 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */ 194 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */ 195 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */ 196 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */ 197 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */ 198 199 kmutex_t strresources; /* protects global resources */ 200 kmutex_t muxifier; /* single-threads multiplexor creation */ 201 202 extern void time_to_wait(clock_t *, clock_t); 203 204 /* 205 * run_queues is no longer used, but is kept in case some 3-d party 206 * module/driver decides to use it. 207 */ 208 int run_queues = 0; 209 210 /* 211 * sq_max_size is the depth of the syncq (in number of messages) before 212 * qfill_syncq() starts QFULL'ing destination queues. As its primary 213 * consumer - IP is no longer D_MTPERMOD, but there may be other 214 * modules/drivers depend on this syncq flow control, we prefer to 215 * choose a large number as the default value. For potential 216 * performance gain, this value is tunable in /etc/system. 217 */ 218 int sq_max_size = 10000; 219 220 /* 221 * the number of ciputctrl structures per syncq and stream we create when 222 * needed. 223 */ 224 int n_ciputctrl; 225 int max_n_ciputctrl = 16; 226 /* 227 * if n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache. 228 */ 229 int min_n_ciputctrl = 2; 230 231 static struct mux_node *mux_nodes; /* mux info for cycle checking */ 232 233 /* 234 * Per-driver/module syncqs 235 * ======================== 236 * 237 * For drivers/modules that use PERMOD or outer syncqs we keep a list of 238 * perdm structures, new entries being added (and new syncqs allocated) when 239 * setq() encounters a module/driver with a streamtab that it hasn't seen 240 * before. 241 * The reason for this mechanism is that some modules and drivers share a 242 * common streamtab and it is necessary for those modules and drivers to also 243 * share a common PERMOD syncq. 244 * 245 * perdm_list --> dm_str == streamtab_1 246 * dm_sq == syncq_1 247 * dm_ref 248 * dm_next --> dm_str == streamtab_2 249 * dm_sq == syncq_2 250 * dm_ref 251 * dm_next --> ... NULL 252 * 253 * The dm_ref field is incremented for each new driver/module that takes 254 * a reference to the perdm structure and hence shares the syncq. 255 * References are held in the fmodsw_impl_t structure for each STREAMS module 256 * or the dev_impl array (indexed by device major number) for each driver. 257 * 258 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL 259 * ^ ^ ^ ^ 260 * | ______________/ | | 261 * | / | | 262 * dev_impl: ...|x|y|... module A module B 263 * 264 * When a module/driver is unloaded the reference count is decremented and, 265 * when it falls to zero, the perdm structure is removed from the list and 266 * the syncq is freed (see rele_dm()). 267 */ 268 perdm_t *perdm_list = NULL; 269 static krwlock_t perdm_rwlock; 270 cdevsw_impl_t *devimpl; 271 272 extern struct qinit strdata; 273 extern struct qinit stwdata; 274 275 static void runservice(queue_t *); 276 static void streams_bufcall_service(void); 277 static void streams_qbkgrnd_service(void); 278 static void streams_sqbkgrnd_service(void); 279 static syncq_t *new_syncq(void); 280 static void free_syncq(syncq_t *); 281 static void outer_insert(syncq_t *, syncq_t *); 282 static void outer_remove(syncq_t *, syncq_t *); 283 static void write_now(syncq_t *); 284 static void clr_qfull(queue_t *); 285 static void enable_svc(queue_t *); 286 static void runbufcalls(void); 287 static void sqenable(syncq_t *); 288 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)()); 289 static void wait_q_syncq(queue_t *); 290 static void backenable_insertedq(queue_t *); 291 292 static void queue_service(queue_t *); 293 static void stream_service(stdata_t *); 294 static void syncq_service(syncq_t *); 295 static void qwriter_outer_service(syncq_t *); 296 static void mblk_free(mblk_t *); 297 #ifdef DEBUG 298 static int qprocsareon(queue_t *); 299 #endif 300 301 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *); 302 static void reset_nfsrv_ptr(queue_t *, queue_t *); 303 304 static void sq_run_events(syncq_t *); 305 static int propagate_syncq(queue_t *); 306 307 static void blocksq(syncq_t *, ushort_t, int); 308 static void unblocksq(syncq_t *, ushort_t, int); 309 static int dropsq(syncq_t *, uint16_t); 310 static void emptysq(syncq_t *); 311 static sqlist_t *sqlist_alloc(struct stdata *, int); 312 static void sqlist_free(sqlist_t *); 313 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t); 314 static void sqlist_insert(sqlist_t *, syncq_t *); 315 static void sqlist_insertall(sqlist_t *, queue_t *); 316 317 static void strsetuio(stdata_t *); 318 319 struct kmem_cache *stream_head_cache; 320 struct kmem_cache *queue_cache; 321 struct kmem_cache *syncq_cache; 322 struct kmem_cache *qband_cache; 323 struct kmem_cache *linkinfo_cache; 324 struct kmem_cache *ciputctrl_cache = NULL; 325 326 static linkinfo_t *linkinfo_list; 327 328 /* 329 * Qinit structure and Module_info structures 330 * for passthru read and write queues 331 */ 332 333 static void pass_wput(queue_t *, mblk_t *); 334 static queue_t *link_addpassthru(stdata_t *); 335 static void link_rempassthru(queue_t *); 336 337 struct module_info passthru_info = { 338 0, 339 "passthru", 340 0, 341 INFPSZ, 342 STRHIGH, 343 STRLOW 344 }; 345 346 struct qinit passthru_rinit = { 347 (int (*)())putnext, 348 NULL, 349 NULL, 350 NULL, 351 NULL, 352 &passthru_info, 353 NULL 354 }; 355 356 struct qinit passthru_winit = { 357 (int (*)()) pass_wput, 358 NULL, 359 NULL, 360 NULL, 361 NULL, 362 &passthru_info, 363 NULL 364 }; 365 366 /* 367 * Special form of assertion: verify that X implies Y i.e. when X is true Y 368 * should also be true. 369 */ 370 #define IMPLY(X, Y) ASSERT(!(X) || (Y)) 371 372 /* 373 * Logical equivalence. Verify that both X and Y are either TRUE or FALSE. 374 */ 375 #define EQUIV(X, Y) { IMPLY(X, Y); IMPLY(Y, X); } 376 377 /* 378 * Verify correctness of list head/tail pointers. 379 */ 380 #define LISTCHECK(head, tail, link) { \ 381 EQUIV(head, tail); \ 382 IMPLY(tail != NULL, tail->link == NULL); \ 383 } 384 385 /* 386 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail' 387 * using a `link' field. 388 */ 389 #define ENQUEUE(el, head, tail, link) { \ 390 ASSERT(el->link == NULL); \ 391 LISTCHECK(head, tail, link); \ 392 if (head == NULL) \ 393 head = el; \ 394 else \ 395 tail->link = el; \ 396 tail = el; \ 397 } 398 399 /* 400 * Dequeue the first element of the list denoted by `head' and `tail' pointers 401 * using a `link' field and put result into `el'. 402 */ 403 #define DQ(el, head, tail, link) { \ 404 LISTCHECK(head, tail, link); \ 405 el = head; \ 406 if (head != NULL) { \ 407 head = head->link; \ 408 if (head == NULL) \ 409 tail = NULL; \ 410 el->link = NULL; \ 411 } \ 412 } 413 414 /* 415 * Remove `el' from the list using `chase' and `curr' pointers and return result 416 * in `succeed'. 417 */ 418 #define RMQ(el, head, tail, link, chase, curr, succeed) { \ 419 LISTCHECK(head, tail, link); \ 420 chase = NULL; \ 421 succeed = 0; \ 422 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \ 423 chase = curr; \ 424 if (curr != NULL) { \ 425 succeed = 1; \ 426 ASSERT(curr == el); \ 427 if (chase != NULL) \ 428 chase->link = curr->link; \ 429 else \ 430 head = curr->link; \ 431 curr->link = NULL; \ 432 if (curr == tail) \ 433 tail = chase; \ 434 } \ 435 LISTCHECK(head, tail, link); \ 436 } 437 438 /* Handling of delayed messages on the inner syncq. */ 439 440 /* 441 * DEBUG versions should use function versions (to simplify tracing) and 442 * non-DEBUG kernels should use macro versions. 443 */ 444 445 /* 446 * Put a queue on the syncq list of queues. 447 * Assumes SQLOCK held. 448 */ 449 #define SQPUT_Q(sq, qp) \ 450 { \ 451 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 452 if (!(qp->q_sqflags & Q_SQQUEUED)) { \ 453 /* The queue should not be linked anywhere */ \ 454 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \ 455 /* Head and tail may only be NULL simultaneously */ \ 456 EQUIV(sq->sq_head, sq->sq_tail); \ 457 /* Queue may be only enqueyed on its syncq */ \ 458 ASSERT(sq == qp->q_syncq); \ 459 /* Check the correctness of SQ_MESSAGES flag */ \ 460 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \ 461 /* Sanity check first/last elements of the list */ \ 462 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\ 463 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\ 464 /* \ 465 * Sanity check of priority field: empty queue should \ 466 * have zero priority \ 467 * and nqueues equal to zero. \ 468 */ \ 469 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \ 470 /* Sanity check of sq_nqueues field */ \ 471 EQUIV(sq->sq_head, sq->sq_nqueues); \ 472 if (sq->sq_head == NULL) { \ 473 sq->sq_head = sq->sq_tail = qp; \ 474 sq->sq_flags |= SQ_MESSAGES; \ 475 } else if (qp->q_spri == 0) { \ 476 qp->q_sqprev = sq->sq_tail; \ 477 sq->sq_tail->q_sqnext = qp; \ 478 sq->sq_tail = qp; \ 479 } else { \ 480 /* \ 481 * Put this queue in priority order: higher \ 482 * priority gets closer to the head. \ 483 */ \ 484 queue_t **qpp = &sq->sq_tail; \ 485 queue_t *qnext = NULL; \ 486 \ 487 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \ 488 qnext = *qpp; \ 489 qpp = &(*qpp)->q_sqprev; \ 490 } \ 491 qp->q_sqnext = qnext; \ 492 qp->q_sqprev = *qpp; \ 493 if (*qpp != NULL) { \ 494 (*qpp)->q_sqnext = qp; \ 495 } else { \ 496 sq->sq_head = qp; \ 497 sq->sq_pri = sq->sq_head->q_spri; \ 498 } \ 499 *qpp = qp; \ 500 } \ 501 qp->q_sqflags |= Q_SQQUEUED; \ 502 qp->q_sqtstamp = lbolt; \ 503 sq->sq_nqueues++; \ 504 } \ 505 } 506 507 /* 508 * Remove a queue from the syncq list 509 * Assumes SQLOCK held. 510 */ 511 #define SQRM_Q(sq, qp) \ 512 { \ 513 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 514 ASSERT(qp->q_sqflags & Q_SQQUEUED); \ 515 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \ 516 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \ 517 /* Check that the queue is actually in the list */ \ 518 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \ 519 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \ 520 ASSERT(sq->sq_nqueues != 0); \ 521 if (qp->q_sqprev == NULL) { \ 522 /* First queue on list, make head q_sqnext */ \ 523 sq->sq_head = qp->q_sqnext; \ 524 } else { \ 525 /* Make prev->next == next */ \ 526 qp->q_sqprev->q_sqnext = qp->q_sqnext; \ 527 } \ 528 if (qp->q_sqnext == NULL) { \ 529 /* Last queue on list, make tail sqprev */ \ 530 sq->sq_tail = qp->q_sqprev; \ 531 } else { \ 532 /* Make next->prev == prev */ \ 533 qp->q_sqnext->q_sqprev = qp->q_sqprev; \ 534 } \ 535 /* clear out references on this queue */ \ 536 qp->q_sqprev = qp->q_sqnext = NULL; \ 537 qp->q_sqflags &= ~Q_SQQUEUED; \ 538 /* If there is nothing queued, clear SQ_MESSAGES */ \ 539 if (sq->sq_head != NULL) { \ 540 sq->sq_pri = sq->sq_head->q_spri; \ 541 } else { \ 542 sq->sq_flags &= ~SQ_MESSAGES; \ 543 sq->sq_pri = 0; \ 544 } \ 545 sq->sq_nqueues--; \ 546 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \ 547 (sq->sq_flags & SQ_QUEUED) == 0); \ 548 } 549 550 /* Hide the definition from the header file. */ 551 #ifdef SQPUT_MP 552 #undef SQPUT_MP 553 #endif 554 555 /* 556 * Put a message on the queue syncq. 557 * Assumes QLOCK held. 558 */ 559 #define SQPUT_MP(qp, mp) \ 560 { \ 561 ASSERT(MUTEX_HELD(QLOCK(qp))); \ 562 ASSERT(qp->q_sqhead == NULL || \ 563 (qp->q_sqtail != NULL && \ 564 qp->q_sqtail->b_next == NULL)); \ 565 qp->q_syncqmsgs++; \ 566 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \ 567 if (qp->q_sqhead == NULL) { \ 568 qp->q_sqhead = qp->q_sqtail = mp; \ 569 } else { \ 570 qp->q_sqtail->b_next = mp; \ 571 qp->q_sqtail = mp; \ 572 } \ 573 ASSERT(qp->q_syncqmsgs > 0); \ 574 } 575 576 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ 577 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 578 if ((sq)->sq_ciputctrl != NULL) { \ 579 int i; \ 580 int nlocks = (sq)->sq_nciputctrl; \ 581 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 582 ASSERT((sq)->sq_type & SQ_CIPUT); \ 583 for (i = 0; i <= nlocks; i++) { \ 584 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 585 cip[i].ciputctrl_count |= SQ_FASTPUT; \ 586 } \ 587 } \ 588 } 589 590 591 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ 592 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 593 if ((sq)->sq_ciputctrl != NULL) { \ 594 int i; \ 595 int nlocks = (sq)->sq_nciputctrl; \ 596 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 597 ASSERT((sq)->sq_type & SQ_CIPUT); \ 598 for (i = 0; i <= nlocks; i++) { \ 599 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 600 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \ 601 } \ 602 } \ 603 } 604 605 /* 606 * Run service procedures for all queues in the stream head. 607 */ 608 #define STR_SERVICE(stp, q) { \ 609 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \ 610 while (stp->sd_qhead != NULL) { \ 611 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \ 612 ASSERT(stp->sd_nqueues > 0); \ 613 stp->sd_nqueues--; \ 614 ASSERT(!(q->q_flag & QINSERVICE)); \ 615 mutex_exit(&stp->sd_qlock); \ 616 queue_service(q); \ 617 mutex_enter(&stp->sd_qlock); \ 618 } \ 619 ASSERT(stp->sd_nqueues == 0); \ 620 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \ 621 } 622 623 /* 624 * constructor/destructor routines for the stream head cache 625 */ 626 /* ARGSUSED */ 627 static int 628 stream_head_constructor(void *buf, void *cdrarg, int kmflags) 629 { 630 stdata_t *stp = buf; 631 632 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL); 633 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL); 634 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL); 635 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL); 636 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL); 637 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL); 638 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL); 639 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL); 640 stp->sd_wrq = NULL; 641 642 return (0); 643 } 644 645 /* ARGSUSED */ 646 static void 647 stream_head_destructor(void *buf, void *cdrarg) 648 { 649 stdata_t *stp = buf; 650 651 mutex_destroy(&stp->sd_lock); 652 mutex_destroy(&stp->sd_reflock); 653 mutex_destroy(&stp->sd_qlock); 654 cv_destroy(&stp->sd_monitor); 655 cv_destroy(&stp->sd_iocmonitor); 656 cv_destroy(&stp->sd_refmonitor); 657 cv_destroy(&stp->sd_qcv); 658 cv_destroy(&stp->sd_zcopy_wait); 659 } 660 661 /* 662 * constructor/destructor routines for the queue cache 663 */ 664 /* ARGSUSED */ 665 static int 666 queue_constructor(void *buf, void *cdrarg, int kmflags) 667 { 668 queinfo_t *qip = buf; 669 queue_t *qp = &qip->qu_rqueue; 670 queue_t *wqp = &qip->qu_wqueue; 671 syncq_t *sq = &qip->qu_syncq; 672 673 qp->q_first = NULL; 674 qp->q_link = NULL; 675 qp->q_count = 0; 676 qp->q_mblkcnt = 0; 677 qp->q_sqhead = NULL; 678 qp->q_sqtail = NULL; 679 qp->q_sqnext = NULL; 680 qp->q_sqprev = NULL; 681 qp->q_sqflags = 0; 682 qp->q_rwcnt = 0; 683 qp->q_spri = 0; 684 685 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL); 686 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL); 687 688 wqp->q_first = NULL; 689 wqp->q_link = NULL; 690 wqp->q_count = 0; 691 wqp->q_mblkcnt = 0; 692 wqp->q_sqhead = NULL; 693 wqp->q_sqtail = NULL; 694 wqp->q_sqnext = NULL; 695 wqp->q_sqprev = NULL; 696 wqp->q_sqflags = 0; 697 wqp->q_rwcnt = 0; 698 wqp->q_spri = 0; 699 700 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL); 701 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL); 702 703 sq->sq_head = NULL; 704 sq->sq_tail = NULL; 705 sq->sq_evhead = NULL; 706 sq->sq_evtail = NULL; 707 sq->sq_callbpend = NULL; 708 sq->sq_outer = NULL; 709 sq->sq_onext = NULL; 710 sq->sq_oprev = NULL; 711 sq->sq_next = NULL; 712 sq->sq_svcflags = 0; 713 sq->sq_servcount = 0; 714 sq->sq_needexcl = 0; 715 sq->sq_nqueues = 0; 716 sq->sq_pri = 0; 717 718 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 719 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 720 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 721 722 return (0); 723 } 724 725 /* ARGSUSED */ 726 static void 727 queue_destructor(void *buf, void *cdrarg) 728 { 729 queinfo_t *qip = buf; 730 queue_t *qp = &qip->qu_rqueue; 731 queue_t *wqp = &qip->qu_wqueue; 732 syncq_t *sq = &qip->qu_syncq; 733 734 ASSERT(qp->q_sqhead == NULL); 735 ASSERT(wqp->q_sqhead == NULL); 736 ASSERT(qp->q_sqnext == NULL); 737 ASSERT(wqp->q_sqnext == NULL); 738 ASSERT(qp->q_rwcnt == 0); 739 ASSERT(wqp->q_rwcnt == 0); 740 741 mutex_destroy(&qp->q_lock); 742 cv_destroy(&qp->q_wait); 743 744 mutex_destroy(&wqp->q_lock); 745 cv_destroy(&wqp->q_wait); 746 747 mutex_destroy(&sq->sq_lock); 748 cv_destroy(&sq->sq_wait); 749 cv_destroy(&sq->sq_exitwait); 750 } 751 752 /* 753 * constructor/destructor routines for the syncq cache 754 */ 755 /* ARGSUSED */ 756 static int 757 syncq_constructor(void *buf, void *cdrarg, int kmflags) 758 { 759 syncq_t *sq = buf; 760 761 bzero(buf, sizeof (syncq_t)); 762 763 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 764 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 765 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 766 767 return (0); 768 } 769 770 /* ARGSUSED */ 771 static void 772 syncq_destructor(void *buf, void *cdrarg) 773 { 774 syncq_t *sq = buf; 775 776 ASSERT(sq->sq_head == NULL); 777 ASSERT(sq->sq_tail == NULL); 778 ASSERT(sq->sq_evhead == NULL); 779 ASSERT(sq->sq_evtail == NULL); 780 ASSERT(sq->sq_callbpend == NULL); 781 ASSERT(sq->sq_callbflags == 0); 782 ASSERT(sq->sq_outer == NULL); 783 ASSERT(sq->sq_onext == NULL); 784 ASSERT(sq->sq_oprev == NULL); 785 ASSERT(sq->sq_next == NULL); 786 ASSERT(sq->sq_needexcl == 0); 787 ASSERT(sq->sq_svcflags == 0); 788 ASSERT(sq->sq_servcount == 0); 789 ASSERT(sq->sq_nqueues == 0); 790 ASSERT(sq->sq_pri == 0); 791 ASSERT(sq->sq_count == 0); 792 ASSERT(sq->sq_rmqcount == 0); 793 ASSERT(sq->sq_cancelid == 0); 794 ASSERT(sq->sq_ciputctrl == NULL); 795 ASSERT(sq->sq_nciputctrl == 0); 796 ASSERT(sq->sq_type == 0); 797 ASSERT(sq->sq_flags == 0); 798 799 mutex_destroy(&sq->sq_lock); 800 cv_destroy(&sq->sq_wait); 801 cv_destroy(&sq->sq_exitwait); 802 } 803 804 /* ARGSUSED */ 805 static int 806 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags) 807 { 808 ciputctrl_t *cip = buf; 809 int i; 810 811 for (i = 0; i < n_ciputctrl; i++) { 812 cip[i].ciputctrl_count = SQ_FASTPUT; 813 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL); 814 } 815 816 return (0); 817 } 818 819 /* ARGSUSED */ 820 static void 821 ciputctrl_destructor(void *buf, void *cdrarg) 822 { 823 ciputctrl_t *cip = buf; 824 int i; 825 826 for (i = 0; i < n_ciputctrl; i++) { 827 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT); 828 mutex_destroy(&cip[i].ciputctrl_lock); 829 } 830 } 831 832 /* 833 * Init routine run from main at boot time. 834 */ 835 void 836 strinit(void) 837 { 838 int i; 839 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); 840 841 /* 842 * Set up mux_node structures. 843 */ 844 mux_nodes = kmem_zalloc((sizeof (struct mux_node) * devcnt), KM_SLEEP); 845 for (i = 0; i < devcnt; i++) 846 mux_nodes[i].mn_imaj = i; 847 848 stream_head_cache = kmem_cache_create("stream_head_cache", 849 sizeof (stdata_t), 0, 850 stream_head_constructor, stream_head_destructor, NULL, 851 NULL, NULL, 0); 852 853 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0, 854 queue_constructor, queue_destructor, NULL, NULL, NULL, 0); 855 856 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0, 857 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0); 858 859 qband_cache = kmem_cache_create("qband_cache", 860 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 861 862 linkinfo_cache = kmem_cache_create("linkinfo_cache", 863 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 864 865 n_ciputctrl = ncpus; 866 n_ciputctrl = 1 << highbit(n_ciputctrl - 1); 867 ASSERT(n_ciputctrl >= 1); 868 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl); 869 if (n_ciputctrl >= min_n_ciputctrl) { 870 ciputctrl_cache = kmem_cache_create("ciputctrl_cache", 871 sizeof (ciputctrl_t) * n_ciputctrl, 872 sizeof (ciputctrl_t), ciputctrl_constructor, 873 ciputctrl_destructor, NULL, NULL, NULL, 0); 874 } 875 876 streams_taskq = system_taskq; 877 878 if (streams_taskq == NULL) 879 panic("strinit: no memory for streams taskq!"); 880 881 bc_bkgrnd_thread = thread_create(NULL, 0, 882 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri); 883 884 streams_qbkgrnd_thread = thread_create(NULL, 0, 885 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 886 887 streams_sqbkgrnd_thread = thread_create(NULL, 0, 888 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 889 890 /* 891 * Create STREAMS kstats. 892 */ 893 str_kstat = kstat_create("streams", 0, "strstat", 894 "net", KSTAT_TYPE_NAMED, 895 sizeof (str_statistics) / sizeof (kstat_named_t), 896 KSTAT_FLAG_VIRTUAL); 897 898 if (str_kstat != NULL) { 899 str_kstat->ks_data = &str_statistics; 900 kstat_install(str_kstat); 901 } 902 903 /* 904 * TPI support routine initialisation. 905 */ 906 tpi_init(); 907 } 908 909 void 910 str_sendsig(vnode_t *vp, int event, uchar_t band, int error) 911 { 912 struct stdata *stp; 913 914 ASSERT(vp->v_stream); 915 stp = vp->v_stream; 916 /* Have to hold sd_lock to prevent siglist from changing */ 917 mutex_enter(&stp->sd_lock); 918 if (stp->sd_sigflags & event) 919 strsendsig(stp->sd_siglist, event, band, error); 920 mutex_exit(&stp->sd_lock); 921 } 922 923 /* 924 * Send the "sevent" set of signals to a process. 925 * This might send more than one signal if the process is registered 926 * for multiple events. The caller should pass in an sevent that only 927 * includes the events for which the process has registered. 928 */ 929 static void 930 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info, 931 uchar_t band, int error) 932 { 933 ASSERT(MUTEX_HELD(&proc->p_lock)); 934 935 info->si_band = 0; 936 info->si_errno = 0; 937 938 if (sevent & S_ERROR) { 939 sevent &= ~S_ERROR; 940 info->si_code = POLL_ERR; 941 info->si_errno = error; 942 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 943 "strsendsig:proc %p info %p", proc, info); 944 sigaddq(proc, NULL, info, KM_NOSLEEP); 945 info->si_errno = 0; 946 } 947 if (sevent & S_HANGUP) { 948 sevent &= ~S_HANGUP; 949 info->si_code = POLL_HUP; 950 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 951 "strsendsig:proc %p info %p", proc, info); 952 sigaddq(proc, NULL, info, KM_NOSLEEP); 953 } 954 if (sevent & S_HIPRI) { 955 sevent &= ~S_HIPRI; 956 info->si_code = POLL_PRI; 957 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 958 "strsendsig:proc %p info %p", proc, info); 959 sigaddq(proc, NULL, info, KM_NOSLEEP); 960 } 961 if (sevent & S_RDBAND) { 962 sevent &= ~S_RDBAND; 963 if (events & S_BANDURG) 964 sigtoproc(proc, NULL, SIGURG); 965 else 966 sigtoproc(proc, NULL, SIGPOLL); 967 } 968 if (sevent & S_WRBAND) { 969 sevent &= ~S_WRBAND; 970 sigtoproc(proc, NULL, SIGPOLL); 971 } 972 if (sevent & S_INPUT) { 973 sevent &= ~S_INPUT; 974 info->si_code = POLL_IN; 975 info->si_band = band; 976 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 977 "strsendsig:proc %p info %p", proc, info); 978 sigaddq(proc, NULL, info, KM_NOSLEEP); 979 info->si_band = 0; 980 } 981 if (sevent & S_OUTPUT) { 982 sevent &= ~S_OUTPUT; 983 info->si_code = POLL_OUT; 984 info->si_band = band; 985 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 986 "strsendsig:proc %p info %p", proc, info); 987 sigaddq(proc, NULL, info, KM_NOSLEEP); 988 info->si_band = 0; 989 } 990 if (sevent & S_MSG) { 991 sevent &= ~S_MSG; 992 info->si_code = POLL_MSG; 993 info->si_band = band; 994 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 995 "strsendsig:proc %p info %p", proc, info); 996 sigaddq(proc, NULL, info, KM_NOSLEEP); 997 info->si_band = 0; 998 } 999 if (sevent & S_RDNORM) { 1000 sevent &= ~S_RDNORM; 1001 sigtoproc(proc, NULL, SIGPOLL); 1002 } 1003 if (sevent != 0) { 1004 panic("strsendsig: unknown event(s) %x", sevent); 1005 } 1006 } 1007 1008 /* 1009 * Send SIGPOLL/SIGURG signal to all processes and process groups 1010 * registered on the given signal list that want a signal for at 1011 * least one of the specified events. 1012 * 1013 * Must be called with exclusive access to siglist (caller holding sd_lock). 1014 * 1015 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding 1016 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure 1017 * while it is in the siglist. 1018 * 1019 * For performance reasons (MP scalability) the code drops pidlock 1020 * when sending signals to a single process. 1021 * When sending to a process group the code holds 1022 * pidlock to prevent the membership in the process group from changing 1023 * while walking the p_pglink list. 1024 */ 1025 void 1026 strsendsig(strsig_t *siglist, int event, uchar_t band, int error) 1027 { 1028 strsig_t *ssp; 1029 k_siginfo_t info; 1030 struct pid *pidp; 1031 proc_t *proc; 1032 1033 info.si_signo = SIGPOLL; 1034 info.si_errno = 0; 1035 for (ssp = siglist; ssp; ssp = ssp->ss_next) { 1036 int sevent; 1037 1038 sevent = ssp->ss_events & event; 1039 if (sevent == 0) 1040 continue; 1041 1042 if ((pidp = ssp->ss_pidp) == NULL) { 1043 /* pid was released but still on event list */ 1044 continue; 1045 } 1046 1047 1048 if (ssp->ss_pid > 0) { 1049 /* 1050 * XXX This unfortunately still generates 1051 * a signal when a fd is closed but 1052 * the proc is active. 1053 */ 1054 ASSERT(ssp->ss_pid == pidp->pid_id); 1055 1056 mutex_enter(&pidlock); 1057 proc = prfind_zone(pidp->pid_id, ALL_ZONES); 1058 if (proc == NULL) { 1059 mutex_exit(&pidlock); 1060 continue; 1061 } 1062 mutex_enter(&proc->p_lock); 1063 mutex_exit(&pidlock); 1064 dosendsig(proc, ssp->ss_events, sevent, &info, 1065 band, error); 1066 mutex_exit(&proc->p_lock); 1067 } else { 1068 /* 1069 * Send to process group. Hold pidlock across 1070 * calls to dosendsig(). 1071 */ 1072 pid_t pgrp = -ssp->ss_pid; 1073 1074 mutex_enter(&pidlock); 1075 proc = pgfind_zone(pgrp, ALL_ZONES); 1076 while (proc != NULL) { 1077 mutex_enter(&proc->p_lock); 1078 dosendsig(proc, ssp->ss_events, sevent, 1079 &info, band, error); 1080 mutex_exit(&proc->p_lock); 1081 proc = proc->p_pglink; 1082 } 1083 mutex_exit(&pidlock); 1084 } 1085 } 1086 } 1087 1088 /* 1089 * Attach a stream device or module. 1090 * qp is a read queue; the new queue goes in so its next 1091 * read ptr is the argument, and the write queue corresponding 1092 * to the argument points to this queue. Return 0 on success, 1093 * or a non-zero errno on failure. 1094 */ 1095 int 1096 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp, 1097 boolean_t is_insert) 1098 { 1099 major_t major; 1100 cdevsw_impl_t *dp; 1101 struct streamtab *str; 1102 queue_t *rq; 1103 queue_t *wrq; 1104 uint32_t qflag; 1105 uint32_t sqtype; 1106 perdm_t *dmp; 1107 int error; 1108 int sflag; 1109 1110 rq = allocq(); 1111 wrq = _WR(rq); 1112 STREAM(rq) = STREAM(wrq) = STREAM(qp); 1113 1114 if (fp != NULL) { 1115 str = fp->f_str; 1116 qflag = fp->f_qflag; 1117 sqtype = fp->f_sqtype; 1118 dmp = fp->f_dmp; 1119 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 1120 sflag = MODOPEN; 1121 1122 /* 1123 * stash away a pointer to the module structure so we can 1124 * unref it in qdetach. 1125 */ 1126 rq->q_fp = fp; 1127 } else { 1128 ASSERT(!is_insert); 1129 1130 major = getmajor(*devp); 1131 dp = &devimpl[major]; 1132 1133 str = dp->d_str; 1134 ASSERT(str == STREAMSTAB(major)); 1135 1136 qflag = dp->d_qflag; 1137 ASSERT(qflag & QISDRV); 1138 sqtype = dp->d_sqtype; 1139 1140 /* create perdm_t if needed */ 1141 if (NEED_DM(dp->d_dmp, qflag)) 1142 dp->d_dmp = hold_dm(str, qflag, sqtype); 1143 1144 dmp = dp->d_dmp; 1145 sflag = 0; 1146 } 1147 1148 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS, 1149 "qattach:qflag == %X(%X)", qflag, *devp); 1150 1151 /* setq might sleep in allocator - avoid holding locks. */ 1152 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE); 1153 1154 /* 1155 * Before calling the module's open routine, set up the q_next 1156 * pointer for inserting a module in the middle of a stream. 1157 * 1158 * Note that we can always set _QINSERTING and set up q_next 1159 * pointer for both inserting and pushing a module. Then there 1160 * is no need for the is_insert parameter. In insertq(), called 1161 * by qprocson(), assume that q_next of the new module always points 1162 * to the correct queue and use it for insertion. Everything should 1163 * work out fine. But in the first release of _I_INSERT, we 1164 * distinguish between inserting and pushing to make sure that 1165 * pushing a module follows the same code path as before. 1166 */ 1167 if (is_insert) { 1168 rq->q_flag |= _QINSERTING; 1169 rq->q_next = qp; 1170 } 1171 1172 /* 1173 * If there is an outer perimeter get exclusive access during 1174 * the open procedure. Bump up the reference count on the queue. 1175 */ 1176 entersq(rq->q_syncq, SQ_OPENCLOSE); 1177 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp); 1178 if (error != 0) 1179 goto failed; 1180 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1181 ASSERT(qprocsareon(rq)); 1182 return (0); 1183 1184 failed: 1185 rq->q_flag &= ~_QINSERTING; 1186 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq) 1187 qprocsoff(rq); 1188 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1189 rq->q_next = wrq->q_next = NULL; 1190 qdetach(rq, 0, 0, crp, B_FALSE); 1191 return (error); 1192 } 1193 1194 /* 1195 * Handle second open of stream. For modules, set the 1196 * last argument to MODOPEN and do not pass any open flags. 1197 * Ignore dummydev since this is not the first open. 1198 */ 1199 int 1200 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp) 1201 { 1202 int error; 1203 dev_t dummydev; 1204 queue_t *wqp = _WR(qp); 1205 1206 ASSERT(qp->q_flag & QREADR); 1207 entersq(qp->q_syncq, SQ_OPENCLOSE); 1208 1209 dummydev = *devp; 1210 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev, 1211 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) { 1212 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1213 mutex_enter(&STREAM(qp)->sd_lock); 1214 qp->q_stream->sd_flag |= STREOPENFAIL; 1215 mutex_exit(&STREAM(qp)->sd_lock); 1216 return (error); 1217 } 1218 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1219 1220 /* 1221 * successful open should have done qprocson() 1222 */ 1223 ASSERT(qprocsareon(_RD(qp))); 1224 return (0); 1225 } 1226 1227 /* 1228 * Detach a stream module or device. 1229 * If clmode == 1 then the module or driver was opened and its 1230 * close routine must be called. If clmode == 0, the module 1231 * or driver was never opened or the open failed, and so its close 1232 * should not be called. 1233 */ 1234 void 1235 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove) 1236 { 1237 queue_t *wqp = _WR(qp); 1238 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB)); 1239 1240 if (STREAM_NEEDSERVICE(STREAM(qp))) 1241 stream_runservice(STREAM(qp)); 1242 1243 if (clmode) { 1244 /* 1245 * Make sure that all the messages on the write side syncq are 1246 * processed and nothing is left. Since we are closing, no new 1247 * messages may appear there. 1248 */ 1249 wait_q_syncq(wqp); 1250 1251 entersq(qp->q_syncq, SQ_OPENCLOSE); 1252 if (is_remove) { 1253 mutex_enter(QLOCK(qp)); 1254 qp->q_flag |= _QREMOVING; 1255 mutex_exit(QLOCK(qp)); 1256 } 1257 (*qp->q_qinfo->qi_qclose)(qp, flag, crp); 1258 /* 1259 * Check that qprocsoff() was actually called. 1260 */ 1261 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE)); 1262 1263 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1264 } else { 1265 disable_svc(qp); 1266 } 1267 1268 /* 1269 * Allow any threads blocked in entersq to proceed and discover 1270 * the QWCLOSE is set. 1271 * Note: This assumes that all users of entersq check QWCLOSE. 1272 * Currently runservice is the only entersq that can happen 1273 * after removeq has finished. 1274 * Removeq will have discarded all messages destined to the closing 1275 * pair of queues from the syncq. 1276 * NOTE: Calling a function inside an assert is unconventional. 1277 * However, it does not cause any problem since flush_syncq() does 1278 * not change any state except when it returns non-zero i.e. 1279 * when the assert will trigger. 1280 */ 1281 ASSERT(flush_syncq(qp->q_syncq, qp) == 0); 1282 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0); 1283 ASSERT((qp->q_flag & QPERMOD) || 1284 ((qp->q_syncq->sq_head == NULL) && 1285 (wqp->q_syncq->sq_head == NULL))); 1286 1287 /* 1288 * Flush the queues before q_next is set to NULL. This is needed 1289 * in order to backenable any downstream queue before we go away. 1290 * Note: we are already removed from the stream so that the 1291 * backenabling will not cause any messages to be delivered to our 1292 * put procedures. 1293 */ 1294 flushq(qp, FLUSHALL); 1295 flushq(wqp, FLUSHALL); 1296 1297 /* 1298 * wait for any pending service processing to complete 1299 */ 1300 wait_svc(qp); 1301 1302 /* Tidy up - removeq only does a half-remove from stream */ 1303 qp->q_next = wqp->q_next = NULL; 1304 ASSERT(!(qp->q_flag & QENAB)); 1305 ASSERT(!(wqp->q_flag & QENAB)); 1306 1307 /* release any fmodsw_impl_t structure held on behalf of the queue */ 1308 1309 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV); 1310 if (qp->q_fp != NULL) 1311 fmodsw_rele(qp->q_fp); 1312 1313 /* freeq removes us from the outer perimeter if any */ 1314 freeq(qp); 1315 } 1316 1317 /* Prevent service procedures from being called */ 1318 void 1319 disable_svc(queue_t *qp) 1320 { 1321 queue_t *wqp = _WR(qp); 1322 1323 ASSERT(qp->q_flag & QREADR); 1324 mutex_enter(QLOCK(qp)); 1325 qp->q_flag |= QWCLOSE; 1326 mutex_exit(QLOCK(qp)); 1327 mutex_enter(QLOCK(wqp)); 1328 wqp->q_flag |= QWCLOSE; 1329 mutex_exit(QLOCK(wqp)); 1330 } 1331 1332 /* allow service procedures to be called again */ 1333 void 1334 enable_svc(queue_t *qp) 1335 { 1336 queue_t *wqp = _WR(qp); 1337 1338 ASSERT(qp->q_flag & QREADR); 1339 mutex_enter(QLOCK(qp)); 1340 qp->q_flag &= ~QWCLOSE; 1341 mutex_exit(QLOCK(qp)); 1342 mutex_enter(QLOCK(wqp)); 1343 wqp->q_flag &= ~QWCLOSE; 1344 mutex_exit(QLOCK(wqp)); 1345 } 1346 1347 /* 1348 * Remove queue from qhead/qtail if it is enabled. 1349 * Only reset QENAB if the queue was removed from the runlist. 1350 * A queue goes through 3 stages: 1351 * It is on the service list and QENAB is set. 1352 * It is removed from the service list but QENAB is still set. 1353 * QENAB gets changed to QINSERVICE. 1354 * QINSERVICE is reset (when the service procedure is done) 1355 * Thus we can not reset QENAB unless we actually removed it from the service 1356 * queue. 1357 */ 1358 void 1359 remove_runlist(queue_t *qp) 1360 { 1361 if (qp->q_flag & QENAB && qhead != NULL) { 1362 queue_t *q_chase; 1363 queue_t *q_curr; 1364 int removed; 1365 1366 mutex_enter(&service_queue); 1367 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed); 1368 mutex_exit(&service_queue); 1369 if (removed) { 1370 STRSTAT(qremoved); 1371 qp->q_flag &= ~QENAB; 1372 } 1373 } 1374 } 1375 1376 1377 /* 1378 * wait for any pending service processing to complete. 1379 * The removal of queues from the runlist is not atomic with the 1380 * clearing of the QENABLED flag and setting the INSERVICE flag. 1381 * consequently it is possible for remove_runlist in strclose 1382 * to not find the queue on the runlist but for it to be QENABLED 1383 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED 1384 * as well as INSERVICE. 1385 */ 1386 void 1387 wait_svc(queue_t *qp) 1388 { 1389 queue_t *wqp = _WR(qp); 1390 1391 ASSERT(qp->q_flag & QREADR); 1392 1393 /* 1394 * Try to remove queues from qhead/qtail list. 1395 */ 1396 if (qhead != NULL) { 1397 remove_runlist(qp); 1398 remove_runlist(wqp); 1399 } 1400 /* 1401 * Wait till the syncqs associated with the queue 1402 * will dissapear from background processing list. 1403 * This only needs to be done for non-PERMOD perimeters since 1404 * for PERMOD perimeters the syncq may be shared and will only be freed 1405 * when the last module/driver is unloaded. 1406 * If for PERMOD perimeters queue was on the syncq list, removeq() 1407 * should call propagate_syncq() or drain_syncq() for it. Both of these 1408 * function remove the queue from its syncq list, so sqthread will not 1409 * try to access the queue. 1410 */ 1411 if (!(qp->q_flag & QPERMOD)) { 1412 syncq_t *rsq = qp->q_syncq; 1413 syncq_t *wsq = wqp->q_syncq; 1414 1415 /* 1416 * Disable rsq and wsq and wait for any background processing of 1417 * syncq to complete. 1418 */ 1419 wait_sq_svc(rsq); 1420 if (wsq != rsq) 1421 wait_sq_svc(wsq); 1422 } 1423 1424 mutex_enter(QLOCK(qp)); 1425 while (qp->q_flag & (QINSERVICE|QENAB)) 1426 cv_wait(&qp->q_wait, QLOCK(qp)); 1427 mutex_exit(QLOCK(qp)); 1428 mutex_enter(QLOCK(wqp)); 1429 while (wqp->q_flag & (QINSERVICE|QENAB)) 1430 cv_wait(&wqp->q_wait, QLOCK(wqp)); 1431 mutex_exit(QLOCK(wqp)); 1432 } 1433 1434 /* 1435 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'. 1436 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may 1437 * also be set, and is passed through to allocb_cred_wait(). 1438 * 1439 * Returns errno on failure, zero on success. 1440 */ 1441 int 1442 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr) 1443 { 1444 mblk_t *tmp; 1445 ssize_t count; 1446 size_t n; 1447 int error = 0; 1448 1449 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K || 1450 (flag & (U_TO_K | K_TO_K)) == K_TO_K); 1451 1452 if (bp->b_datap->db_type == M_IOCTL) { 1453 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1454 } else { 1455 ASSERT(bp->b_datap->db_type == M_COPYIN); 1456 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1457 } 1458 /* 1459 * strdoioctl validates ioc_count, so if this assert fails it 1460 * cannot be due to user error. 1461 */ 1462 ASSERT(count >= 0); 1463 1464 while (count > 0) { 1465 n = MIN(MAXIOCBSZ, count); 1466 if ((tmp = allocb_cred_wait(n, (flag & STR_NOSIG), &error, 1467 cr)) == NULL) { 1468 return (error); 1469 } 1470 error = strcopyin(arg, tmp->b_wptr, n, flag & (U_TO_K|K_TO_K)); 1471 if (error != 0) { 1472 freeb(tmp); 1473 return (error); 1474 } 1475 arg += n; 1476 DB_CPID(tmp) = curproc->p_pid; 1477 tmp->b_wptr += n; 1478 count -= n; 1479 bp = (bp->b_cont = tmp); 1480 } 1481 1482 return (0); 1483 } 1484 1485 /* 1486 * Copy ioctl data to user-land. Return non-zero errno on failure, 1487 * 0 for success. 1488 */ 1489 int 1490 getiocd(mblk_t *bp, char *arg, int copymode) 1491 { 1492 ssize_t count; 1493 size_t n; 1494 int error; 1495 1496 if (bp->b_datap->db_type == M_IOCACK) 1497 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1498 else { 1499 ASSERT(bp->b_datap->db_type == M_COPYOUT); 1500 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1501 } 1502 ASSERT(count >= 0); 1503 1504 for (bp = bp->b_cont; bp && count; 1505 count -= n, bp = bp->b_cont, arg += n) { 1506 n = MIN(count, bp->b_wptr - bp->b_rptr); 1507 error = strcopyout(bp->b_rptr, arg, n, copymode); 1508 if (error) 1509 return (error); 1510 } 1511 ASSERT(count == 0); 1512 return (0); 1513 } 1514 1515 /* 1516 * Allocate a linkinfo entry given the write queue of the 1517 * bottom module of the top stream and the write queue of the 1518 * stream head of the bottom stream. 1519 */ 1520 linkinfo_t * 1521 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown) 1522 { 1523 linkinfo_t *linkp; 1524 1525 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP); 1526 1527 linkp->li_lblk.l_qtop = qup; 1528 linkp->li_lblk.l_qbot = qdown; 1529 linkp->li_fpdown = fpdown; 1530 1531 mutex_enter(&strresources); 1532 linkp->li_next = linkinfo_list; 1533 linkp->li_prev = NULL; 1534 if (linkp->li_next) 1535 linkp->li_next->li_prev = linkp; 1536 linkinfo_list = linkp; 1537 linkp->li_lblk.l_index = ++lnk_id; 1538 ASSERT(lnk_id != 0); /* this should never wrap in practice */ 1539 mutex_exit(&strresources); 1540 1541 return (linkp); 1542 } 1543 1544 /* 1545 * Free a linkinfo entry. 1546 */ 1547 void 1548 lbfree(linkinfo_t *linkp) 1549 { 1550 mutex_enter(&strresources); 1551 if (linkp->li_next) 1552 linkp->li_next->li_prev = linkp->li_prev; 1553 if (linkp->li_prev) 1554 linkp->li_prev->li_next = linkp->li_next; 1555 else 1556 linkinfo_list = linkp->li_next; 1557 mutex_exit(&strresources); 1558 1559 kmem_cache_free(linkinfo_cache, linkp); 1560 } 1561 1562 /* 1563 * Check for a potential linking cycle. 1564 * Return 1 if a link will result in a cycle, 1565 * and 0 otherwise. 1566 */ 1567 int 1568 linkcycle(stdata_t *upstp, stdata_t *lostp) 1569 { 1570 struct mux_node *np; 1571 struct mux_edge *ep; 1572 int i; 1573 major_t lomaj; 1574 major_t upmaj; 1575 /* 1576 * if the lower stream is a pipe/FIFO, return, since link 1577 * cycles can not happen on pipes/FIFOs 1578 */ 1579 if (lostp->sd_vnode->v_type == VFIFO) 1580 return (0); 1581 1582 for (i = 0; i < devcnt; i++) { 1583 np = &mux_nodes[i]; 1584 MUX_CLEAR(np); 1585 } 1586 lomaj = getmajor(lostp->sd_vnode->v_rdev); 1587 upmaj = getmajor(upstp->sd_vnode->v_rdev); 1588 np = &mux_nodes[lomaj]; 1589 for (;;) { 1590 if (!MUX_DIDVISIT(np)) { 1591 if (np->mn_imaj == upmaj) 1592 return (1); 1593 if (np->mn_outp == NULL) { 1594 MUX_VISIT(np); 1595 if (np->mn_originp == NULL) 1596 return (0); 1597 np = np->mn_originp; 1598 continue; 1599 } 1600 MUX_VISIT(np); 1601 np->mn_startp = np->mn_outp; 1602 } else { 1603 if (np->mn_startp == NULL) { 1604 if (np->mn_originp == NULL) 1605 return (0); 1606 else { 1607 np = np->mn_originp; 1608 continue; 1609 } 1610 } 1611 /* 1612 * If ep->me_nodep is a FIFO (me_nodep == NULL), 1613 * ignore the edge and move on. ep->me_nodep gets 1614 * set to NULL in mux_addedge() if it is a FIFO. 1615 * 1616 */ 1617 ep = np->mn_startp; 1618 np->mn_startp = ep->me_nextp; 1619 if (ep->me_nodep == NULL) 1620 continue; 1621 ep->me_nodep->mn_originp = np; 1622 np = ep->me_nodep; 1623 } 1624 } 1625 } 1626 1627 /* 1628 * Find linkinfo entry corresponding to the parameters. 1629 */ 1630 linkinfo_t * 1631 findlinks(stdata_t *stp, int index, int type) 1632 { 1633 linkinfo_t *linkp; 1634 struct mux_edge *mep; 1635 struct mux_node *mnp; 1636 queue_t *qup; 1637 1638 mutex_enter(&strresources); 1639 if ((type & LINKTYPEMASK) == LINKNORMAL) { 1640 qup = getendq(stp->sd_wrq); 1641 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1642 if ((qup == linkp->li_lblk.l_qtop) && 1643 (!index || (index == linkp->li_lblk.l_index))) { 1644 mutex_exit(&strresources); 1645 return (linkp); 1646 } 1647 } 1648 } else { 1649 ASSERT((type & LINKTYPEMASK) == LINKPERSIST); 1650 mnp = &mux_nodes[getmajor(stp->sd_vnode->v_rdev)]; 1651 mep = mnp->mn_outp; 1652 while (mep) { 1653 if ((index == 0) || (index == mep->me_muxid)) 1654 break; 1655 mep = mep->me_nextp; 1656 } 1657 if (!mep) { 1658 mutex_exit(&strresources); 1659 return (NULL); 1660 } 1661 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1662 if ((!linkp->li_lblk.l_qtop) && 1663 (mep->me_muxid == linkp->li_lblk.l_index)) { 1664 mutex_exit(&strresources); 1665 return (linkp); 1666 } 1667 } 1668 } 1669 mutex_exit(&strresources); 1670 return (NULL); 1671 } 1672 1673 /* 1674 * Given a queue ptr, follow the chain of q_next pointers until you reach the 1675 * last queue on the chain and return it. 1676 */ 1677 queue_t * 1678 getendq(queue_t *q) 1679 { 1680 ASSERT(q != NULL); 1681 while (_SAMESTR(q)) 1682 q = q->q_next; 1683 return (q); 1684 } 1685 1686 /* 1687 * wait for the syncq count to drop to zero. 1688 * sq could be either outer or inner. 1689 */ 1690 1691 static void 1692 wait_syncq(syncq_t *sq) 1693 { 1694 uint16_t count; 1695 1696 mutex_enter(SQLOCK(sq)); 1697 count = sq->sq_count; 1698 SQ_PUTLOCKS_ENTER(sq); 1699 SUM_SQ_PUTCOUNTS(sq, count); 1700 while (count != 0) { 1701 sq->sq_flags |= SQ_WANTWAKEUP; 1702 SQ_PUTLOCKS_EXIT(sq); 1703 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1704 count = sq->sq_count; 1705 SQ_PUTLOCKS_ENTER(sq); 1706 SUM_SQ_PUTCOUNTS(sq, count); 1707 } 1708 SQ_PUTLOCKS_EXIT(sq); 1709 mutex_exit(SQLOCK(sq)); 1710 } 1711 1712 /* 1713 * Wait while there are any messages for the queue in its syncq. 1714 */ 1715 static void 1716 wait_q_syncq(queue_t *q) 1717 { 1718 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1719 syncq_t *sq = q->q_syncq; 1720 1721 mutex_enter(SQLOCK(sq)); 1722 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1723 sq->sq_flags |= SQ_WANTWAKEUP; 1724 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1725 } 1726 mutex_exit(SQLOCK(sq)); 1727 } 1728 } 1729 1730 1731 int 1732 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp, 1733 int lhlink) 1734 { 1735 struct stdata *stp; 1736 struct strioctl strioc; 1737 struct linkinfo *linkp; 1738 struct stdata *stpdown; 1739 struct streamtab *str; 1740 queue_t *passq; 1741 syncq_t *passyncq; 1742 queue_t *rq; 1743 cdevsw_impl_t *dp; 1744 uint32_t qflag; 1745 uint32_t sqtype; 1746 perdm_t *dmp; 1747 int error = 0; 1748 1749 stp = vp->v_stream; 1750 TRACE_1(TR_FAC_STREAMS_FR, 1751 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp); 1752 /* 1753 * Test for invalid upper stream 1754 */ 1755 if (stp->sd_flag & STRHUP) { 1756 return (ENXIO); 1757 } 1758 if (vp->v_type == VFIFO) { 1759 return (EINVAL); 1760 } 1761 if (stp->sd_strtab == NULL) { 1762 return (EINVAL); 1763 } 1764 if (!stp->sd_strtab->st_muxwinit) { 1765 return (EINVAL); 1766 } 1767 if (fpdown == NULL) { 1768 return (EBADF); 1769 } 1770 if (getmajor(stp->sd_vnode->v_rdev) >= devcnt) { 1771 return (EINVAL); 1772 } 1773 mutex_enter(&muxifier); 1774 if (stp->sd_flag & STPLEX) { 1775 mutex_exit(&muxifier); 1776 return (ENXIO); 1777 } 1778 1779 /* 1780 * Test for invalid lower stream. 1781 * The check for the v_type != VFIFO and having a major 1782 * number not >= devcnt is done to avoid problems with 1783 * adding mux_node entry past the end of mux_nodes[]. 1784 * For FIFO's we don't add an entry so this isn't a 1785 * problem. 1786 */ 1787 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) || 1788 (stpdown == stp) || (stpdown->sd_flag & 1789 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) || 1790 ((stpdown->sd_vnode->v_type != VFIFO) && 1791 (getmajor(stpdown->sd_vnode->v_rdev) >= devcnt)) || 1792 linkcycle(stp, stpdown)) { 1793 mutex_exit(&muxifier); 1794 return (EINVAL); 1795 } 1796 TRACE_1(TR_FAC_STREAMS_FR, 1797 TR_STPDOWN, "stpdown:%p", stpdown); 1798 rq = getendq(stp->sd_wrq); 1799 if (cmd == I_PLINK) 1800 rq = NULL; 1801 1802 linkp = alloclink(rq, stpdown->sd_wrq, fpdown); 1803 1804 strioc.ic_cmd = cmd; 1805 strioc.ic_timout = INFTIM; 1806 strioc.ic_len = sizeof (struct linkblk); 1807 strioc.ic_dp = (char *)&linkp->li_lblk; 1808 1809 /* 1810 * STRPLUMB protects plumbing changes and should be set before 1811 * link_addpassthru()/link_rempassthru() are called, so it is set here 1812 * and cleared in the end of mlink when passthru queue is removed. 1813 * Setting of STRPLUMB prevents reopens of the stream while passthru 1814 * queue is in-place (it is not a proper module and doesn't have open 1815 * entry point). 1816 * 1817 * STPLEX prevents any threads from entering the stream from above. It 1818 * can't be set before the call to link_addpassthru() because putnext 1819 * from below may cause stream head I/O routines to be called and these 1820 * routines assert that STPLEX is not set. After link_addpassthru() 1821 * nothing may come from below since the pass queue syncq is blocked. 1822 * Note also that STPLEX should be cleared before the call to 1823 * link_remmpassthru() since when messages start flowing to the stream 1824 * head (e.g. because of message propagation from the pass queue) stream 1825 * head I/O routines may be called with STPLEX flag set. 1826 * 1827 * When STPLEX is set, nothing may come into the stream from above and 1828 * it is safe to do a setq which will change stream head. So, the 1829 * correct sequence of actions is: 1830 * 1831 * 1) Set STRPLUMB 1832 * 2) Call link_addpassthru() 1833 * 3) Set STPLEX 1834 * 4) Call setq and update the stream state 1835 * 5) Clear STPLEX 1836 * 6) Call link_rempassthru() 1837 * 7) Clear STRPLUMB 1838 * 1839 * The same sequence applies to munlink() code. 1840 */ 1841 mutex_enter(&stpdown->sd_lock); 1842 stpdown->sd_flag |= STRPLUMB; 1843 mutex_exit(&stpdown->sd_lock); 1844 /* 1845 * Add passthru queue below lower mux. This will block 1846 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 1847 */ 1848 passq = link_addpassthru(stpdown); 1849 1850 mutex_enter(&stpdown->sd_lock); 1851 stpdown->sd_flag |= STPLEX; 1852 mutex_exit(&stpdown->sd_lock); 1853 1854 rq = _RD(stpdown->sd_wrq); 1855 /* 1856 * There may be messages in the streamhead's syncq due to messages 1857 * that arrived before link_addpassthru() was done. To avoid 1858 * background processing of the syncq happening simultaneous with 1859 * setq processing, we disable the streamhead syncq and wait until 1860 * existing background thread finishes working on it. 1861 */ 1862 wait_sq_svc(rq->q_syncq); 1863 passyncq = passq->q_syncq; 1864 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1865 blocksq(passyncq, SQ_BLOCKED, 0); 1866 1867 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 1868 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 1869 rq->q_ptr = _WR(rq)->q_ptr = NULL; 1870 1871 /* setq might sleep in allocator - avoid holding locks. */ 1872 /* Note: we are holding muxifier here. */ 1873 1874 str = stp->sd_strtab; 1875 dp = &devimpl[getmajor(vp->v_rdev)]; 1876 ASSERT(dp->d_str == str); 1877 1878 qflag = dp->d_qflag; 1879 sqtype = dp->d_sqtype; 1880 1881 /* create perdm_t if needed */ 1882 if (NEED_DM(dp->d_dmp, qflag)) 1883 dp->d_dmp = hold_dm(str, qflag, sqtype); 1884 1885 dmp = dp->d_dmp; 1886 1887 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype, 1888 B_TRUE); 1889 1890 /* 1891 * XXX Remove any "odd" messages from the queue. 1892 * Keep only M_DATA, M_PROTO, M_PCPROTO. 1893 */ 1894 error = strdoioctl(stp, &strioc, FNATIVE, 1895 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 1896 if (error != 0) { 1897 lbfree(linkp); 1898 1899 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1900 blocksq(passyncq, SQ_BLOCKED, 0); 1901 /* 1902 * Restore the stream head queue and then remove 1903 * the passq. Turn off STPLEX before we turn on 1904 * the stream by removing the passq. 1905 */ 1906 rq->q_ptr = _WR(rq)->q_ptr = stpdown; 1907 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, 1908 B_TRUE); 1909 1910 mutex_enter(&stpdown->sd_lock); 1911 stpdown->sd_flag &= ~STPLEX; 1912 mutex_exit(&stpdown->sd_lock); 1913 1914 link_rempassthru(passq); 1915 1916 mutex_enter(&stpdown->sd_lock); 1917 stpdown->sd_flag &= ~STRPLUMB; 1918 /* Wakeup anyone waiting for STRPLUMB to clear. */ 1919 cv_broadcast(&stpdown->sd_monitor); 1920 mutex_exit(&stpdown->sd_lock); 1921 1922 mutex_exit(&muxifier); 1923 return (error); 1924 } 1925 mutex_enter(&fpdown->f_tlock); 1926 fpdown->f_count++; 1927 mutex_exit(&fpdown->f_tlock); 1928 1929 /* 1930 * if we've made it here the linkage is all set up so we should also 1931 * set up the layered driver linkages 1932 */ 1933 1934 ASSERT((cmd == I_LINK) || (cmd == I_PLINK)); 1935 if (cmd == I_LINK) { 1936 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL); 1937 } else { 1938 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST); 1939 } 1940 1941 link_rempassthru(passq); 1942 1943 mux_addedge(stp, stpdown, linkp->li_lblk.l_index); 1944 1945 /* 1946 * Mark the upper stream as having dependent links 1947 * so that strclose can clean it up. 1948 */ 1949 if (cmd == I_LINK) { 1950 mutex_enter(&stp->sd_lock); 1951 stp->sd_flag |= STRHASLINKS; 1952 mutex_exit(&stp->sd_lock); 1953 } 1954 /* 1955 * Wake up any other processes that may have been 1956 * waiting on the lower stream. These will all 1957 * error out. 1958 */ 1959 mutex_enter(&stpdown->sd_lock); 1960 /* The passthru module is removed so we may release STRPLUMB */ 1961 stpdown->sd_flag &= ~STRPLUMB; 1962 cv_broadcast(&rq->q_wait); 1963 cv_broadcast(&_WR(rq)->q_wait); 1964 cv_broadcast(&stpdown->sd_monitor); 1965 mutex_exit(&stpdown->sd_lock); 1966 mutex_exit(&muxifier); 1967 *rvalp = linkp->li_lblk.l_index; 1968 return (0); 1969 } 1970 1971 int 1972 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink) 1973 { 1974 int ret; 1975 struct file *fpdown; 1976 1977 fpdown = getf(arg); 1978 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink); 1979 if (fpdown != NULL) 1980 releasef(arg); 1981 return (ret); 1982 } 1983 1984 /* 1985 * Unlink a multiplexor link. Stp is the controlling stream for the 1986 * link, and linkp points to the link's entry in the linkinfo list. 1987 * The muxifier lock must be held on entry and is dropped on exit. 1988 * 1989 * NOTE : Currently it is assumed that mux would process all the messages 1990 * sitting on it's queue before ACKing the UNLINK. It is the responsibility 1991 * of the mux to handle all the messages that arrive before UNLINK. 1992 * If the mux has to send down messages on its lower stream before 1993 * ACKing I_UNLINK, then it *should* know to handle messages even 1994 * after the UNLINK is acked (actually it should be able to handle till we 1995 * re-block the read side of the pass queue here). If the mux does not 1996 * open up the lower stream, any messages that arrive during UNLINK 1997 * will be put in the stream head. In the case of lower stream opening 1998 * up, some messages might land in the stream head depending on when 1999 * the message arrived and when the read side of the pass queue was 2000 * re-blocked. 2001 */ 2002 int 2003 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp) 2004 { 2005 struct strioctl strioc; 2006 struct stdata *stpdown; 2007 queue_t *rq, *wrq; 2008 queue_t *passq; 2009 syncq_t *passyncq; 2010 int error = 0; 2011 file_t *fpdown; 2012 2013 ASSERT(MUTEX_HELD(&muxifier)); 2014 2015 stpdown = linkp->li_fpdown->f_vnode->v_stream; 2016 2017 /* 2018 * See the comment in mlink() concerning STRPLUMB/STPLEX flags. 2019 */ 2020 mutex_enter(&stpdown->sd_lock); 2021 stpdown->sd_flag |= STRPLUMB; 2022 mutex_exit(&stpdown->sd_lock); 2023 2024 /* 2025 * Add passthru queue below lower mux. This will block 2026 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 2027 */ 2028 passq = link_addpassthru(stpdown); 2029 2030 if ((flag & LINKTYPEMASK) == LINKNORMAL) 2031 strioc.ic_cmd = I_UNLINK; 2032 else 2033 strioc.ic_cmd = I_PUNLINK; 2034 strioc.ic_timout = INFTIM; 2035 strioc.ic_len = sizeof (struct linkblk); 2036 strioc.ic_dp = (char *)&linkp->li_lblk; 2037 2038 error = strdoioctl(stp, &strioc, FNATIVE, 2039 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 2040 2041 /* 2042 * If there was an error and this is not called via strclose, 2043 * return to the user. Otherwise, pretend there was no error 2044 * and close the link. 2045 */ 2046 if (error) { 2047 if (flag & LINKCLOSE) { 2048 cmn_err(CE_WARN, "KERNEL: munlink: could not perform " 2049 "unlink ioctl, closing anyway (%d)\n", error); 2050 } else { 2051 link_rempassthru(passq); 2052 mutex_enter(&stpdown->sd_lock); 2053 stpdown->sd_flag &= ~STRPLUMB; 2054 cv_broadcast(&stpdown->sd_monitor); 2055 mutex_exit(&stpdown->sd_lock); 2056 mutex_exit(&muxifier); 2057 return (error); 2058 } 2059 } 2060 2061 mux_rmvedge(stp, linkp->li_lblk.l_index); 2062 fpdown = linkp->li_fpdown; 2063 lbfree(linkp); 2064 2065 /* 2066 * We go ahead and drop muxifier here--it's a nasty global lock that 2067 * can slow others down. It's okay to since attempts to mlink() this 2068 * stream will be stopped because STPLEX is still set in the stdata 2069 * structure, and munlink() is stopped because mux_rmvedge() and 2070 * lbfree() have removed it from mux_nodes[] and linkinfo_list, 2071 * respectively. Note that we defer the closef() of fpdown until 2072 * after we drop muxifier since strclose() can call munlinkall(). 2073 */ 2074 mutex_exit(&muxifier); 2075 2076 wrq = stpdown->sd_wrq; 2077 rq = _RD(wrq); 2078 2079 /* 2080 * Get rid of outstanding service procedure runs, before we make 2081 * it a stream head, since a stream head doesn't have any service 2082 * procedure. 2083 */ 2084 disable_svc(rq); 2085 wait_svc(rq); 2086 2087 /* 2088 * Since we don't disable the syncq for QPERMOD, we wait for whatever 2089 * is queued up to be finished. mux should take care that nothing is 2090 * send down to this queue. We should do it now as we're going to block 2091 * passyncq if it was unblocked. 2092 */ 2093 if (wrq->q_flag & QPERMOD) { 2094 syncq_t *sq = wrq->q_syncq; 2095 2096 mutex_enter(SQLOCK(sq)); 2097 while (wrq->q_sqflags & Q_SQQUEUED) { 2098 sq->sq_flags |= SQ_WANTWAKEUP; 2099 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2100 } 2101 mutex_exit(SQLOCK(sq)); 2102 } 2103 passyncq = passq->q_syncq; 2104 if (!(passyncq->sq_flags & SQ_BLOCKED)) { 2105 2106 syncq_t *sq, *outer; 2107 2108 /* 2109 * Messages could be flowing from underneath. We will 2110 * block the read side of the passq. This would be 2111 * sufficient for QPAIR and QPERQ muxes to ensure 2112 * that no data is flowing up into this queue 2113 * and hence no thread active in this instance of 2114 * lower mux. But for QPERMOD and QMTOUTPERIM there 2115 * could be messages on the inner and outer/inner 2116 * syncqs respectively. We will wait for them to drain. 2117 * Because passq is blocked messages end up in the syncq 2118 * And qfill_syncq could possibly end up setting QFULL 2119 * which will access the rq->q_flag. Hence, we have to 2120 * acquire the QLOCK in setq. 2121 * 2122 * XXX Messages can also flow from top into this 2123 * queue though the unlink is over (Ex. some instance 2124 * in putnext() called from top that has still not 2125 * accessed this queue. And also putq(lowerq) ?). 2126 * Solution : How about blocking the l_qtop queue ? 2127 * Do we really care about such pure D_MP muxes ? 2128 */ 2129 2130 blocksq(passyncq, SQ_BLOCKED, 0); 2131 2132 sq = rq->q_syncq; 2133 if ((outer = sq->sq_outer) != NULL) { 2134 2135 /* 2136 * We have to just wait for the outer sq_count 2137 * drop to zero. As this does not prevent new 2138 * messages to enter the outer perimeter, this 2139 * is subject to starvation. 2140 * 2141 * NOTE :Because of blocksq above, messages could 2142 * be in the inner syncq only because of some 2143 * thread holding the outer perimeter exclusively. 2144 * Hence it would be sufficient to wait for the 2145 * exclusive holder of the outer perimeter to drain 2146 * the inner and outer syncqs. But we will not depend 2147 * on this feature and hence check the inner syncqs 2148 * separately. 2149 */ 2150 wait_syncq(outer); 2151 } 2152 2153 2154 /* 2155 * There could be messages destined for 2156 * this queue. Let the exclusive holder 2157 * drain it. 2158 */ 2159 2160 wait_syncq(sq); 2161 ASSERT((rq->q_flag & QPERMOD) || 2162 ((rq->q_syncq->sq_head == NULL) && 2163 (_WR(rq)->q_syncq->sq_head == NULL))); 2164 } 2165 2166 /* 2167 * We haven't taken care of QPERMOD case yet. QPERMOD is a special 2168 * case as we don't disable its syncq or remove it off the syncq 2169 * service list. 2170 */ 2171 if (rq->q_flag & QPERMOD) { 2172 syncq_t *sq = rq->q_syncq; 2173 2174 mutex_enter(SQLOCK(sq)); 2175 while (rq->q_sqflags & Q_SQQUEUED) { 2176 sq->sq_flags |= SQ_WANTWAKEUP; 2177 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2178 } 2179 mutex_exit(SQLOCK(sq)); 2180 } 2181 2182 /* 2183 * flush_syncq changes states only when there is some messages to 2184 * free. ie when it returns non-zero value to return. 2185 */ 2186 ASSERT(flush_syncq(rq->q_syncq, rq) == 0); 2187 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0); 2188 2189 /* 2190 * No body else should know about this queue now. 2191 * If the mux did not process the messages before 2192 * acking the I_UNLINK, free them now. 2193 */ 2194 2195 flushq(rq, FLUSHALL); 2196 flushq(_WR(rq), FLUSHALL); 2197 2198 /* 2199 * Convert the mux lower queue into a stream head queue. 2200 * Turn off STPLEX before we turn on the stream by removing the passq. 2201 */ 2202 rq->q_ptr = wrq->q_ptr = stpdown; 2203 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE); 2204 2205 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 2206 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 2207 2208 enable_svc(rq); 2209 2210 /* 2211 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still 2212 * needs to be set to prevent reopen() of the stream - such reopen may 2213 * try to call non-existent pass queue open routine and panic. 2214 */ 2215 mutex_enter(&stpdown->sd_lock); 2216 stpdown->sd_flag &= ~STPLEX; 2217 mutex_exit(&stpdown->sd_lock); 2218 2219 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) || 2220 ((flag & LINKTYPEMASK) == LINKPERSIST)); 2221 2222 /* clean up the layered driver linkages */ 2223 if ((flag & LINKTYPEMASK) == LINKNORMAL) { 2224 ldi_munlink_fp(stp, fpdown, LINKNORMAL); 2225 } else { 2226 ldi_munlink_fp(stp, fpdown, LINKPERSIST); 2227 } 2228 2229 link_rempassthru(passq); 2230 2231 /* 2232 * Now all plumbing changes are finished and STRPLUMB is no 2233 * longer needed. 2234 */ 2235 mutex_enter(&stpdown->sd_lock); 2236 stpdown->sd_flag &= ~STRPLUMB; 2237 cv_broadcast(&stpdown->sd_monitor); 2238 mutex_exit(&stpdown->sd_lock); 2239 2240 (void) closef(fpdown); 2241 return (0); 2242 } 2243 2244 /* 2245 * Unlink all multiplexor links for which stp is the controlling stream. 2246 * Return 0, or a non-zero errno on failure. 2247 */ 2248 int 2249 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp) 2250 { 2251 linkinfo_t *linkp; 2252 int error = 0; 2253 2254 mutex_enter(&muxifier); 2255 while (linkp = findlinks(stp, 0, flag)) { 2256 /* 2257 * munlink() releases the muxifier lock. 2258 */ 2259 if (error = munlink(stp, linkp, flag, crp, rvalp)) 2260 return (error); 2261 mutex_enter(&muxifier); 2262 } 2263 mutex_exit(&muxifier); 2264 return (0); 2265 } 2266 2267 /* 2268 * A multiplexor link has been made. Add an 2269 * edge to the directed graph. 2270 */ 2271 void 2272 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid) 2273 { 2274 struct mux_node *np; 2275 struct mux_edge *ep; 2276 major_t upmaj; 2277 major_t lomaj; 2278 2279 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2280 lomaj = getmajor(lostp->sd_vnode->v_rdev); 2281 np = &mux_nodes[upmaj]; 2282 if (np->mn_outp) { 2283 ep = np->mn_outp; 2284 while (ep->me_nextp) 2285 ep = ep->me_nextp; 2286 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2287 ep = ep->me_nextp; 2288 } else { 2289 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2290 ep = np->mn_outp; 2291 } 2292 ep->me_nextp = NULL; 2293 ep->me_muxid = muxid; 2294 if (lostp->sd_vnode->v_type == VFIFO) 2295 ep->me_nodep = NULL; 2296 else 2297 ep->me_nodep = &mux_nodes[lomaj]; 2298 } 2299 2300 /* 2301 * A multiplexor link has been removed. Remove the 2302 * edge in the directed graph. 2303 */ 2304 void 2305 mux_rmvedge(stdata_t *upstp, int muxid) 2306 { 2307 struct mux_node *np; 2308 struct mux_edge *ep; 2309 struct mux_edge *pep = NULL; 2310 major_t upmaj; 2311 2312 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2313 np = &mux_nodes[upmaj]; 2314 ASSERT(np->mn_outp != NULL); 2315 ep = np->mn_outp; 2316 while (ep) { 2317 if (ep->me_muxid == muxid) { 2318 if (pep) 2319 pep->me_nextp = ep->me_nextp; 2320 else 2321 np->mn_outp = ep->me_nextp; 2322 kmem_free(ep, sizeof (struct mux_edge)); 2323 return; 2324 } 2325 pep = ep; 2326 ep = ep->me_nextp; 2327 } 2328 ASSERT(0); /* should not reach here */ 2329 } 2330 2331 /* 2332 * Translate the device flags (from conf.h) to the corresponding 2333 * qflag and sq_flag (type) values. 2334 */ 2335 int 2336 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp, 2337 uint32_t *sqtypep) 2338 { 2339 uint32_t qflag = 0; 2340 uint32_t sqtype = 0; 2341 2342 if (devflag & _D_OLD) 2343 goto bad; 2344 2345 /* Inner perimeter presence and scope */ 2346 switch (devflag & D_MTINNER_MASK) { 2347 case D_MP: 2348 qflag |= QMTSAFE; 2349 sqtype |= SQ_CI; 2350 break; 2351 case D_MTPERQ|D_MP: 2352 qflag |= QPERQ; 2353 break; 2354 case D_MTQPAIR|D_MP: 2355 qflag |= QPAIR; 2356 break; 2357 case D_MTPERMOD|D_MP: 2358 qflag |= QPERMOD; 2359 break; 2360 default: 2361 goto bad; 2362 } 2363 2364 /* Outer perimeter */ 2365 if (devflag & D_MTOUTPERIM) { 2366 switch (devflag & D_MTINNER_MASK) { 2367 case D_MP: 2368 case D_MTPERQ|D_MP: 2369 case D_MTQPAIR|D_MP: 2370 break; 2371 default: 2372 goto bad; 2373 } 2374 qflag |= QMTOUTPERIM; 2375 } 2376 2377 /* Inner perimeter modifiers */ 2378 if (devflag & D_MTINNER_MOD) { 2379 switch (devflag & D_MTINNER_MASK) { 2380 case D_MP: 2381 goto bad; 2382 default: 2383 break; 2384 } 2385 if (devflag & D_MTPUTSHARED) 2386 sqtype |= SQ_CIPUT; 2387 if (devflag & _D_MTOCSHARED) { 2388 /* 2389 * The code in putnext assumes that it has the 2390 * highest concurrency by not checking sq_count. 2391 * Thus _D_MTOCSHARED can only be supported when 2392 * D_MTPUTSHARED is set. 2393 */ 2394 if (!(devflag & D_MTPUTSHARED)) 2395 goto bad; 2396 sqtype |= SQ_CIOC; 2397 } 2398 if (devflag & _D_MTCBSHARED) { 2399 /* 2400 * The code in putnext assumes that it has the 2401 * highest concurrency by not checking sq_count. 2402 * Thus _D_MTCBSHARED can only be supported when 2403 * D_MTPUTSHARED is set. 2404 */ 2405 if (!(devflag & D_MTPUTSHARED)) 2406 goto bad; 2407 sqtype |= SQ_CICB; 2408 } 2409 if (devflag & _D_MTSVCSHARED) { 2410 /* 2411 * The code in putnext assumes that it has the 2412 * highest concurrency by not checking sq_count. 2413 * Thus _D_MTSVCSHARED can only be supported when 2414 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is 2415 * supported only for QPERMOD. 2416 */ 2417 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD)) 2418 goto bad; 2419 sqtype |= SQ_CISVC; 2420 } 2421 } 2422 2423 /* Default outer perimeter concurrency */ 2424 sqtype |= SQ_CO; 2425 2426 /* Outer perimeter modifiers */ 2427 if (devflag & D_MTOCEXCL) { 2428 if (!(devflag & D_MTOUTPERIM)) { 2429 /* No outer perimeter */ 2430 goto bad; 2431 } 2432 sqtype &= ~SQ_COOC; 2433 } 2434 2435 /* Synchronous Streams extended qinit structure */ 2436 if (devflag & D_SYNCSTR) 2437 qflag |= QSYNCSTR; 2438 2439 *qflagp = qflag; 2440 *sqtypep = sqtype; 2441 return (0); 2442 2443 bad: 2444 cmn_err(CE_WARN, 2445 "stropen: bad MT flags (0x%x) in driver '%s'", 2446 (int)(qflag & D_MTSAFETY_MASK), 2447 stp->st_rdinit->qi_minfo->mi_idname); 2448 2449 return (EINVAL); 2450 } 2451 2452 /* 2453 * Set the interface values for a pair of queues (qinit structure, 2454 * packet sizes, water marks). 2455 * setq assumes that the caller does not have a claim (entersq or claimq) 2456 * on the queue. 2457 */ 2458 void 2459 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit, 2460 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed) 2461 { 2462 queue_t *wq; 2463 syncq_t *sq, *outer; 2464 2465 ASSERT(rq->q_flag & QREADR); 2466 ASSERT((qflag & QMT_TYPEMASK) != 0); 2467 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 2468 2469 wq = _WR(rq); 2470 rq->q_qinfo = rinit; 2471 rq->q_hiwat = rinit->qi_minfo->mi_hiwat; 2472 rq->q_lowat = rinit->qi_minfo->mi_lowat; 2473 rq->q_minpsz = rinit->qi_minfo->mi_minpsz; 2474 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz; 2475 wq->q_qinfo = winit; 2476 wq->q_hiwat = winit->qi_minfo->mi_hiwat; 2477 wq->q_lowat = winit->qi_minfo->mi_lowat; 2478 wq->q_minpsz = winit->qi_minfo->mi_minpsz; 2479 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz; 2480 2481 /* Remove old syncqs */ 2482 sq = rq->q_syncq; 2483 outer = sq->sq_outer; 2484 if (outer != NULL) { 2485 ASSERT(wq->q_syncq->sq_outer == outer); 2486 outer_remove(outer, rq->q_syncq); 2487 if (wq->q_syncq != rq->q_syncq) 2488 outer_remove(outer, wq->q_syncq); 2489 } 2490 ASSERT(sq->sq_outer == NULL); 2491 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2492 2493 if (sq != SQ(rq)) { 2494 if (!(rq->q_flag & QPERMOD)) 2495 free_syncq(sq); 2496 if (wq->q_syncq == rq->q_syncq) 2497 wq->q_syncq = NULL; 2498 rq->q_syncq = NULL; 2499 } 2500 if (wq->q_syncq != NULL && wq->q_syncq != sq && 2501 wq->q_syncq != SQ(rq)) { 2502 free_syncq(wq->q_syncq); 2503 wq->q_syncq = NULL; 2504 } 2505 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL && 2506 rq->q_syncq->sq_tail == NULL)); 2507 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL && 2508 wq->q_syncq->sq_tail == NULL)); 2509 2510 if (!(rq->q_flag & QPERMOD) && 2511 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) { 2512 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2513 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl, 2514 rq->q_syncq->sq_nciputctrl, 0); 2515 ASSERT(ciputctrl_cache != NULL); 2516 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl); 2517 rq->q_syncq->sq_ciputctrl = NULL; 2518 rq->q_syncq->sq_nciputctrl = 0; 2519 } 2520 2521 if (!(wq->q_flag & QPERMOD) && 2522 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) { 2523 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2524 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl, 2525 wq->q_syncq->sq_nciputctrl, 0); 2526 ASSERT(ciputctrl_cache != NULL); 2527 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl); 2528 wq->q_syncq->sq_ciputctrl = NULL; 2529 wq->q_syncq->sq_nciputctrl = 0; 2530 } 2531 2532 sq = SQ(rq); 2533 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 2534 ASSERT(sq->sq_outer == NULL); 2535 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2536 2537 /* 2538 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS 2539 * bits in sq_flag based on the sqtype. 2540 */ 2541 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); 2542 2543 rq->q_syncq = wq->q_syncq = sq; 2544 sq->sq_type = sqtype; 2545 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); 2546 2547 /* 2548 * We are making sq_svcflags zero, 2549 * resetting SQ_DISABLED in case it was set by 2550 * wait_svc() in the munlink path. 2551 * 2552 */ 2553 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); 2554 sq->sq_svcflags = 0; 2555 2556 /* 2557 * We need to acquire the lock here for the mlink and munlink case, 2558 * where canputnext, backenable, etc can access the q_flag. 2559 */ 2560 if (lock_needed) { 2561 mutex_enter(QLOCK(rq)); 2562 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2563 mutex_exit(QLOCK(rq)); 2564 mutex_enter(QLOCK(wq)); 2565 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2566 mutex_exit(QLOCK(wq)); 2567 } else { 2568 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2569 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2570 } 2571 2572 if (qflag & QPERQ) { 2573 /* Allocate a separate syncq for the write side */ 2574 sq = new_syncq(); 2575 sq->sq_type = rq->q_syncq->sq_type; 2576 sq->sq_flags = rq->q_syncq->sq_flags; 2577 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2578 sq->sq_oprev == NULL); 2579 wq->q_syncq = sq; 2580 } 2581 if (qflag & QPERMOD) { 2582 sq = dmp->dm_sq; 2583 2584 /* 2585 * Assert that we do have an inner perimeter syncq and that it 2586 * does not have an outer perimeter associated with it. 2587 */ 2588 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2589 sq->sq_oprev == NULL); 2590 rq->q_syncq = wq->q_syncq = sq; 2591 } 2592 if (qflag & QMTOUTPERIM) { 2593 outer = dmp->dm_sq; 2594 2595 ASSERT(outer->sq_outer == NULL); 2596 outer_insert(outer, rq->q_syncq); 2597 if (wq->q_syncq != rq->q_syncq) 2598 outer_insert(outer, wq->q_syncq); 2599 } 2600 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2601 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2602 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2603 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2604 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK)); 2605 2606 /* 2607 * Initialize struio() types. 2608 */ 2609 rq->q_struiot = 2610 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE; 2611 wq->q_struiot = 2612 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE; 2613 } 2614 2615 perdm_t * 2616 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype) 2617 { 2618 syncq_t *sq; 2619 perdm_t **pp; 2620 perdm_t *p; 2621 perdm_t *dmp; 2622 2623 ASSERT(str != NULL); 2624 ASSERT(qflag & (QPERMOD | QMTOUTPERIM)); 2625 2626 rw_enter(&perdm_rwlock, RW_READER); 2627 for (p = perdm_list; p != NULL; p = p->dm_next) { 2628 if (p->dm_str == str) { /* found one */ 2629 atomic_add_32(&(p->dm_ref), 1); 2630 rw_exit(&perdm_rwlock); 2631 return (p); 2632 } 2633 } 2634 rw_exit(&perdm_rwlock); 2635 2636 sq = new_syncq(); 2637 if (qflag & QPERMOD) { 2638 sq->sq_type = sqtype | SQ_PERMOD; 2639 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; 2640 } else { 2641 ASSERT(qflag & QMTOUTPERIM); 2642 sq->sq_onext = sq->sq_oprev = sq; 2643 } 2644 2645 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP); 2646 dmp->dm_sq = sq; 2647 dmp->dm_str = str; 2648 dmp->dm_ref = 1; 2649 dmp->dm_next = NULL; 2650 2651 rw_enter(&perdm_rwlock, RW_WRITER); 2652 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) { 2653 if (p->dm_str == str) { /* already present */ 2654 p->dm_ref++; 2655 rw_exit(&perdm_rwlock); 2656 free_syncq(sq); 2657 kmem_free(dmp, sizeof (perdm_t)); 2658 return (p); 2659 } 2660 } 2661 2662 *pp = dmp; 2663 rw_exit(&perdm_rwlock); 2664 return (dmp); 2665 } 2666 2667 void 2668 rele_dm(perdm_t *dmp) 2669 { 2670 perdm_t **pp; 2671 perdm_t *p; 2672 2673 rw_enter(&perdm_rwlock, RW_WRITER); 2674 ASSERT(dmp->dm_ref > 0); 2675 2676 if (--dmp->dm_ref > 0) { 2677 rw_exit(&perdm_rwlock); 2678 return; 2679 } 2680 2681 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) 2682 if (p == dmp) 2683 break; 2684 ASSERT(p == dmp); 2685 *pp = p->dm_next; 2686 rw_exit(&perdm_rwlock); 2687 2688 /* 2689 * Wait for any background processing that relies on the 2690 * syncq to complete before it is freed. 2691 */ 2692 wait_sq_svc(p->dm_sq); 2693 free_syncq(p->dm_sq); 2694 kmem_free(p, sizeof (perdm_t)); 2695 } 2696 2697 /* 2698 * Make a protocol message given control and data buffers. 2699 * n.b., this can block; be careful of what locks you hold when calling it. 2700 * 2701 * If sd_maxblk is less than *iosize this routine can fail part way through 2702 * (due to an allocation failure). In this case on return *iosize will contain 2703 * the amount that was consumed. Otherwise *iosize will not be modified 2704 * i.e. it will contain the amount that was consumed. 2705 */ 2706 int 2707 strmakemsg( 2708 struct strbuf *mctl, 2709 ssize_t *iosize, 2710 struct uio *uiop, 2711 stdata_t *stp, 2712 int32_t flag, 2713 mblk_t **mpp) 2714 { 2715 mblk_t *mpctl = NULL; 2716 mblk_t *mpdata = NULL; 2717 int error; 2718 2719 ASSERT(uiop != NULL); 2720 2721 *mpp = NULL; 2722 /* Create control part, if any */ 2723 if ((mctl != NULL) && (mctl->len >= 0)) { 2724 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl); 2725 if (error) 2726 return (error); 2727 } 2728 /* Create data part, if any */ 2729 if (*iosize >= 0) { 2730 error = strmakedata(iosize, uiop, stp, flag, &mpdata); 2731 if (error) { 2732 freemsg(mpctl); 2733 return (error); 2734 } 2735 } 2736 if (mpctl != NULL) { 2737 if (mpdata != NULL) 2738 linkb(mpctl, mpdata); 2739 *mpp = mpctl; 2740 } else { 2741 *mpp = mpdata; 2742 } 2743 return (0); 2744 } 2745 2746 /* 2747 * Make the control part of a protocol message given a control buffer. 2748 * n.b., this can block; be careful of what locks you hold when calling it. 2749 */ 2750 int 2751 strmakectl( 2752 struct strbuf *mctl, 2753 int32_t flag, 2754 int32_t fflag, 2755 mblk_t **mpp) 2756 { 2757 mblk_t *bp = NULL; 2758 unsigned char msgtype; 2759 int error = 0; 2760 2761 *mpp = NULL; 2762 /* 2763 * Create control part of message, if any. 2764 */ 2765 if ((mctl != NULL) && (mctl->len >= 0)) { 2766 caddr_t base; 2767 int ctlcount; 2768 int allocsz; 2769 2770 if (flag & RS_HIPRI) 2771 msgtype = M_PCPROTO; 2772 else 2773 msgtype = M_PROTO; 2774 2775 ctlcount = mctl->len; 2776 base = mctl->buf; 2777 2778 /* 2779 * Give modules a better chance to reuse M_PROTO/M_PCPROTO 2780 * blocks by increasing the size to something more usable. 2781 */ 2782 allocsz = MAX(ctlcount, 64); 2783 2784 /* 2785 * Range checking has already been done; simply try 2786 * to allocate a message block for the ctl part. 2787 */ 2788 while (!(bp = allocb(allocsz, BPRI_MED))) { 2789 if (fflag & (FNDELAY|FNONBLOCK)) 2790 return (EAGAIN); 2791 if (error = strwaitbuf(allocsz, BPRI_MED)) 2792 return (error); 2793 } 2794 2795 bp->b_datap->db_type = msgtype; 2796 if (copyin(base, bp->b_wptr, ctlcount)) { 2797 freeb(bp); 2798 return (EFAULT); 2799 } 2800 bp->b_wptr += ctlcount; 2801 } 2802 *mpp = bp; 2803 return (0); 2804 } 2805 2806 /* 2807 * Make a protocol message given data buffers. 2808 * n.b., this can block; be careful of what locks you hold when calling it. 2809 * 2810 * If sd_maxblk is less than *iosize this routine can fail part way through 2811 * (due to an allocation failure). In this case on return *iosize will contain 2812 * the amount that was consumed. Otherwise *iosize will not be modified 2813 * i.e. it will contain the amount that was consumed. 2814 */ 2815 int 2816 strmakedata( 2817 ssize_t *iosize, 2818 struct uio *uiop, 2819 stdata_t *stp, 2820 int32_t flag, 2821 mblk_t **mpp) 2822 { 2823 mblk_t *mp = NULL; 2824 mblk_t *bp; 2825 int wroff = (int)stp->sd_wroff; 2826 int error = 0; 2827 ssize_t maxblk; 2828 ssize_t count = *iosize; 2829 cred_t *cr = CRED(); 2830 2831 *mpp = NULL; 2832 if (count < 0) 2833 return (0); 2834 2835 maxblk = stp->sd_maxblk; 2836 if (maxblk == INFPSZ) 2837 maxblk = count; 2838 2839 /* 2840 * Create data part of message, if any. 2841 */ 2842 do { 2843 ssize_t size; 2844 dblk_t *dp; 2845 2846 ASSERT(uiop); 2847 2848 size = MIN(count, maxblk); 2849 2850 while ((bp = allocb_cred(size + wroff, cr)) == NULL) { 2851 error = EAGAIN; 2852 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 2853 (error = strwaitbuf(size + wroff, BPRI_MED)) != 0) { 2854 if (count == *iosize) { 2855 freemsg(mp); 2856 return (error); 2857 } else { 2858 *iosize -= count; 2859 *mpp = mp; 2860 return (0); 2861 } 2862 } 2863 } 2864 dp = bp->b_datap; 2865 dp->db_cpid = curproc->p_pid; 2866 ASSERT(wroff <= dp->db_lim - bp->b_wptr); 2867 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff; 2868 2869 if (flag & STRUIO_POSTPONE) { 2870 /* 2871 * Setup the stream uio portion of the 2872 * dblk for subsequent use by struioget(). 2873 */ 2874 dp->db_struioflag = STRUIO_SPEC; 2875 dp->db_cksumstart = 0; 2876 dp->db_cksumstuff = 0; 2877 dp->db_cksumend = size; 2878 *(long long *)dp->db_struioun.data = 0ll; 2879 } else { 2880 if (stp->sd_copyflag & STRCOPYCACHED) 2881 uiop->uio_extflg |= UIO_COPY_CACHED; 2882 2883 if (size != 0) { 2884 error = uiomove(bp->b_wptr, size, UIO_WRITE, 2885 uiop); 2886 if (error != 0) { 2887 freeb(bp); 2888 freemsg(mp); 2889 return (error); 2890 } 2891 } 2892 } 2893 2894 bp->b_wptr += size; 2895 count -= size; 2896 2897 if (mp == NULL) 2898 mp = bp; 2899 else 2900 linkb(mp, bp); 2901 } while (count > 0); 2902 2903 *mpp = mp; 2904 return (0); 2905 } 2906 2907 /* 2908 * Wait for a buffer to become available. Return non-zero errno 2909 * if not able to wait, 0 if buffer is probably there. 2910 */ 2911 int 2912 strwaitbuf(size_t size, int pri) 2913 { 2914 bufcall_id_t id; 2915 2916 mutex_enter(&bcall_monitor); 2917 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast, 2918 &ttoproc(curthread)->p_flag_cv)) == 0) { 2919 mutex_exit(&bcall_monitor); 2920 return (ENOSR); 2921 } 2922 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) { 2923 unbufcall(id); 2924 mutex_exit(&bcall_monitor); 2925 return (EINTR); 2926 } 2927 unbufcall(id); 2928 mutex_exit(&bcall_monitor); 2929 return (0); 2930 } 2931 2932 /* 2933 * This function waits for a read or write event to happen on a stream. 2934 * fmode can specify FNDELAY and/or FNONBLOCK. 2935 * The timeout is in ms with -1 meaning infinite. 2936 * The flag values work as follows: 2937 * READWAIT Check for read side errors, send M_READ 2938 * GETWAIT Check for read side errors, no M_READ 2939 * WRITEWAIT Check for write side errors. 2940 * NOINTR Do not return error if nonblocking or timeout. 2941 * STR_NOERROR Ignore all errors except STPLEX. 2942 * STR_NOSIG Ignore/hold signals during the duration of the call. 2943 * STR_PEEK Pass through the strgeterr(). 2944 */ 2945 int 2946 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout, 2947 int *done) 2948 { 2949 int slpflg, errs; 2950 int error; 2951 kcondvar_t *sleepon; 2952 mblk_t *mp; 2953 ssize_t *rd_count; 2954 clock_t rval; 2955 2956 ASSERT(MUTEX_HELD(&stp->sd_lock)); 2957 if ((flag & READWAIT) || (flag & GETWAIT)) { 2958 slpflg = RSLEEP; 2959 sleepon = &_RD(stp->sd_wrq)->q_wait; 2960 errs = STRDERR|STPLEX; 2961 } else { 2962 slpflg = WSLEEP; 2963 sleepon = &stp->sd_wrq->q_wait; 2964 errs = STWRERR|STRHUP|STPLEX; 2965 } 2966 if (flag & STR_NOERROR) 2967 errs = STPLEX; 2968 2969 if (stp->sd_wakeq & slpflg) { 2970 /* 2971 * A strwakeq() is pending, no need to sleep. 2972 */ 2973 stp->sd_wakeq &= ~slpflg; 2974 *done = 0; 2975 return (0); 2976 } 2977 2978 if (fmode & (FNDELAY|FNONBLOCK)) { 2979 if (!(flag & NOINTR)) 2980 error = EAGAIN; 2981 else 2982 error = 0; 2983 *done = 1; 2984 return (error); 2985 } 2986 2987 if (stp->sd_flag & errs) { 2988 /* 2989 * Check for errors before going to sleep since the 2990 * caller might not have checked this while holding 2991 * sd_lock. 2992 */ 2993 error = strgeterr(stp, errs, (flag & STR_PEEK)); 2994 if (error != 0) { 2995 *done = 1; 2996 return (error); 2997 } 2998 } 2999 3000 /* 3001 * If any module downstream has requested read notification 3002 * by setting SNDMREAD flag using M_SETOPTS, send a message 3003 * down stream. 3004 */ 3005 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) { 3006 mutex_exit(&stp->sd_lock); 3007 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED, 3008 (flag & STR_NOSIG), &error))) { 3009 mutex_enter(&stp->sd_lock); 3010 *done = 1; 3011 return (error); 3012 } 3013 mp->b_datap->db_type = M_READ; 3014 rd_count = (ssize_t *)mp->b_wptr; 3015 *rd_count = count; 3016 mp->b_wptr += sizeof (ssize_t); 3017 /* 3018 * Send the number of bytes requested by the 3019 * read as the argument to M_READ. 3020 */ 3021 stream_willservice(stp); 3022 putnext(stp->sd_wrq, mp); 3023 stream_runservice(stp); 3024 mutex_enter(&stp->sd_lock); 3025 3026 /* 3027 * If any data arrived due to inline processing 3028 * of putnext(), don't sleep. 3029 */ 3030 if (_RD(stp->sd_wrq)->q_first != NULL) { 3031 *done = 0; 3032 return (0); 3033 } 3034 } 3035 3036 stp->sd_flag |= slpflg; 3037 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2, 3038 "strwaitq sleeps (2):%p, %X, %lX, %X, %p", 3039 stp, flag, count, fmode, done); 3040 3041 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG); 3042 if (rval > 0) { 3043 /* EMPTY */ 3044 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2, 3045 "strwaitq awakes(2):%X, %X, %X, %X, %X", 3046 stp, flag, count, fmode, done); 3047 } else if (rval == 0) { 3048 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2, 3049 "strwaitq interrupt #2:%p, %X, %lX, %X, %p", 3050 stp, flag, count, fmode, done); 3051 stp->sd_flag &= ~slpflg; 3052 cv_broadcast(sleepon); 3053 if (!(flag & NOINTR)) 3054 error = EINTR; 3055 else 3056 error = 0; 3057 *done = 1; 3058 return (error); 3059 } else { 3060 /* timeout */ 3061 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME, 3062 "strwaitq timeout:%p, %X, %lX, %X, %p", 3063 stp, flag, count, fmode, done); 3064 *done = 1; 3065 if (!(flag & NOINTR)) 3066 return (ETIME); 3067 else 3068 return (0); 3069 } 3070 /* 3071 * If the caller implements delayed errors (i.e. queued after data) 3072 * we can not check for errors here since data as well as an 3073 * error might have arrived at the stream head. We return to 3074 * have the caller check the read queue before checking for errors. 3075 */ 3076 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) { 3077 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3078 if (error != 0) { 3079 *done = 1; 3080 return (error); 3081 } 3082 } 3083 *done = 0; 3084 return (0); 3085 } 3086 3087 /* 3088 * Perform job control discipline access checks. 3089 * Return 0 for success and the errno for failure. 3090 */ 3091 3092 #define cantsend(p, t, sig) \ 3093 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig)) 3094 3095 int 3096 straccess(struct stdata *stp, enum jcaccess mode) 3097 { 3098 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */ 3099 kthread_t *t = curthread; 3100 proc_t *p = ttoproc(t); 3101 sess_t *sp; 3102 3103 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO) 3104 return (0); 3105 3106 mutex_enter(&p->p_lock); 3107 sp = p->p_sessp; 3108 3109 for (;;) { 3110 /* 3111 * If this is not the calling process's controlling terminal 3112 * or if the calling process is already in the foreground 3113 * then allow access. 3114 */ 3115 if (sp->s_dev != stp->sd_vnode->v_rdev || 3116 p->p_pgidp == stp->sd_pgidp) { 3117 mutex_exit(&p->p_lock); 3118 return (0); 3119 } 3120 3121 /* 3122 * Check to see if controlling terminal has been deallocated. 3123 */ 3124 if (sp->s_vp == NULL) { 3125 if (!cantsend(p, t, SIGHUP)) 3126 sigtoproc(p, t, SIGHUP); 3127 mutex_exit(&p->p_lock); 3128 return (EIO); 3129 } 3130 3131 if (mode == JCGETP) { 3132 mutex_exit(&p->p_lock); 3133 return (0); 3134 } 3135 3136 if (mode == JCREAD) { 3137 if (p->p_detached || cantsend(p, t, SIGTTIN)) { 3138 mutex_exit(&p->p_lock); 3139 return (EIO); 3140 } 3141 mutex_exit(&p->p_lock); 3142 pgsignal(p->p_pgidp, SIGTTIN); 3143 mutex_enter(&p->p_lock); 3144 } else { /* mode == JCWRITE or JCSETP */ 3145 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) || 3146 cantsend(p, t, SIGTTOU)) { 3147 mutex_exit(&p->p_lock); 3148 return (0); 3149 } 3150 if (p->p_detached) { 3151 mutex_exit(&p->p_lock); 3152 return (EIO); 3153 } 3154 mutex_exit(&p->p_lock); 3155 pgsignal(p->p_pgidp, SIGTTOU); 3156 mutex_enter(&p->p_lock); 3157 } 3158 3159 /* 3160 * We call cv_wait_sig_swap() to cause the appropriate 3161 * action for the jobcontrol signal to take place. 3162 * If the signal is being caught, we will take the 3163 * EINTR error return. Otherwise, the default action 3164 * of causing the process to stop will take place. 3165 * In this case, we rely on the periodic cv_broadcast() on 3166 * &lbolt_cv to wake us up to loop around and test again. 3167 * We can't get here if the signal is ignored or 3168 * if the current thread is blocking the signal. 3169 */ 3170 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) { 3171 mutex_exit(&p->p_lock); 3172 return (EINTR); 3173 } 3174 } 3175 } 3176 3177 /* 3178 * Return size of message of block type (bp->b_datap->db_type) 3179 */ 3180 size_t 3181 xmsgsize(mblk_t *bp) 3182 { 3183 unsigned char type; 3184 size_t count = 0; 3185 3186 type = bp->b_datap->db_type; 3187 3188 for (; bp; bp = bp->b_cont) { 3189 if (type != bp->b_datap->db_type) 3190 break; 3191 ASSERT(bp->b_wptr >= bp->b_rptr); 3192 count += bp->b_wptr - bp->b_rptr; 3193 } 3194 return (count); 3195 } 3196 3197 /* 3198 * Allocate a stream head. 3199 */ 3200 struct stdata * 3201 shalloc(queue_t *qp) 3202 { 3203 stdata_t *stp; 3204 3205 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP); 3206 3207 stp->sd_wrq = _WR(qp); 3208 stp->sd_strtab = NULL; 3209 stp->sd_iocid = 0; 3210 stp->sd_mate = NULL; 3211 stp->sd_freezer = NULL; 3212 stp->sd_refcnt = 0; 3213 stp->sd_wakeq = 0; 3214 stp->sd_anchor = 0; 3215 stp->sd_struiowrq = NULL; 3216 stp->sd_struiordq = NULL; 3217 stp->sd_struiodnak = 0; 3218 stp->sd_struionak = NULL; 3219 #ifdef C2_AUDIT 3220 stp->sd_t_audit_data = NULL; 3221 #endif 3222 stp->sd_rput_opt = 0; 3223 stp->sd_wput_opt = 0; 3224 stp->sd_read_opt = 0; 3225 stp->sd_rprotofunc = strrput_proto; 3226 stp->sd_rmiscfunc = strrput_misc; 3227 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL; 3228 stp->sd_ciputctrl = NULL; 3229 stp->sd_nciputctrl = 0; 3230 stp->sd_qhead = NULL; 3231 stp->sd_qtail = NULL; 3232 stp->sd_servid = NULL; 3233 stp->sd_nqueues = 0; 3234 stp->sd_svcflags = 0; 3235 stp->sd_copyflag = 0; 3236 return (stp); 3237 } 3238 3239 /* 3240 * Free a stream head. 3241 */ 3242 void 3243 shfree(stdata_t *stp) 3244 { 3245 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock)); 3246 3247 stp->sd_wrq = NULL; 3248 3249 mutex_enter(&stp->sd_qlock); 3250 while (stp->sd_svcflags & STRS_SCHEDULED) { 3251 STRSTAT(strwaits); 3252 cv_wait(&stp->sd_qcv, &stp->sd_qlock); 3253 } 3254 mutex_exit(&stp->sd_qlock); 3255 3256 if (stp->sd_ciputctrl != NULL) { 3257 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1); 3258 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl, 3259 stp->sd_nciputctrl, 0); 3260 ASSERT(ciputctrl_cache != NULL); 3261 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl); 3262 stp->sd_ciputctrl = NULL; 3263 stp->sd_nciputctrl = 0; 3264 } 3265 ASSERT(stp->sd_qhead == NULL); 3266 ASSERT(stp->sd_qtail == NULL); 3267 ASSERT(stp->sd_nqueues == 0); 3268 kmem_cache_free(stream_head_cache, stp); 3269 } 3270 3271 /* 3272 * Allocate a pair of queues and a syncq for the pair 3273 */ 3274 queue_t * 3275 allocq(void) 3276 { 3277 queinfo_t *qip; 3278 queue_t *qp, *wqp; 3279 syncq_t *sq; 3280 3281 qip = kmem_cache_alloc(queue_cache, KM_SLEEP); 3282 3283 qp = &qip->qu_rqueue; 3284 wqp = &qip->qu_wqueue; 3285 sq = &qip->qu_syncq; 3286 3287 qp->q_last = NULL; 3288 qp->q_next = NULL; 3289 qp->q_ptr = NULL; 3290 qp->q_flag = QUSE | QREADR; 3291 qp->q_bandp = NULL; 3292 qp->q_stream = NULL; 3293 qp->q_syncq = sq; 3294 qp->q_nband = 0; 3295 qp->q_nfsrv = NULL; 3296 qp->q_draining = 0; 3297 qp->q_syncqmsgs = 0; 3298 qp->q_spri = 0; 3299 qp->q_qtstamp = 0; 3300 qp->q_sqtstamp = 0; 3301 qp->q_fp = NULL; 3302 3303 wqp->q_last = NULL; 3304 wqp->q_next = NULL; 3305 wqp->q_ptr = NULL; 3306 wqp->q_flag = QUSE; 3307 wqp->q_bandp = NULL; 3308 wqp->q_stream = NULL; 3309 wqp->q_syncq = sq; 3310 wqp->q_nband = 0; 3311 wqp->q_nfsrv = NULL; 3312 wqp->q_draining = 0; 3313 wqp->q_syncqmsgs = 0; 3314 wqp->q_qtstamp = 0; 3315 wqp->q_sqtstamp = 0; 3316 wqp->q_spri = 0; 3317 3318 sq->sq_count = 0; 3319 sq->sq_rmqcount = 0; 3320 sq->sq_flags = 0; 3321 sq->sq_type = 0; 3322 sq->sq_callbflags = 0; 3323 sq->sq_cancelid = 0; 3324 sq->sq_ciputctrl = NULL; 3325 sq->sq_nciputctrl = 0; 3326 sq->sq_needexcl = 0; 3327 sq->sq_svcflags = 0; 3328 3329 return (qp); 3330 } 3331 3332 /* 3333 * Free a pair of queues and the "attached" syncq. 3334 * Discard any messages left on the syncq(s), remove the syncq(s) from the 3335 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq. 3336 */ 3337 void 3338 freeq(queue_t *qp) 3339 { 3340 qband_t *qbp, *nqbp; 3341 syncq_t *sq, *outer; 3342 queue_t *wqp = _WR(qp); 3343 3344 ASSERT(qp->q_flag & QREADR); 3345 3346 (void) flush_syncq(qp->q_syncq, qp); 3347 (void) flush_syncq(wqp->q_syncq, wqp); 3348 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0); 3349 3350 outer = qp->q_syncq->sq_outer; 3351 if (outer != NULL) { 3352 outer_remove(outer, qp->q_syncq); 3353 if (wqp->q_syncq != qp->q_syncq) 3354 outer_remove(outer, wqp->q_syncq); 3355 } 3356 /* 3357 * Free any syncqs that are outside what allocq returned. 3358 */ 3359 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD)) 3360 free_syncq(qp->q_syncq); 3361 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp)) 3362 free_syncq(wqp->q_syncq); 3363 3364 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3365 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3366 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 3367 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp))); 3368 sq = SQ(qp); 3369 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 3370 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 3371 ASSERT(sq->sq_outer == NULL); 3372 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 3373 ASSERT(sq->sq_callbpend == NULL); 3374 ASSERT(sq->sq_needexcl == 0); 3375 3376 if (sq->sq_ciputctrl != NULL) { 3377 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 3378 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 3379 sq->sq_nciputctrl, 0); 3380 ASSERT(ciputctrl_cache != NULL); 3381 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 3382 sq->sq_ciputctrl = NULL; 3383 sq->sq_nciputctrl = 0; 3384 } 3385 3386 ASSERT(qp->q_first == NULL && wqp->q_first == NULL); 3387 ASSERT(qp->q_count == 0 && wqp->q_count == 0); 3388 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0); 3389 3390 qp->q_flag &= ~QUSE; 3391 wqp->q_flag &= ~QUSE; 3392 3393 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */ 3394 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */ 3395 3396 qbp = qp->q_bandp; 3397 while (qbp) { 3398 nqbp = qbp->qb_next; 3399 freeband(qbp); 3400 qbp = nqbp; 3401 } 3402 qbp = wqp->q_bandp; 3403 while (qbp) { 3404 nqbp = qbp->qb_next; 3405 freeband(qbp); 3406 qbp = nqbp; 3407 } 3408 kmem_cache_free(queue_cache, qp); 3409 } 3410 3411 /* 3412 * Allocate a qband structure. 3413 */ 3414 qband_t * 3415 allocband(void) 3416 { 3417 qband_t *qbp; 3418 3419 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP); 3420 if (qbp == NULL) 3421 return (NULL); 3422 3423 qbp->qb_next = NULL; 3424 qbp->qb_count = 0; 3425 qbp->qb_mblkcnt = 0; 3426 qbp->qb_first = NULL; 3427 qbp->qb_last = NULL; 3428 qbp->qb_flag = 0; 3429 3430 return (qbp); 3431 } 3432 3433 /* 3434 * Free a qband structure. 3435 */ 3436 void 3437 freeband(qband_t *qbp) 3438 { 3439 kmem_cache_free(qband_cache, qbp); 3440 } 3441 3442 /* 3443 * Just like putnextctl(9F), except that allocb_wait() is used. 3444 * 3445 * Consolidation Private, and of course only callable from the stream head or 3446 * routines that may block. 3447 */ 3448 int 3449 putnextctl_wait(queue_t *q, int type) 3450 { 3451 mblk_t *bp; 3452 int error; 3453 3454 if ((datamsg(type) && (type != M_DELAY)) || 3455 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL) 3456 return (0); 3457 3458 bp->b_datap->db_type = (unsigned char)type; 3459 putnext(q, bp); 3460 return (1); 3461 } 3462 3463 /* 3464 * run any possible bufcalls. 3465 */ 3466 void 3467 runbufcalls(void) 3468 { 3469 strbufcall_t *bcp; 3470 3471 mutex_enter(&bcall_monitor); 3472 mutex_enter(&strbcall_lock); 3473 3474 if (strbcalls.bc_head) { 3475 size_t count; 3476 int nevent; 3477 3478 /* 3479 * count how many events are on the list 3480 * now so we can check to avoid looping 3481 * in low memory situations 3482 */ 3483 nevent = 0; 3484 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next) 3485 nevent++; 3486 3487 /* 3488 * get estimate of available memory from kmem_avail(). 3489 * awake all bufcall functions waiting for 3490 * memory whose request could be satisfied 3491 * by 'count' memory and let 'em fight for it. 3492 */ 3493 count = kmem_avail(); 3494 while ((bcp = strbcalls.bc_head) != NULL && nevent) { 3495 STRSTAT(bufcalls); 3496 --nevent; 3497 if (bcp->bc_size <= count) { 3498 bcp->bc_executor = curthread; 3499 mutex_exit(&strbcall_lock); 3500 (*bcp->bc_func)(bcp->bc_arg); 3501 mutex_enter(&strbcall_lock); 3502 bcp->bc_executor = NULL; 3503 cv_broadcast(&bcall_cv); 3504 strbcalls.bc_head = bcp->bc_next; 3505 kmem_free(bcp, sizeof (strbufcall_t)); 3506 } else { 3507 /* 3508 * too big, try again later - note 3509 * that nevent was decremented above 3510 * so we won't retry this one on this 3511 * iteration of the loop 3512 */ 3513 if (bcp->bc_next != NULL) { 3514 strbcalls.bc_head = bcp->bc_next; 3515 bcp->bc_next = NULL; 3516 strbcalls.bc_tail->bc_next = bcp; 3517 strbcalls.bc_tail = bcp; 3518 } 3519 } 3520 } 3521 if (strbcalls.bc_head == NULL) 3522 strbcalls.bc_tail = NULL; 3523 } 3524 3525 mutex_exit(&strbcall_lock); 3526 mutex_exit(&bcall_monitor); 3527 } 3528 3529 3530 /* 3531 * actually run queue's service routine. 3532 */ 3533 static void 3534 runservice(queue_t *q) 3535 { 3536 qband_t *qbp; 3537 3538 ASSERT(q->q_qinfo->qi_srvp); 3539 again: 3540 entersq(q->q_syncq, SQ_SVC); 3541 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START, 3542 "runservice starts:%p", q); 3543 3544 if (!(q->q_flag & QWCLOSE)) 3545 (*q->q_qinfo->qi_srvp)(q); 3546 3547 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END, 3548 "runservice ends:(%p)", q); 3549 3550 leavesq(q->q_syncq, SQ_SVC); 3551 3552 mutex_enter(QLOCK(q)); 3553 if (q->q_flag & QENAB) { 3554 q->q_flag &= ~QENAB; 3555 mutex_exit(QLOCK(q)); 3556 goto again; 3557 } 3558 q->q_flag &= ~QINSERVICE; 3559 q->q_flag &= ~QBACK; 3560 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next) 3561 qbp->qb_flag &= ~QB_BACK; 3562 /* 3563 * Wakeup thread waiting for the service procedure 3564 * to be run (strclose and qdetach). 3565 */ 3566 cv_broadcast(&q->q_wait); 3567 3568 mutex_exit(QLOCK(q)); 3569 } 3570 3571 /* 3572 * Background processing of bufcalls. 3573 */ 3574 void 3575 streams_bufcall_service(void) 3576 { 3577 callb_cpr_t cprinfo; 3578 3579 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr, 3580 "streams_bufcall_service"); 3581 3582 mutex_enter(&strbcall_lock); 3583 3584 for (;;) { 3585 if (strbcalls.bc_head != NULL && kmem_avail() > 0) { 3586 mutex_exit(&strbcall_lock); 3587 runbufcalls(); 3588 mutex_enter(&strbcall_lock); 3589 } 3590 if (strbcalls.bc_head != NULL) { 3591 clock_t wt, tick; 3592 3593 STRSTAT(bcwaits); 3594 /* Wait for memory to become available */ 3595 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3596 tick = SEC_TO_TICK(60); 3597 time_to_wait(&wt, tick); 3598 (void) cv_timedwait(&memavail_cv, &strbcall_lock, wt); 3599 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3600 } 3601 3602 /* Wait for new work to arrive */ 3603 if (strbcalls.bc_head == NULL) { 3604 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3605 cv_wait(&strbcall_cv, &strbcall_lock); 3606 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3607 } 3608 } 3609 } 3610 3611 /* 3612 * Background processing of streams background tasks which failed 3613 * taskq_dispatch. 3614 */ 3615 static void 3616 streams_qbkgrnd_service(void) 3617 { 3618 callb_cpr_t cprinfo; 3619 queue_t *q; 3620 3621 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3622 "streams_bkgrnd_service"); 3623 3624 mutex_enter(&service_queue); 3625 3626 for (;;) { 3627 /* 3628 * Wait for work to arrive. 3629 */ 3630 while ((freebs_list == NULL) && (qhead == NULL)) { 3631 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3632 cv_wait(&services_to_run, &service_queue); 3633 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3634 } 3635 /* 3636 * Handle all pending freebs requests to free memory. 3637 */ 3638 while (freebs_list != NULL) { 3639 mblk_t *mp = freebs_list; 3640 freebs_list = mp->b_next; 3641 mutex_exit(&service_queue); 3642 mblk_free(mp); 3643 mutex_enter(&service_queue); 3644 } 3645 /* 3646 * Run pending queues. 3647 */ 3648 while (qhead != NULL) { 3649 DQ(q, qhead, qtail, q_link); 3650 ASSERT(q != NULL); 3651 mutex_exit(&service_queue); 3652 queue_service(q); 3653 mutex_enter(&service_queue); 3654 } 3655 ASSERT(qhead == NULL && qtail == NULL); 3656 } 3657 } 3658 3659 /* 3660 * Background processing of streams background tasks which failed 3661 * taskq_dispatch. 3662 */ 3663 static void 3664 streams_sqbkgrnd_service(void) 3665 { 3666 callb_cpr_t cprinfo; 3667 syncq_t *sq; 3668 3669 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3670 "streams_sqbkgrnd_service"); 3671 3672 mutex_enter(&service_queue); 3673 3674 for (;;) { 3675 /* 3676 * Wait for work to arrive. 3677 */ 3678 while (sqhead == NULL) { 3679 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3680 cv_wait(&syncqs_to_run, &service_queue); 3681 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3682 } 3683 3684 /* 3685 * Run pending syncqs. 3686 */ 3687 while (sqhead != NULL) { 3688 DQ(sq, sqhead, sqtail, sq_next); 3689 ASSERT(sq != NULL); 3690 ASSERT(sq->sq_svcflags & SQ_BGTHREAD); 3691 mutex_exit(&service_queue); 3692 syncq_service(sq); 3693 mutex_enter(&service_queue); 3694 } 3695 } 3696 } 3697 3698 /* 3699 * Disable the syncq and wait for background syncq processing to complete. 3700 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the 3701 * list. 3702 */ 3703 void 3704 wait_sq_svc(syncq_t *sq) 3705 { 3706 mutex_enter(SQLOCK(sq)); 3707 sq->sq_svcflags |= SQ_DISABLED; 3708 if (sq->sq_svcflags & SQ_BGTHREAD) { 3709 syncq_t *sq_chase; 3710 syncq_t *sq_curr; 3711 int removed; 3712 3713 ASSERT(sq->sq_servcount == 1); 3714 mutex_enter(&service_queue); 3715 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); 3716 mutex_exit(&service_queue); 3717 if (removed) { 3718 sq->sq_svcflags &= ~SQ_BGTHREAD; 3719 sq->sq_servcount = 0; 3720 STRSTAT(sqremoved); 3721 goto done; 3722 } 3723 } 3724 while (sq->sq_servcount != 0) { 3725 sq->sq_flags |= SQ_WANTWAKEUP; 3726 cv_wait(&sq->sq_wait, SQLOCK(sq)); 3727 } 3728 done: 3729 mutex_exit(SQLOCK(sq)); 3730 } 3731 3732 /* 3733 * Put a syncq on the list of syncq's to be serviced by the sqthread. 3734 * Add the argument to the end of the sqhead list and set the flag 3735 * indicating this syncq has been enabled. If it has already been 3736 * enabled, don't do anything. 3737 * This routine assumes that SQLOCK is held. 3738 * NOTE that the lock order is to have the SQLOCK first, 3739 * so if the service_syncq lock is held, we need to release it 3740 * before aquiring the SQLOCK (mostly relevant for the background 3741 * thread, and this seems to be common among the STREAMS global locks). 3742 * Note the the sq_svcflags are protected by the SQLOCK. 3743 */ 3744 void 3745 sqenable(syncq_t *sq) 3746 { 3747 /* 3748 * This is probably not important except for where I believe it 3749 * is being called. At that point, it should be held (and it 3750 * is a pain to release it just for this routine, so don't do 3751 * it). 3752 */ 3753 ASSERT(MUTEX_HELD(SQLOCK(sq))); 3754 3755 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); 3756 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); 3757 3758 /* 3759 * Do not put on list if background thread is scheduled or 3760 * syncq is disabled. 3761 */ 3762 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) 3763 return; 3764 3765 /* 3766 * Check whether we should enable sq at all. 3767 * Non PERMOD syncqs may be drained by at most one thread. 3768 * PERMOD syncqs may be drained by several threads but we limit the 3769 * total amount to the lesser of 3770 * Number of queues on the squeue and 3771 * Number of CPUs. 3772 */ 3773 if (sq->sq_servcount != 0) { 3774 if (((sq->sq_type & SQ_PERMOD) == 0) || 3775 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { 3776 STRSTAT(sqtoomany); 3777 return; 3778 } 3779 } 3780 3781 sq->sq_tstamp = lbolt; 3782 STRSTAT(sqenables); 3783 3784 /* Attempt a taskq dispatch */ 3785 sq->sq_servid = (void *)taskq_dispatch(streams_taskq, 3786 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); 3787 if (sq->sq_servid != NULL) { 3788 sq->sq_servcount++; 3789 return; 3790 } 3791 3792 /* 3793 * This taskq dispatch failed, but a previous one may have succeeded. 3794 * Don't try to schedule on the background thread whilst there is 3795 * outstanding taskq processing. 3796 */ 3797 if (sq->sq_servcount != 0) 3798 return; 3799 3800 /* 3801 * System is low on resources and can't perform a non-sleeping 3802 * dispatch. Schedule the syncq for a background thread and mark the 3803 * syncq to avoid any further taskq dispatch attempts. 3804 */ 3805 mutex_enter(&service_queue); 3806 STRSTAT(taskqfails); 3807 ENQUEUE(sq, sqhead, sqtail, sq_next); 3808 sq->sq_svcflags |= SQ_BGTHREAD; 3809 sq->sq_servcount = 1; 3810 cv_signal(&syncqs_to_run); 3811 mutex_exit(&service_queue); 3812 } 3813 3814 /* 3815 * Note: fifo_close() depends on the mblk_t on the queue being freed 3816 * asynchronously. The asynchronous freeing of messages breaks the 3817 * recursive call chain of fifo_close() while there are I_SENDFD type of 3818 * messages refering other file pointers on the queue. Then when 3819 * closing pipes it can avoid stack overflow in case of daisy-chained 3820 * pipes, and also avoid deadlock in case of fifonode_t pairs (which 3821 * share the same fifolock_t). 3822 */ 3823 3824 /* ARGSUSED */ 3825 void 3826 freebs_enqueue(mblk_t *mp, dblk_t *dbp) 3827 { 3828 ASSERT(dbp->db_mblk == mp); 3829 3830 /* 3831 * Check data sanity. The dblock should have non-empty free function. 3832 * It is better to panic here then later when the dblock is freed 3833 * asynchronously when the context is lost. 3834 */ 3835 if (dbp->db_frtnp->free_func == NULL) { 3836 panic("freebs_enqueue: dblock %p has a NULL free callback", 3837 (void *) dbp); 3838 } 3839 3840 STRSTAT(freebs); 3841 if (taskq_dispatch(streams_taskq, (task_func_t *)mblk_free, mp, 3842 TQ_NOSLEEP) == NULL) { 3843 /* 3844 * System is low on resources and can't perform a non-sleeping 3845 * dispatch. Schedule for a background thread. 3846 */ 3847 mutex_enter(&service_queue); 3848 STRSTAT(taskqfails); 3849 mp->b_next = freebs_list; 3850 freebs_list = mp; 3851 cv_signal(&services_to_run); 3852 mutex_exit(&service_queue); 3853 } 3854 } 3855 3856 /* 3857 * Set the QBACK or QB_BACK flag in the given queue for 3858 * the given priority band. 3859 */ 3860 void 3861 setqback(queue_t *q, unsigned char pri) 3862 { 3863 int i; 3864 qband_t *qbp; 3865 qband_t **qbpp; 3866 3867 ASSERT(MUTEX_HELD(QLOCK(q))); 3868 if (pri != 0) { 3869 if (pri > q->q_nband) { 3870 qbpp = &q->q_bandp; 3871 while (*qbpp) 3872 qbpp = &(*qbpp)->qb_next; 3873 while (pri > q->q_nband) { 3874 if ((*qbpp = allocband()) == NULL) { 3875 cmn_err(CE_WARN, 3876 "setqback: can't allocate qband\n"); 3877 return; 3878 } 3879 (*qbpp)->qb_hiwat = q->q_hiwat; 3880 (*qbpp)->qb_lowat = q->q_lowat; 3881 q->q_nband++; 3882 qbpp = &(*qbpp)->qb_next; 3883 } 3884 } 3885 qbp = q->q_bandp; 3886 i = pri; 3887 while (--i) 3888 qbp = qbp->qb_next; 3889 qbp->qb_flag |= QB_BACK; 3890 } else { 3891 q->q_flag |= QBACK; 3892 } 3893 } 3894 3895 int 3896 strcopyin(void *from, void *to, size_t len, int copyflag) 3897 { 3898 if (copyflag & U_TO_K) { 3899 ASSERT((copyflag & K_TO_K) == 0); 3900 if (copyin(from, to, len)) 3901 return (EFAULT); 3902 } else { 3903 ASSERT(copyflag & K_TO_K); 3904 bcopy(from, to, len); 3905 } 3906 return (0); 3907 } 3908 3909 int 3910 strcopyout(void *from, void *to, size_t len, int copyflag) 3911 { 3912 if (copyflag & U_TO_K) { 3913 if (copyout(from, to, len)) 3914 return (EFAULT); 3915 } else { 3916 ASSERT(copyflag & K_TO_K); 3917 bcopy(from, to, len); 3918 } 3919 return (0); 3920 } 3921 3922 /* 3923 * strsignal_nolock() posts a signal to the process(es) at the stream head. 3924 * It assumes that the stream head lock is already held, whereas strsignal() 3925 * acquires the lock first. This routine was created because a few callers 3926 * release the stream head lock before calling only to re-acquire it after 3927 * it returns. 3928 */ 3929 void 3930 strsignal_nolock(stdata_t *stp, int sig, int32_t band) 3931 { 3932 ASSERT(MUTEX_HELD(&stp->sd_lock)); 3933 switch (sig) { 3934 case SIGPOLL: 3935 if (stp->sd_sigflags & S_MSG) 3936 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 3937 break; 3938 3939 default: 3940 if (stp->sd_pgidp) { 3941 pgsignal(stp->sd_pgidp, sig); 3942 } 3943 break; 3944 } 3945 } 3946 3947 void 3948 strsignal(stdata_t *stp, int sig, int32_t band) 3949 { 3950 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG, 3951 "strsignal:%p, %X, %X", stp, sig, band); 3952 3953 mutex_enter(&stp->sd_lock); 3954 switch (sig) { 3955 case SIGPOLL: 3956 if (stp->sd_sigflags & S_MSG) 3957 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 3958 break; 3959 3960 default: 3961 if (stp->sd_pgidp) { 3962 pgsignal(stp->sd_pgidp, sig); 3963 } 3964 break; 3965 } 3966 mutex_exit(&stp->sd_lock); 3967 } 3968 3969 void 3970 strhup(stdata_t *stp) 3971 { 3972 pollwakeup(&stp->sd_pollist, POLLHUP); 3973 mutex_enter(&stp->sd_lock); 3974 if (stp->sd_sigflags & S_HANGUP) 3975 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0); 3976 mutex_exit(&stp->sd_lock); 3977 } 3978 3979 void 3980 stralloctty(sess_t *sp, stdata_t *stp) 3981 { 3982 mutex_enter(&stp->sd_lock); 3983 mutex_enter(&pidlock); 3984 stp->sd_sidp = sp->s_sidp; 3985 stp->sd_pgidp = sp->s_sidp; 3986 PID_HOLD(stp->sd_pgidp); 3987 PID_HOLD(stp->sd_sidp); 3988 mutex_exit(&pidlock); 3989 mutex_exit(&stp->sd_lock); 3990 } 3991 3992 void 3993 strfreectty(stdata_t *stp) 3994 { 3995 mutex_enter(&stp->sd_lock); 3996 pgsignal(stp->sd_pgidp, SIGHUP); 3997 mutex_enter(&pidlock); 3998 PID_RELE(stp->sd_pgidp); 3999 PID_RELE(stp->sd_sidp); 4000 stp->sd_pgidp = NULL; 4001 stp->sd_sidp = NULL; 4002 mutex_exit(&pidlock); 4003 mutex_exit(&stp->sd_lock); 4004 if (!(stp->sd_flag & STRHUP)) 4005 strhup(stp); 4006 } 4007 4008 void 4009 strctty(stdata_t *stp) 4010 { 4011 extern vnode_t *makectty(); 4012 proc_t *p = curproc; 4013 sess_t *sp = p->p_sessp; 4014 4015 mutex_enter(&stp->sd_lock); 4016 /* 4017 * No need to hold the session lock or do a TTYHOLD, 4018 * because this is the only thread that can be the 4019 * session leader and not have a controlling tty. 4020 */ 4021 if ((stp->sd_flag & (STRHUP|STRDERR|STWRERR|STPLEX)) == 0 && 4022 stp->sd_sidp == NULL && /* not allocated as ctty */ 4023 sp->s_sidp == p->p_pidp && /* session leader */ 4024 sp->s_flag != SESS_CLOSE && /* session is not closing */ 4025 sp->s_vp == NULL) { /* without ctty */ 4026 mutex_exit(&stp->sd_lock); 4027 ASSERT(stp->sd_pgidp == NULL); 4028 alloctty(p, makectty(stp->sd_vnode)); 4029 stralloctty(sp, stp); 4030 mutex_enter(&stp->sd_lock); 4031 stp->sd_flag |= STRISTTY; /* just to be sure */ 4032 } 4033 mutex_exit(&stp->sd_lock); 4034 } 4035 4036 /* 4037 * enable first back queue with svc procedure. 4038 * Use pri == -1 to avoid the setqback 4039 */ 4040 void 4041 backenable(queue_t *q, uchar_t pri) 4042 { 4043 queue_t *nq; 4044 4045 /* 4046 * our presence might not prevent other modules in our own 4047 * stream from popping/pushing since the caller of getq might not 4048 * have a claim on the queue (some drivers do a getq on somebody 4049 * else's queue - they know that the queue itself is not going away 4050 * but the framework has to guarantee q_next in that stream.) 4051 */ 4052 claimstr(q); 4053 4054 /* find nearest back queue with service proc */ 4055 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) { 4056 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq)); 4057 } 4058 4059 if (nq) { 4060 kthread_t *freezer; 4061 /* 4062 * backenable can be called either with no locks held 4063 * or with the stream frozen (the latter occurs when a module 4064 * calls rmvq with the stream frozen.) If the stream is frozen 4065 * by the caller the caller will hold all qlocks in the stream. 4066 */ 4067 freezer = STREAM(q)->sd_freezer; 4068 if (freezer != curthread) { 4069 mutex_enter(QLOCK(nq)); 4070 } 4071 #ifdef DEBUG 4072 else { 4073 ASSERT(frozenstr(q)); 4074 ASSERT(MUTEX_HELD(QLOCK(q))); 4075 ASSERT(MUTEX_HELD(QLOCK(nq))); 4076 } 4077 #endif 4078 setqback(nq, pri); 4079 qenable_locked(nq); 4080 if (freezer != curthread) 4081 mutex_exit(QLOCK(nq)); 4082 } 4083 releasestr(q); 4084 } 4085 4086 /* 4087 * Return the appropriate errno when one of flags_to_check is set 4088 * in sd_flags. Uses the exported error routines if they are set. 4089 * Will return 0 if non error is set (or if the exported error routines 4090 * do not return an error). 4091 * 4092 * If there is both a read and write error to check we prefer the read error. 4093 * Also, give preference to recorded errno's over the error functions. 4094 * The flags that are handled are: 4095 * STPLEX return EINVAL 4096 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST) 4097 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST) 4098 * STRHUP return sd_werror 4099 * 4100 * If the caller indicates that the operation is a peek a nonpersistent error 4101 * is not cleared. 4102 */ 4103 int 4104 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek) 4105 { 4106 int32_t sd_flag = stp->sd_flag & flags_to_check; 4107 int error = 0; 4108 4109 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4110 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0); 4111 if (sd_flag & STPLEX) 4112 error = EINVAL; 4113 else if (sd_flag & STRDERR) { 4114 error = stp->sd_rerror; 4115 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) { 4116 /* 4117 * Read errors are non-persistent i.e. discarded once 4118 * returned to a non-peeking caller, 4119 */ 4120 stp->sd_rerror = 0; 4121 stp->sd_flag &= ~STRDERR; 4122 } 4123 if (error == 0 && stp->sd_rderrfunc != NULL) { 4124 int clearerr = 0; 4125 4126 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek, 4127 &clearerr); 4128 if (clearerr) { 4129 stp->sd_flag &= ~STRDERR; 4130 stp->sd_rderrfunc = NULL; 4131 } 4132 } 4133 } else if (sd_flag & STWRERR) { 4134 error = stp->sd_werror; 4135 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) { 4136 /* 4137 * Write errors are non-persistent i.e. discarded once 4138 * returned to a non-peeking caller, 4139 */ 4140 stp->sd_werror = 0; 4141 stp->sd_flag &= ~STWRERR; 4142 } 4143 if (error == 0 && stp->sd_wrerrfunc != NULL) { 4144 int clearerr = 0; 4145 4146 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek, 4147 &clearerr); 4148 if (clearerr) { 4149 stp->sd_flag &= ~STWRERR; 4150 stp->sd_wrerrfunc = NULL; 4151 } 4152 } 4153 } else if (sd_flag & STRHUP) { 4154 /* sd_werror set when STRHUP */ 4155 error = stp->sd_werror; 4156 } 4157 return (error); 4158 } 4159 4160 4161 /* 4162 * single-thread open/close/push/pop 4163 * for twisted streams also 4164 */ 4165 int 4166 strstartplumb(stdata_t *stp, int flag, int cmd) 4167 { 4168 int waited = 1; 4169 int error = 0; 4170 4171 if (STRMATED(stp)) { 4172 struct stdata *stmatep = stp->sd_mate; 4173 4174 STRLOCKMATES(stp); 4175 while (waited) { 4176 waited = 0; 4177 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4178 if ((cmd == I_POP) && 4179 (flag & (FNDELAY|FNONBLOCK))) { 4180 STRUNLOCKMATES(stp); 4181 return (EAGAIN); 4182 } 4183 waited = 1; 4184 mutex_exit(&stp->sd_lock); 4185 if (!cv_wait_sig(&stmatep->sd_monitor, 4186 &stmatep->sd_lock)) { 4187 mutex_exit(&stmatep->sd_lock); 4188 return (EINTR); 4189 } 4190 mutex_exit(&stmatep->sd_lock); 4191 STRLOCKMATES(stp); 4192 } 4193 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4194 if ((cmd == I_POP) && 4195 (flag & (FNDELAY|FNONBLOCK))) { 4196 STRUNLOCKMATES(stp); 4197 return (EAGAIN); 4198 } 4199 waited = 1; 4200 mutex_exit(&stmatep->sd_lock); 4201 if (!cv_wait_sig(&stp->sd_monitor, 4202 &stp->sd_lock)) { 4203 mutex_exit(&stp->sd_lock); 4204 return (EINTR); 4205 } 4206 mutex_exit(&stp->sd_lock); 4207 STRLOCKMATES(stp); 4208 } 4209 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4210 error = strgeterr(stp, 4211 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4212 if (error != 0) { 4213 STRUNLOCKMATES(stp); 4214 return (error); 4215 } 4216 } 4217 } 4218 stp->sd_flag |= STRPLUMB; 4219 STRUNLOCKMATES(stp); 4220 } else { 4221 mutex_enter(&stp->sd_lock); 4222 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4223 if (((cmd == I_POP) || (cmd == _I_REMOVE)) && 4224 (flag & (FNDELAY|FNONBLOCK))) { 4225 mutex_exit(&stp->sd_lock); 4226 return (EAGAIN); 4227 } 4228 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) { 4229 mutex_exit(&stp->sd_lock); 4230 return (EINTR); 4231 } 4232 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4233 error = strgeterr(stp, 4234 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4235 if (error != 0) { 4236 mutex_exit(&stp->sd_lock); 4237 return (error); 4238 } 4239 } 4240 } 4241 stp->sd_flag |= STRPLUMB; 4242 mutex_exit(&stp->sd_lock); 4243 } 4244 return (0); 4245 } 4246 4247 /* 4248 * Complete the plumbing operation associated with stream `stp'. 4249 */ 4250 void 4251 strendplumb(stdata_t *stp) 4252 { 4253 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4254 ASSERT(stp->sd_flag & STRPLUMB); 4255 stp->sd_flag &= ~STRPLUMB; 4256 cv_broadcast(&stp->sd_monitor); 4257 } 4258 4259 /* 4260 * This describes how the STREAMS framework handles synchronization 4261 * during open/push and close/pop. 4262 * The key interfaces for open and close are qprocson and qprocsoff, 4263 * respectively. While the close case in general is harder both open 4264 * have close have significant similarities. 4265 * 4266 * During close the STREAMS framework has to both ensure that there 4267 * are no stale references to the queue pair (and syncq) that 4268 * are being closed and also provide the guarantees that are documented 4269 * in qprocsoff(9F). 4270 * If there are stale references to the queue that is closing it can 4271 * result in kernel memory corruption or kernel panics. 4272 * 4273 * Note that is it up to the module/driver to ensure that it itself 4274 * does not have any stale references to the closing queues once its close 4275 * routine returns. This includes: 4276 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines 4277 * associated with the queues. For timeout and bufcall callbacks the 4278 * module/driver also has to ensure (or wait for) any callbacks that 4279 * are in progress. 4280 * - If the module/driver is using esballoc it has to ensure that any 4281 * esballoc free functions do not refer to a queue that has closed. 4282 * (Note that in general the close routine can not wait for the esballoc'ed 4283 * messages to be freed since that can cause a deadlock.) 4284 * - Cancelling any interrupts that refer to the closing queues and 4285 * also ensuring that there are no interrupts in progress that will 4286 * refer to the closing queues once the close routine returns. 4287 * - For multiplexors removing any driver global state that refers to 4288 * the closing queue and also ensuring that there are no threads in 4289 * the multiplexor that has picked up a queue pointer but not yet 4290 * finished using it. 4291 * 4292 * In addition, a driver/module can only reference the q_next pointer 4293 * in its open, close, put, or service procedures or in a 4294 * qtimeout/qbufcall callback procedure executing "on" the correct 4295 * stream. Thus it can not reference the q_next pointer in an interrupt 4296 * routine or a timeout, bufcall or esballoc callback routine. Likewise 4297 * it can not reference q_next of a different queue e.g. in a mux that 4298 * passes messages from one queues put/service procedure to another queue. 4299 * In all the cases when the driver/module can not access the q_next 4300 * field it must use the *next* versions e.g. canputnext instead of 4301 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...). 4302 * 4303 * 4304 * Assuming that the driver/module conforms to the above constraints 4305 * the STREAMS framework has to avoid stale references to q_next for all 4306 * the framework internal cases which include (but are not limited to): 4307 * - Threads in canput/canputnext/backenable and elsewhere that are 4308 * walking q_next. 4309 * - Messages on a syncq that have a reference to the queue through b_queue. 4310 * - Messages on an outer perimeter (syncq) that have a reference to the 4311 * queue through b_queue. 4312 * - Threads that use q_nfsrv (e.g. canput) to find a queue. 4313 * Note that only canput and bcanput use q_nfsrv without any locking. 4314 * 4315 * The STREAMS framework providing the qprocsoff(9F) guarantees means that 4316 * after qprocsoff returns, the framework has to ensure that no threads can 4317 * enter the put or service routines for the closing read or write-side queue. 4318 * In addition to preventing "direct" entry into the put procedures 4319 * the framework also has to prevent messages being drained from 4320 * the syncq or the outer perimeter. 4321 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only 4322 * mechanism to prevent qwriter(PERIM_OUTER) from running after 4323 * qprocsoff has returned. 4324 * Note that if a module/driver uses put(9F) on one of its own queues 4325 * it is up to the module/driver to ensure that the put() doesn't 4326 * get called when the queue is closing. 4327 * 4328 * 4329 * The framework aspects of the above "contract" is implemented by 4330 * qprocsoff, removeq, and strlock: 4331 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from 4332 * entering the service procedures. 4333 * - strlock acquires the sd_lock and sd_reflock to prevent putnext, 4334 * canputnext, backenable etc from dereferencing the q_next that will 4335 * soon change. 4336 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext 4337 * or other q_next walker that uses claimstr/releasestr to finish. 4338 * - optionally for every syncq in the stream strlock acquires all the 4339 * sq_lock's and waits for all sq_counts to drop to a value that indicates 4340 * that no thread executes in the put or service procedures and that no 4341 * thread is draining into the module/driver. This ensures that no 4342 * open, close, put, service, or qtimeout/qbufcall callback procedure is 4343 * currently executing hence no such thread can end up with the old stale 4344 * q_next value and no canput/backenable can have the old stale 4345 * q_nfsrv/q_next. 4346 * - qdetach (wait_svc) makes sure that any scheduled or running threads 4347 * have either finished or observed the QWCLOSE flag and gone away. 4348 */ 4349 4350 4351 /* 4352 * Get all the locks necessary to change q_next. 4353 * 4354 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the 4355 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that 4356 * the only threads inside the sqncq are threads currently calling removeq(). 4357 * Since threads calling removeq() are in the process of removing their queues 4358 * from the stream, we do not need to worry about them accessing a stale q_next 4359 * pointer and thus we do not need to wait for them to exit (in fact, waiting 4360 * for them can cause deadlock). 4361 * 4362 * This routine is subject to starvation since it does not set any flag to 4363 * prevent threads from entering a module in the stream(i.e. sq_count can 4364 * increase on some syncq while it is waiting on some other syncq.) 4365 * 4366 * Assumes that only one thread attempts to call strlock for a given 4367 * stream. If this is not the case the two threads would deadlock. 4368 * This assumption is guaranteed since strlock is only called by insertq 4369 * and removeq and streams plumbing changes are single-threaded for 4370 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags. 4371 * 4372 * For pipes, it is not difficult to atomically designate a pair of streams 4373 * to be mated. Once mated atomically by the framework the twisted pair remain 4374 * configured that way until dismantled atomically by the framework. 4375 * When plumbing takes place on a twisted stream it is necessary to ensure that 4376 * this operation is done exclusively on the twisted stream since two such 4377 * operations, each initiated on different ends of the pipe will deadlock 4378 * waiting for each other to complete. 4379 * 4380 * On entry, no locks should be held. 4381 * The locks acquired and held by strlock depends on a few factors. 4382 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired 4383 * and held on exit and all sq_count are at an acceptable level. 4384 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with 4385 * sd_refcnt being zero. 4386 */ 4387 4388 static void 4389 strlock(struct stdata *stp, sqlist_t *sqlist) 4390 { 4391 syncql_t *sql, *sql2; 4392 retry: 4393 /* 4394 * Wait for any claimstr to go away. 4395 */ 4396 if (STRMATED(stp)) { 4397 struct stdata *stp1, *stp2; 4398 4399 STRLOCKMATES(stp); 4400 /* 4401 * Note that the selection of locking order is not 4402 * important, just that they are always aquired in 4403 * the same order. To assure this, we choose this 4404 * order based on the value of the pointer, and since 4405 * the pointer will not change for the life of this 4406 * pair, we will always grab the locks in the same 4407 * order (and hence, prevent deadlocks). 4408 */ 4409 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) { 4410 stp1 = stp; 4411 stp2 = stp->sd_mate; 4412 } else { 4413 stp2 = stp; 4414 stp1 = stp->sd_mate; 4415 } 4416 mutex_enter(&stp1->sd_reflock); 4417 if (stp1->sd_refcnt > 0) { 4418 STRUNLOCKMATES(stp); 4419 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock); 4420 mutex_exit(&stp1->sd_reflock); 4421 goto retry; 4422 } 4423 mutex_enter(&stp2->sd_reflock); 4424 if (stp2->sd_refcnt > 0) { 4425 STRUNLOCKMATES(stp); 4426 mutex_exit(&stp1->sd_reflock); 4427 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock); 4428 mutex_exit(&stp2->sd_reflock); 4429 goto retry; 4430 } 4431 STREAM_PUTLOCKS_ENTER(stp1); 4432 STREAM_PUTLOCKS_ENTER(stp2); 4433 } else { 4434 mutex_enter(&stp->sd_lock); 4435 mutex_enter(&stp->sd_reflock); 4436 while (stp->sd_refcnt > 0) { 4437 mutex_exit(&stp->sd_lock); 4438 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock); 4439 if (mutex_tryenter(&stp->sd_lock) == 0) { 4440 mutex_exit(&stp->sd_reflock); 4441 mutex_enter(&stp->sd_lock); 4442 mutex_enter(&stp->sd_reflock); 4443 } 4444 } 4445 STREAM_PUTLOCKS_ENTER(stp); 4446 } 4447 4448 if (sqlist == NULL) 4449 return; 4450 4451 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4452 syncq_t *sq = sql->sql_sq; 4453 uint16_t count; 4454 4455 mutex_enter(SQLOCK(sq)); 4456 count = sq->sq_count; 4457 ASSERT(sq->sq_rmqcount <= count); 4458 SQ_PUTLOCKS_ENTER(sq); 4459 SUM_SQ_PUTCOUNTS(sq, count); 4460 if (count == sq->sq_rmqcount) 4461 continue; 4462 4463 /* Failed - drop all locks that we have acquired so far */ 4464 if (STRMATED(stp)) { 4465 STREAM_PUTLOCKS_EXIT(stp); 4466 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4467 STRUNLOCKMATES(stp); 4468 mutex_exit(&stp->sd_reflock); 4469 mutex_exit(&stp->sd_mate->sd_reflock); 4470 } else { 4471 STREAM_PUTLOCKS_EXIT(stp); 4472 mutex_exit(&stp->sd_lock); 4473 mutex_exit(&stp->sd_reflock); 4474 } 4475 for (sql2 = sqlist->sqlist_head; sql2 != sql; 4476 sql2 = sql2->sql_next) { 4477 SQ_PUTLOCKS_EXIT(sql2->sql_sq); 4478 mutex_exit(SQLOCK(sql2->sql_sq)); 4479 } 4480 4481 /* 4482 * The wait loop below may starve when there are many threads 4483 * claiming the syncq. This is especially a problem with permod 4484 * syncqs (IP). To lessen the impact of the problem we increment 4485 * sq_needexcl and clear fastbits so that putnexts will slow 4486 * down and call sqenable instead of draining right away. 4487 */ 4488 sq->sq_needexcl++; 4489 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4490 while (count > sq->sq_rmqcount) { 4491 sq->sq_flags |= SQ_WANTWAKEUP; 4492 SQ_PUTLOCKS_EXIT(sq); 4493 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4494 count = sq->sq_count; 4495 SQ_PUTLOCKS_ENTER(sq); 4496 SUM_SQ_PUTCOUNTS(sq, count); 4497 } 4498 sq->sq_needexcl--; 4499 if (sq->sq_needexcl == 0) 4500 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4501 SQ_PUTLOCKS_EXIT(sq); 4502 ASSERT(count == sq->sq_rmqcount); 4503 mutex_exit(SQLOCK(sq)); 4504 goto retry; 4505 } 4506 } 4507 4508 /* 4509 * Drop all the locks that strlock acquired. 4510 */ 4511 static void 4512 strunlock(struct stdata *stp, sqlist_t *sqlist) 4513 { 4514 syncql_t *sql; 4515 4516 if (STRMATED(stp)) { 4517 STREAM_PUTLOCKS_EXIT(stp); 4518 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4519 STRUNLOCKMATES(stp); 4520 mutex_exit(&stp->sd_reflock); 4521 mutex_exit(&stp->sd_mate->sd_reflock); 4522 } else { 4523 STREAM_PUTLOCKS_EXIT(stp); 4524 mutex_exit(&stp->sd_lock); 4525 mutex_exit(&stp->sd_reflock); 4526 } 4527 4528 if (sqlist == NULL) 4529 return; 4530 4531 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4532 SQ_PUTLOCKS_EXIT(sql->sql_sq); 4533 mutex_exit(SQLOCK(sql->sql_sq)); 4534 } 4535 } 4536 4537 /* 4538 * When the module has service procedure, we need check if the next 4539 * module which has service procedure is in flow control to trigger 4540 * the backenable. 4541 */ 4542 static void 4543 backenable_insertedq(queue_t *q) 4544 { 4545 qband_t *qbp; 4546 4547 claimstr(q); 4548 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) { 4549 if (q->q_next->q_nfsrv->q_flag & QWANTW) 4550 backenable(q, 0); 4551 4552 qbp = q->q_next->q_nfsrv->q_bandp; 4553 for (; qbp != NULL; qbp = qbp->qb_next) 4554 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL) 4555 backenable(q, qbp->qb_first->b_band); 4556 } 4557 releasestr(q); 4558 } 4559 4560 /* 4561 * Given two read queues, insert a new single one after another. 4562 * 4563 * This routine acquires all the necessary locks in order to change 4564 * q_next and related pointer using strlock(). 4565 * It depends on the stream head ensuring that there are no concurrent 4566 * insertq or removeq on the same stream. The stream head ensures this 4567 * using the flags STWOPEN, STRCLOSE, and STRPLUMB. 4568 * 4569 * Note that no syncq locks are held during the q_next change. This is 4570 * applied to all streams since, unlike removeq, there is no problem of stale 4571 * pointers when adding a module to the stream. Thus drivers/modules that do a 4572 * canput(rq->q_next) would never get a closed/freed queue pointer even if we 4573 * applied this optimization to all streams. 4574 */ 4575 void 4576 insertq(struct stdata *stp, queue_t *new) 4577 { 4578 queue_t *after; 4579 queue_t *wafter; 4580 queue_t *wnew = _WR(new); 4581 boolean_t have_fifo = B_FALSE; 4582 4583 if (new->q_flag & _QINSERTING) { 4584 ASSERT(stp->sd_vnode->v_type != VFIFO); 4585 after = new->q_next; 4586 wafter = _WR(new->q_next); 4587 } else { 4588 after = _RD(stp->sd_wrq); 4589 wafter = stp->sd_wrq; 4590 } 4591 4592 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ, 4593 "insertq:%p, %p", after, new); 4594 ASSERT(after->q_flag & QREADR); 4595 ASSERT(new->q_flag & QREADR); 4596 4597 strlock(stp, NULL); 4598 4599 /* Do we have a FIFO? */ 4600 if (wafter->q_next == after) { 4601 have_fifo = B_TRUE; 4602 wnew->q_next = new; 4603 } else { 4604 wnew->q_next = wafter->q_next; 4605 } 4606 new->q_next = after; 4607 4608 set_nfsrv_ptr(new, wnew, after, wafter); 4609 /* 4610 * set_nfsrv_ptr() needs to know if this is an insertion or not, 4611 * so only reset this flag after calling it. 4612 */ 4613 new->q_flag &= ~_QINSERTING; 4614 4615 if (have_fifo) { 4616 wafter->q_next = wnew; 4617 } else { 4618 if (wafter->q_next) 4619 _OTHERQ(wafter->q_next)->q_next = new; 4620 wafter->q_next = wnew; 4621 } 4622 4623 set_qend(new); 4624 /* The QEND flag might have to be updated for the upstream guy */ 4625 set_qend(after); 4626 4627 ASSERT(_SAMESTR(new) == O_SAMESTR(new)); 4628 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew)); 4629 ASSERT(_SAMESTR(after) == O_SAMESTR(after)); 4630 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter)); 4631 strsetuio(stp); 4632 4633 /* 4634 * If this was a module insertion, bump the push count. 4635 */ 4636 if (!(new->q_flag & QISDRV)) 4637 stp->sd_pushcnt++; 4638 4639 strunlock(stp, NULL); 4640 4641 /* check if the write Q needs backenable */ 4642 backenable_insertedq(wnew); 4643 4644 /* check if the read Q needs backenable */ 4645 backenable_insertedq(new); 4646 } 4647 4648 /* 4649 * Given a read queue, unlink it from any neighbors. 4650 * 4651 * This routine acquires all the necessary locks in order to 4652 * change q_next and related pointers and also guard against 4653 * stale references (e.g. through q_next) to the queue that 4654 * is being removed. It also plays part of the role in ensuring 4655 * that the module's/driver's put procedure doesn't get called 4656 * after qprocsoff returns. 4657 * 4658 * Removeq depends on the stream head ensuring that there are 4659 * no concurrent insertq or removeq on the same stream. The 4660 * stream head ensures this using the flags STWOPEN, STRCLOSE and 4661 * STRPLUMB. 4662 * 4663 * The set of locks needed to remove the queue is different in 4664 * different cases: 4665 * 4666 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after 4667 * waiting for the syncq reference count to drop to 0 indicating that no 4668 * non-close threads are present anywhere in the stream. This ensures that any 4669 * module/driver can reference q_next in its open, close, put, or service 4670 * procedures. 4671 * 4672 * The sq_rmqcount counter tracks the number of threads inside removeq(). 4673 * strlock() ensures that there is either no threads executing inside perimeter 4674 * or there is only a thread calling qprocsoff(). 4675 * 4676 * strlock() compares the value of sq_count with the number of threads inside 4677 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup 4678 * any threads waiting in strlock() when the sq_rmqcount increases. 4679 */ 4680 4681 void 4682 removeq(queue_t *qp) 4683 { 4684 queue_t *wqp = _WR(qp); 4685 struct stdata *stp = STREAM(qp); 4686 sqlist_t *sqlist = NULL; 4687 boolean_t isdriver; 4688 int moved; 4689 syncq_t *sq = qp->q_syncq; 4690 syncq_t *wsq = wqp->q_syncq; 4691 4692 ASSERT(stp); 4693 4694 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ, 4695 "removeq:%p %p", qp, wqp); 4696 ASSERT(qp->q_flag&QREADR); 4697 4698 /* 4699 * For queues using Synchronous streams, we must wait for all threads in 4700 * rwnext() to drain out before proceeding. 4701 */ 4702 if (qp->q_flag & QSYNCSTR) { 4703 /* First, we need wakeup any threads blocked in rwnext() */ 4704 mutex_enter(SQLOCK(sq)); 4705 if (sq->sq_flags & SQ_WANTWAKEUP) { 4706 sq->sq_flags &= ~SQ_WANTWAKEUP; 4707 cv_broadcast(&sq->sq_wait); 4708 } 4709 mutex_exit(SQLOCK(sq)); 4710 4711 if (wsq != sq) { 4712 mutex_enter(SQLOCK(wsq)); 4713 if (wsq->sq_flags & SQ_WANTWAKEUP) { 4714 wsq->sq_flags &= ~SQ_WANTWAKEUP; 4715 cv_broadcast(&wsq->sq_wait); 4716 } 4717 mutex_exit(SQLOCK(wsq)); 4718 } 4719 4720 mutex_enter(QLOCK(qp)); 4721 while (qp->q_rwcnt > 0) { 4722 qp->q_flag |= QWANTRMQSYNC; 4723 cv_wait(&qp->q_wait, QLOCK(qp)); 4724 } 4725 mutex_exit(QLOCK(qp)); 4726 4727 mutex_enter(QLOCK(wqp)); 4728 while (wqp->q_rwcnt > 0) { 4729 wqp->q_flag |= QWANTRMQSYNC; 4730 cv_wait(&wqp->q_wait, QLOCK(wqp)); 4731 } 4732 mutex_exit(QLOCK(wqp)); 4733 } 4734 4735 mutex_enter(SQLOCK(sq)); 4736 sq->sq_rmqcount++; 4737 if (sq->sq_flags & SQ_WANTWAKEUP) { 4738 sq->sq_flags &= ~SQ_WANTWAKEUP; 4739 cv_broadcast(&sq->sq_wait); 4740 } 4741 mutex_exit(SQLOCK(sq)); 4742 4743 isdriver = (qp->q_flag & QISDRV); 4744 4745 sqlist = sqlist_build(qp, stp, STRMATED(stp)); 4746 strlock(stp, sqlist); 4747 4748 reset_nfsrv_ptr(qp, wqp); 4749 4750 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp); 4751 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp); 4752 /* Do we have a FIFO? */ 4753 if (wqp->q_next == qp) { 4754 stp->sd_wrq->q_next = _RD(stp->sd_wrq); 4755 } else { 4756 if (wqp->q_next) 4757 backq(qp)->q_next = qp->q_next; 4758 if (qp->q_next) 4759 backq(wqp)->q_next = wqp->q_next; 4760 } 4761 4762 /* The QEND flag might have to be updated for the upstream guy */ 4763 if (qp->q_next) 4764 set_qend(qp->q_next); 4765 4766 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq)); 4767 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq))); 4768 4769 /* 4770 * Move any messages destined for the put procedures to the next 4771 * syncq in line. Otherwise free them. 4772 */ 4773 moved = 0; 4774 /* 4775 * Quick check to see whether there are any messages or events. 4776 */ 4777 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS)) 4778 moved += propagate_syncq(qp); 4779 if (wqp->q_syncqmsgs != 0 || 4780 (wqp->q_syncq->sq_flags & SQ_EVENTS)) 4781 moved += propagate_syncq(wqp); 4782 4783 strsetuio(stp); 4784 4785 /* 4786 * If this was a module removal, decrement the push count. 4787 */ 4788 if (!isdriver) 4789 stp->sd_pushcnt--; 4790 4791 strunlock(stp, sqlist); 4792 sqlist_free(sqlist); 4793 4794 /* 4795 * Make sure any messages that were propagated are drained. 4796 * Also clear any QFULL bit caused by messages that were propagated. 4797 */ 4798 4799 if (qp->q_next != NULL) { 4800 clr_qfull(qp); 4801 /* 4802 * For the driver calling qprocsoff, propagate_syncq 4803 * frees all the messages instead of putting it in 4804 * the stream head 4805 */ 4806 if (!isdriver && (moved > 0)) 4807 emptysq(qp->q_next->q_syncq); 4808 } 4809 if (wqp->q_next != NULL) { 4810 clr_qfull(wqp); 4811 /* 4812 * We come here for any pop of a module except for the 4813 * case of driver being removed. We don't call emptysq 4814 * if we did not move any messages. This will avoid holding 4815 * PERMOD syncq locks in emptysq 4816 */ 4817 if (moved > 0) 4818 emptysq(wqp->q_next->q_syncq); 4819 } 4820 4821 mutex_enter(SQLOCK(sq)); 4822 sq->sq_rmqcount--; 4823 mutex_exit(SQLOCK(sq)); 4824 } 4825 4826 /* 4827 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or 4828 * SQ_WRITER) on a syncq. 4829 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the 4830 * sync queue and waits until sq_count reaches maxcnt. 4831 * 4832 * if maxcnt is -1 there's no need to grab sq_putlocks since the caller 4833 * does not care about putnext threads that are in the middle of calling put 4834 * entry points. 4835 * 4836 * This routine is used for both inner and outer syncqs. 4837 */ 4838 static void 4839 blocksq(syncq_t *sq, ushort_t flag, int maxcnt) 4840 { 4841 uint16_t count = 0; 4842 4843 mutex_enter(SQLOCK(sq)); 4844 /* 4845 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset. 4846 * SQ_FROZEN will be set if there is a frozen stream that has a 4847 * queue which also refers to this "shared" syncq. 4848 * SQ_BLOCKED will be set if there is "off" queue which also 4849 * refers to this "shared" syncq. 4850 */ 4851 if (maxcnt != -1) { 4852 count = sq->sq_count; 4853 SQ_PUTLOCKS_ENTER(sq); 4854 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4855 SUM_SQ_PUTCOUNTS(sq, count); 4856 } 4857 sq->sq_needexcl++; 4858 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 4859 4860 while ((sq->sq_flags & flag) || 4861 (maxcnt != -1 && count > (unsigned)maxcnt)) { 4862 sq->sq_flags |= SQ_WANTWAKEUP; 4863 if (maxcnt != -1) { 4864 SQ_PUTLOCKS_EXIT(sq); 4865 } 4866 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4867 if (maxcnt != -1) { 4868 count = sq->sq_count; 4869 SQ_PUTLOCKS_ENTER(sq); 4870 SUM_SQ_PUTCOUNTS(sq, count); 4871 } 4872 } 4873 sq->sq_needexcl--; 4874 sq->sq_flags |= flag; 4875 ASSERT(maxcnt == -1 || count == maxcnt); 4876 if (maxcnt != -1) { 4877 if (sq->sq_needexcl == 0) { 4878 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4879 } 4880 SQ_PUTLOCKS_EXIT(sq); 4881 } else if (sq->sq_needexcl == 0) { 4882 SQ_PUTCOUNT_SETFAST(sq); 4883 } 4884 4885 mutex_exit(SQLOCK(sq)); 4886 } 4887 4888 /* 4889 * Reset a flag that was set with blocksq. 4890 * 4891 * Can not use this routine to reset SQ_WRITER. 4892 * 4893 * If "isouter" is set then the syncq is assumed to be an outer perimeter 4894 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread 4895 * to handle the queued qwriter operations. 4896 * 4897 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4898 * sq_putlocks are used. 4899 */ 4900 static void 4901 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) 4902 { 4903 uint16_t flags; 4904 4905 mutex_enter(SQLOCK(sq)); 4906 ASSERT(resetflag != SQ_WRITER); 4907 ASSERT(sq->sq_flags & resetflag); 4908 flags = sq->sq_flags & ~resetflag; 4909 sq->sq_flags = flags; 4910 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) { 4911 if (flags & SQ_WANTWAKEUP) { 4912 flags &= ~SQ_WANTWAKEUP; 4913 cv_broadcast(&sq->sq_wait); 4914 } 4915 sq->sq_flags = flags; 4916 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 4917 if (!isouter) { 4918 /* drain_syncq drops SQLOCK */ 4919 drain_syncq(sq); 4920 return; 4921 } 4922 } 4923 } 4924 mutex_exit(SQLOCK(sq)); 4925 } 4926 4927 /* 4928 * Reset a flag that was set with blocksq. 4929 * Does not drain the syncq. Use emptysq() for that. 4930 * Returns 1 if SQ_QUEUED is set. Otherwise 0. 4931 * 4932 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4933 * sq_putlocks are used. 4934 */ 4935 static int 4936 dropsq(syncq_t *sq, uint16_t resetflag) 4937 { 4938 uint16_t flags; 4939 4940 mutex_enter(SQLOCK(sq)); 4941 ASSERT(sq->sq_flags & resetflag); 4942 flags = sq->sq_flags & ~resetflag; 4943 if (flags & SQ_WANTWAKEUP) { 4944 flags &= ~SQ_WANTWAKEUP; 4945 cv_broadcast(&sq->sq_wait); 4946 } 4947 sq->sq_flags = flags; 4948 mutex_exit(SQLOCK(sq)); 4949 if (flags & SQ_QUEUED) 4950 return (1); 4951 return (0); 4952 } 4953 4954 /* 4955 * Empty all the messages on a syncq. 4956 * 4957 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 4958 * sq_putlocks are used. 4959 */ 4960 static void 4961 emptysq(syncq_t *sq) 4962 { 4963 uint16_t flags; 4964 4965 mutex_enter(SQLOCK(sq)); 4966 flags = sq->sq_flags; 4967 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 4968 /* 4969 * To prevent potential recursive invocation of drain_syncq we 4970 * do not call drain_syncq if count is non-zero. 4971 */ 4972 if (sq->sq_count == 0) { 4973 /* drain_syncq() drops SQLOCK */ 4974 drain_syncq(sq); 4975 return; 4976 } else 4977 sqenable(sq); 4978 } 4979 mutex_exit(SQLOCK(sq)); 4980 } 4981 4982 /* 4983 * Ordered insert while removing duplicates. 4984 */ 4985 static void 4986 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp) 4987 { 4988 syncql_t *sqlp, **prev_sqlpp, *new_sqlp; 4989 4990 prev_sqlpp = &sqlist->sqlist_head; 4991 while ((sqlp = *prev_sqlpp) != NULL) { 4992 if (sqlp->sql_sq >= sqp) { 4993 if (sqlp->sql_sq == sqp) /* duplicate */ 4994 return; 4995 break; 4996 } 4997 prev_sqlpp = &sqlp->sql_next; 4998 } 4999 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++]; 5000 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size); 5001 new_sqlp->sql_next = sqlp; 5002 new_sqlp->sql_sq = sqp; 5003 *prev_sqlpp = new_sqlp; 5004 } 5005 5006 /* 5007 * Walk the write side queues until we hit either the driver 5008 * or a twist in the stream (_SAMESTR will return false in both 5009 * these cases) then turn around and walk the read side queues 5010 * back up to the stream head. 5011 */ 5012 static void 5013 sqlist_insertall(sqlist_t *sqlist, queue_t *q) 5014 { 5015 while (q != NULL) { 5016 sqlist_insert(sqlist, q->q_syncq); 5017 5018 if (_SAMESTR(q)) 5019 q = q->q_next; 5020 else if (!(q->q_flag & QREADR)) 5021 q = _RD(q); 5022 else 5023 q = NULL; 5024 } 5025 } 5026 5027 /* 5028 * Allocate and build a list of all syncqs in a stream and the syncq(s) 5029 * associated with the "q" parameter. The resulting list is sorted in a 5030 * canonical order and is free of duplicates. 5031 * Assumes the passed queue is a _RD(q). 5032 */ 5033 static sqlist_t * 5034 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist) 5035 { 5036 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP); 5037 5038 /* 5039 * start with the current queue/qpair 5040 */ 5041 ASSERT(q->q_flag & QREADR); 5042 5043 sqlist_insert(sqlist, q->q_syncq); 5044 sqlist_insert(sqlist, _WR(q)->q_syncq); 5045 5046 sqlist_insertall(sqlist, stp->sd_wrq); 5047 if (do_twist) 5048 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq); 5049 5050 return (sqlist); 5051 } 5052 5053 static sqlist_t * 5054 sqlist_alloc(struct stdata *stp, int kmflag) 5055 { 5056 size_t sqlist_size; 5057 sqlist_t *sqlist; 5058 5059 /* 5060 * Allocate 2 syncql_t's for each pushed module. Note that 5061 * the sqlist_t structure already has 4 syncql_t's built in: 5062 * 2 for the stream head, and 2 for the driver/other stream head. 5063 */ 5064 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt + 5065 sizeof (sqlist_t); 5066 if (STRMATED(stp)) 5067 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt; 5068 sqlist = kmem_alloc(sqlist_size, kmflag); 5069 5070 sqlist->sqlist_head = NULL; 5071 sqlist->sqlist_size = sqlist_size; 5072 sqlist->sqlist_index = 0; 5073 5074 return (sqlist); 5075 } 5076 5077 /* 5078 * Free the list created by sqlist_alloc() 5079 */ 5080 static void 5081 sqlist_free(sqlist_t *sqlist) 5082 { 5083 kmem_free(sqlist, sqlist->sqlist_size); 5084 } 5085 5086 /* 5087 * Prevent any new entries into any syncq in this stream. 5088 * Used by freezestr. 5089 */ 5090 void 5091 strblock(queue_t *q) 5092 { 5093 struct stdata *stp; 5094 syncql_t *sql; 5095 sqlist_t *sqlist; 5096 5097 q = _RD(q); 5098 5099 stp = STREAM(q); 5100 ASSERT(stp != NULL); 5101 5102 /* 5103 * Get a sorted list with all the duplicates removed containing 5104 * all the syncqs referenced by this stream. 5105 */ 5106 sqlist = sqlist_build(q, stp, B_FALSE); 5107 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5108 blocksq(sql->sql_sq, SQ_FROZEN, -1); 5109 sqlist_free(sqlist); 5110 } 5111 5112 /* 5113 * Release the block on new entries into this stream 5114 */ 5115 void 5116 strunblock(queue_t *q) 5117 { 5118 struct stdata *stp; 5119 syncql_t *sql; 5120 sqlist_t *sqlist; 5121 int drain_needed; 5122 5123 q = _RD(q); 5124 5125 /* 5126 * Get a sorted list with all the duplicates removed containing 5127 * all the syncqs referenced by this stream. 5128 * Have to drop the SQ_FROZEN flag on all the syncqs before 5129 * starting to drain them; otherwise the draining might 5130 * cause a freezestr in some module on the stream (which 5131 * would deadlock.) 5132 */ 5133 stp = STREAM(q); 5134 ASSERT(stp != NULL); 5135 sqlist = sqlist_build(q, stp, B_FALSE); 5136 drain_needed = 0; 5137 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5138 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN); 5139 if (drain_needed) { 5140 for (sql = sqlist->sqlist_head; sql != NULL; 5141 sql = sql->sql_next) 5142 emptysq(sql->sql_sq); 5143 } 5144 sqlist_free(sqlist); 5145 } 5146 5147 #ifdef DEBUG 5148 static int 5149 qprocsareon(queue_t *rq) 5150 { 5151 if (rq->q_next == NULL) 5152 return (0); 5153 return (_WR(rq->q_next)->q_next == _WR(rq)); 5154 } 5155 5156 int 5157 qclaimed(queue_t *q) 5158 { 5159 uint_t count; 5160 5161 count = q->q_syncq->sq_count; 5162 SUM_SQ_PUTCOUNTS(q->q_syncq, count); 5163 return (count != 0); 5164 } 5165 5166 /* 5167 * Check if anyone has frozen this stream with freezestr 5168 */ 5169 int 5170 frozenstr(queue_t *q) 5171 { 5172 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0); 5173 } 5174 #endif /* DEBUG */ 5175 5176 /* 5177 * Enter a queue. 5178 * Obsoleted interface. Should not be used. 5179 */ 5180 void 5181 enterq(queue_t *q) 5182 { 5183 entersq(q->q_syncq, SQ_CALLBACK); 5184 } 5185 5186 void 5187 leaveq(queue_t *q) 5188 { 5189 leavesq(q->q_syncq, SQ_CALLBACK); 5190 } 5191 5192 /* 5193 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits 5194 * to check. 5195 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter 5196 * calls and the running of open, close and service procedures. 5197 * 5198 * if c_inner bit is set no need to grab sq_putlocks since we don't care 5199 * if other threads have entered or are entering put entry point. 5200 * 5201 * if c_inner bit is set it might have been posible to use 5202 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize 5203 * open/close path for IP) but since the count may need to be decremented in 5204 * qwait() we wouldn't know which counter to decrement. Currently counter is 5205 * selected by current cpu_seqid and current CPU can change at any moment. XXX 5206 * in the future we might use curthread id bits to select the counter and this 5207 * would stay constant across routine calls. 5208 */ 5209 void 5210 entersq(syncq_t *sq, int entrypoint) 5211 { 5212 uint16_t count = 0; 5213 uint16_t flags; 5214 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 5215 uint16_t type; 5216 uint_t c_inner = entrypoint & SQ_CI; 5217 uint_t c_outer = entrypoint & SQ_CO; 5218 5219 /* 5220 * Increment ref count to keep closes out of this queue. 5221 */ 5222 ASSERT(sq); 5223 ASSERT(c_inner && c_outer); 5224 mutex_enter(SQLOCK(sq)); 5225 flags = sq->sq_flags; 5226 type = sq->sq_type; 5227 if (!(type & c_inner)) { 5228 /* Make sure all putcounts now use slowlock. */ 5229 count = sq->sq_count; 5230 SQ_PUTLOCKS_ENTER(sq); 5231 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5232 SUM_SQ_PUTCOUNTS(sq, count); 5233 sq->sq_needexcl++; 5234 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5235 waitflags |= SQ_MESSAGES; 5236 } 5237 /* 5238 * Wait until we can enter the inner perimeter. 5239 * If we want exclusive access we wait until sq_count is 0. 5240 * We have to do this before entering the outer perimeter in order 5241 * to preserve put/close message ordering. 5242 */ 5243 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) { 5244 sq->sq_flags = flags | SQ_WANTWAKEUP; 5245 if (!(type & c_inner)) { 5246 SQ_PUTLOCKS_EXIT(sq); 5247 } 5248 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5249 if (!(type & c_inner)) { 5250 count = sq->sq_count; 5251 SQ_PUTLOCKS_ENTER(sq); 5252 SUM_SQ_PUTCOUNTS(sq, count); 5253 } 5254 flags = sq->sq_flags; 5255 } 5256 5257 if (!(type & c_inner)) { 5258 ASSERT(sq->sq_needexcl > 0); 5259 sq->sq_needexcl--; 5260 if (sq->sq_needexcl == 0) { 5261 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5262 } 5263 } 5264 5265 /* Check if we need to enter the outer perimeter */ 5266 if (!(type & c_outer)) { 5267 /* 5268 * We have to enter the outer perimeter exclusively before 5269 * we can increment sq_count to avoid deadlock. This implies 5270 * that we have to re-check sq_flags and sq_count. 5271 * 5272 * is it possible to have c_inner set when c_outer is not set? 5273 */ 5274 if (!(type & c_inner)) { 5275 SQ_PUTLOCKS_EXIT(sq); 5276 } 5277 mutex_exit(SQLOCK(sq)); 5278 outer_enter(sq->sq_outer, SQ_GOAWAY); 5279 mutex_enter(SQLOCK(sq)); 5280 flags = sq->sq_flags; 5281 /* 5282 * there should be no need to recheck sq_putcounts 5283 * because outer_enter() has already waited for them to clear 5284 * after setting SQ_WRITER. 5285 */ 5286 count = sq->sq_count; 5287 #ifdef DEBUG 5288 /* 5289 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead 5290 * of doing an ASSERT internally. Others should do 5291 * something like 5292 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0); 5293 * without the need to #ifdef DEBUG it. 5294 */ 5295 SUMCHECK_SQ_PUTCOUNTS(sq, 0); 5296 #endif 5297 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) || 5298 (!(type & c_inner) && count != 0)) { 5299 sq->sq_flags = flags | SQ_WANTWAKEUP; 5300 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5301 count = sq->sq_count; 5302 flags = sq->sq_flags; 5303 } 5304 } 5305 5306 sq->sq_count++; 5307 ASSERT(sq->sq_count != 0); /* Wraparound */ 5308 if (!(type & c_inner)) { 5309 /* Exclusive entry */ 5310 ASSERT(sq->sq_count == 1); 5311 sq->sq_flags |= SQ_EXCL; 5312 if (type & c_outer) { 5313 SQ_PUTLOCKS_EXIT(sq); 5314 } 5315 } 5316 mutex_exit(SQLOCK(sq)); 5317 } 5318 5319 /* 5320 * leave a syncq. announce to framework that closes may proceed. 5321 * c_inner and c_outer specifies which concurrency bits 5322 * to check. 5323 * 5324 * must never be called from driver or module put entry point. 5325 * 5326 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5327 * sq_putlocks are used. 5328 */ 5329 void 5330 leavesq(syncq_t *sq, int entrypoint) 5331 { 5332 uint16_t flags; 5333 uint16_t type; 5334 uint_t c_outer = entrypoint & SQ_CO; 5335 #ifdef DEBUG 5336 uint_t c_inner = entrypoint & SQ_CI; 5337 #endif 5338 5339 /* 5340 * decrement ref count, drain the syncq if possible, and wake up 5341 * any waiting close. 5342 */ 5343 ASSERT(sq); 5344 ASSERT(c_inner && c_outer); 5345 mutex_enter(SQLOCK(sq)); 5346 flags = sq->sq_flags; 5347 type = sq->sq_type; 5348 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) { 5349 5350 if (flags & SQ_WANTWAKEUP) { 5351 flags &= ~SQ_WANTWAKEUP; 5352 cv_broadcast(&sq->sq_wait); 5353 } 5354 if (flags & SQ_WANTEXWAKEUP) { 5355 flags &= ~SQ_WANTEXWAKEUP; 5356 cv_broadcast(&sq->sq_exitwait); 5357 } 5358 5359 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) { 5360 /* 5361 * The syncq needs to be drained. "Exit" the syncq 5362 * before calling drain_syncq. 5363 */ 5364 ASSERT(sq->sq_count != 0); 5365 sq->sq_count--; 5366 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5367 sq->sq_flags = flags & ~SQ_EXCL; 5368 drain_syncq(sq); 5369 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 5370 /* Check if we need to exit the outer perimeter */ 5371 /* XXX will this ever be true? */ 5372 if (!(type & c_outer)) 5373 outer_exit(sq->sq_outer); 5374 return; 5375 } 5376 } 5377 ASSERT(sq->sq_count != 0); 5378 sq->sq_count--; 5379 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5380 sq->sq_flags = flags & ~SQ_EXCL; 5381 mutex_exit(SQLOCK(sq)); 5382 5383 /* Check if we need to exit the outer perimeter */ 5384 if (!(sq->sq_type & c_outer)) 5385 outer_exit(sq->sq_outer); 5386 } 5387 5388 /* 5389 * Prevent q_next from changing in this stream by incrementing sq_count. 5390 * 5391 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5392 * sq_putlocks are used. 5393 */ 5394 void 5395 claimq(queue_t *qp) 5396 { 5397 syncq_t *sq = qp->q_syncq; 5398 5399 mutex_enter(SQLOCK(sq)); 5400 sq->sq_count++; 5401 ASSERT(sq->sq_count != 0); /* Wraparound */ 5402 mutex_exit(SQLOCK(sq)); 5403 } 5404 5405 /* 5406 * Undo claimq. 5407 * 5408 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 5409 * sq_putlocks are used. 5410 */ 5411 void 5412 releaseq(queue_t *qp) 5413 { 5414 syncq_t *sq = qp->q_syncq; 5415 uint16_t flags; 5416 5417 mutex_enter(SQLOCK(sq)); 5418 ASSERT(sq->sq_count > 0); 5419 sq->sq_count--; 5420 5421 flags = sq->sq_flags; 5422 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) { 5423 if (flags & SQ_WANTWAKEUP) { 5424 flags &= ~SQ_WANTWAKEUP; 5425 cv_broadcast(&sq->sq_wait); 5426 } 5427 sq->sq_flags = flags; 5428 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5429 /* 5430 * To prevent potential recursive invocation of 5431 * drain_syncq we do not call drain_syncq if count is 5432 * non-zero. 5433 */ 5434 if (sq->sq_count == 0) { 5435 drain_syncq(sq); 5436 return; 5437 } else 5438 sqenable(sq); 5439 } 5440 } 5441 mutex_exit(SQLOCK(sq)); 5442 } 5443 5444 /* 5445 * Prevent q_next from changing in this stream by incrementing sd_refcnt. 5446 */ 5447 void 5448 claimstr(queue_t *qp) 5449 { 5450 struct stdata *stp = STREAM(qp); 5451 5452 mutex_enter(&stp->sd_reflock); 5453 stp->sd_refcnt++; 5454 ASSERT(stp->sd_refcnt != 0); /* Wraparound */ 5455 mutex_exit(&stp->sd_reflock); 5456 } 5457 5458 /* 5459 * Undo claimstr. 5460 */ 5461 void 5462 releasestr(queue_t *qp) 5463 { 5464 struct stdata *stp = STREAM(qp); 5465 5466 mutex_enter(&stp->sd_reflock); 5467 ASSERT(stp->sd_refcnt != 0); 5468 if (--stp->sd_refcnt == 0) 5469 cv_broadcast(&stp->sd_refmonitor); 5470 mutex_exit(&stp->sd_reflock); 5471 } 5472 5473 static syncq_t * 5474 new_syncq(void) 5475 { 5476 return (kmem_cache_alloc(syncq_cache, KM_SLEEP)); 5477 } 5478 5479 static void 5480 free_syncq(syncq_t *sq) 5481 { 5482 ASSERT(sq->sq_head == NULL); 5483 ASSERT(sq->sq_outer == NULL); 5484 ASSERT(sq->sq_callbpend == NULL); 5485 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || 5486 (sq->sq_onext == sq && sq->sq_oprev == sq)); 5487 5488 if (sq->sq_ciputctrl != NULL) { 5489 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 5490 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 5491 sq->sq_nciputctrl, 0); 5492 ASSERT(ciputctrl_cache != NULL); 5493 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 5494 } 5495 5496 sq->sq_tail = NULL; 5497 sq->sq_evhead = NULL; 5498 sq->sq_evtail = NULL; 5499 sq->sq_ciputctrl = NULL; 5500 sq->sq_nciputctrl = 0; 5501 sq->sq_count = 0; 5502 sq->sq_rmqcount = 0; 5503 sq->sq_callbflags = 0; 5504 sq->sq_cancelid = 0; 5505 sq->sq_next = NULL; 5506 sq->sq_needexcl = 0; 5507 sq->sq_svcflags = 0; 5508 sq->sq_nqueues = 0; 5509 sq->sq_pri = 0; 5510 sq->sq_onext = NULL; 5511 sq->sq_oprev = NULL; 5512 sq->sq_flags = 0; 5513 sq->sq_type = 0; 5514 sq->sq_servcount = 0; 5515 5516 kmem_cache_free(syncq_cache, sq); 5517 } 5518 5519 /* Outer perimeter code */ 5520 5521 /* 5522 * The outer syncq uses the fields and flags in the syncq slightly 5523 * differently from the inner syncqs. 5524 * sq_count Incremented when there are pending or running 5525 * writers at the outer perimeter to prevent the set of 5526 * inner syncqs that belong to the outer perimeter from 5527 * changing. 5528 * sq_head/tail List of deferred qwriter(OUTER) operations. 5529 * 5530 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while 5531 * inner syncqs are added to or removed from the 5532 * outer perimeter. 5533 * SQ_QUEUED sq_head/tail has messages or eventsqueued. 5534 * 5535 * SQ_WRITER A thread is currently traversing all the inner syncqs 5536 * setting the SQ_WRITER flag. 5537 */ 5538 5539 /* 5540 * Get write access at the outer perimeter. 5541 * Note that read access is done by entersq, putnext, and put by simply 5542 * incrementing sq_count in the inner syncq. 5543 * 5544 * Waits until "flags" is no longer set in the outer to prevent multiple 5545 * threads from having write access at the same time. SQ_WRITER has to be part 5546 * of "flags". 5547 * 5548 * Increases sq_count on the outer syncq to keep away outer_insert/remove 5549 * until the outer_exit is finished. 5550 * 5551 * outer_enter is vulnerable to starvation since it does not prevent new 5552 * threads from entering the inner syncqs while it is waiting for sq_count to 5553 * go to zero. 5554 */ 5555 void 5556 outer_enter(syncq_t *outer, uint16_t flags) 5557 { 5558 syncq_t *sq; 5559 int wait_needed; 5560 uint16_t count; 5561 5562 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5563 outer->sq_oprev != NULL); 5564 ASSERT(flags & SQ_WRITER); 5565 5566 retry: 5567 mutex_enter(SQLOCK(outer)); 5568 while (outer->sq_flags & flags) { 5569 outer->sq_flags |= SQ_WANTWAKEUP; 5570 cv_wait(&outer->sq_wait, SQLOCK(outer)); 5571 } 5572 5573 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5574 outer->sq_flags |= SQ_WRITER; 5575 outer->sq_count++; 5576 ASSERT(outer->sq_count != 0); /* wraparound */ 5577 wait_needed = 0; 5578 /* 5579 * Set SQ_WRITER on all the inner syncqs while holding 5580 * the SQLOCK on the outer syncq. This ensures that the changing 5581 * of SQ_WRITER is atomic under the outer SQLOCK. 5582 */ 5583 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5584 mutex_enter(SQLOCK(sq)); 5585 count = sq->sq_count; 5586 SQ_PUTLOCKS_ENTER(sq); 5587 sq->sq_flags |= SQ_WRITER; 5588 SUM_SQ_PUTCOUNTS(sq, count); 5589 if (count != 0) 5590 wait_needed = 1; 5591 SQ_PUTLOCKS_EXIT(sq); 5592 mutex_exit(SQLOCK(sq)); 5593 } 5594 mutex_exit(SQLOCK(outer)); 5595 5596 /* 5597 * Get everybody out of the syncqs sequentially. 5598 * Note that we don't actually need to aqiure the PUTLOCKS, since 5599 * we have already cleared the fastbit, and set QWRITER. By 5600 * definition, the count can not increase since putnext will 5601 * take the slowlock path (and the purpose of aquiring the 5602 * putlocks was to make sure it didn't increase while we were 5603 * waiting). 5604 * 5605 * Note that we still aquire the PUTLOCKS to be safe. 5606 */ 5607 if (wait_needed) { 5608 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5609 mutex_enter(SQLOCK(sq)); 5610 count = sq->sq_count; 5611 SQ_PUTLOCKS_ENTER(sq); 5612 SUM_SQ_PUTCOUNTS(sq, count); 5613 while (count != 0) { 5614 sq->sq_flags |= SQ_WANTWAKEUP; 5615 SQ_PUTLOCKS_EXIT(sq); 5616 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5617 count = sq->sq_count; 5618 SQ_PUTLOCKS_ENTER(sq); 5619 SUM_SQ_PUTCOUNTS(sq, count); 5620 } 5621 SQ_PUTLOCKS_EXIT(sq); 5622 mutex_exit(SQLOCK(sq)); 5623 } 5624 /* 5625 * Verify that none of the flags got set while we 5626 * were waiting for the sq_counts to drop. 5627 * If this happens we exit and retry entering the 5628 * outer perimeter. 5629 */ 5630 mutex_enter(SQLOCK(outer)); 5631 if (outer->sq_flags & (flags & ~SQ_WRITER)) { 5632 mutex_exit(SQLOCK(outer)); 5633 outer_exit(outer); 5634 goto retry; 5635 } 5636 mutex_exit(SQLOCK(outer)); 5637 } 5638 } 5639 5640 /* 5641 * Drop the write access at the outer perimeter. 5642 * Read access is dropped implicitly (by putnext, put, and leavesq) by 5643 * decrementing sq_count. 5644 */ 5645 void 5646 outer_exit(syncq_t *outer) 5647 { 5648 syncq_t *sq; 5649 int drain_needed; 5650 uint16_t flags; 5651 5652 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5653 outer->sq_oprev != NULL); 5654 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer))); 5655 5656 /* 5657 * Atomically (from the perspective of threads calling become_writer) 5658 * drop the write access at the outer perimeter by holding 5659 * SQLOCK(outer) across all the dropsq calls and the resetting of 5660 * SQ_WRITER. 5661 * This defines a locking order between the outer perimeter 5662 * SQLOCK and the inner perimeter SQLOCKs. 5663 */ 5664 mutex_enter(SQLOCK(outer)); 5665 flags = outer->sq_flags; 5666 ASSERT(outer->sq_flags & SQ_WRITER); 5667 if (flags & SQ_QUEUED) { 5668 write_now(outer); 5669 flags = outer->sq_flags; 5670 } 5671 5672 /* 5673 * sq_onext is stable since sq_count has not yet been decreased. 5674 * Reset the SQ_WRITER flags in all syncqs. 5675 * After dropping SQ_WRITER on the outer syncq we empty all the 5676 * inner syncqs. 5677 */ 5678 drain_needed = 0; 5679 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5680 drain_needed += dropsq(sq, SQ_WRITER); 5681 ASSERT(!(outer->sq_flags & SQ_QUEUED)); 5682 flags &= ~SQ_WRITER; 5683 if (drain_needed) { 5684 outer->sq_flags = flags; 5685 mutex_exit(SQLOCK(outer)); 5686 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5687 emptysq(sq); 5688 mutex_enter(SQLOCK(outer)); 5689 flags = outer->sq_flags; 5690 } 5691 if (flags & SQ_WANTWAKEUP) { 5692 flags &= ~SQ_WANTWAKEUP; 5693 cv_broadcast(&outer->sq_wait); 5694 } 5695 outer->sq_flags = flags; 5696 ASSERT(outer->sq_count > 0); 5697 outer->sq_count--; 5698 mutex_exit(SQLOCK(outer)); 5699 } 5700 5701 /* 5702 * Add another syncq to an outer perimeter. 5703 * Block out all other access to the outer perimeter while it is being 5704 * changed using blocksq. 5705 * Assumes that the caller has *not* done an outer_enter. 5706 * 5707 * Vulnerable to starvation in blocksq. 5708 */ 5709 static void 5710 outer_insert(syncq_t *outer, syncq_t *sq) 5711 { 5712 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5713 outer->sq_oprev != NULL); 5714 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 5715 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ 5716 5717 /* Get exclusive access to the outer perimeter list */ 5718 blocksq(outer, SQ_BLOCKED, 0); 5719 ASSERT(outer->sq_flags & SQ_BLOCKED); 5720 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5721 5722 mutex_enter(SQLOCK(sq)); 5723 sq->sq_outer = outer; 5724 outer->sq_onext->sq_oprev = sq; 5725 sq->sq_onext = outer->sq_onext; 5726 outer->sq_onext = sq; 5727 sq->sq_oprev = outer; 5728 mutex_exit(SQLOCK(sq)); 5729 unblocksq(outer, SQ_BLOCKED, 1); 5730 } 5731 5732 /* 5733 * Remove a syncq from an outer perimeter. 5734 * Block out all other access to the outer perimeter while it is being 5735 * changed using blocksq. 5736 * Assumes that the caller has *not* done an outer_enter. 5737 * 5738 * Vulnerable to starvation in blocksq. 5739 */ 5740 static void 5741 outer_remove(syncq_t *outer, syncq_t *sq) 5742 { 5743 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5744 outer->sq_oprev != NULL); 5745 ASSERT(sq->sq_outer == outer); 5746 5747 /* Get exclusive access to the outer perimeter list */ 5748 blocksq(outer, SQ_BLOCKED, 0); 5749 ASSERT(outer->sq_flags & SQ_BLOCKED); 5750 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5751 5752 mutex_enter(SQLOCK(sq)); 5753 sq->sq_outer = NULL; 5754 sq->sq_onext->sq_oprev = sq->sq_oprev; 5755 sq->sq_oprev->sq_onext = sq->sq_onext; 5756 sq->sq_oprev = sq->sq_onext = NULL; 5757 mutex_exit(SQLOCK(sq)); 5758 unblocksq(outer, SQ_BLOCKED, 1); 5759 } 5760 5761 /* 5762 * Queue a deferred qwriter(OUTER) callback for this outer perimeter. 5763 * If this is the first callback for this outer perimeter then add 5764 * this outer perimeter to the list of outer perimeters that 5765 * the qwriter_outer_thread will process. 5766 * 5767 * Increments sq_count in the outer syncq to prevent the membership 5768 * of the outer perimeter (in terms of inner syncqs) to change while 5769 * the callback is pending. 5770 */ 5771 static void 5772 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp) 5773 { 5774 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5775 5776 mp->b_prev = (mblk_t *)func; 5777 mp->b_queue = q; 5778 mp->b_next = NULL; 5779 outer->sq_count++; /* Decremented when dequeued */ 5780 ASSERT(outer->sq_count != 0); /* Wraparound */ 5781 if (outer->sq_evhead == NULL) { 5782 /* First message. */ 5783 outer->sq_evhead = outer->sq_evtail = mp; 5784 outer->sq_flags |= SQ_EVENTS; 5785 mutex_exit(SQLOCK(outer)); 5786 STRSTAT(qwr_outer); 5787 (void) taskq_dispatch(streams_taskq, 5788 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP); 5789 } else { 5790 ASSERT(outer->sq_flags & SQ_EVENTS); 5791 outer->sq_evtail->b_next = mp; 5792 outer->sq_evtail = mp; 5793 mutex_exit(SQLOCK(outer)); 5794 } 5795 } 5796 5797 /* 5798 * Try and upgrade to write access at the outer perimeter. If this can 5799 * not be done without blocking then queue the callback to be done 5800 * by the qwriter_outer_thread. 5801 * 5802 * This routine can only be called from put or service procedures plus 5803 * asynchronous callback routines that have properly entered to 5804 * queue (with entersq.) Thus qwriter(OUTER) assumes the caller has one claim 5805 * on the syncq associated with q. 5806 */ 5807 void 5808 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)()) 5809 { 5810 syncq_t *osq, *sq, *outer; 5811 int failed; 5812 uint16_t flags; 5813 5814 osq = q->q_syncq; 5815 outer = osq->sq_outer; 5816 if (outer == NULL) 5817 panic("qwriter(PERIM_OUTER): no outer perimeter"); 5818 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5819 outer->sq_oprev != NULL); 5820 5821 mutex_enter(SQLOCK(outer)); 5822 flags = outer->sq_flags; 5823 /* 5824 * If some thread is traversing sq_next, or if we are blocked by 5825 * outer_insert or outer_remove, or if the we already have queued 5826 * callbacks, then queue this callback for later processing. 5827 * 5828 * Also queue the qwriter for an interrupt thread in order 5829 * to reduce the time spent running at high IPL. 5830 * to identify there are events. 5831 */ 5832 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) { 5833 /* 5834 * Queue the become_writer request. 5835 * The queueing is atomic under SQLOCK(outer) in order 5836 * to synchronize with outer_exit. 5837 * queue_writer will drop the outer SQLOCK 5838 */ 5839 if (flags & SQ_BLOCKED) { 5840 /* Must set SQ_WRITER on inner perimeter */ 5841 mutex_enter(SQLOCK(osq)); 5842 osq->sq_flags |= SQ_WRITER; 5843 mutex_exit(SQLOCK(osq)); 5844 } else { 5845 if (!(flags & SQ_WRITER)) { 5846 /* 5847 * The outer could have been SQ_BLOCKED thus 5848 * SQ_WRITER might not be set on the inner. 5849 */ 5850 mutex_enter(SQLOCK(osq)); 5851 osq->sq_flags |= SQ_WRITER; 5852 mutex_exit(SQLOCK(osq)); 5853 } 5854 ASSERT(osq->sq_flags & SQ_WRITER); 5855 } 5856 queue_writer(outer, func, q, mp); 5857 return; 5858 } 5859 /* 5860 * We are half-way to exclusive access to the outer perimeter. 5861 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove 5862 * while the inner syncqs are traversed. 5863 */ 5864 outer->sq_count++; 5865 ASSERT(outer->sq_count != 0); /* wraparound */ 5866 flags |= SQ_WRITER; 5867 /* 5868 * Check if we can run the function immediately. Mark all 5869 * syncqs with the writer flag to prevent new entries into 5870 * put and service procedures. 5871 * 5872 * Set SQ_WRITER on all the inner syncqs while holding 5873 * the SQLOCK on the outer syncq. This ensures that the changing 5874 * of SQ_WRITER is atomic under the outer SQLOCK. 5875 */ 5876 failed = 0; 5877 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5878 uint16_t count; 5879 uint_t maxcnt = (sq == osq) ? 1 : 0; 5880 5881 mutex_enter(SQLOCK(sq)); 5882 count = sq->sq_count; 5883 SQ_PUTLOCKS_ENTER(sq); 5884 SUM_SQ_PUTCOUNTS(sq, count); 5885 if (sq->sq_count > maxcnt) 5886 failed = 1; 5887 sq->sq_flags |= SQ_WRITER; 5888 SQ_PUTLOCKS_EXIT(sq); 5889 mutex_exit(SQLOCK(sq)); 5890 } 5891 if (failed) { 5892 /* 5893 * Some other thread has a read claim on the outer perimeter. 5894 * Queue the callback for deferred processing. 5895 * 5896 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER 5897 * so that other qwriter(OUTER) calls will queue their 5898 * callbacks as well. queue_writer increments sq_count so we 5899 * decrement to compensate for the our increment. 5900 * 5901 * Dropping SQ_WRITER enables the writer thread to work 5902 * on this outer perimeter. 5903 */ 5904 outer->sq_flags = flags; 5905 queue_writer(outer, func, q, mp); 5906 /* queue_writer dropper the lock */ 5907 mutex_enter(SQLOCK(outer)); 5908 ASSERT(outer->sq_count > 0); 5909 outer->sq_count--; 5910 ASSERT(outer->sq_flags & SQ_WRITER); 5911 flags = outer->sq_flags; 5912 flags &= ~SQ_WRITER; 5913 if (flags & SQ_WANTWAKEUP) { 5914 flags &= ~SQ_WANTWAKEUP; 5915 cv_broadcast(&outer->sq_wait); 5916 } 5917 outer->sq_flags = flags; 5918 mutex_exit(SQLOCK(outer)); 5919 return; 5920 } else { 5921 outer->sq_flags = flags; 5922 mutex_exit(SQLOCK(outer)); 5923 } 5924 5925 /* Can run it immediately */ 5926 (*func)(q, mp); 5927 5928 outer_exit(outer); 5929 } 5930 5931 /* 5932 * Dequeue all writer callbacks from the outer perimeter and run them. 5933 */ 5934 static void 5935 write_now(syncq_t *outer) 5936 { 5937 mblk_t *mp; 5938 queue_t *q; 5939 void (*func)(); 5940 5941 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5942 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5943 outer->sq_oprev != NULL); 5944 while ((mp = outer->sq_evhead) != NULL) { 5945 /* 5946 * queues cannot be placed on the queuelist on the outer 5947 * perimiter. 5948 */ 5949 ASSERT(!(outer->sq_flags & SQ_MESSAGES)); 5950 ASSERT((outer->sq_flags & SQ_EVENTS)); 5951 5952 outer->sq_evhead = mp->b_next; 5953 if (outer->sq_evhead == NULL) { 5954 outer->sq_evtail = NULL; 5955 outer->sq_flags &= ~SQ_EVENTS; 5956 } 5957 ASSERT(outer->sq_count != 0); 5958 outer->sq_count--; /* Incremented when enqueued. */ 5959 mutex_exit(SQLOCK(outer)); 5960 /* 5961 * Drop the message if the queue is closing. 5962 * Make sure that the queue is "claimed" when the callback 5963 * is run in order to satisfy various ASSERTs. 5964 */ 5965 q = mp->b_queue; 5966 func = (void (*)())mp->b_prev; 5967 ASSERT(func != NULL); 5968 mp->b_next = mp->b_prev = NULL; 5969 if (q->q_flag & QWCLOSE) { 5970 freemsg(mp); 5971 } else { 5972 claimq(q); 5973 (*func)(q, mp); 5974 releaseq(q); 5975 } 5976 mutex_enter(SQLOCK(outer)); 5977 } 5978 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5979 } 5980 5981 /* 5982 * The list of messages on the inner syncq is effectively hashed 5983 * by destination queue. These destination queues are doubly 5984 * linked lists (hopefully) in priority order. Messages are then 5985 * put on the queue referenced by the q_sqhead/q_sqtail elements. 5986 * Additional messages are linked together by the b_next/b_prev 5987 * elements in the mblk, with (similar to putq()) the first message 5988 * having a NULL b_prev and the last message having a NULL b_next. 5989 * 5990 * Events, such as qwriter callbacks, are put onto a list in FIFO 5991 * order referenced by sq_evhead, and sq_evtail. This is a singly 5992 * linked list, and messages here MUST be processed in the order queued. 5993 */ 5994 5995 /* 5996 * Run the events on the syncq event list (sq_evhead). 5997 * Assumes there is only one claim on the syncq, it is 5998 * already exclusive (SQ_EXCL set), and the SQLOCK held. 5999 * Messages here are processed in order, with the SQ_EXCL bit 6000 * held all the way through till the last message is processed. 6001 */ 6002 void 6003 sq_run_events(syncq_t *sq) 6004 { 6005 mblk_t *bp; 6006 queue_t *qp; 6007 uint16_t flags = sq->sq_flags; 6008 void (*func)(); 6009 6010 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6011 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6012 sq->sq_oprev == NULL) || 6013 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6014 sq->sq_oprev != NULL)); 6015 6016 ASSERT(flags & SQ_EXCL); 6017 ASSERT(sq->sq_count == 1); 6018 6019 /* 6020 * We need to process all of the events on this list. It 6021 * is possible that new events will be added while we are 6022 * away processing a callback, so on every loop, we start 6023 * back at the beginning of the list. 6024 */ 6025 /* 6026 * We have to reaccess sq_evhead since there is a 6027 * possibility of a new entry while we were running 6028 * the callback. 6029 */ 6030 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { 6031 ASSERT(bp->b_queue->q_syncq == sq); 6032 ASSERT(sq->sq_flags & SQ_EVENTS); 6033 6034 qp = bp->b_queue; 6035 func = (void (*)())bp->b_prev; 6036 ASSERT(func != NULL); 6037 6038 /* 6039 * Messages from the event queue must be taken off in 6040 * FIFO order. 6041 */ 6042 ASSERT(sq->sq_evhead == bp); 6043 sq->sq_evhead = bp->b_next; 6044 6045 if (bp->b_next == NULL) { 6046 /* Deleting last */ 6047 ASSERT(sq->sq_evtail == bp); 6048 sq->sq_evtail = NULL; 6049 sq->sq_flags &= ~SQ_EVENTS; 6050 } 6051 bp->b_prev = bp->b_next = NULL; 6052 ASSERT(bp->b_datap->db_ref != 0); 6053 6054 mutex_exit(SQLOCK(sq)); 6055 6056 (*func)(qp, bp); 6057 6058 mutex_enter(SQLOCK(sq)); 6059 /* 6060 * re-read the flags, since they could have changed. 6061 */ 6062 flags = sq->sq_flags; 6063 ASSERT(flags & SQ_EXCL); 6064 } 6065 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); 6066 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6067 6068 if (flags & SQ_WANTWAKEUP) { 6069 flags &= ~SQ_WANTWAKEUP; 6070 cv_broadcast(&sq->sq_wait); 6071 } 6072 if (flags & SQ_WANTEXWAKEUP) { 6073 flags &= ~SQ_WANTEXWAKEUP; 6074 cv_broadcast(&sq->sq_exitwait); 6075 } 6076 sq->sq_flags = flags; 6077 } 6078 6079 /* 6080 * Put messages on the event list. 6081 * If we can go exclusive now, do so and process the event list, otherwise 6082 * let the last claim service this list (or wake the sqthread). 6083 * This procedure assumes SQLOCK is held. To run the event list, it 6084 * must be called with no claims. 6085 */ 6086 static void 6087 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) 6088 { 6089 uint16_t count; 6090 6091 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6092 ASSERT(func != NULL); 6093 6094 /* 6095 * This is a callback. Add it to the list of callbacks 6096 * and see about upgrading. 6097 */ 6098 mp->b_prev = (mblk_t *)func; 6099 mp->b_queue = q; 6100 mp->b_next = NULL; 6101 if (sq->sq_evhead == NULL) { 6102 sq->sq_evhead = sq->sq_evtail = mp; 6103 sq->sq_flags |= SQ_EVENTS; 6104 } else { 6105 ASSERT(sq->sq_evtail != NULL); 6106 ASSERT(sq->sq_evtail->b_next == NULL); 6107 ASSERT(sq->sq_flags & SQ_EVENTS); 6108 sq->sq_evtail->b_next = mp; 6109 sq->sq_evtail = mp; 6110 } 6111 /* 6112 * We have set SQ_EVENTS, so threads will have to 6113 * unwind out of the perimiter, and new entries will 6114 * not grab a putlock. But we still need to know 6115 * how many threads have already made a claim to the 6116 * syncq, so grab the putlocks, and sum the counts. 6117 * If there are no claims on the syncq, we can upgrade 6118 * to exclusive, and run the event list. 6119 * NOTE: We hold the SQLOCK, so we can just grab the 6120 * putlocks. 6121 */ 6122 count = sq->sq_count; 6123 SQ_PUTLOCKS_ENTER(sq); 6124 SUM_SQ_PUTCOUNTS(sq, count); 6125 /* 6126 * We have no claim, so we need to check if there 6127 * are no others, then we can upgrade. 6128 */ 6129 /* 6130 * There are currently no claims on 6131 * the syncq by this thread (at least on this entry). The thread who has 6132 * the claim should drain syncq. 6133 */ 6134 if (count > 0) { 6135 /* 6136 * Can't upgrade - other threads inside. 6137 */ 6138 SQ_PUTLOCKS_EXIT(sq); 6139 mutex_exit(SQLOCK(sq)); 6140 return; 6141 } 6142 /* 6143 * Need to set SQ_EXCL and make a claim on the syncq. 6144 */ 6145 ASSERT((sq->sq_flags & SQ_EXCL) == 0); 6146 sq->sq_flags |= SQ_EXCL; 6147 ASSERT(sq->sq_count == 0); 6148 sq->sq_count++; 6149 SQ_PUTLOCKS_EXIT(sq); 6150 6151 /* Process the events list */ 6152 sq_run_events(sq); 6153 6154 /* 6155 * Release our claim... 6156 */ 6157 sq->sq_count--; 6158 6159 /* 6160 * And release SQ_EXCL. 6161 * We don't need to acquire the putlocks to release 6162 * SQ_EXCL, since we are exclusive, and hold the SQLOCK. 6163 */ 6164 sq->sq_flags &= ~SQ_EXCL; 6165 6166 /* 6167 * sq_run_events should have released SQ_EXCL 6168 */ 6169 ASSERT(!(sq->sq_flags & SQ_EXCL)); 6170 6171 /* 6172 * If anything happened while we were running the 6173 * events (or was there before), we need to process 6174 * them now. We shouldn't be exclusive sine we 6175 * released the perimiter above (plus, we asserted 6176 * for it). 6177 */ 6178 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) 6179 drain_syncq(sq); 6180 else 6181 mutex_exit(SQLOCK(sq)); 6182 } 6183 6184 /* 6185 * Perform delayed processing. The caller has to make sure that it is safe 6186 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are 6187 * set.) 6188 * 6189 * Assume that the caller has NO claims on the syncq. However, a claim 6190 * on the syncq does not indicate that a thread is draining the syncq. 6191 * There may be more claims on the syncq than there are threads draining 6192 * (i.e. #_threads_draining <= sq_count) 6193 * 6194 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set 6195 * in order to preserve qwriter(OUTER) ordering constraints. 6196 * 6197 * sq_putcount only needs to be checked when dispatching the queued 6198 * writer call for CIPUT sync queue, but this is handled in sq_run_events. 6199 */ 6200 void 6201 drain_syncq(syncq_t *sq) 6202 { 6203 queue_t *qp; 6204 uint16_t count; 6205 uint16_t type = sq->sq_type; 6206 uint16_t flags = sq->sq_flags; 6207 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; 6208 6209 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6210 "drain_syncq start:%p", sq); 6211 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6212 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6213 sq->sq_oprev == NULL) || 6214 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6215 sq->sq_oprev != NULL)); 6216 6217 /* 6218 * Drop SQ_SERVICE flag. 6219 */ 6220 if (bg_service) 6221 sq->sq_svcflags &= ~SQ_SERVICE; 6222 6223 /* 6224 * If SQ_EXCL is set, someone else is processing this syncq - let him 6225 * finish the job. 6226 */ 6227 if (flags & SQ_EXCL) { 6228 if (bg_service) { 6229 ASSERT(sq->sq_servcount != 0); 6230 sq->sq_servcount--; 6231 } 6232 mutex_exit(SQLOCK(sq)); 6233 return; 6234 } 6235 6236 /* 6237 * This routine can be called by a background thread if 6238 * it was scheduled by a hi-priority thread. SO, if there are 6239 * NOT messages queued, return (remember, we have the SQLOCK, 6240 * and it cannot change until we release it). Wakeup any waiters also. 6241 */ 6242 if (!(flags & SQ_QUEUED)) { 6243 if (flags & SQ_WANTWAKEUP) { 6244 flags &= ~SQ_WANTWAKEUP; 6245 cv_broadcast(&sq->sq_wait); 6246 } 6247 if (flags & SQ_WANTEXWAKEUP) { 6248 flags &= ~SQ_WANTEXWAKEUP; 6249 cv_broadcast(&sq->sq_exitwait); 6250 } 6251 sq->sq_flags = flags; 6252 if (bg_service) { 6253 ASSERT(sq->sq_servcount != 0); 6254 sq->sq_servcount--; 6255 } 6256 mutex_exit(SQLOCK(sq)); 6257 return; 6258 } 6259 6260 /* 6261 * If this is not a concurrent put perimiter, we need to 6262 * become exclusive to drain. Also, if not CIPUT, we would 6263 * not have acquired a putlock, so we don't need to check 6264 * the putcounts. If not entering with a claim, we test 6265 * for sq_count == 0. 6266 */ 6267 type = sq->sq_type; 6268 if (!(type & SQ_CIPUT)) { 6269 if (sq->sq_count > 1) { 6270 if (bg_service) { 6271 ASSERT(sq->sq_servcount != 0); 6272 sq->sq_servcount--; 6273 } 6274 mutex_exit(SQLOCK(sq)); 6275 return; 6276 } 6277 sq->sq_flags |= SQ_EXCL; 6278 } 6279 6280 /* 6281 * This is where we make a claim to the syncq. 6282 * This can either be done by incrementing a putlock, or 6283 * the sq_count. But since we already have the SQLOCK 6284 * here, we just bump the sq_count. 6285 * 6286 * Note that after we make a claim, we need to let the code 6287 * fall through to the end of this routine to clean itself 6288 * up. A return in the while loop will put the syncq in a 6289 * very bad state. 6290 */ 6291 sq->sq_count++; 6292 ASSERT(sq->sq_count != 0); /* wraparound */ 6293 6294 while ((flags = sq->sq_flags) & SQ_QUEUED) { 6295 /* 6296 * If we are told to stayaway or went exclusive, 6297 * we are done. 6298 */ 6299 if (flags & (SQ_STAYAWAY)) { 6300 break; 6301 } 6302 6303 /* 6304 * If there are events to run, do so. 6305 * We have one claim to the syncq, so if there are 6306 * more than one, other threads are running. 6307 */ 6308 if (sq->sq_evhead != NULL) { 6309 ASSERT(sq->sq_flags & SQ_EVENTS); 6310 6311 count = sq->sq_count; 6312 SQ_PUTLOCKS_ENTER(sq); 6313 SUM_SQ_PUTCOUNTS(sq, count); 6314 if (count > 1) { 6315 SQ_PUTLOCKS_EXIT(sq); 6316 /* Can't upgrade - other threads inside */ 6317 break; 6318 } 6319 ASSERT((flags & SQ_EXCL) == 0); 6320 sq->sq_flags = flags | SQ_EXCL; 6321 SQ_PUTLOCKS_EXIT(sq); 6322 /* 6323 * we have the only claim, run the events, 6324 * sq_run_events will clear the SQ_EXCL flag. 6325 */ 6326 sq_run_events(sq); 6327 6328 /* 6329 * If this is a CIPUT perimiter, we need 6330 * to drop the SQ_EXCL flag so we can properly 6331 * continue draining the syncq. 6332 */ 6333 if (type & SQ_CIPUT) { 6334 ASSERT(sq->sq_flags & SQ_EXCL); 6335 sq->sq_flags &= ~SQ_EXCL; 6336 } 6337 6338 /* 6339 * And go back to the beginning just in case 6340 * anything changed while we were away. 6341 */ 6342 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); 6343 continue; 6344 } 6345 6346 ASSERT(sq->sq_evhead == NULL); 6347 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6348 6349 /* 6350 * Find the queue that is not draining. 6351 * 6352 * q_draining is protected by QLOCK which we do not hold. 6353 * But if it was set, then a thread was draining, and if it gets 6354 * cleared, then it was because the thread has successfully 6355 * drained the syncq, or a GOAWAY state occured. For the GOAWAY 6356 * state to happen, a thread needs the SQLOCK which we hold, and 6357 * if there was such a flag, we whould have already seen it. 6358 */ 6359 6360 for (qp = sq->sq_head; 6361 qp != NULL && (qp->q_draining || 6362 (qp->q_sqflags & Q_SQDRAINING)); 6363 qp = qp->q_sqnext) 6364 ; 6365 6366 if (qp == NULL) 6367 break; 6368 6369 /* 6370 * We have a queue to work on, and we hold the 6371 * SQLOCK and one claim, call qdrain_syncq. 6372 * This means we need to release the SQLOCK and 6373 * aquire the QLOCK (OK since we have a claim). 6374 * Note that qdrain_syncq will actually dequeue 6375 * this queue from the sq_head list when it is 6376 * convinced all the work is done and release 6377 * the QLOCK before returning. 6378 */ 6379 qp->q_sqflags |= Q_SQDRAINING; 6380 mutex_exit(SQLOCK(sq)); 6381 mutex_enter(QLOCK(qp)); 6382 qdrain_syncq(sq, qp); 6383 mutex_enter(SQLOCK(sq)); 6384 6385 /* The queue is drained */ 6386 ASSERT(qp->q_sqflags & Q_SQDRAINING); 6387 qp->q_sqflags &= ~Q_SQDRAINING; 6388 /* 6389 * NOTE: After this point qp should not be used since it may be 6390 * closed. 6391 */ 6392 } 6393 6394 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6395 flags = sq->sq_flags; 6396 6397 /* 6398 * sq->sq_head cannot change because we hold the 6399 * sqlock. However, a thread CAN decide that it is no longer 6400 * going to drain that queue. However, this should be due to 6401 * a GOAWAY state, and we should see that here. 6402 * 6403 * This loop is not very efficient. One solution may be adding a second 6404 * pointer to the "draining" queue, but it is difficult to do when 6405 * queues are inserted in the middle due to priority ordering. Another 6406 * possibility is to yank the queue out of the sq list and put it onto 6407 * the "draining list" and then put it back if it can't be drained. 6408 */ 6409 6410 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || 6411 (type & SQ_CI) || sq->sq_head->q_draining); 6412 6413 /* Drop SQ_EXCL for non-CIPUT perimiters */ 6414 if (!(type & SQ_CIPUT)) 6415 flags &= ~SQ_EXCL; 6416 ASSERT((flags & SQ_EXCL) == 0); 6417 6418 /* Wake up any waiters. */ 6419 if (flags & SQ_WANTWAKEUP) { 6420 flags &= ~SQ_WANTWAKEUP; 6421 cv_broadcast(&sq->sq_wait); 6422 } 6423 if (flags & SQ_WANTEXWAKEUP) { 6424 flags &= ~SQ_WANTEXWAKEUP; 6425 cv_broadcast(&sq->sq_exitwait); 6426 } 6427 sq->sq_flags = flags; 6428 6429 ASSERT(sq->sq_count != 0); 6430 /* Release our claim. */ 6431 sq->sq_count--; 6432 6433 if (bg_service) { 6434 ASSERT(sq->sq_servcount != 0); 6435 sq->sq_servcount--; 6436 } 6437 6438 mutex_exit(SQLOCK(sq)); 6439 6440 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6441 "drain_syncq end:%p", sq); 6442 } 6443 6444 6445 /* 6446 * 6447 * qdrain_syncq can be called (currently) from only one of two places: 6448 * drain_syncq 6449 * putnext (or some variation of it). 6450 * and eventually 6451 * qwait(_sig) 6452 * 6453 * If called from drain_syncq, we found it in the list 6454 * of queue's needing service, so there is work to be done (or it 6455 * wouldn't be on the list). 6456 * 6457 * If called from some putnext variation, it was because the 6458 * perimiter is open, but messages are blocking a putnext and 6459 * there is not a thread working on it. Now a thread could start 6460 * working on it while we are getting ready to do so ourself, but 6461 * the thread would set the q_draining flag, and we can spin out. 6462 * 6463 * As for qwait(_sig), I think I shall let it continue to call 6464 * drain_syncq directly (after all, it will get here eventually). 6465 * 6466 * qdrain_syncq has to terminate when: 6467 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering 6468 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering 6469 * 6470 * ASSUMES: 6471 * One claim 6472 * QLOCK held 6473 * SQLOCK not held 6474 * Will release QLOCK before returning 6475 */ 6476 void 6477 qdrain_syncq(syncq_t *sq, queue_t *q) 6478 { 6479 mblk_t *bp; 6480 boolean_t do_clr; 6481 #ifdef DEBUG 6482 uint16_t count; 6483 #endif 6484 6485 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6486 "drain_syncq start:%p", sq); 6487 ASSERT(q->q_syncq == sq); 6488 ASSERT(MUTEX_HELD(QLOCK(q))); 6489 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6490 /* 6491 * For non-CIPUT perimiters, we should be called with the 6492 * exclusive bit set already. For non-CIPUT perimiters we 6493 * will be doing a concurrent drain, so it better not be set. 6494 */ 6495 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); 6496 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); 6497 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); 6498 /* 6499 * All outer pointers are set, or none of them are 6500 */ 6501 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6502 sq->sq_oprev == NULL) || 6503 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6504 sq->sq_oprev != NULL)); 6505 #ifdef DEBUG 6506 count = sq->sq_count; 6507 /* 6508 * This is OK without the putlocks, because we have one 6509 * claim either from the sq_count, or a putcount. We could 6510 * get an erroneous value from other counts, but ours won't 6511 * change, so one way or another, we will have at least a 6512 * value of one. 6513 */ 6514 SUM_SQ_PUTCOUNTS(sq, count); 6515 ASSERT(count >= 1); 6516 #endif /* DEBUG */ 6517 6518 /* 6519 * The first thing to do here, is find out if a thread is already 6520 * draining this queue or the queue is closing. If so, we are done, 6521 * just return. Also, if there are no messages, we are done as well. 6522 * Note that we check the q_sqhead since there is s window of 6523 * opportunity for us to enter here because Q_SQQUEUED was set, but is 6524 * not anymore. 6525 */ 6526 if (q->q_draining || (q->q_sqhead == NULL)) { 6527 mutex_exit(QLOCK(q)); 6528 return; 6529 } 6530 6531 /* 6532 * If the perimiter is exclusive, there is nothing we can 6533 * do right now, go away. 6534 * Note that there is nothing to prevent this case from changing 6535 * right after this check, but the spin-out will catch it. 6536 */ 6537 6538 /* Tell other threads that we are draining this queue */ 6539 q->q_draining = 1; /* Protected by QLOCK */ 6540 6541 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) { 6542 6543 /* 6544 * Because we can enter this routine just because 6545 * a putnext is blocked, we need to spin out if 6546 * the perimiter wants to go exclusive as well 6547 * as just blocked. We need to spin out also if 6548 * events are queued on the syncq. 6549 * Don't check for SQ_EXCL, because non-CIPUT 6550 * perimiters would set it, and it can't become 6551 * exclusive while we hold a claim. 6552 */ 6553 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { 6554 break; 6555 } 6556 6557 #ifdef DEBUG 6558 /* 6559 * Since we are in qdrain_syncq, we already know the queue, 6560 * but for sanity, we want to check this against the qp that 6561 * was passed in by bp->b_queue. 6562 */ 6563 6564 ASSERT(bp->b_queue == q); 6565 ASSERT(bp->b_queue->q_syncq == sq); 6566 bp->b_queue = NULL; 6567 6568 /* 6569 * We would have the following check in the DEBUG code: 6570 * 6571 * if (bp->b_prev != NULL) { 6572 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp); 6573 * } 6574 * 6575 * This can't be done, however, since IP modifies qinfo 6576 * structure at run-time (switching between IPv4 qinfo and IPv6 6577 * qinfo), invalidating the check. 6578 * So the assignment to func is left here, but the ASSERT itself 6579 * is removed until the whole issue is resolved. 6580 */ 6581 #endif 6582 ASSERT(q->q_sqhead == bp); 6583 q->q_sqhead = bp->b_next; 6584 bp->b_prev = bp->b_next = NULL; 6585 ASSERT(q->q_syncqmsgs > 0); 6586 mutex_exit(QLOCK(q)); 6587 6588 ASSERT(bp->b_datap->db_ref != 0); 6589 6590 (void) (*q->q_qinfo->qi_putp)(q, bp); 6591 6592 mutex_enter(QLOCK(q)); 6593 /* 6594 * We should decrement q_syncqmsgs only after executing the 6595 * put procedure to avoid a possible race with putnext(). 6596 * In putnext() though it sees Q_SQQUEUED is set, there is 6597 * an optimization which allows putnext to call the put 6598 * procedure directly if (q_syncqmsgs == 0) and thus 6599 * a message reodering could otherwise occur. 6600 */ 6601 q->q_syncqmsgs--; 6602 6603 /* 6604 * Clear QFULL in the next service procedure queue if 6605 * this is the last message destined to that queue. 6606 * 6607 * It would make better sense to have some sort of 6608 * tunable for the low water mark, but these symantics 6609 * are not yet defined. So, alas, we use a constant. 6610 */ 6611 do_clr = (q->q_syncqmsgs == 0); 6612 mutex_exit(QLOCK(q)); 6613 6614 if (do_clr) 6615 clr_qfull(q); 6616 6617 mutex_enter(QLOCK(q)); 6618 /* 6619 * Always clear SQ_EXCL when CIPUT in order to handle 6620 * qwriter(INNER). 6621 */ 6622 /* 6623 * The putp() can call qwriter and get exclusive access 6624 * IFF this is the only claim. So, we need to test for 6625 * this possibility so we can aquire the mutex and clear 6626 * the bit. 6627 */ 6628 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { 6629 mutex_enter(SQLOCK(sq)); 6630 sq->sq_flags &= ~SQ_EXCL; 6631 mutex_exit(SQLOCK(sq)); 6632 } 6633 } 6634 6635 /* 6636 * We should either have no queues on the syncq, or we were 6637 * told to goaway by a waiter (which we will wake up at the 6638 * end of this function). 6639 */ 6640 ASSERT((q->q_sqhead == NULL) || 6641 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); 6642 6643 ASSERT(MUTEX_HELD(QLOCK(q))); 6644 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6645 6646 /* 6647 * Remove the q from the syncq list if all the messages are 6648 * drained. 6649 */ 6650 if (q->q_sqhead == NULL) { 6651 mutex_enter(SQLOCK(sq)); 6652 if (q->q_sqflags & Q_SQQUEUED) 6653 SQRM_Q(sq, q); 6654 mutex_exit(SQLOCK(sq)); 6655 /* 6656 * Since the queue is removed from the list, reset its priority. 6657 */ 6658 q->q_spri = 0; 6659 } 6660 6661 /* 6662 * Remember, the q_draining flag is used to let another 6663 * thread know that there is a thread currently draining 6664 * the messages for a queue. Since we are now done with 6665 * this queue (even if there may be messages still there), 6666 * we need to clear this flag so some thread will work 6667 * on it if needed. 6668 */ 6669 ASSERT(q->q_draining); 6670 q->q_draining = 0; 6671 6672 /* called with a claim, so OK to drop all locks. */ 6673 mutex_exit(QLOCK(q)); 6674 6675 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6676 "drain_syncq end:%p", sq); 6677 } 6678 /* END OF QDRAIN_SYNCQ */ 6679 6680 6681 /* 6682 * This is the mate to qdrain_syncq, except that it is putting the 6683 * message onto the the queue instead draining. Since the 6684 * message is destined for the queue that is selected, there is 6685 * no need to identify the function because the message is 6686 * intended for the put routine for the queue. But this 6687 * routine will do it anyway just in case (but only for debug kernels). 6688 * 6689 * After the message is enqueued on the syncq, it calls putnext_tail() 6690 * which will schedule a background thread to actually process the message. 6691 * 6692 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and 6693 * SQLOCK(sq) and QLOCK(q) are not held. 6694 */ 6695 void 6696 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) 6697 { 6698 queue_t *fq = NULL; 6699 6700 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6701 ASSERT(MUTEX_NOT_HELD(QLOCK(q))); 6702 ASSERT(sq->sq_count > 0); 6703 ASSERT(q->q_syncq == sq); 6704 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6705 sq->sq_oprev == NULL) || 6706 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6707 sq->sq_oprev != NULL)); 6708 6709 mutex_enter(QLOCK(q)); 6710 6711 /* 6712 * Set QFULL in next service procedure queue (that cares) if not 6713 * already set and if there are already more messages on the syncq 6714 * than sq_max_size. If sq_max_size is 0, no flow control will be 6715 * asserted on any syncq. 6716 * 6717 * The fq here is the next queue with a service procedure. 6718 * This is where we would fail canputnext, so this is where we 6719 * need to set QFULL. 6720 * 6721 * LOCKING HIERARCHY: In the case when fq != q we need to 6722 * a) Take QLOCK(fq) to set QFULL flag and 6723 * b) Take sd_reflock in the case of the hot stream to update 6724 * sd_refcnt. 6725 * We already have QLOCK at this point. To avoid cross-locks with 6726 * freezestr() which grabs all QLOCKs and with strlock() which grabs 6727 * both SQLOCK and sd_reflock, we need to drop respective locks first. 6728 */ 6729 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) && 6730 (q->q_syncqmsgs > sq_max_size)) { 6731 if ((fq = q->q_nfsrv) == q) { 6732 fq->q_flag |= QFULL; 6733 } else { 6734 mutex_exit(QLOCK(q)); 6735 mutex_enter(QLOCK(fq)); 6736 fq->q_flag |= QFULL; 6737 mutex_exit(QLOCK(fq)); 6738 mutex_enter(QLOCK(q)); 6739 } 6740 } 6741 6742 #ifdef DEBUG 6743 /* 6744 * This is used for debug in the qfill_syncq/qdrain_syncq case 6745 * to trace the queue that the message is intended for. Note 6746 * that the original use was to identify the queue and function 6747 * to call on the drain. In the new syncq, we have the context 6748 * of the queue that we are draining, so call it's putproc and 6749 * don't rely on the saved values. But for debug this is still 6750 * usefull information. 6751 */ 6752 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp; 6753 mp->b_queue = q; 6754 mp->b_next = NULL; 6755 #endif 6756 ASSERT(q->q_syncq == sq); 6757 /* 6758 * Enqueue the message on the list. 6759 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to 6760 * protect it. So its ok to acquire SQLOCK after SQPUT_MP(). 6761 */ 6762 SQPUT_MP(q, mp); 6763 mutex_enter(SQLOCK(sq)); 6764 6765 /* 6766 * And queue on syncq for scheduling, if not already queued. 6767 * Note that we need the SQLOCK for this, and for testing flags 6768 * at the end to see if we will drain. So grab it now, and 6769 * release it before we call qdrain_syncq or return. 6770 */ 6771 if (!(q->q_sqflags & Q_SQQUEUED)) { 6772 q->q_spri = curthread->t_pri; 6773 SQPUT_Q(sq, q); 6774 } 6775 #ifdef DEBUG 6776 else { 6777 /* 6778 * All of these conditions MUST be true! 6779 */ 6780 ASSERT(sq->sq_tail != NULL); 6781 if (sq->sq_tail == sq->sq_head) { 6782 ASSERT((q->q_sqprev == NULL) && 6783 (q->q_sqnext == NULL)); 6784 } else { 6785 ASSERT((q->q_sqprev != NULL) || 6786 (q->q_sqnext != NULL)); 6787 } 6788 ASSERT(sq->sq_flags & SQ_QUEUED); 6789 ASSERT(q->q_syncqmsgs != 0); 6790 ASSERT(q->q_sqflags & Q_SQQUEUED); 6791 } 6792 #endif 6793 mutex_exit(QLOCK(q)); 6794 /* 6795 * SQLOCK is still held, so sq_count can be safely decremented. 6796 */ 6797 sq->sq_count--; 6798 6799 putnext_tail(sq, q, 0); 6800 /* Should not reference sq or q after this point. */ 6801 } 6802 6803 /* End of qfill_syncq */ 6804 6805 /* 6806 * Remove all messages from a syncq (if qp is NULL) or remove all messages 6807 * that would be put into qp by drain_syncq. 6808 * Used when deleting the syncq (qp == NULL) or when detaching 6809 * a queue (qp != NULL). 6810 * Return non-zero if one or more messages were freed. 6811 * 6812 * no need to grab sq_putlocks here. See comment in strsubr.h that explains when 6813 * sq_putlocks are used. 6814 * 6815 * NOTE: This function assumes that it is called from the close() context and 6816 * that all the queues in the syncq are going aay. For this reason it doesn't 6817 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is 6818 * currently valid, but it is useful to rethink this function to behave properly 6819 * in other cases. 6820 */ 6821 int 6822 flush_syncq(syncq_t *sq, queue_t *qp) 6823 { 6824 mblk_t *bp, *mp_head, *mp_next, *mp_prev; 6825 queue_t *q; 6826 int ret = 0; 6827 6828 mutex_enter(SQLOCK(sq)); 6829 6830 /* 6831 * Before we leave, we need to make sure there are no 6832 * events listed for this queue. All events for this queue 6833 * will just be freed. 6834 */ 6835 if (qp != NULL && sq->sq_evhead != NULL) { 6836 ASSERT(sq->sq_flags & SQ_EVENTS); 6837 6838 mp_prev = NULL; 6839 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { 6840 mp_next = bp->b_next; 6841 if (bp->b_queue == qp) { 6842 /* Delete this message */ 6843 if (mp_prev != NULL) { 6844 mp_prev->b_next = mp_next; 6845 /* 6846 * Update sq_evtail if the last element 6847 * is removed. 6848 */ 6849 if (bp == sq->sq_evtail) { 6850 ASSERT(mp_next == NULL); 6851 sq->sq_evtail = mp_prev; 6852 } 6853 } else 6854 sq->sq_evhead = mp_next; 6855 if (sq->sq_evhead == NULL) 6856 sq->sq_flags &= ~SQ_EVENTS; 6857 bp->b_prev = bp->b_next = NULL; 6858 freemsg(bp); 6859 ret++; 6860 } else { 6861 mp_prev = bp; 6862 } 6863 } 6864 } 6865 6866 /* 6867 * Walk sq_head and: 6868 * - match qp if qp is set, remove it's messages 6869 * - all if qp is not set 6870 */ 6871 q = sq->sq_head; 6872 while (q != NULL) { 6873 ASSERT(q->q_syncq == sq); 6874 if ((qp == NULL) || (qp == q)) { 6875 /* 6876 * Yank the messages as a list off the queue 6877 */ 6878 mp_head = q->q_sqhead; 6879 /* 6880 * We do not have QLOCK(q) here (which is safe due to 6881 * assumptions mentioned above). To obtain the lock we 6882 * need to release SQLOCK which may allow lots of things 6883 * to change upon us. This place requires more analysis. 6884 */ 6885 q->q_sqhead = q->q_sqtail = NULL; 6886 ASSERT(mp_head->b_queue && 6887 mp_head->b_queue->q_syncq == sq); 6888 6889 /* 6890 * Free each of the messages. 6891 */ 6892 for (bp = mp_head; bp != NULL; bp = mp_next) { 6893 mp_next = bp->b_next; 6894 bp->b_prev = bp->b_next = NULL; 6895 freemsg(bp); 6896 ret++; 6897 } 6898 /* 6899 * Now remove the queue from the syncq. 6900 */ 6901 ASSERT(q->q_sqflags & Q_SQQUEUED); 6902 SQRM_Q(sq, q); 6903 q->q_spri = 0; 6904 q->q_syncqmsgs = 0; 6905 6906 /* 6907 * If qp was specified, we are done with it and are 6908 * going to drop SQLOCK(sq) and return. We wakeup syncq 6909 * waiters while we still have the SQLOCK. 6910 */ 6911 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { 6912 sq->sq_flags &= ~SQ_WANTWAKEUP; 6913 cv_broadcast(&sq->sq_wait); 6914 } 6915 /* Drop SQLOCK across clr_qfull */ 6916 mutex_exit(SQLOCK(sq)); 6917 6918 /* 6919 * We avoid doing the test that drain_syncq does and 6920 * unconditionally clear qfull for every flushed 6921 * message. Since flush_syncq is only called during 6922 * close this should not be a problem. 6923 */ 6924 clr_qfull(q); 6925 if (qp != NULL) { 6926 return (ret); 6927 } else { 6928 mutex_enter(SQLOCK(sq)); 6929 /* 6930 * The head was removed by SQRM_Q above. 6931 * reread the new head and flush it. 6932 */ 6933 q = sq->sq_head; 6934 } 6935 } else { 6936 q = q->q_sqnext; 6937 } 6938 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6939 } 6940 6941 if (sq->sq_flags & SQ_WANTWAKEUP) { 6942 sq->sq_flags &= ~SQ_WANTWAKEUP; 6943 cv_broadcast(&sq->sq_wait); 6944 } 6945 6946 mutex_exit(SQLOCK(sq)); 6947 return (ret); 6948 } 6949 6950 /* 6951 * Propagate all messages from a syncq to the next syncq that are associated 6952 * with the specified queue. If the queue is attached to a driver or if the 6953 * messages have been added due to a qwriter(PERIM_INNER), free the messages. 6954 * 6955 * Assumes that the stream is strlock()'ed. We don't come here if there 6956 * are no messages to propagate. 6957 * 6958 * NOTE : If the queue is attached to a driver, all the messages are freed 6959 * as there is no point in propagating the messages from the driver syncq 6960 * to the closing stream head which will in turn get freed later. 6961 */ 6962 static int 6963 propagate_syncq(queue_t *qp) 6964 { 6965 mblk_t *bp, *head, *tail, *prev, *next; 6966 syncq_t *sq; 6967 queue_t *nqp; 6968 syncq_t *nsq; 6969 boolean_t isdriver; 6970 int moved = 0; 6971 uint16_t flags; 6972 pri_t priority = curthread->t_pri; 6973 #ifdef DEBUG 6974 void (*func)(); 6975 #endif 6976 6977 sq = qp->q_syncq; 6978 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6979 /* debug macro */ 6980 SQ_PUTLOCKS_HELD(sq); 6981 /* 6982 * As entersq() does not increment the sq_count for 6983 * the write side, check sq_count for non-QPERQ 6984 * perimeters alone. 6985 */ 6986 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); 6987 6988 /* 6989 * propagate_syncq() can be called because of either messages on the 6990 * queue syncq or because on events on the queue syncq. Do actual 6991 * message propagations if there are any messages. 6992 */ 6993 if (qp->q_syncqmsgs) { 6994 isdriver = (qp->q_flag & QISDRV); 6995 6996 if (!isdriver) { 6997 nqp = qp->q_next; 6998 nsq = nqp->q_syncq; 6999 ASSERT(MUTEX_HELD(SQLOCK(nsq))); 7000 /* debug macro */ 7001 SQ_PUTLOCKS_HELD(nsq); 7002 #ifdef DEBUG 7003 func = (void (*)())nqp->q_qinfo->qi_putp; 7004 #endif 7005 } 7006 7007 SQRM_Q(sq, qp); 7008 priority = MAX(qp->q_spri, priority); 7009 qp->q_spri = 0; 7010 head = qp->q_sqhead; 7011 tail = qp->q_sqtail; 7012 qp->q_sqhead = qp->q_sqtail = NULL; 7013 qp->q_syncqmsgs = 0; 7014 7015 /* 7016 * Walk the list of messages, and free them if this is a driver, 7017 * otherwise reset the b_prev and b_queue value to the new putp. 7018 * Afterward, we will just add the head to the end of the next 7019 * syncq, and point the tail to the end of this one. 7020 */ 7021 7022 for (bp = head; bp != NULL; bp = next) { 7023 next = bp->b_next; 7024 if (isdriver) { 7025 bp->b_prev = bp->b_next = NULL; 7026 freemsg(bp); 7027 continue; 7028 } 7029 /* Change the q values for this message */ 7030 bp->b_queue = nqp; 7031 #ifdef DEBUG 7032 bp->b_prev = (mblk_t *)func; 7033 #endif 7034 moved++; 7035 } 7036 /* 7037 * Attach list of messages to the end of the new queue (if there 7038 * is a list of messages). 7039 */ 7040 7041 if (!isdriver && head != NULL) { 7042 ASSERT(tail != NULL); 7043 if (nqp->q_sqhead == NULL) { 7044 nqp->q_sqhead = head; 7045 } else { 7046 ASSERT(nqp->q_sqtail != NULL); 7047 nqp->q_sqtail->b_next = head; 7048 } 7049 nqp->q_sqtail = tail; 7050 /* 7051 * When messages are moved from high priority queue to 7052 * another queue, the destination queue priority is 7053 * upgraded. 7054 */ 7055 7056 if (priority > nqp->q_spri) 7057 nqp->q_spri = priority; 7058 7059 SQPUT_Q(nsq, nqp); 7060 7061 nqp->q_syncqmsgs += moved; 7062 ASSERT(nqp->q_syncqmsgs != 0); 7063 } 7064 } 7065 7066 /* 7067 * Before we leave, we need to make sure there are no 7068 * events listed for this queue. All events for this queue 7069 * will just be freed. 7070 */ 7071 if (sq->sq_evhead != NULL) { 7072 ASSERT(sq->sq_flags & SQ_EVENTS); 7073 prev = NULL; 7074 for (bp = sq->sq_evhead; bp != NULL; bp = next) { 7075 next = bp->b_next; 7076 if (bp->b_queue == qp) { 7077 /* Delete this message */ 7078 if (prev != NULL) { 7079 prev->b_next = next; 7080 /* 7081 * Update sq_evtail if the last element 7082 * is removed. 7083 */ 7084 if (bp == sq->sq_evtail) { 7085 ASSERT(next == NULL); 7086 sq->sq_evtail = prev; 7087 } 7088 } else 7089 sq->sq_evhead = next; 7090 if (sq->sq_evhead == NULL) 7091 sq->sq_flags &= ~SQ_EVENTS; 7092 bp->b_prev = bp->b_next = NULL; 7093 freemsg(bp); 7094 } else { 7095 prev = bp; 7096 } 7097 } 7098 } 7099 7100 flags = sq->sq_flags; 7101 7102 /* Wake up any waiter before leaving. */ 7103 if (flags & SQ_WANTWAKEUP) { 7104 flags &= ~SQ_WANTWAKEUP; 7105 cv_broadcast(&sq->sq_wait); 7106 } 7107 sq->sq_flags = flags; 7108 7109 return (moved); 7110 } 7111 7112 /* 7113 * Try and upgrade to exclusive access at the inner perimeter. If this can 7114 * not be done without blocking then request will be queued on the syncq 7115 * and drain_syncq will run it later. 7116 * 7117 * This routine can only be called from put or service procedures plus 7118 * asynchronous callback routines that have properly entered to 7119 * queue (with entersq.) Thus qwriter_inner assumes the caller has one claim 7120 * on the syncq associated with q. 7121 */ 7122 void 7123 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)()) 7124 { 7125 syncq_t *sq = q->q_syncq; 7126 uint16_t count; 7127 7128 mutex_enter(SQLOCK(sq)); 7129 count = sq->sq_count; 7130 SQ_PUTLOCKS_ENTER(sq); 7131 SUM_SQ_PUTCOUNTS(sq, count); 7132 ASSERT(count >= 1); 7133 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); 7134 7135 if (count == 1) { 7136 /* 7137 * Can upgrade. This case also handles nested qwriter calls 7138 * (when the qwriter callback function calls qwriter). In that 7139 * case SQ_EXCL is already set. 7140 */ 7141 sq->sq_flags |= SQ_EXCL; 7142 SQ_PUTLOCKS_EXIT(sq); 7143 mutex_exit(SQLOCK(sq)); 7144 (*func)(q, mp); 7145 /* 7146 * Assumes that leavesq, putnext, and drain_syncq will reset 7147 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on 7148 * until putnext, leavesq, or drain_syncq drops it. 7149 * That way we handle nested qwriter(INNER) without dropping 7150 * SQ_EXCL until the outermost qwriter callback routine is 7151 * done. 7152 */ 7153 return; 7154 } 7155 SQ_PUTLOCKS_EXIT(sq); 7156 sqfill_events(sq, q, mp, func); 7157 } 7158 7159 /* 7160 * Synchronous callback support functions 7161 */ 7162 7163 /* 7164 * Allocate a callback parameter structure. 7165 * Assumes that caller initializes the flags and the id. 7166 * Acquires SQLOCK(sq) if non-NULL is returned. 7167 */ 7168 callbparams_t * 7169 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) 7170 { 7171 callbparams_t *cbp; 7172 size_t size = sizeof (callbparams_t); 7173 7174 cbp = kmem_alloc(size, kmflags & ~KM_PANIC); 7175 7176 /* 7177 * Only try tryhard allocation if the caller is ready to panic. 7178 * Otherwise just fail. 7179 */ 7180 if (cbp == NULL) { 7181 if (kmflags & KM_PANIC) 7182 cbp = kmem_alloc_tryhard(sizeof (callbparams_t), 7183 &size, kmflags); 7184 else 7185 return (NULL); 7186 } 7187 7188 ASSERT(size >= sizeof (callbparams_t)); 7189 cbp->cbp_size = size; 7190 cbp->cbp_sq = sq; 7191 cbp->cbp_func = func; 7192 cbp->cbp_arg = arg; 7193 mutex_enter(SQLOCK(sq)); 7194 cbp->cbp_next = sq->sq_callbpend; 7195 sq->sq_callbpend = cbp; 7196 return (cbp); 7197 } 7198 7199 void 7200 callbparams_free(syncq_t *sq, callbparams_t *cbp) 7201 { 7202 callbparams_t **pp, *p; 7203 7204 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7205 7206 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7207 if (p == cbp) { 7208 *pp = p->cbp_next; 7209 kmem_free(p, p->cbp_size); 7210 return; 7211 } 7212 } 7213 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7214 "callbparams_free: not found\n")); 7215 } 7216 7217 void 7218 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) 7219 { 7220 callbparams_t **pp, *p; 7221 7222 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7223 7224 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7225 if (p->cbp_id == id && p->cbp_flags == flag) { 7226 *pp = p->cbp_next; 7227 kmem_free(p, p->cbp_size); 7228 return; 7229 } 7230 } 7231 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7232 "callbparams_free_id: not found\n")); 7233 } 7234 7235 /* 7236 * Callback wrapper function used by once-only callbacks that can be 7237 * cancelled (qtimeout and qbufcall) 7238 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be 7239 * cancelled by the qun* functions. 7240 */ 7241 void 7242 qcallbwrapper(void *arg) 7243 { 7244 callbparams_t *cbp = arg; 7245 syncq_t *sq; 7246 uint16_t count = 0; 7247 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 7248 uint16_t type; 7249 7250 sq = cbp->cbp_sq; 7251 mutex_enter(SQLOCK(sq)); 7252 type = sq->sq_type; 7253 if (!(type & SQ_CICB)) { 7254 count = sq->sq_count; 7255 SQ_PUTLOCKS_ENTER(sq); 7256 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 7257 SUM_SQ_PUTCOUNTS(sq, count); 7258 sq->sq_needexcl++; 7259 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 7260 waitflags |= SQ_MESSAGES; 7261 } 7262 /* Can not handle exlusive entry at outer perimeter */ 7263 ASSERT(type & SQ_COCB); 7264 7265 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { 7266 if ((sq->sq_callbflags & cbp->cbp_flags) && 7267 (sq->sq_cancelid == cbp->cbp_id)) { 7268 /* timeout has been cancelled */ 7269 sq->sq_callbflags |= SQ_CALLB_BYPASSED; 7270 callbparams_free(sq, cbp); 7271 if (!(type & SQ_CICB)) { 7272 ASSERT(sq->sq_needexcl > 0); 7273 sq->sq_needexcl--; 7274 if (sq->sq_needexcl == 0) { 7275 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7276 } 7277 SQ_PUTLOCKS_EXIT(sq); 7278 } 7279 mutex_exit(SQLOCK(sq)); 7280 return; 7281 } 7282 sq->sq_flags |= SQ_WANTWAKEUP; 7283 if (!(type & SQ_CICB)) { 7284 SQ_PUTLOCKS_EXIT(sq); 7285 } 7286 cv_wait(&sq->sq_wait, SQLOCK(sq)); 7287 if (!(type & SQ_CICB)) { 7288 count = sq->sq_count; 7289 SQ_PUTLOCKS_ENTER(sq); 7290 SUM_SQ_PUTCOUNTS(sq, count); 7291 } 7292 } 7293 7294 sq->sq_count++; 7295 ASSERT(sq->sq_count != 0); /* Wraparound */ 7296 if (!(type & SQ_CICB)) { 7297 ASSERT(count == 0); 7298 sq->sq_flags |= SQ_EXCL; 7299 ASSERT(sq->sq_needexcl > 0); 7300 sq->sq_needexcl--; 7301 if (sq->sq_needexcl == 0) { 7302 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7303 } 7304 SQ_PUTLOCKS_EXIT(sq); 7305 } 7306 7307 mutex_exit(SQLOCK(sq)); 7308 7309 cbp->cbp_func(cbp->cbp_arg); 7310 7311 /* 7312 * We drop the lock only for leavesq to re-acquire it. 7313 * Possible optimization is inline of leavesq. 7314 */ 7315 mutex_enter(SQLOCK(sq)); 7316 callbparams_free(sq, cbp); 7317 mutex_exit(SQLOCK(sq)); 7318 leavesq(sq, SQ_CALLBACK); 7319 } 7320 7321 /* 7322 * no need to grab sq_putlocks here. See comment in strsubr.h that 7323 * explains when sq_putlocks are used. 7324 * 7325 * sq_count (or one of the sq_putcounts) has already been 7326 * decremented by the caller, and if SQ_QUEUED, we need to call 7327 * drain_syncq (the global syncq drain). 7328 * If putnext_tail is called with the SQ_EXCL bit set, we are in 7329 * one of two states, non-CIPUT perimiter, and we need to clear 7330 * it, or we went exclusive in the put procedure. In any case, 7331 * we want to clear the bit now, and it is probably easier to do 7332 * this at the beginning of this function (remember, we hold 7333 * the SQLOCK). Lastly, if there are other messages queued 7334 * on the syncq (and not for our destination), enable the syncq 7335 * for background work. 7336 */ 7337 7338 /* ARGSUSED */ 7339 void 7340 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) 7341 { 7342 uint16_t flags = sq->sq_flags; 7343 7344 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7345 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 7346 7347 /* Clear SQ_EXCL if set in passflags */ 7348 if (passflags & SQ_EXCL) { 7349 flags &= ~SQ_EXCL; 7350 } 7351 if (flags & SQ_WANTWAKEUP) { 7352 flags &= ~SQ_WANTWAKEUP; 7353 cv_broadcast(&sq->sq_wait); 7354 } 7355 if (flags & SQ_WANTEXWAKEUP) { 7356 flags &= ~SQ_WANTEXWAKEUP; 7357 cv_broadcast(&sq->sq_exitwait); 7358 } 7359 sq->sq_flags = flags; 7360 7361 /* 7362 * We have cleared SQ_EXCL if we were asked to, and started 7363 * the wakeup process for waiters. If there are no writers 7364 * then we need to drain the syncq if we were told to, or 7365 * enable the background thread to do it. 7366 */ 7367 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) { 7368 if ((passflags & SQ_QUEUED) || 7369 (sq->sq_svcflags & SQ_DISABLED)) { 7370 /* drain_syncq will take care of events in the list */ 7371 drain_syncq(sq); 7372 return; 7373 } else if (flags & SQ_QUEUED) { 7374 sqenable(sq); 7375 } 7376 } 7377 /* Drop the SQLOCK on exit */ 7378 mutex_exit(SQLOCK(sq)); 7379 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END, 7380 "putnext_end:(%p, %p, %p) done", NULL, qp, sq); 7381 } 7382 7383 void 7384 set_qend(queue_t *q) 7385 { 7386 mutex_enter(QLOCK(q)); 7387 if (!O_SAMESTR(q)) 7388 q->q_flag |= QEND; 7389 else 7390 q->q_flag &= ~QEND; 7391 mutex_exit(QLOCK(q)); 7392 q = _OTHERQ(q); 7393 mutex_enter(QLOCK(q)); 7394 if (!O_SAMESTR(q)) 7395 q->q_flag |= QEND; 7396 else 7397 q->q_flag &= ~QEND; 7398 mutex_exit(QLOCK(q)); 7399 } 7400 7401 7402 void 7403 clr_qfull(queue_t *q) 7404 { 7405 queue_t *oq = q; 7406 7407 q = q->q_nfsrv; 7408 /* Fast check if there is any work to do before getting the lock. */ 7409 if ((q->q_flag & (QFULL|QWANTW)) == 0) { 7410 return; 7411 } 7412 7413 /* 7414 * Do not reset QFULL (and backenable) if the q_count is the reason 7415 * for QFULL being set. 7416 */ 7417 mutex_enter(QLOCK(q)); 7418 /* 7419 * If both q_count and q_mblkcnt are less than the hiwat mark 7420 */ 7421 if ((q->q_count < q->q_hiwat) && (q->q_mblkcnt < q->q_hiwat)) { 7422 q->q_flag &= ~QFULL; 7423 /* 7424 * A little more confusing, how about this way: 7425 * if someone wants to write, 7426 * AND 7427 * both counts are less than the lowat mark 7428 * OR 7429 * the lowat mark is zero 7430 * THEN 7431 * backenable 7432 */ 7433 if ((q->q_flag & QWANTW) && 7434 (((q->q_count < q->q_lowat) && 7435 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) { 7436 q->q_flag &= ~QWANTW; 7437 mutex_exit(QLOCK(q)); 7438 backenable(oq, 0); 7439 } else 7440 mutex_exit(QLOCK(q)); 7441 } else 7442 mutex_exit(QLOCK(q)); 7443 } 7444 7445 /* 7446 * Set the forward service procedure pointer. 7447 * 7448 * Called at insert-time to cache a queue's next forward service procedure in 7449 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted 7450 * has a service procedure then q_nfsrv points to itself. If the queue to be 7451 * inserted does not have a service procedure, then q_nfsrv points to the next 7452 * queue forward that has a service procedure. If the queue is at the logical 7453 * end of the stream (driver for write side, stream head for the read side) 7454 * and does not have a service procedure, then q_nfsrv also points to itself. 7455 */ 7456 void 7457 set_nfsrv_ptr( 7458 queue_t *rnew, /* read queue pointer to new module */ 7459 queue_t *wnew, /* write queue pointer to new module */ 7460 queue_t *prev_rq, /* read queue pointer to the module above */ 7461 queue_t *prev_wq) /* write queue pointer to the module above */ 7462 { 7463 queue_t *qp; 7464 7465 if (prev_wq->q_next == NULL) { 7466 /* 7467 * Insert the driver, initialize the driver and stream head. 7468 * In this case, prev_rq/prev_wq should be the stream head. 7469 * _I_INSERT does not allow inserting a driver. Make sure 7470 * that it is not an insertion. 7471 */ 7472 ASSERT(!(rnew->q_flag & _QINSERTING)); 7473 wnew->q_nfsrv = wnew; 7474 if (rnew->q_qinfo->qi_srvp) 7475 rnew->q_nfsrv = rnew; 7476 else 7477 rnew->q_nfsrv = prev_rq; 7478 prev_rq->q_nfsrv = prev_rq; 7479 prev_wq->q_nfsrv = prev_wq; 7480 } else { 7481 /* 7482 * set up read side q_nfsrv pointer. This MUST be done 7483 * before setting the write side, because the setting of 7484 * the write side for a fifo may depend on it. 7485 * 7486 * Suppose we have a fifo that only has pipemod pushed. 7487 * pipemod has no read or write service procedures, so 7488 * nfsrv for both pipemod queues points to prev_rq (the 7489 * stream read head). Now push bufmod (which has only a 7490 * read service procedure). Doing the write side first, 7491 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which 7492 * is WRONG; the next queue forward from wnew with a 7493 * service procedure will be rnew, not the stream read head. 7494 * Since the downstream queue (which in the case of a fifo 7495 * is the read queue rnew) can affect upstream queues, it 7496 * needs to be done first. Setting up the read side first 7497 * sets nfsrv for both pipemod queues to rnew and then 7498 * when the write side is set up, wnew-q_nfsrv will also 7499 * point to rnew. 7500 */ 7501 if (rnew->q_qinfo->qi_srvp) { 7502 /* 7503 * use _OTHERQ() because, if this is a pipe, next 7504 * module may have been pushed from other end and 7505 * q_next could be a read queue. 7506 */ 7507 qp = _OTHERQ(prev_wq->q_next); 7508 while (qp && qp->q_nfsrv != qp) { 7509 qp->q_nfsrv = rnew; 7510 qp = backq(qp); 7511 } 7512 rnew->q_nfsrv = rnew; 7513 } else 7514 rnew->q_nfsrv = prev_rq->q_nfsrv; 7515 7516 /* set up write side q_nfsrv pointer */ 7517 if (wnew->q_qinfo->qi_srvp) { 7518 wnew->q_nfsrv = wnew; 7519 7520 /* 7521 * For insertion, need to update nfsrv of the modules 7522 * above which do not have a service routine. 7523 */ 7524 if (rnew->q_flag & _QINSERTING) { 7525 for (qp = prev_wq; 7526 qp != NULL && qp->q_nfsrv != qp; 7527 qp = backq(qp)) { 7528 qp->q_nfsrv = wnew->q_nfsrv; 7529 } 7530 } 7531 } else { 7532 if (prev_wq->q_next == prev_rq) 7533 /* 7534 * Since prev_wq/prev_rq are the middle of a 7535 * fifo, wnew/rnew will also be the middle of 7536 * a fifo and wnew's nfsrv is same as rnew's. 7537 */ 7538 wnew->q_nfsrv = rnew->q_nfsrv; 7539 else 7540 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv; 7541 } 7542 } 7543 } 7544 7545 /* 7546 * Reset the forward service procedure pointer; called at remove-time. 7547 */ 7548 void 7549 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp) 7550 { 7551 queue_t *tmp_qp; 7552 7553 /* Reset the write side q_nfsrv pointer for _I_REMOVE */ 7554 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) { 7555 for (tmp_qp = backq(wqp); 7556 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp; 7557 tmp_qp = backq(tmp_qp)) { 7558 tmp_qp->q_nfsrv = wqp->q_nfsrv; 7559 } 7560 } 7561 7562 /* reset the read side q_nfsrv pointer */ 7563 if (rqp->q_qinfo->qi_srvp) { 7564 if (wqp->q_next) { /* non-driver case */ 7565 tmp_qp = _OTHERQ(wqp->q_next); 7566 while (tmp_qp && tmp_qp->q_nfsrv == rqp) { 7567 /* Note that rqp->q_next cannot be NULL */ 7568 ASSERT(rqp->q_next != NULL); 7569 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv; 7570 tmp_qp = backq(tmp_qp); 7571 } 7572 } 7573 } 7574 } 7575 7576 /* 7577 * This routine should be called after all stream geometry changes to update 7578 * the stream head cached struio() rd/wr queue pointers. Note must be called 7579 * with the streamlock()ed. 7580 * 7581 * Note: only enables Synchronous STREAMS for a side of a Stream which has 7582 * an explicit synchronous barrier module queue. That is, a queue that 7583 * has specified a struio() type. 7584 */ 7585 static void 7586 strsetuio(stdata_t *stp) 7587 { 7588 queue_t *wrq; 7589 7590 if (stp->sd_flag & STPLEX) { 7591 /* 7592 * Not stremahead, but a mux, so no Synchronous STREAMS. 7593 */ 7594 stp->sd_struiowrq = NULL; 7595 stp->sd_struiordq = NULL; 7596 return; 7597 } 7598 /* 7599 * Scan the write queue(s) while synchronous 7600 * until we find a qinfo uio type specified. 7601 */ 7602 wrq = stp->sd_wrq->q_next; 7603 while (wrq) { 7604 if (wrq->q_struiot == STRUIOT_NONE) { 7605 wrq = 0; 7606 break; 7607 } 7608 if (wrq->q_struiot != STRUIOT_DONTCARE) 7609 break; 7610 if (! _SAMESTR(wrq)) { 7611 wrq = 0; 7612 break; 7613 } 7614 wrq = wrq->q_next; 7615 } 7616 stp->sd_struiowrq = wrq; 7617 /* 7618 * Scan the read queue(s) while synchronous 7619 * until we find a qinfo uio type specified. 7620 */ 7621 wrq = stp->sd_wrq->q_next; 7622 while (wrq) { 7623 if (_RD(wrq)->q_struiot == STRUIOT_NONE) { 7624 wrq = 0; 7625 break; 7626 } 7627 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE) 7628 break; 7629 if (! _SAMESTR(wrq)) { 7630 wrq = 0; 7631 break; 7632 } 7633 wrq = wrq->q_next; 7634 } 7635 stp->sd_struiordq = wrq ? _RD(wrq) : 0; 7636 } 7637 7638 /* 7639 * pass_wput, unblocks the passthru queues, so that 7640 * messages can arrive at muxs lower read queue, before 7641 * I_LINK/I_UNLINK is acked/nacked. 7642 */ 7643 static void 7644 pass_wput(queue_t *q, mblk_t *mp) 7645 { 7646 syncq_t *sq; 7647 7648 sq = _RD(q)->q_syncq; 7649 if (sq->sq_flags & SQ_BLOCKED) 7650 unblocksq(sq, SQ_BLOCKED, 0); 7651 putnext(q, mp); 7652 } 7653 7654 /* 7655 * Set up queues for the link/unlink. 7656 * Create a new queue and block it and then insert it 7657 * below the stream head on the lower stream. 7658 * This prevents any messages from arriving during the setq 7659 * as well as while the mux is processing the LINK/I_UNLINK. 7660 * The blocked passq is unblocked once the LINK/I_UNLINK has 7661 * been acked or nacked or if a message is generated and sent 7662 * down muxs write put procedure. 7663 * see pass_wput(). 7664 * 7665 * After the new queue is inserted, all messages coming from below are 7666 * blocked. The call to strlock will ensure that all activity in the stream head 7667 * read queue syncq is stopped (sq_count drops to zero). 7668 */ 7669 static queue_t * 7670 link_addpassthru(stdata_t *stpdown) 7671 { 7672 queue_t *passq; 7673 sqlist_t sqlist; 7674 7675 passq = allocq(); 7676 STREAM(passq) = STREAM(_WR(passq)) = stpdown; 7677 /* setq might sleep in allocator - avoid holding locks. */ 7678 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ, 7679 SQ_CI|SQ_CO, B_FALSE); 7680 claimq(passq); 7681 blocksq(passq->q_syncq, SQ_BLOCKED, 1); 7682 insertq(STREAM(passq), passq); 7683 7684 /* 7685 * Use strlock() to wait for the stream head sq_count to drop to zero 7686 * since we are going to change q_ptr in the stream head. Note that 7687 * insertq() doesn't wait for any syncq counts to drop to zero. 7688 */ 7689 sqlist.sqlist_head = NULL; 7690 sqlist.sqlist_index = 0; 7691 sqlist.sqlist_size = sizeof (sqlist_t); 7692 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq); 7693 strlock(stpdown, &sqlist); 7694 strunlock(stpdown, &sqlist); 7695 7696 releaseq(passq); 7697 return (passq); 7698 } 7699 7700 /* 7701 * Let messages flow up into the mux by removing 7702 * the passq. 7703 */ 7704 static void 7705 link_rempassthru(queue_t *passq) 7706 { 7707 claimq(passq); 7708 removeq(passq); 7709 releaseq(passq); 7710 freeq(passq); 7711 } 7712 7713 /* 7714 * wait for an event with optional timeout and optional return if 7715 * a signal is sent to the thread 7716 * tim: -1 : no timeout 7717 * otherwise the value is relative time in milliseconds to wait 7718 * nosig: if 0 then signals will be ignored, otherwise signals 7719 * will terminate wait 7720 * returns >0 on success, 0 if signal was encountered, -1 if timeout 7721 * was reached. 7722 */ 7723 clock_t 7724 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs) 7725 { 7726 clock_t ret, now, tick; 7727 7728 if (tim < 0) { 7729 if (nosigs) { 7730 cv_wait(cvp, mp); 7731 ret = 1; 7732 } else { 7733 ret = cv_wait_sig(cvp, mp); 7734 } 7735 } else if (tim > 0) { 7736 /* 7737 * convert milliseconds to clock ticks 7738 */ 7739 tick = MSEC_TO_TICK_ROUNDUP(tim); 7740 time_to_wait(&now, tick); 7741 if (nosigs) { 7742 ret = cv_timedwait(cvp, mp, now); 7743 } else { 7744 ret = cv_timedwait_sig(cvp, mp, now); 7745 } 7746 } else { 7747 ret = -1; 7748 } 7749 return (ret); 7750 } 7751 7752 /* 7753 * Wait until the stream head can determine if it is at the mark but 7754 * don't wait forever to prevent a race condition between the "mark" state 7755 * in the stream head and any mark state in the caller/user of this routine. 7756 * 7757 * This is used by sockets and for a socket it would be incorrect 7758 * to return a failure for SIOCATMARK when there is no data in the receive 7759 * queue and the marked urgent data is traveling up the stream. 7760 * 7761 * This routine waits until the mark is known by waiting for one of these 7762 * three events: 7763 * The stream head read queue becoming non-empty (including an EOF) 7764 * The STRATMARK flag being set. (Due to a MSGMARKNEXT message.) 7765 * The STRNOTATMARK flag being set (which indicates that the transport 7766 * has sent a MSGNOTMARKNEXT message to indicate that it is not at 7767 * the mark). 7768 * 7769 * The routine returns 1 if the stream is at the mark; 0 if it can 7770 * be determined that the stream is not at the mark. 7771 * If the wait times out and it can't determine 7772 * whether or not the stream might be at the mark the routine will return -1. 7773 * 7774 * Note: This routine should only be used when a mark is pending i.e., 7775 * in the socket case the SIGURG has been posted. 7776 * Note2: This can not wakeup just because synchronous streams indicate 7777 * that data is available since it is not possible to use the synchronous 7778 * streams interfaces to determine the b_flag value for the data queued below 7779 * the stream head. 7780 */ 7781 int 7782 strwaitmark(vnode_t *vp) 7783 { 7784 struct stdata *stp = vp->v_stream; 7785 queue_t *rq = _RD(stp->sd_wrq); 7786 int mark; 7787 7788 mutex_enter(&stp->sd_lock); 7789 while (rq->q_first == NULL && 7790 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) { 7791 stp->sd_flag |= RSLEEP; 7792 7793 /* Wait for 100 milliseconds for any state change. */ 7794 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) { 7795 mutex_exit(&stp->sd_lock); 7796 return (-1); 7797 } 7798 } 7799 if (stp->sd_flag & STRATMARK) 7800 mark = 1; 7801 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK)) 7802 mark = 1; 7803 else 7804 mark = 0; 7805 7806 mutex_exit(&stp->sd_lock); 7807 return (mark); 7808 } 7809 7810 /* 7811 * Set a read side error. If persist is set change the socket error 7812 * to persistent. If errfunc is set install the function as the exported 7813 * error handler. 7814 */ 7815 void 7816 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 7817 { 7818 struct stdata *stp = vp->v_stream; 7819 7820 mutex_enter(&stp->sd_lock); 7821 stp->sd_rerror = error; 7822 if (error == 0 && errfunc == NULL) 7823 stp->sd_flag &= ~STRDERR; 7824 else 7825 stp->sd_flag |= STRDERR; 7826 if (persist) { 7827 stp->sd_flag &= ~STRDERRNONPERSIST; 7828 } else { 7829 stp->sd_flag |= STRDERRNONPERSIST; 7830 } 7831 stp->sd_rderrfunc = errfunc; 7832 if (error != 0 || errfunc != NULL) { 7833 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 7834 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 7835 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 7836 7837 mutex_exit(&stp->sd_lock); 7838 pollwakeup(&stp->sd_pollist, POLLERR); 7839 mutex_enter(&stp->sd_lock); 7840 7841 if (stp->sd_sigflags & S_ERROR) 7842 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 7843 } 7844 mutex_exit(&stp->sd_lock); 7845 } 7846 7847 /* 7848 * Set a write side error. If persist is set change the socket error 7849 * to persistent. 7850 */ 7851 void 7852 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 7853 { 7854 struct stdata *stp = vp->v_stream; 7855 7856 mutex_enter(&stp->sd_lock); 7857 stp->sd_werror = error; 7858 if (error == 0 && errfunc == NULL) 7859 stp->sd_flag &= ~STWRERR; 7860 else 7861 stp->sd_flag |= STWRERR; 7862 if (persist) { 7863 stp->sd_flag &= ~STWRERRNONPERSIST; 7864 } else { 7865 stp->sd_flag |= STWRERRNONPERSIST; 7866 } 7867 stp->sd_wrerrfunc = errfunc; 7868 if (error != 0 || errfunc != NULL) { 7869 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 7870 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 7871 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 7872 7873 mutex_exit(&stp->sd_lock); 7874 pollwakeup(&stp->sd_pollist, POLLERR); 7875 mutex_enter(&stp->sd_lock); 7876 7877 if (stp->sd_sigflags & S_ERROR) 7878 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 7879 } 7880 mutex_exit(&stp->sd_lock); 7881 } 7882 7883 /* 7884 * Make the stream return 0 (EOF) when all data has been read. 7885 * No effect on write side. 7886 */ 7887 void 7888 strseteof(vnode_t *vp, int eof) 7889 { 7890 struct stdata *stp = vp->v_stream; 7891 7892 mutex_enter(&stp->sd_lock); 7893 if (!eof) { 7894 stp->sd_flag &= ~STREOF; 7895 mutex_exit(&stp->sd_lock); 7896 return; 7897 } 7898 stp->sd_flag |= STREOF; 7899 if (stp->sd_flag & RSLEEP) { 7900 stp->sd_flag &= ~RSLEEP; 7901 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); 7902 } 7903 7904 mutex_exit(&stp->sd_lock); 7905 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM); 7906 mutex_enter(&stp->sd_lock); 7907 7908 if (stp->sd_sigflags & (S_INPUT|S_RDNORM)) 7909 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0); 7910 mutex_exit(&stp->sd_lock); 7911 } 7912 7913 void 7914 strflushrq(vnode_t *vp, int flag) 7915 { 7916 struct stdata *stp = vp->v_stream; 7917 7918 mutex_enter(&stp->sd_lock); 7919 flushq(_RD(stp->sd_wrq), flag); 7920 mutex_exit(&stp->sd_lock); 7921 } 7922 7923 void 7924 strsetrputhooks(vnode_t *vp, uint_t flags, 7925 msgfunc_t protofunc, msgfunc_t miscfunc) 7926 { 7927 struct stdata *stp = vp->v_stream; 7928 7929 mutex_enter(&stp->sd_lock); 7930 7931 if (protofunc == NULL) 7932 stp->sd_rprotofunc = strrput_proto; 7933 else 7934 stp->sd_rprotofunc = protofunc; 7935 7936 if (miscfunc == NULL) 7937 stp->sd_rmiscfunc = strrput_misc; 7938 else 7939 stp->sd_rmiscfunc = miscfunc; 7940 7941 if (flags & SH_CONSOL_DATA) 7942 stp->sd_rput_opt |= SR_CONSOL_DATA; 7943 else 7944 stp->sd_rput_opt &= ~SR_CONSOL_DATA; 7945 7946 if (flags & SH_SIGALLDATA) 7947 stp->sd_rput_opt |= SR_SIGALLDATA; 7948 else 7949 stp->sd_rput_opt &= ~SR_SIGALLDATA; 7950 7951 if (flags & SH_IGN_ZEROLEN) 7952 stp->sd_rput_opt |= SR_IGN_ZEROLEN; 7953 else 7954 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN; 7955 7956 mutex_exit(&stp->sd_lock); 7957 } 7958 7959 void 7960 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime) 7961 { 7962 struct stdata *stp = vp->v_stream; 7963 7964 mutex_enter(&stp->sd_lock); 7965 stp->sd_closetime = closetime; 7966 7967 if (flags & SH_SIGPIPE) 7968 stp->sd_wput_opt |= SW_SIGPIPE; 7969 else 7970 stp->sd_wput_opt &= ~SW_SIGPIPE; 7971 if (flags & SH_RECHECK_ERR) 7972 stp->sd_wput_opt |= SW_RECHECK_ERR; 7973 else 7974 stp->sd_wput_opt &= ~SW_RECHECK_ERR; 7975 7976 mutex_exit(&stp->sd_lock); 7977 } 7978 7979 /* Used within framework when the queue is already locked */ 7980 void 7981 qenable_locked(queue_t *q) 7982 { 7983 stdata_t *stp = STREAM(q); 7984 7985 ASSERT(MUTEX_HELD(QLOCK(q))); 7986 7987 if (!q->q_qinfo->qi_srvp) 7988 return; 7989 7990 /* 7991 * Do not place on run queue if already enabled or closing. 7992 */ 7993 if (q->q_flag & (QWCLOSE|QENAB)) 7994 return; 7995 7996 /* 7997 * mark queue enabled and place on run list if it is not already being 7998 * serviced. If it is serviced, the runservice() function will detect 7999 * that QENAB is set and call service procedure before clearing 8000 * QINSERVICE flag. 8001 */ 8002 q->q_flag |= QENAB; 8003 if (q->q_flag & QINSERVICE) 8004 return; 8005 8006 /* Record the time of qenable */ 8007 q->q_qtstamp = lbolt; 8008 8009 /* 8010 * Put the queue in the stp list and schedule it for background 8011 * processing if it is not already scheduled or if stream head does not 8012 * intent to process it in the foreground later by setting 8013 * STRS_WILLSERVICE flag. 8014 */ 8015 mutex_enter(&stp->sd_qlock); 8016 /* 8017 * If there are already something on the list, stp flags should show 8018 * intention to drain it. 8019 */ 8020 IMPLY(STREAM_NEEDSERVICE(stp), 8021 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))); 8022 8023 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link); 8024 stp->sd_nqueues++; 8025 8026 /* 8027 * If no one will drain this stream we are the first producer and 8028 * need to schedule it for background thread. 8029 */ 8030 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) { 8031 /* 8032 * No one will service this stream later, so we have to 8033 * schedule it now. 8034 */ 8035 STRSTAT(stenables); 8036 stp->sd_svcflags |= STRS_SCHEDULED; 8037 stp->sd_servid = (void *)taskq_dispatch(streams_taskq, 8038 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE); 8039 8040 if (stp->sd_servid == NULL) { 8041 /* 8042 * Task queue failed so fail over to the backup 8043 * servicing thread. 8044 */ 8045 STRSTAT(taskqfails); 8046 /* 8047 * It is safe to clear STRS_SCHEDULED flag because it 8048 * was set by this thread above. 8049 */ 8050 stp->sd_svcflags &= ~STRS_SCHEDULED; 8051 8052 /* 8053 * Failover scheduling is protected by service_queue 8054 * lock. 8055 */ 8056 mutex_enter(&service_queue); 8057 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q)); 8058 ASSERT(q->q_link == NULL); 8059 /* 8060 * Append the queue to qhead/qtail list. 8061 */ 8062 if (qhead == NULL) 8063 qhead = q; 8064 else 8065 qtail->q_link = q; 8066 qtail = q; 8067 /* 8068 * Clear stp queue list. 8069 */ 8070 stp->sd_qhead = stp->sd_qtail = NULL; 8071 stp->sd_nqueues = 0; 8072 /* 8073 * Wakeup background queue processing thread. 8074 */ 8075 cv_signal(&services_to_run); 8076 mutex_exit(&service_queue); 8077 } 8078 } 8079 mutex_exit(&stp->sd_qlock); 8080 } 8081 8082 static void 8083 queue_service(queue_t *q) 8084 { 8085 /* 8086 * The queue in the list should have 8087 * QENAB flag set and should not have 8088 * QINSERVICE flag set. QINSERVICE is 8089 * set when the queue is dequeued and 8090 * qenable_locked doesn't enqueue a 8091 * queue with QINSERVICE set. 8092 */ 8093 8094 ASSERT(!(q->q_flag & QINSERVICE)); 8095 ASSERT((q->q_flag & QENAB)); 8096 mutex_enter(QLOCK(q)); 8097 q->q_flag &= ~QENAB; 8098 q->q_flag |= QINSERVICE; 8099 mutex_exit(QLOCK(q)); 8100 runservice(q); 8101 } 8102 8103 static void 8104 syncq_service(syncq_t *sq) 8105 { 8106 STRSTAT(syncqservice); 8107 mutex_enter(SQLOCK(sq)); 8108 ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); 8109 ASSERT(sq->sq_servcount != 0); 8110 ASSERT(sq->sq_next == NULL); 8111 8112 /* if we came here from the background thread, clear the flag */ 8113 if (sq->sq_svcflags & SQ_BGTHREAD) 8114 sq->sq_svcflags &= ~SQ_BGTHREAD; 8115 8116 /* let drain_syncq know that it's being called in the background */ 8117 sq->sq_svcflags |= SQ_SERVICE; 8118 drain_syncq(sq); 8119 } 8120 8121 static void 8122 qwriter_outer_service(syncq_t *outer) 8123 { 8124 /* 8125 * Note that SQ_WRITER is used on the outer perimeter 8126 * to signal that a qwriter(OUTER) is either investigating 8127 * running or that it is actually running a function. 8128 */ 8129 outer_enter(outer, SQ_BLOCKED|SQ_WRITER); 8130 8131 /* 8132 * All inner syncq are empty and have SQ_WRITER set 8133 * to block entering the outer perimeter. 8134 * 8135 * We do not need to explicitly call write_now since 8136 * outer_exit does it for us. 8137 */ 8138 outer_exit(outer); 8139 } 8140 8141 static void 8142 mblk_free(mblk_t *mp) 8143 { 8144 dblk_t *dbp = mp->b_datap; 8145 frtn_t *frp = dbp->db_frtnp; 8146 8147 mp->b_next = NULL; 8148 if (dbp->db_fthdr != NULL) 8149 str_ftfree(dbp); 8150 8151 ASSERT(dbp->db_fthdr == NULL); 8152 frp->free_func(frp->free_arg); 8153 ASSERT(dbp->db_mblk == mp); 8154 8155 if (dbp->db_credp != NULL) { 8156 crfree(dbp->db_credp); 8157 dbp->db_credp = NULL; 8158 } 8159 dbp->db_cpid = -1; 8160 dbp->db_struioflag = 0; 8161 dbp->db_struioun.cksum.flags = 0; 8162 8163 kmem_cache_free(dbp->db_cache, dbp); 8164 } 8165 8166 /* 8167 * Background processing of the stream queue list. 8168 */ 8169 static void 8170 stream_service(stdata_t *stp) 8171 { 8172 queue_t *q; 8173 8174 mutex_enter(&stp->sd_qlock); 8175 8176 STR_SERVICE(stp, q); 8177 8178 stp->sd_svcflags &= ~STRS_SCHEDULED; 8179 stp->sd_servid = NULL; 8180 cv_signal(&stp->sd_qcv); 8181 mutex_exit(&stp->sd_qlock); 8182 } 8183 8184 /* 8185 * Foreground processing of the stream queue list. 8186 */ 8187 void 8188 stream_runservice(stdata_t *stp) 8189 { 8190 queue_t *q; 8191 8192 mutex_enter(&stp->sd_qlock); 8193 STRSTAT(rservice); 8194 /* 8195 * We are going to drain this stream queue list, so qenable_locked will 8196 * not schedule it until we finish. 8197 */ 8198 stp->sd_svcflags |= STRS_WILLSERVICE; 8199 8200 STR_SERVICE(stp, q); 8201 8202 stp->sd_svcflags &= ~STRS_WILLSERVICE; 8203 mutex_exit(&stp->sd_qlock); 8204 /* 8205 * Help backup background thread to drain the qhead/qtail list. 8206 */ 8207 while (qhead != NULL) { 8208 STRSTAT(qhelps); 8209 mutex_enter(&service_queue); 8210 DQ(q, qhead, qtail, q_link); 8211 mutex_exit(&service_queue); 8212 if (q != NULL) 8213 queue_service(q); 8214 } 8215 } 8216 8217 void 8218 stream_willservice(stdata_t *stp) 8219 { 8220 mutex_enter(&stp->sd_qlock); 8221 stp->sd_svcflags |= STRS_WILLSERVICE; 8222 mutex_exit(&stp->sd_qlock); 8223 } 8224 8225 /* 8226 * Replace the cred currently in the mblk with a different one. 8227 */ 8228 void 8229 mblk_setcred(mblk_t *mp, cred_t *cr) 8230 { 8231 cred_t *ocr = DB_CRED(mp); 8232 8233 ASSERT(cr != NULL); 8234 8235 if (cr != ocr) { 8236 crhold(mp->b_datap->db_credp = cr); 8237 if (ocr != NULL) 8238 crfree(ocr); 8239 } 8240 } 8241 8242 int 8243 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8244 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value, 8245 uint32_t flags, int km_flags) 8246 { 8247 int rc = 0; 8248 8249 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8250 if (mp->b_datap->db_type == M_DATA) { 8251 /* Associate values for M_DATA type */ 8252 mp->b_datap->db_cksumstart = (intptr_t)start; 8253 mp->b_datap->db_cksumstuff = (intptr_t)stuff; 8254 mp->b_datap->db_cksumend = (intptr_t)end; 8255 mp->b_datap->db_struioun.cksum.flags = flags; 8256 mp->b_datap->db_cksum16 = (uint16_t)value; 8257 8258 } else { 8259 pattrinfo_t pa_info; 8260 8261 ASSERT(mmd != NULL); 8262 8263 pa_info.type = PATTR_HCKSUM; 8264 pa_info.len = sizeof (pattr_hcksum_t); 8265 8266 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) { 8267 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf; 8268 8269 hck->hcksum_start_offset = start; 8270 hck->hcksum_stuff_offset = stuff; 8271 hck->hcksum_end_offset = end; 8272 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value; 8273 hck->hcksum_flags = flags; 8274 } 8275 } 8276 return (rc); 8277 } 8278 8279 void 8280 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8281 uint32_t *start, uint32_t *stuff, uint32_t *end, 8282 uint32_t *value, uint32_t *flags) 8283 { 8284 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8285 if (mp->b_datap->db_type == M_DATA) { 8286 if (flags != NULL) { 8287 *flags = mp->b_datap->db_struioun.cksum.flags; 8288 if (*flags & HCK_PARTIALCKSUM) { 8289 if (start != NULL) 8290 *start = (uint32_t) 8291 mp->b_datap->db_cksumstart; 8292 if (stuff != NULL) 8293 *stuff = (uint32_t) 8294 mp->b_datap->db_cksumstuff; 8295 if (end != NULL) 8296 *end = 8297 (uint32_t)mp->b_datap->db_cksumend; 8298 if (value != NULL) 8299 *value = 8300 (uint32_t)mp->b_datap->db_cksum16; 8301 } 8302 } 8303 } else { 8304 pattrinfo_t hck_attr = {PATTR_HCKSUM}; 8305 8306 ASSERT(mmd != NULL); 8307 8308 /* get hardware checksum attribute */ 8309 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) { 8310 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf; 8311 8312 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t)); 8313 if (flags != NULL) 8314 *flags = hck->hcksum_flags; 8315 if (start != NULL) 8316 *start = hck->hcksum_start_offset; 8317 if (stuff != NULL) 8318 *stuff = hck->hcksum_stuff_offset; 8319 if (end != NULL) 8320 *end = hck->hcksum_end_offset; 8321 if (value != NULL) 8322 *value = (uint32_t) 8323 hck->hcksum_cksum_val.inet_cksum; 8324 } 8325 } 8326 } 8327 8328 /* 8329 * Checksum buffer *bp for len bytes with psum partial checksum, 8330 * or 0 if none, and return the 16 bit partial checksum. 8331 */ 8332 unsigned 8333 bcksum(uchar_t *bp, int len, unsigned int psum) 8334 { 8335 int odd = len & 1; 8336 extern unsigned int ip_ocsum(); 8337 8338 if (((intptr_t)bp & 1) == 0 && !odd) { 8339 /* 8340 * Bp is 16 bit aligned and len is multiple of 16 bit word. 8341 */ 8342 return (ip_ocsum((ushort_t *)bp, len >> 1, psum)); 8343 } 8344 if (((intptr_t)bp & 1) != 0) { 8345 /* 8346 * Bp isn't 16 bit aligned. 8347 */ 8348 unsigned int tsum; 8349 8350 #ifdef _LITTLE_ENDIAN 8351 psum += *bp; 8352 #else 8353 psum += *bp << 8; 8354 #endif 8355 len--; 8356 bp++; 8357 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0); 8358 psum += (tsum << 8) & 0xffff | (tsum >> 8); 8359 if (len & 1) { 8360 bp += len - 1; 8361 #ifdef _LITTLE_ENDIAN 8362 psum += *bp << 8; 8363 #else 8364 psum += *bp; 8365 #endif 8366 } 8367 } else { 8368 /* 8369 * Bp is 16 bit aligned. 8370 */ 8371 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum); 8372 if (odd) { 8373 bp += len - 1; 8374 #ifdef _LITTLE_ENDIAN 8375 psum += *bp; 8376 #else 8377 psum += *bp << 8; 8378 #endif 8379 } 8380 } 8381 /* 8382 * Normalize psum to 16 bits before returning the new partial 8383 * checksum. The max psum value before normalization is 0x3FDFE. 8384 */ 8385 return ((psum >> 16) + (psum & 0xFFFF)); 8386 } 8387 8388 boolean_t 8389 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd) 8390 { 8391 boolean_t rc; 8392 8393 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8394 if (DB_TYPE(mp) == M_DATA) { 8395 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0); 8396 } else { 8397 pattrinfo_t zcopy_attr = {PATTR_ZCOPY}; 8398 8399 ASSERT(mmd != NULL); 8400 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL); 8401 } 8402 return (rc); 8403 } 8404 8405 void 8406 freemsgchain(mblk_t *mp) 8407 { 8408 mblk_t *next; 8409 8410 while (mp != NULL) { 8411 next = mp->b_next; 8412 mp->b_next = NULL; 8413 8414 freemsg(mp); 8415 mp = next; 8416 } 8417 } 8418 8419 mblk_t * 8420 copymsgchain(mblk_t *mp) 8421 { 8422 mblk_t *nmp = NULL; 8423 mblk_t **nmpp = &nmp; 8424 8425 for (; mp != NULL; mp = mp->b_next) { 8426 if ((*nmpp = copymsg(mp)) == NULL) { 8427 freemsgchain(nmp); 8428 return (NULL); 8429 } 8430 8431 nmpp = &((*nmpp)->b_next); 8432 } 8433 8434 return (nmp); 8435 } 8436 8437 /* NOTE: Do not add code after this point. */ 8438 #undef QLOCK 8439 8440 /* 8441 * replacement for QLOCK macro for those that can't use it. 8442 */ 8443 kmutex_t * 8444 QLOCK(queue_t *q) 8445 { 8446 return (&(q)->q_lock); 8447 } 8448 8449 /* 8450 * Dummy runqueues/queuerun functions functions for backwards compatibility. 8451 */ 8452 #undef runqueues 8453 void 8454 runqueues(void) 8455 { 8456 } 8457 8458 #undef queuerun 8459 void 8460 queuerun(void) 8461 { 8462 } 8463