1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 22 /* All Rights Reserved */ 23 24 25 /* 26 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 27 * Use is subject to license terms. 28 */ 29 30 #include <sys/types.h> 31 #include <sys/sysmacros.h> 32 #include <sys/param.h> 33 #include <sys/errno.h> 34 #include <sys/signal.h> 35 #include <sys/proc.h> 36 #include <sys/conf.h> 37 #include <sys/cred.h> 38 #include <sys/user.h> 39 #include <sys/vnode.h> 40 #include <sys/file.h> 41 #include <sys/session.h> 42 #include <sys/stream.h> 43 #include <sys/strsubr.h> 44 #include <sys/stropts.h> 45 #include <sys/poll.h> 46 #include <sys/systm.h> 47 #include <sys/cpuvar.h> 48 #include <sys/uio.h> 49 #include <sys/cmn_err.h> 50 #include <sys/priocntl.h> 51 #include <sys/procset.h> 52 #include <sys/vmem.h> 53 #include <sys/bitmap.h> 54 #include <sys/kmem.h> 55 #include <sys/siginfo.h> 56 #include <sys/vtrace.h> 57 #include <sys/callb.h> 58 #include <sys/debug.h> 59 #include <sys/modctl.h> 60 #include <sys/vmsystm.h> 61 #include <vm/page.h> 62 #include <sys/atomic.h> 63 #include <sys/suntpi.h> 64 #include <sys/strlog.h> 65 #include <sys/promif.h> 66 #include <sys/project.h> 67 #include <sys/vm.h> 68 #include <sys/taskq.h> 69 #include <sys/sunddi.h> 70 #include <sys/sunldi_impl.h> 71 #include <sys/strsun.h> 72 #include <sys/isa_defs.h> 73 #include <sys/multidata.h> 74 #include <sys/pattr.h> 75 #include <sys/strft.h> 76 #include <sys/fs/snode.h> 77 #include <sys/zone.h> 78 #include <sys/open.h> 79 #include <sys/sunldi.h> 80 #include <sys/sad.h> 81 #include <sys/netstack.h> 82 83 #define O_SAMESTR(q) (((q)->q_next) && \ 84 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR))) 85 86 /* 87 * WARNING: 88 * The variables and routines in this file are private, belonging 89 * to the STREAMS subsystem. These should not be used by modules 90 * or drivers. Compatibility will not be guaranteed. 91 */ 92 93 /* 94 * Id value used to distinguish between different multiplexor links. 95 */ 96 static int32_t lnk_id = 0; 97 98 #define STREAMS_LOPRI MINCLSYSPRI 99 static pri_t streams_lopri = STREAMS_LOPRI; 100 101 #define STRSTAT(x) (str_statistics.x.value.ui64++) 102 typedef struct str_stat { 103 kstat_named_t sqenables; 104 kstat_named_t stenables; 105 kstat_named_t syncqservice; 106 kstat_named_t freebs; 107 kstat_named_t qwr_outer; 108 kstat_named_t rservice; 109 kstat_named_t strwaits; 110 kstat_named_t taskqfails; 111 kstat_named_t bufcalls; 112 kstat_named_t qhelps; 113 kstat_named_t qremoved; 114 kstat_named_t sqremoved; 115 kstat_named_t bcwaits; 116 kstat_named_t sqtoomany; 117 } str_stat_t; 118 119 static str_stat_t str_statistics = { 120 { "sqenables", KSTAT_DATA_UINT64 }, 121 { "stenables", KSTAT_DATA_UINT64 }, 122 { "syncqservice", KSTAT_DATA_UINT64 }, 123 { "freebs", KSTAT_DATA_UINT64 }, 124 { "qwr_outer", KSTAT_DATA_UINT64 }, 125 { "rservice", KSTAT_DATA_UINT64 }, 126 { "strwaits", KSTAT_DATA_UINT64 }, 127 { "taskqfails", KSTAT_DATA_UINT64 }, 128 { "bufcalls", KSTAT_DATA_UINT64 }, 129 { "qhelps", KSTAT_DATA_UINT64 }, 130 { "qremoved", KSTAT_DATA_UINT64 }, 131 { "sqremoved", KSTAT_DATA_UINT64 }, 132 { "bcwaits", KSTAT_DATA_UINT64 }, 133 { "sqtoomany", KSTAT_DATA_UINT64 }, 134 }; 135 136 static kstat_t *str_kstat; 137 138 /* 139 * qrunflag was used previously to control background scheduling of queues. It 140 * is not used anymore, but kept here in case some module still wants to access 141 * it via qready() and setqsched macros. 142 */ 143 char qrunflag; /* Unused */ 144 145 /* 146 * Most of the streams scheduling is done via task queues. Task queues may fail 147 * for non-sleep dispatches, so there are two backup threads servicing failed 148 * requests for queues and syncqs. Both of these threads also service failed 149 * dispatches freebs requests. Queues are put in the list specified by `qhead' 150 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs 151 * requests are put into `freebs_list' which has no tail pointer. All three 152 * lists are protected by a single `service_queue' lock and use 153 * `services_to_run' condition variable for signaling background threads. Use of 154 * a single lock should not be a problem because it is only used under heavy 155 * loads when task queues start to fail and at that time it may be a good idea 156 * to throttle scheduling requests. 157 * 158 * NOTE: queues and syncqs should be scheduled by two separate threads because 159 * queue servicing may be blocked waiting for a syncq which may be also 160 * scheduled for background execution. This may create a deadlock when only one 161 * thread is used for both. 162 */ 163 164 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */ 165 166 static kmutex_t service_queue; /* protects all of servicing vars */ 167 static kcondvar_t services_to_run; /* wake up background service thread */ 168 static kcondvar_t syncqs_to_run; /* wake up background service thread */ 169 170 /* 171 * List of queues scheduled for background processing due to lack of resources 172 * in the task queues. Protected by service_queue lock; 173 */ 174 static struct queue *qhead; 175 static struct queue *qtail; 176 177 /* 178 * Same list for syncqs 179 */ 180 static syncq_t *sqhead; 181 static syncq_t *sqtail; 182 183 static mblk_t *freebs_list; /* list of buffers to free */ 184 185 /* 186 * Backup threads for servicing queues and syncqs 187 */ 188 kthread_t *streams_qbkgrnd_thread; 189 kthread_t *streams_sqbkgrnd_thread; 190 191 /* 192 * Bufcalls related variables. 193 */ 194 struct bclist strbcalls; /* list of waiting bufcalls */ 195 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */ 196 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */ 197 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */ 198 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */ 199 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */ 200 201 kmutex_t strresources; /* protects global resources */ 202 kmutex_t muxifier; /* single-threads multiplexor creation */ 203 204 static void *str_stack_init(netstackid_t stackid, netstack_t *ns); 205 static void str_stack_shutdown(netstackid_t stackid, void *arg); 206 static void str_stack_fini(netstackid_t stackid, void *arg); 207 208 /* 209 * run_queues is no longer used, but is kept in case some 3rd party 210 * module/driver decides to use it. 211 */ 212 int run_queues = 0; 213 214 /* 215 * sq_max_size is the depth of the syncq (in number of messages) before 216 * qfill_syncq() starts QFULL'ing destination queues. As its primary 217 * consumer - IP is no longer D_MTPERMOD, but there may be other 218 * modules/drivers depend on this syncq flow control, we prefer to 219 * choose a large number as the default value. For potential 220 * performance gain, this value is tunable in /etc/system. 221 */ 222 int sq_max_size = 10000; 223 224 /* 225 * The number of ciputctrl structures per syncq and stream we create when 226 * needed. 227 */ 228 int n_ciputctrl; 229 int max_n_ciputctrl = 16; 230 /* 231 * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache. 232 */ 233 int min_n_ciputctrl = 2; 234 235 /* 236 * Per-driver/module syncqs 237 * ======================== 238 * 239 * For drivers/modules that use PERMOD or outer syncqs we keep a list of 240 * perdm structures, new entries being added (and new syncqs allocated) when 241 * setq() encounters a module/driver with a streamtab that it hasn't seen 242 * before. 243 * The reason for this mechanism is that some modules and drivers share a 244 * common streamtab and it is necessary for those modules and drivers to also 245 * share a common PERMOD syncq. 246 * 247 * perdm_list --> dm_str == streamtab_1 248 * dm_sq == syncq_1 249 * dm_ref 250 * dm_next --> dm_str == streamtab_2 251 * dm_sq == syncq_2 252 * dm_ref 253 * dm_next --> ... NULL 254 * 255 * The dm_ref field is incremented for each new driver/module that takes 256 * a reference to the perdm structure and hence shares the syncq. 257 * References are held in the fmodsw_impl_t structure for each STREAMS module 258 * or the dev_impl array (indexed by device major number) for each driver. 259 * 260 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL 261 * ^ ^ ^ ^ 262 * | ______________/ | | 263 * | / | | 264 * dev_impl: ...|x|y|... module A module B 265 * 266 * When a module/driver is unloaded the reference count is decremented and, 267 * when it falls to zero, the perdm structure is removed from the list and 268 * the syncq is freed (see rele_dm()). 269 */ 270 perdm_t *perdm_list = NULL; 271 static krwlock_t perdm_rwlock; 272 cdevsw_impl_t *devimpl; 273 274 extern struct qinit strdata; 275 extern struct qinit stwdata; 276 277 static void runservice(queue_t *); 278 static void streams_bufcall_service(void); 279 static void streams_qbkgrnd_service(void); 280 static void streams_sqbkgrnd_service(void); 281 static syncq_t *new_syncq(void); 282 static void free_syncq(syncq_t *); 283 static void outer_insert(syncq_t *, syncq_t *); 284 static void outer_remove(syncq_t *, syncq_t *); 285 static void write_now(syncq_t *); 286 static void clr_qfull(queue_t *); 287 static void runbufcalls(void); 288 static void sqenable(syncq_t *); 289 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)()); 290 static void wait_q_syncq(queue_t *); 291 static void backenable_insertedq(queue_t *); 292 293 static void queue_service(queue_t *); 294 static void stream_service(stdata_t *); 295 static void syncq_service(syncq_t *); 296 static void qwriter_outer_service(syncq_t *); 297 static void mblk_free(mblk_t *); 298 #ifdef DEBUG 299 static int qprocsareon(queue_t *); 300 #endif 301 302 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *); 303 static void reset_nfsrv_ptr(queue_t *, queue_t *); 304 void set_qfull(queue_t *); 305 306 static void sq_run_events(syncq_t *); 307 static int propagate_syncq(queue_t *); 308 309 static void blocksq(syncq_t *, ushort_t, int); 310 static void unblocksq(syncq_t *, ushort_t, int); 311 static int dropsq(syncq_t *, uint16_t); 312 static void emptysq(syncq_t *); 313 static sqlist_t *sqlist_alloc(struct stdata *, int); 314 static void sqlist_free(sqlist_t *); 315 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t); 316 static void sqlist_insert(sqlist_t *, syncq_t *); 317 static void sqlist_insertall(sqlist_t *, queue_t *); 318 319 static void strsetuio(stdata_t *); 320 321 struct kmem_cache *stream_head_cache; 322 struct kmem_cache *queue_cache; 323 struct kmem_cache *syncq_cache; 324 struct kmem_cache *qband_cache; 325 struct kmem_cache *linkinfo_cache; 326 struct kmem_cache *ciputctrl_cache = NULL; 327 328 static linkinfo_t *linkinfo_list; 329 330 /* Global esballoc throttling queue */ 331 static esb_queue_t system_esbq; 332 333 /* Array of esballoc throttling queues, of length esbq_nelem */ 334 static esb_queue_t *volatile system_esbq_array; 335 static int esbq_nelem; 336 static kmutex_t esbq_lock; 337 static int esbq_log2_cpus_per_q = 0; 338 339 /* Scale the system_esbq length by setting number of CPUs per queue. */ 340 uint_t esbq_cpus_per_q = 1; 341 342 /* 343 * esballoc tunable parameters. 344 */ 345 int esbq_max_qlen = 0x16; /* throttled queue length */ 346 clock_t esbq_timeout = 0x8; /* timeout to process esb queue */ 347 348 /* 349 * Routines to handle esballoc queueing. 350 */ 351 static void esballoc_process_queue(esb_queue_t *); 352 static void esballoc_enqueue_mblk(mblk_t *); 353 static void esballoc_timer(void *); 354 static void esballoc_set_timer(esb_queue_t *, clock_t); 355 static void esballoc_mblk_free(mblk_t *); 356 357 /* 358 * Qinit structure and Module_info structures 359 * for passthru read and write queues 360 */ 361 362 static void pass_wput(queue_t *, mblk_t *); 363 static queue_t *link_addpassthru(stdata_t *); 364 static void link_rempassthru(queue_t *); 365 366 struct module_info passthru_info = { 367 0, 368 "passthru", 369 0, 370 INFPSZ, 371 STRHIGH, 372 STRLOW 373 }; 374 375 struct qinit passthru_rinit = { 376 (int (*)())putnext, 377 NULL, 378 NULL, 379 NULL, 380 NULL, 381 &passthru_info, 382 NULL 383 }; 384 385 struct qinit passthru_winit = { 386 (int (*)()) pass_wput, 387 NULL, 388 NULL, 389 NULL, 390 NULL, 391 &passthru_info, 392 NULL 393 }; 394 395 /* 396 * Verify correctness of list head/tail pointers. 397 */ 398 #define LISTCHECK(head, tail, link) { \ 399 EQUIV(head, tail); \ 400 IMPLY(tail != NULL, tail->link == NULL); \ 401 } 402 403 /* 404 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail' 405 * using a `link' field. 406 */ 407 #define ENQUEUE(el, head, tail, link) { \ 408 ASSERT(el->link == NULL); \ 409 LISTCHECK(head, tail, link); \ 410 if (head == NULL) \ 411 head = el; \ 412 else \ 413 tail->link = el; \ 414 tail = el; \ 415 } 416 417 /* 418 * Dequeue the first element of the list denoted by `head' and `tail' pointers 419 * using a `link' field and put result into `el'. 420 */ 421 #define DQ(el, head, tail, link) { \ 422 LISTCHECK(head, tail, link); \ 423 el = head; \ 424 if (head != NULL) { \ 425 head = head->link; \ 426 if (head == NULL) \ 427 tail = NULL; \ 428 el->link = NULL; \ 429 } \ 430 } 431 432 /* 433 * Remove `el' from the list using `chase' and `curr' pointers and return result 434 * in `succeed'. 435 */ 436 #define RMQ(el, head, tail, link, chase, curr, succeed) { \ 437 LISTCHECK(head, tail, link); \ 438 chase = NULL; \ 439 succeed = 0; \ 440 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \ 441 chase = curr; \ 442 if (curr != NULL) { \ 443 succeed = 1; \ 444 ASSERT(curr == el); \ 445 if (chase != NULL) \ 446 chase->link = curr->link; \ 447 else \ 448 head = curr->link; \ 449 curr->link = NULL; \ 450 if (curr == tail) \ 451 tail = chase; \ 452 } \ 453 LISTCHECK(head, tail, link); \ 454 } 455 456 /* Handling of delayed messages on the inner syncq. */ 457 458 /* 459 * DEBUG versions should use function versions (to simplify tracing) and 460 * non-DEBUG kernels should use macro versions. 461 */ 462 463 /* 464 * Put a queue on the syncq list of queues. 465 * Assumes SQLOCK held. 466 */ 467 #define SQPUT_Q(sq, qp) \ 468 { \ 469 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 470 if (!(qp->q_sqflags & Q_SQQUEUED)) { \ 471 /* The queue should not be linked anywhere */ \ 472 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \ 473 /* Head and tail may only be NULL simultaneously */ \ 474 EQUIV(sq->sq_head, sq->sq_tail); \ 475 /* Queue may be only enqueued on its syncq */ \ 476 ASSERT(sq == qp->q_syncq); \ 477 /* Check the correctness of SQ_MESSAGES flag */ \ 478 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \ 479 /* Sanity check first/last elements of the list */ \ 480 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\ 481 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\ 482 /* \ 483 * Sanity check of priority field: empty queue should \ 484 * have zero priority \ 485 * and nqueues equal to zero. \ 486 */ \ 487 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \ 488 /* Sanity check of sq_nqueues field */ \ 489 EQUIV(sq->sq_head, sq->sq_nqueues); \ 490 if (sq->sq_head == NULL) { \ 491 sq->sq_head = sq->sq_tail = qp; \ 492 sq->sq_flags |= SQ_MESSAGES; \ 493 } else if (qp->q_spri == 0) { \ 494 qp->q_sqprev = sq->sq_tail; \ 495 sq->sq_tail->q_sqnext = qp; \ 496 sq->sq_tail = qp; \ 497 } else { \ 498 /* \ 499 * Put this queue in priority order: higher \ 500 * priority gets closer to the head. \ 501 */ \ 502 queue_t **qpp = &sq->sq_tail; \ 503 queue_t *qnext = NULL; \ 504 \ 505 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \ 506 qnext = *qpp; \ 507 qpp = &(*qpp)->q_sqprev; \ 508 } \ 509 qp->q_sqnext = qnext; \ 510 qp->q_sqprev = *qpp; \ 511 if (*qpp != NULL) { \ 512 (*qpp)->q_sqnext = qp; \ 513 } else { \ 514 sq->sq_head = qp; \ 515 sq->sq_pri = sq->sq_head->q_spri; \ 516 } \ 517 *qpp = qp; \ 518 } \ 519 qp->q_sqflags |= Q_SQQUEUED; \ 520 qp->q_sqtstamp = ddi_get_lbolt(); \ 521 sq->sq_nqueues++; \ 522 } \ 523 } 524 525 /* 526 * Remove a queue from the syncq list 527 * Assumes SQLOCK held. 528 */ 529 #define SQRM_Q(sq, qp) \ 530 { \ 531 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 532 ASSERT(qp->q_sqflags & Q_SQQUEUED); \ 533 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \ 534 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \ 535 /* Check that the queue is actually in the list */ \ 536 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \ 537 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \ 538 ASSERT(sq->sq_nqueues != 0); \ 539 if (qp->q_sqprev == NULL) { \ 540 /* First queue on list, make head q_sqnext */ \ 541 sq->sq_head = qp->q_sqnext; \ 542 } else { \ 543 /* Make prev->next == next */ \ 544 qp->q_sqprev->q_sqnext = qp->q_sqnext; \ 545 } \ 546 if (qp->q_sqnext == NULL) { \ 547 /* Last queue on list, make tail sqprev */ \ 548 sq->sq_tail = qp->q_sqprev; \ 549 } else { \ 550 /* Make next->prev == prev */ \ 551 qp->q_sqnext->q_sqprev = qp->q_sqprev; \ 552 } \ 553 /* clear out references on this queue */ \ 554 qp->q_sqprev = qp->q_sqnext = NULL; \ 555 qp->q_sqflags &= ~Q_SQQUEUED; \ 556 /* If there is nothing queued, clear SQ_MESSAGES */ \ 557 if (sq->sq_head != NULL) { \ 558 sq->sq_pri = sq->sq_head->q_spri; \ 559 } else { \ 560 sq->sq_flags &= ~SQ_MESSAGES; \ 561 sq->sq_pri = 0; \ 562 } \ 563 sq->sq_nqueues--; \ 564 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \ 565 (sq->sq_flags & SQ_QUEUED) == 0); \ 566 } 567 568 /* Hide the definition from the header file. */ 569 #ifdef SQPUT_MP 570 #undef SQPUT_MP 571 #endif 572 573 /* 574 * Put a message on the queue syncq. 575 * Assumes QLOCK held. 576 */ 577 #define SQPUT_MP(qp, mp) \ 578 { \ 579 ASSERT(MUTEX_HELD(QLOCK(qp))); \ 580 ASSERT(qp->q_sqhead == NULL || \ 581 (qp->q_sqtail != NULL && \ 582 qp->q_sqtail->b_next == NULL)); \ 583 qp->q_syncqmsgs++; \ 584 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \ 585 if (qp->q_sqhead == NULL) { \ 586 qp->q_sqhead = qp->q_sqtail = mp; \ 587 } else { \ 588 qp->q_sqtail->b_next = mp; \ 589 qp->q_sqtail = mp; \ 590 } \ 591 ASSERT(qp->q_syncqmsgs > 0); \ 592 set_qfull(qp); \ 593 } 594 595 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ 596 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 597 if ((sq)->sq_ciputctrl != NULL) { \ 598 int i; \ 599 int nlocks = (sq)->sq_nciputctrl; \ 600 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 601 ASSERT((sq)->sq_type & SQ_CIPUT); \ 602 for (i = 0; i <= nlocks; i++) { \ 603 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 604 cip[i].ciputctrl_count |= SQ_FASTPUT; \ 605 } \ 606 } \ 607 } 608 609 610 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ 611 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 612 if ((sq)->sq_ciputctrl != NULL) { \ 613 int i; \ 614 int nlocks = (sq)->sq_nciputctrl; \ 615 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 616 ASSERT((sq)->sq_type & SQ_CIPUT); \ 617 for (i = 0; i <= nlocks; i++) { \ 618 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 619 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \ 620 } \ 621 } \ 622 } 623 624 /* 625 * Run service procedures for all queues in the stream head. 626 */ 627 #define STR_SERVICE(stp, q) { \ 628 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \ 629 while (stp->sd_qhead != NULL) { \ 630 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \ 631 ASSERT(stp->sd_nqueues > 0); \ 632 stp->sd_nqueues--; \ 633 ASSERT(!(q->q_flag & QINSERVICE)); \ 634 mutex_exit(&stp->sd_qlock); \ 635 queue_service(q); \ 636 mutex_enter(&stp->sd_qlock); \ 637 } \ 638 ASSERT(stp->sd_nqueues == 0); \ 639 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \ 640 } 641 642 /* 643 * Constructor/destructor routines for the stream head cache 644 */ 645 /* ARGSUSED */ 646 static int 647 stream_head_constructor(void *buf, void *cdrarg, int kmflags) 648 { 649 stdata_t *stp = buf; 650 651 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL); 652 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL); 653 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL); 654 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL); 655 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL); 656 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL); 657 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL); 658 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL); 659 stp->sd_wrq = NULL; 660 661 return (0); 662 } 663 664 /* ARGSUSED */ 665 static void 666 stream_head_destructor(void *buf, void *cdrarg) 667 { 668 stdata_t *stp = buf; 669 670 mutex_destroy(&stp->sd_lock); 671 mutex_destroy(&stp->sd_reflock); 672 mutex_destroy(&stp->sd_qlock); 673 cv_destroy(&stp->sd_monitor); 674 cv_destroy(&stp->sd_iocmonitor); 675 cv_destroy(&stp->sd_refmonitor); 676 cv_destroy(&stp->sd_qcv); 677 cv_destroy(&stp->sd_zcopy_wait); 678 } 679 680 /* 681 * Constructor/destructor routines for the queue cache 682 */ 683 /* ARGSUSED */ 684 static int 685 queue_constructor(void *buf, void *cdrarg, int kmflags) 686 { 687 queinfo_t *qip = buf; 688 queue_t *qp = &qip->qu_rqueue; 689 queue_t *wqp = &qip->qu_wqueue; 690 syncq_t *sq = &qip->qu_syncq; 691 692 qp->q_first = NULL; 693 qp->q_link = NULL; 694 qp->q_count = 0; 695 qp->q_mblkcnt = 0; 696 qp->q_sqhead = NULL; 697 qp->q_sqtail = NULL; 698 qp->q_sqnext = NULL; 699 qp->q_sqprev = NULL; 700 qp->q_sqflags = 0; 701 qp->q_rwcnt = 0; 702 qp->q_spri = 0; 703 704 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL); 705 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL); 706 707 wqp->q_first = NULL; 708 wqp->q_link = NULL; 709 wqp->q_count = 0; 710 wqp->q_mblkcnt = 0; 711 wqp->q_sqhead = NULL; 712 wqp->q_sqtail = NULL; 713 wqp->q_sqnext = NULL; 714 wqp->q_sqprev = NULL; 715 wqp->q_sqflags = 0; 716 wqp->q_rwcnt = 0; 717 wqp->q_spri = 0; 718 719 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL); 720 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL); 721 722 sq->sq_head = NULL; 723 sq->sq_tail = NULL; 724 sq->sq_evhead = NULL; 725 sq->sq_evtail = NULL; 726 sq->sq_callbpend = NULL; 727 sq->sq_outer = NULL; 728 sq->sq_onext = NULL; 729 sq->sq_oprev = NULL; 730 sq->sq_next = NULL; 731 sq->sq_svcflags = 0; 732 sq->sq_servcount = 0; 733 sq->sq_needexcl = 0; 734 sq->sq_nqueues = 0; 735 sq->sq_pri = 0; 736 737 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 738 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 739 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 740 741 return (0); 742 } 743 744 /* ARGSUSED */ 745 static void 746 queue_destructor(void *buf, void *cdrarg) 747 { 748 queinfo_t *qip = buf; 749 queue_t *qp = &qip->qu_rqueue; 750 queue_t *wqp = &qip->qu_wqueue; 751 syncq_t *sq = &qip->qu_syncq; 752 753 ASSERT(qp->q_sqhead == NULL); 754 ASSERT(wqp->q_sqhead == NULL); 755 ASSERT(qp->q_sqnext == NULL); 756 ASSERT(wqp->q_sqnext == NULL); 757 ASSERT(qp->q_rwcnt == 0); 758 ASSERT(wqp->q_rwcnt == 0); 759 760 mutex_destroy(&qp->q_lock); 761 cv_destroy(&qp->q_wait); 762 763 mutex_destroy(&wqp->q_lock); 764 cv_destroy(&wqp->q_wait); 765 766 mutex_destroy(&sq->sq_lock); 767 cv_destroy(&sq->sq_wait); 768 cv_destroy(&sq->sq_exitwait); 769 } 770 771 /* 772 * Constructor/destructor routines for the syncq cache 773 */ 774 /* ARGSUSED */ 775 static int 776 syncq_constructor(void *buf, void *cdrarg, int kmflags) 777 { 778 syncq_t *sq = buf; 779 780 bzero(buf, sizeof (syncq_t)); 781 782 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 783 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 784 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 785 786 return (0); 787 } 788 789 /* ARGSUSED */ 790 static void 791 syncq_destructor(void *buf, void *cdrarg) 792 { 793 syncq_t *sq = buf; 794 795 ASSERT(sq->sq_head == NULL); 796 ASSERT(sq->sq_tail == NULL); 797 ASSERT(sq->sq_evhead == NULL); 798 ASSERT(sq->sq_evtail == NULL); 799 ASSERT(sq->sq_callbpend == NULL); 800 ASSERT(sq->sq_callbflags == 0); 801 ASSERT(sq->sq_outer == NULL); 802 ASSERT(sq->sq_onext == NULL); 803 ASSERT(sq->sq_oprev == NULL); 804 ASSERT(sq->sq_next == NULL); 805 ASSERT(sq->sq_needexcl == 0); 806 ASSERT(sq->sq_svcflags == 0); 807 ASSERT(sq->sq_servcount == 0); 808 ASSERT(sq->sq_nqueues == 0); 809 ASSERT(sq->sq_pri == 0); 810 ASSERT(sq->sq_count == 0); 811 ASSERT(sq->sq_rmqcount == 0); 812 ASSERT(sq->sq_cancelid == 0); 813 ASSERT(sq->sq_ciputctrl == NULL); 814 ASSERT(sq->sq_nciputctrl == 0); 815 ASSERT(sq->sq_type == 0); 816 ASSERT(sq->sq_flags == 0); 817 818 mutex_destroy(&sq->sq_lock); 819 cv_destroy(&sq->sq_wait); 820 cv_destroy(&sq->sq_exitwait); 821 } 822 823 /* ARGSUSED */ 824 static int 825 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags) 826 { 827 ciputctrl_t *cip = buf; 828 int i; 829 830 for (i = 0; i < n_ciputctrl; i++) { 831 cip[i].ciputctrl_count = SQ_FASTPUT; 832 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL); 833 } 834 835 return (0); 836 } 837 838 /* ARGSUSED */ 839 static void 840 ciputctrl_destructor(void *buf, void *cdrarg) 841 { 842 ciputctrl_t *cip = buf; 843 int i; 844 845 for (i = 0; i < n_ciputctrl; i++) { 846 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT); 847 mutex_destroy(&cip[i].ciputctrl_lock); 848 } 849 } 850 851 /* 852 * Init routine run from main at boot time. 853 */ 854 void 855 strinit(void) 856 { 857 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); 858 859 stream_head_cache = kmem_cache_create("stream_head_cache", 860 sizeof (stdata_t), 0, 861 stream_head_constructor, stream_head_destructor, NULL, 862 NULL, NULL, 0); 863 864 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0, 865 queue_constructor, queue_destructor, NULL, NULL, NULL, 0); 866 867 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0, 868 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0); 869 870 qband_cache = kmem_cache_create("qband_cache", 871 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 872 873 linkinfo_cache = kmem_cache_create("linkinfo_cache", 874 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 875 876 n_ciputctrl = ncpus; 877 n_ciputctrl = 1 << highbit(n_ciputctrl - 1); 878 ASSERT(n_ciputctrl >= 1); 879 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl); 880 if (n_ciputctrl >= min_n_ciputctrl) { 881 ciputctrl_cache = kmem_cache_create("ciputctrl_cache", 882 sizeof (ciputctrl_t) * n_ciputctrl, 883 sizeof (ciputctrl_t), ciputctrl_constructor, 884 ciputctrl_destructor, NULL, NULL, NULL, 0); 885 } 886 887 streams_taskq = system_taskq; 888 889 if (streams_taskq == NULL) 890 panic("strinit: no memory for streams taskq!"); 891 892 bc_bkgrnd_thread = thread_create(NULL, 0, 893 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri); 894 895 streams_qbkgrnd_thread = thread_create(NULL, 0, 896 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 897 898 streams_sqbkgrnd_thread = thread_create(NULL, 0, 899 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 900 901 /* 902 * Create STREAMS kstats. 903 */ 904 str_kstat = kstat_create("streams", 0, "strstat", 905 "net", KSTAT_TYPE_NAMED, 906 sizeof (str_statistics) / sizeof (kstat_named_t), 907 KSTAT_FLAG_VIRTUAL); 908 909 if (str_kstat != NULL) { 910 str_kstat->ks_data = &str_statistics; 911 kstat_install(str_kstat); 912 } 913 914 /* 915 * TPI support routine initialisation. 916 */ 917 tpi_init(); 918 919 /* 920 * Handle to have autopush and persistent link information per 921 * zone. 922 * Note: uses shutdown hook instead of destroy hook so that the 923 * persistent links can be torn down before the destroy hooks 924 * in the TCP/IP stack are called. 925 */ 926 netstack_register(NS_STR, str_stack_init, str_stack_shutdown, 927 str_stack_fini); 928 } 929 930 void 931 str_sendsig(vnode_t *vp, int event, uchar_t band, int error) 932 { 933 struct stdata *stp; 934 935 ASSERT(vp->v_stream); 936 stp = vp->v_stream; 937 /* Have to hold sd_lock to prevent siglist from changing */ 938 mutex_enter(&stp->sd_lock); 939 if (stp->sd_sigflags & event) 940 strsendsig(stp->sd_siglist, event, band, error); 941 mutex_exit(&stp->sd_lock); 942 } 943 944 /* 945 * Send the "sevent" set of signals to a process. 946 * This might send more than one signal if the process is registered 947 * for multiple events. The caller should pass in an sevent that only 948 * includes the events for which the process has registered. 949 */ 950 static void 951 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info, 952 uchar_t band, int error) 953 { 954 ASSERT(MUTEX_HELD(&proc->p_lock)); 955 956 info->si_band = 0; 957 info->si_errno = 0; 958 959 if (sevent & S_ERROR) { 960 sevent &= ~S_ERROR; 961 info->si_code = POLL_ERR; 962 info->si_errno = error; 963 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 964 "strsendsig:proc %p info %p", proc, info); 965 sigaddq(proc, NULL, info, KM_NOSLEEP); 966 info->si_errno = 0; 967 } 968 if (sevent & S_HANGUP) { 969 sevent &= ~S_HANGUP; 970 info->si_code = POLL_HUP; 971 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 972 "strsendsig:proc %p info %p", proc, info); 973 sigaddq(proc, NULL, info, KM_NOSLEEP); 974 } 975 if (sevent & S_HIPRI) { 976 sevent &= ~S_HIPRI; 977 info->si_code = POLL_PRI; 978 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 979 "strsendsig:proc %p info %p", proc, info); 980 sigaddq(proc, NULL, info, KM_NOSLEEP); 981 } 982 if (sevent & S_RDBAND) { 983 sevent &= ~S_RDBAND; 984 if (events & S_BANDURG) 985 sigtoproc(proc, NULL, SIGURG); 986 else 987 sigtoproc(proc, NULL, SIGPOLL); 988 } 989 if (sevent & S_WRBAND) { 990 sevent &= ~S_WRBAND; 991 sigtoproc(proc, NULL, SIGPOLL); 992 } 993 if (sevent & S_INPUT) { 994 sevent &= ~S_INPUT; 995 info->si_code = POLL_IN; 996 info->si_band = band; 997 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 998 "strsendsig:proc %p info %p", proc, info); 999 sigaddq(proc, NULL, info, KM_NOSLEEP); 1000 info->si_band = 0; 1001 } 1002 if (sevent & S_OUTPUT) { 1003 sevent &= ~S_OUTPUT; 1004 info->si_code = POLL_OUT; 1005 info->si_band = band; 1006 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1007 "strsendsig:proc %p info %p", proc, info); 1008 sigaddq(proc, NULL, info, KM_NOSLEEP); 1009 info->si_band = 0; 1010 } 1011 if (sevent & S_MSG) { 1012 sevent &= ~S_MSG; 1013 info->si_code = POLL_MSG; 1014 info->si_band = band; 1015 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1016 "strsendsig:proc %p info %p", proc, info); 1017 sigaddq(proc, NULL, info, KM_NOSLEEP); 1018 info->si_band = 0; 1019 } 1020 if (sevent & S_RDNORM) { 1021 sevent &= ~S_RDNORM; 1022 sigtoproc(proc, NULL, SIGPOLL); 1023 } 1024 if (sevent != 0) { 1025 panic("strsendsig: unknown event(s) %x", sevent); 1026 } 1027 } 1028 1029 /* 1030 * Send SIGPOLL/SIGURG signal to all processes and process groups 1031 * registered on the given signal list that want a signal for at 1032 * least one of the specified events. 1033 * 1034 * Must be called with exclusive access to siglist (caller holding sd_lock). 1035 * 1036 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding 1037 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure 1038 * while it is in the siglist. 1039 * 1040 * For performance reasons (MP scalability) the code drops pidlock 1041 * when sending signals to a single process. 1042 * When sending to a process group the code holds 1043 * pidlock to prevent the membership in the process group from changing 1044 * while walking the p_pglink list. 1045 */ 1046 void 1047 strsendsig(strsig_t *siglist, int event, uchar_t band, int error) 1048 { 1049 strsig_t *ssp; 1050 k_siginfo_t info; 1051 struct pid *pidp; 1052 proc_t *proc; 1053 1054 info.si_signo = SIGPOLL; 1055 info.si_errno = 0; 1056 for (ssp = siglist; ssp; ssp = ssp->ss_next) { 1057 int sevent; 1058 1059 sevent = ssp->ss_events & event; 1060 if (sevent == 0) 1061 continue; 1062 1063 if ((pidp = ssp->ss_pidp) == NULL) { 1064 /* pid was released but still on event list */ 1065 continue; 1066 } 1067 1068 1069 if (ssp->ss_pid > 0) { 1070 /* 1071 * XXX This unfortunately still generates 1072 * a signal when a fd is closed but 1073 * the proc is active. 1074 */ 1075 ASSERT(ssp->ss_pid == pidp->pid_id); 1076 1077 mutex_enter(&pidlock); 1078 proc = prfind_zone(pidp->pid_id, ALL_ZONES); 1079 if (proc == NULL) { 1080 mutex_exit(&pidlock); 1081 continue; 1082 } 1083 mutex_enter(&proc->p_lock); 1084 mutex_exit(&pidlock); 1085 dosendsig(proc, ssp->ss_events, sevent, &info, 1086 band, error); 1087 mutex_exit(&proc->p_lock); 1088 } else { 1089 /* 1090 * Send to process group. Hold pidlock across 1091 * calls to dosendsig(). 1092 */ 1093 pid_t pgrp = -ssp->ss_pid; 1094 1095 mutex_enter(&pidlock); 1096 proc = pgfind_zone(pgrp, ALL_ZONES); 1097 while (proc != NULL) { 1098 mutex_enter(&proc->p_lock); 1099 dosendsig(proc, ssp->ss_events, sevent, 1100 &info, band, error); 1101 mutex_exit(&proc->p_lock); 1102 proc = proc->p_pglink; 1103 } 1104 mutex_exit(&pidlock); 1105 } 1106 } 1107 } 1108 1109 /* 1110 * Attach a stream device or module. 1111 * qp is a read queue; the new queue goes in so its next 1112 * read ptr is the argument, and the write queue corresponding 1113 * to the argument points to this queue. Return 0 on success, 1114 * or a non-zero errno on failure. 1115 */ 1116 int 1117 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp, 1118 boolean_t is_insert) 1119 { 1120 major_t major; 1121 cdevsw_impl_t *dp; 1122 struct streamtab *str; 1123 queue_t *rq; 1124 queue_t *wrq; 1125 uint32_t qflag; 1126 uint32_t sqtype; 1127 perdm_t *dmp; 1128 int error; 1129 int sflag; 1130 1131 rq = allocq(); 1132 wrq = _WR(rq); 1133 STREAM(rq) = STREAM(wrq) = STREAM(qp); 1134 1135 if (fp != NULL) { 1136 str = fp->f_str; 1137 qflag = fp->f_qflag; 1138 sqtype = fp->f_sqtype; 1139 dmp = fp->f_dmp; 1140 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 1141 sflag = MODOPEN; 1142 1143 /* 1144 * stash away a pointer to the module structure so we can 1145 * unref it in qdetach. 1146 */ 1147 rq->q_fp = fp; 1148 } else { 1149 ASSERT(!is_insert); 1150 1151 major = getmajor(*devp); 1152 dp = &devimpl[major]; 1153 1154 str = dp->d_str; 1155 ASSERT(str == STREAMSTAB(major)); 1156 1157 qflag = dp->d_qflag; 1158 ASSERT(qflag & QISDRV); 1159 sqtype = dp->d_sqtype; 1160 1161 /* create perdm_t if needed */ 1162 if (NEED_DM(dp->d_dmp, qflag)) 1163 dp->d_dmp = hold_dm(str, qflag, sqtype); 1164 1165 dmp = dp->d_dmp; 1166 sflag = 0; 1167 } 1168 1169 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS, 1170 "qattach:qflag == %X(%X)", qflag, *devp); 1171 1172 /* setq might sleep in allocator - avoid holding locks. */ 1173 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE); 1174 1175 /* 1176 * Before calling the module's open routine, set up the q_next 1177 * pointer for inserting a module in the middle of a stream. 1178 * 1179 * Note that we can always set _QINSERTING and set up q_next 1180 * pointer for both inserting and pushing a module. Then there 1181 * is no need for the is_insert parameter. In insertq(), called 1182 * by qprocson(), assume that q_next of the new module always points 1183 * to the correct queue and use it for insertion. Everything should 1184 * work out fine. But in the first release of _I_INSERT, we 1185 * distinguish between inserting and pushing to make sure that 1186 * pushing a module follows the same code path as before. 1187 */ 1188 if (is_insert) { 1189 rq->q_flag |= _QINSERTING; 1190 rq->q_next = qp; 1191 } 1192 1193 /* 1194 * If there is an outer perimeter get exclusive access during 1195 * the open procedure. Bump up the reference count on the queue. 1196 */ 1197 entersq(rq->q_syncq, SQ_OPENCLOSE); 1198 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp); 1199 if (error != 0) 1200 goto failed; 1201 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1202 ASSERT(qprocsareon(rq)); 1203 return (0); 1204 1205 failed: 1206 rq->q_flag &= ~_QINSERTING; 1207 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq) 1208 qprocsoff(rq); 1209 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1210 rq->q_next = wrq->q_next = NULL; 1211 qdetach(rq, 0, 0, crp, B_FALSE); 1212 return (error); 1213 } 1214 1215 /* 1216 * Handle second open of stream. For modules, set the 1217 * last argument to MODOPEN and do not pass any open flags. 1218 * Ignore dummydev since this is not the first open. 1219 */ 1220 int 1221 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp) 1222 { 1223 int error; 1224 dev_t dummydev; 1225 queue_t *wqp = _WR(qp); 1226 1227 ASSERT(qp->q_flag & QREADR); 1228 entersq(qp->q_syncq, SQ_OPENCLOSE); 1229 1230 dummydev = *devp; 1231 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev, 1232 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) { 1233 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1234 mutex_enter(&STREAM(qp)->sd_lock); 1235 qp->q_stream->sd_flag |= STREOPENFAIL; 1236 mutex_exit(&STREAM(qp)->sd_lock); 1237 return (error); 1238 } 1239 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1240 1241 /* 1242 * successful open should have done qprocson() 1243 */ 1244 ASSERT(qprocsareon(_RD(qp))); 1245 return (0); 1246 } 1247 1248 /* 1249 * Detach a stream module or device. 1250 * If clmode == 1 then the module or driver was opened and its 1251 * close routine must be called. If clmode == 0, the module 1252 * or driver was never opened or the open failed, and so its close 1253 * should not be called. 1254 */ 1255 void 1256 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove) 1257 { 1258 queue_t *wqp = _WR(qp); 1259 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB)); 1260 1261 if (STREAM_NEEDSERVICE(STREAM(qp))) 1262 stream_runservice(STREAM(qp)); 1263 1264 if (clmode) { 1265 /* 1266 * Make sure that all the messages on the write side syncq are 1267 * processed and nothing is left. Since we are closing, no new 1268 * messages may appear there. 1269 */ 1270 wait_q_syncq(wqp); 1271 1272 entersq(qp->q_syncq, SQ_OPENCLOSE); 1273 if (is_remove) { 1274 mutex_enter(QLOCK(qp)); 1275 qp->q_flag |= _QREMOVING; 1276 mutex_exit(QLOCK(qp)); 1277 } 1278 (*qp->q_qinfo->qi_qclose)(qp, flag, crp); 1279 /* 1280 * Check that qprocsoff() was actually called. 1281 */ 1282 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE)); 1283 1284 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1285 } else { 1286 disable_svc(qp); 1287 } 1288 1289 /* 1290 * Allow any threads blocked in entersq to proceed and discover 1291 * the QWCLOSE is set. 1292 * Note: This assumes that all users of entersq check QWCLOSE. 1293 * Currently runservice is the only entersq that can happen 1294 * after removeq has finished. 1295 * Removeq will have discarded all messages destined to the closing 1296 * pair of queues from the syncq. 1297 * NOTE: Calling a function inside an assert is unconventional. 1298 * However, it does not cause any problem since flush_syncq() does 1299 * not change any state except when it returns non-zero i.e. 1300 * when the assert will trigger. 1301 */ 1302 ASSERT(flush_syncq(qp->q_syncq, qp) == 0); 1303 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0); 1304 ASSERT((qp->q_flag & QPERMOD) || 1305 ((qp->q_syncq->sq_head == NULL) && 1306 (wqp->q_syncq->sq_head == NULL))); 1307 1308 /* release any fmodsw_impl_t structure held on behalf of the queue */ 1309 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV); 1310 if (qp->q_fp != NULL) 1311 fmodsw_rele(qp->q_fp); 1312 1313 /* freeq removes us from the outer perimeter if any */ 1314 freeq(qp); 1315 } 1316 1317 /* Prevent service procedures from being called */ 1318 void 1319 disable_svc(queue_t *qp) 1320 { 1321 queue_t *wqp = _WR(qp); 1322 1323 ASSERT(qp->q_flag & QREADR); 1324 mutex_enter(QLOCK(qp)); 1325 qp->q_flag |= QWCLOSE; 1326 mutex_exit(QLOCK(qp)); 1327 mutex_enter(QLOCK(wqp)); 1328 wqp->q_flag |= QWCLOSE; 1329 mutex_exit(QLOCK(wqp)); 1330 } 1331 1332 /* Allow service procedures to be called again */ 1333 void 1334 enable_svc(queue_t *qp) 1335 { 1336 queue_t *wqp = _WR(qp); 1337 1338 ASSERT(qp->q_flag & QREADR); 1339 mutex_enter(QLOCK(qp)); 1340 qp->q_flag &= ~QWCLOSE; 1341 mutex_exit(QLOCK(qp)); 1342 mutex_enter(QLOCK(wqp)); 1343 wqp->q_flag &= ~QWCLOSE; 1344 mutex_exit(QLOCK(wqp)); 1345 } 1346 1347 /* 1348 * Remove queue from qhead/qtail if it is enabled. 1349 * Only reset QENAB if the queue was removed from the runlist. 1350 * A queue goes through 3 stages: 1351 * It is on the service list and QENAB is set. 1352 * It is removed from the service list but QENAB is still set. 1353 * QENAB gets changed to QINSERVICE. 1354 * QINSERVICE is reset (when the service procedure is done) 1355 * Thus we can not reset QENAB unless we actually removed it from the service 1356 * queue. 1357 */ 1358 void 1359 remove_runlist(queue_t *qp) 1360 { 1361 if (qp->q_flag & QENAB && qhead != NULL) { 1362 queue_t *q_chase; 1363 queue_t *q_curr; 1364 int removed; 1365 1366 mutex_enter(&service_queue); 1367 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed); 1368 mutex_exit(&service_queue); 1369 if (removed) { 1370 STRSTAT(qremoved); 1371 qp->q_flag &= ~QENAB; 1372 } 1373 } 1374 } 1375 1376 1377 /* 1378 * Wait for any pending service processing to complete. 1379 * The removal of queues from the runlist is not atomic with the 1380 * clearing of the QENABLED flag and setting the INSERVICE flag. 1381 * consequently it is possible for remove_runlist in strclose 1382 * to not find the queue on the runlist but for it to be QENABLED 1383 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED 1384 * as well as INSERVICE. 1385 */ 1386 void 1387 wait_svc(queue_t *qp) 1388 { 1389 queue_t *wqp = _WR(qp); 1390 1391 ASSERT(qp->q_flag & QREADR); 1392 1393 /* 1394 * Try to remove queues from qhead/qtail list. 1395 */ 1396 if (qhead != NULL) { 1397 remove_runlist(qp); 1398 remove_runlist(wqp); 1399 } 1400 /* 1401 * Wait till the syncqs associated with the queue disappear from the 1402 * background processing list. 1403 * This only needs to be done for non-PERMOD perimeters since 1404 * for PERMOD perimeters the syncq may be shared and will only be freed 1405 * when the last module/driver is unloaded. 1406 * If for PERMOD perimeters queue was on the syncq list, removeq() 1407 * should call propagate_syncq() or drain_syncq() for it. Both of these 1408 * functions remove the queue from its syncq list, so sqthread will not 1409 * try to access the queue. 1410 */ 1411 if (!(qp->q_flag & QPERMOD)) { 1412 syncq_t *rsq = qp->q_syncq; 1413 syncq_t *wsq = wqp->q_syncq; 1414 1415 /* 1416 * Disable rsq and wsq and wait for any background processing of 1417 * syncq to complete. 1418 */ 1419 wait_sq_svc(rsq); 1420 if (wsq != rsq) 1421 wait_sq_svc(wsq); 1422 } 1423 1424 mutex_enter(QLOCK(qp)); 1425 while (qp->q_flag & (QINSERVICE|QENAB)) 1426 cv_wait(&qp->q_wait, QLOCK(qp)); 1427 mutex_exit(QLOCK(qp)); 1428 mutex_enter(QLOCK(wqp)); 1429 while (wqp->q_flag & (QINSERVICE|QENAB)) 1430 cv_wait(&wqp->q_wait, QLOCK(wqp)); 1431 mutex_exit(QLOCK(wqp)); 1432 } 1433 1434 /* 1435 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'. 1436 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may 1437 * also be set, and is passed through to allocb_cred_wait(). 1438 * 1439 * Returns errno on failure, zero on success. 1440 */ 1441 int 1442 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr) 1443 { 1444 mblk_t *tmp; 1445 ssize_t count; 1446 int error = 0; 1447 1448 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K || 1449 (flag & (U_TO_K | K_TO_K)) == K_TO_K); 1450 1451 if (bp->b_datap->db_type == M_IOCTL) { 1452 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1453 } else { 1454 ASSERT(bp->b_datap->db_type == M_COPYIN); 1455 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1456 } 1457 /* 1458 * strdoioctl validates ioc_count, so if this assert fails it 1459 * cannot be due to user error. 1460 */ 1461 ASSERT(count >= 0); 1462 1463 if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr, 1464 curproc->p_pid)) == NULL) { 1465 return (error); 1466 } 1467 error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K)); 1468 if (error != 0) { 1469 freeb(tmp); 1470 return (error); 1471 } 1472 DB_CPID(tmp) = curproc->p_pid; 1473 tmp->b_wptr += count; 1474 bp->b_cont = tmp; 1475 1476 return (0); 1477 } 1478 1479 /* 1480 * Copy ioctl data to user-land. Return non-zero errno on failure, 1481 * 0 for success. 1482 */ 1483 int 1484 getiocd(mblk_t *bp, char *arg, int copymode) 1485 { 1486 ssize_t count; 1487 size_t n; 1488 int error; 1489 1490 if (bp->b_datap->db_type == M_IOCACK) 1491 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1492 else { 1493 ASSERT(bp->b_datap->db_type == M_COPYOUT); 1494 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1495 } 1496 ASSERT(count >= 0); 1497 1498 for (bp = bp->b_cont; bp && count; 1499 count -= n, bp = bp->b_cont, arg += n) { 1500 n = MIN(count, bp->b_wptr - bp->b_rptr); 1501 error = strcopyout(bp->b_rptr, arg, n, copymode); 1502 if (error) 1503 return (error); 1504 } 1505 ASSERT(count == 0); 1506 return (0); 1507 } 1508 1509 /* 1510 * Allocate a linkinfo entry given the write queue of the 1511 * bottom module of the top stream and the write queue of the 1512 * stream head of the bottom stream. 1513 */ 1514 linkinfo_t * 1515 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown) 1516 { 1517 linkinfo_t *linkp; 1518 1519 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP); 1520 1521 linkp->li_lblk.l_qtop = qup; 1522 linkp->li_lblk.l_qbot = qdown; 1523 linkp->li_fpdown = fpdown; 1524 1525 mutex_enter(&strresources); 1526 linkp->li_next = linkinfo_list; 1527 linkp->li_prev = NULL; 1528 if (linkp->li_next) 1529 linkp->li_next->li_prev = linkp; 1530 linkinfo_list = linkp; 1531 linkp->li_lblk.l_index = ++lnk_id; 1532 ASSERT(lnk_id != 0); /* this should never wrap in practice */ 1533 mutex_exit(&strresources); 1534 1535 return (linkp); 1536 } 1537 1538 /* 1539 * Free a linkinfo entry. 1540 */ 1541 void 1542 lbfree(linkinfo_t *linkp) 1543 { 1544 mutex_enter(&strresources); 1545 if (linkp->li_next) 1546 linkp->li_next->li_prev = linkp->li_prev; 1547 if (linkp->li_prev) 1548 linkp->li_prev->li_next = linkp->li_next; 1549 else 1550 linkinfo_list = linkp->li_next; 1551 mutex_exit(&strresources); 1552 1553 kmem_cache_free(linkinfo_cache, linkp); 1554 } 1555 1556 /* 1557 * Check for a potential linking cycle. 1558 * Return 1 if a link will result in a cycle, 1559 * and 0 otherwise. 1560 */ 1561 int 1562 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss) 1563 { 1564 struct mux_node *np; 1565 struct mux_edge *ep; 1566 int i; 1567 major_t lomaj; 1568 major_t upmaj; 1569 /* 1570 * if the lower stream is a pipe/FIFO, return, since link 1571 * cycles can not happen on pipes/FIFOs 1572 */ 1573 if (lostp->sd_vnode->v_type == VFIFO) 1574 return (0); 1575 1576 for (i = 0; i < ss->ss_devcnt; i++) { 1577 np = &ss->ss_mux_nodes[i]; 1578 MUX_CLEAR(np); 1579 } 1580 lomaj = getmajor(lostp->sd_vnode->v_rdev); 1581 upmaj = getmajor(upstp->sd_vnode->v_rdev); 1582 np = &ss->ss_mux_nodes[lomaj]; 1583 for (;;) { 1584 if (!MUX_DIDVISIT(np)) { 1585 if (np->mn_imaj == upmaj) 1586 return (1); 1587 if (np->mn_outp == NULL) { 1588 MUX_VISIT(np); 1589 if (np->mn_originp == NULL) 1590 return (0); 1591 np = np->mn_originp; 1592 continue; 1593 } 1594 MUX_VISIT(np); 1595 np->mn_startp = np->mn_outp; 1596 } else { 1597 if (np->mn_startp == NULL) { 1598 if (np->mn_originp == NULL) 1599 return (0); 1600 else { 1601 np = np->mn_originp; 1602 continue; 1603 } 1604 } 1605 /* 1606 * If ep->me_nodep is a FIFO (me_nodep == NULL), 1607 * ignore the edge and move on. ep->me_nodep gets 1608 * set to NULL in mux_addedge() if it is a FIFO. 1609 * 1610 */ 1611 ep = np->mn_startp; 1612 np->mn_startp = ep->me_nextp; 1613 if (ep->me_nodep == NULL) 1614 continue; 1615 ep->me_nodep->mn_originp = np; 1616 np = ep->me_nodep; 1617 } 1618 } 1619 } 1620 1621 /* 1622 * Find linkinfo entry corresponding to the parameters. 1623 */ 1624 linkinfo_t * 1625 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss) 1626 { 1627 linkinfo_t *linkp; 1628 struct mux_edge *mep; 1629 struct mux_node *mnp; 1630 queue_t *qup; 1631 1632 mutex_enter(&strresources); 1633 if ((type & LINKTYPEMASK) == LINKNORMAL) { 1634 qup = getendq(stp->sd_wrq); 1635 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1636 if ((qup == linkp->li_lblk.l_qtop) && 1637 (!index || (index == linkp->li_lblk.l_index))) { 1638 mutex_exit(&strresources); 1639 return (linkp); 1640 } 1641 } 1642 } else { 1643 ASSERT((type & LINKTYPEMASK) == LINKPERSIST); 1644 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)]; 1645 mep = mnp->mn_outp; 1646 while (mep) { 1647 if ((index == 0) || (index == mep->me_muxid)) 1648 break; 1649 mep = mep->me_nextp; 1650 } 1651 if (!mep) { 1652 mutex_exit(&strresources); 1653 return (NULL); 1654 } 1655 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1656 if ((!linkp->li_lblk.l_qtop) && 1657 (mep->me_muxid == linkp->li_lblk.l_index)) { 1658 mutex_exit(&strresources); 1659 return (linkp); 1660 } 1661 } 1662 } 1663 mutex_exit(&strresources); 1664 return (NULL); 1665 } 1666 1667 /* 1668 * Given a queue ptr, follow the chain of q_next pointers until you reach the 1669 * last queue on the chain and return it. 1670 */ 1671 queue_t * 1672 getendq(queue_t *q) 1673 { 1674 ASSERT(q != NULL); 1675 while (_SAMESTR(q)) 1676 q = q->q_next; 1677 return (q); 1678 } 1679 1680 /* 1681 * Wait for the syncq count to drop to zero. 1682 * sq could be either outer or inner. 1683 */ 1684 1685 static void 1686 wait_syncq(syncq_t *sq) 1687 { 1688 uint16_t count; 1689 1690 mutex_enter(SQLOCK(sq)); 1691 count = sq->sq_count; 1692 SQ_PUTLOCKS_ENTER(sq); 1693 SUM_SQ_PUTCOUNTS(sq, count); 1694 while (count != 0) { 1695 sq->sq_flags |= SQ_WANTWAKEUP; 1696 SQ_PUTLOCKS_EXIT(sq); 1697 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1698 count = sq->sq_count; 1699 SQ_PUTLOCKS_ENTER(sq); 1700 SUM_SQ_PUTCOUNTS(sq, count); 1701 } 1702 SQ_PUTLOCKS_EXIT(sq); 1703 mutex_exit(SQLOCK(sq)); 1704 } 1705 1706 /* 1707 * Wait while there are any messages for the queue in its syncq. 1708 */ 1709 static void 1710 wait_q_syncq(queue_t *q) 1711 { 1712 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1713 syncq_t *sq = q->q_syncq; 1714 1715 mutex_enter(SQLOCK(sq)); 1716 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1717 sq->sq_flags |= SQ_WANTWAKEUP; 1718 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1719 } 1720 mutex_exit(SQLOCK(sq)); 1721 } 1722 } 1723 1724 1725 int 1726 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp, 1727 int lhlink) 1728 { 1729 struct stdata *stp; 1730 struct strioctl strioc; 1731 struct linkinfo *linkp; 1732 struct stdata *stpdown; 1733 struct streamtab *str; 1734 queue_t *passq; 1735 syncq_t *passyncq; 1736 queue_t *rq; 1737 cdevsw_impl_t *dp; 1738 uint32_t qflag; 1739 uint32_t sqtype; 1740 perdm_t *dmp; 1741 int error = 0; 1742 netstack_t *ns; 1743 str_stack_t *ss; 1744 1745 stp = vp->v_stream; 1746 TRACE_1(TR_FAC_STREAMS_FR, 1747 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp); 1748 /* 1749 * Test for invalid upper stream 1750 */ 1751 if (stp->sd_flag & STRHUP) { 1752 return (ENXIO); 1753 } 1754 if (vp->v_type == VFIFO) { 1755 return (EINVAL); 1756 } 1757 if (stp->sd_strtab == NULL) { 1758 return (EINVAL); 1759 } 1760 if (!stp->sd_strtab->st_muxwinit) { 1761 return (EINVAL); 1762 } 1763 if (fpdown == NULL) { 1764 return (EBADF); 1765 } 1766 ns = netstack_find_by_cred(crp); 1767 ASSERT(ns != NULL); 1768 ss = ns->netstack_str; 1769 ASSERT(ss != NULL); 1770 1771 if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) { 1772 netstack_rele(ss->ss_netstack); 1773 return (EINVAL); 1774 } 1775 mutex_enter(&muxifier); 1776 if (stp->sd_flag & STPLEX) { 1777 mutex_exit(&muxifier); 1778 netstack_rele(ss->ss_netstack); 1779 return (ENXIO); 1780 } 1781 1782 /* 1783 * Test for invalid lower stream. 1784 * The check for the v_type != VFIFO and having a major 1785 * number not >= devcnt is done to avoid problems with 1786 * adding mux_node entry past the end of mux_nodes[]. 1787 * For FIFO's we don't add an entry so this isn't a 1788 * problem. 1789 */ 1790 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) || 1791 (stpdown == stp) || (stpdown->sd_flag & 1792 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) || 1793 ((stpdown->sd_vnode->v_type != VFIFO) && 1794 (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) || 1795 linkcycle(stp, stpdown, ss)) { 1796 mutex_exit(&muxifier); 1797 netstack_rele(ss->ss_netstack); 1798 return (EINVAL); 1799 } 1800 TRACE_1(TR_FAC_STREAMS_FR, 1801 TR_STPDOWN, "stpdown:%p", stpdown); 1802 rq = getendq(stp->sd_wrq); 1803 if (cmd == I_PLINK) 1804 rq = NULL; 1805 1806 linkp = alloclink(rq, stpdown->sd_wrq, fpdown); 1807 1808 strioc.ic_cmd = cmd; 1809 strioc.ic_timout = INFTIM; 1810 strioc.ic_len = sizeof (struct linkblk); 1811 strioc.ic_dp = (char *)&linkp->li_lblk; 1812 1813 /* 1814 * STRPLUMB protects plumbing changes and should be set before 1815 * link_addpassthru()/link_rempassthru() are called, so it is set here 1816 * and cleared in the end of mlink when passthru queue is removed. 1817 * Setting of STRPLUMB prevents reopens of the stream while passthru 1818 * queue is in-place (it is not a proper module and doesn't have open 1819 * entry point). 1820 * 1821 * STPLEX prevents any threads from entering the stream from above. It 1822 * can't be set before the call to link_addpassthru() because putnext 1823 * from below may cause stream head I/O routines to be called and these 1824 * routines assert that STPLEX is not set. After link_addpassthru() 1825 * nothing may come from below since the pass queue syncq is blocked. 1826 * Note also that STPLEX should be cleared before the call to 1827 * link_rempassthru() since when messages start flowing to the stream 1828 * head (e.g. because of message propagation from the pass queue) stream 1829 * head I/O routines may be called with STPLEX flag set. 1830 * 1831 * When STPLEX is set, nothing may come into the stream from above and 1832 * it is safe to do a setq which will change stream head. So, the 1833 * correct sequence of actions is: 1834 * 1835 * 1) Set STRPLUMB 1836 * 2) Call link_addpassthru() 1837 * 3) Set STPLEX 1838 * 4) Call setq and update the stream state 1839 * 5) Clear STPLEX 1840 * 6) Call link_rempassthru() 1841 * 7) Clear STRPLUMB 1842 * 1843 * The same sequence applies to munlink() code. 1844 */ 1845 mutex_enter(&stpdown->sd_lock); 1846 stpdown->sd_flag |= STRPLUMB; 1847 mutex_exit(&stpdown->sd_lock); 1848 /* 1849 * Add passthru queue below lower mux. This will block 1850 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 1851 */ 1852 passq = link_addpassthru(stpdown); 1853 1854 mutex_enter(&stpdown->sd_lock); 1855 stpdown->sd_flag |= STPLEX; 1856 mutex_exit(&stpdown->sd_lock); 1857 1858 rq = _RD(stpdown->sd_wrq); 1859 /* 1860 * There may be messages in the streamhead's syncq due to messages 1861 * that arrived before link_addpassthru() was done. To avoid 1862 * background processing of the syncq happening simultaneous with 1863 * setq processing, we disable the streamhead syncq and wait until 1864 * existing background thread finishes working on it. 1865 */ 1866 wait_sq_svc(rq->q_syncq); 1867 passyncq = passq->q_syncq; 1868 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1869 blocksq(passyncq, SQ_BLOCKED, 0); 1870 1871 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 1872 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 1873 rq->q_ptr = _WR(rq)->q_ptr = NULL; 1874 1875 /* setq might sleep in allocator - avoid holding locks. */ 1876 /* Note: we are holding muxifier here. */ 1877 1878 str = stp->sd_strtab; 1879 dp = &devimpl[getmajor(vp->v_rdev)]; 1880 ASSERT(dp->d_str == str); 1881 1882 qflag = dp->d_qflag; 1883 sqtype = dp->d_sqtype; 1884 1885 /* create perdm_t if needed */ 1886 if (NEED_DM(dp->d_dmp, qflag)) 1887 dp->d_dmp = hold_dm(str, qflag, sqtype); 1888 1889 dmp = dp->d_dmp; 1890 1891 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype, 1892 B_TRUE); 1893 1894 /* 1895 * XXX Remove any "odd" messages from the queue. 1896 * Keep only M_DATA, M_PROTO, M_PCPROTO. 1897 */ 1898 error = strdoioctl(stp, &strioc, FNATIVE, 1899 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 1900 if (error != 0) { 1901 lbfree(linkp); 1902 1903 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1904 blocksq(passyncq, SQ_BLOCKED, 0); 1905 /* 1906 * Restore the stream head queue and then remove 1907 * the passq. Turn off STPLEX before we turn on 1908 * the stream by removing the passq. 1909 */ 1910 rq->q_ptr = _WR(rq)->q_ptr = stpdown; 1911 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, 1912 B_TRUE); 1913 1914 mutex_enter(&stpdown->sd_lock); 1915 stpdown->sd_flag &= ~STPLEX; 1916 mutex_exit(&stpdown->sd_lock); 1917 1918 link_rempassthru(passq); 1919 1920 mutex_enter(&stpdown->sd_lock); 1921 stpdown->sd_flag &= ~STRPLUMB; 1922 /* Wakeup anyone waiting for STRPLUMB to clear. */ 1923 cv_broadcast(&stpdown->sd_monitor); 1924 mutex_exit(&stpdown->sd_lock); 1925 1926 mutex_exit(&muxifier); 1927 netstack_rele(ss->ss_netstack); 1928 return (error); 1929 } 1930 mutex_enter(&fpdown->f_tlock); 1931 fpdown->f_count++; 1932 mutex_exit(&fpdown->f_tlock); 1933 1934 /* 1935 * if we've made it here the linkage is all set up so we should also 1936 * set up the layered driver linkages 1937 */ 1938 1939 ASSERT((cmd == I_LINK) || (cmd == I_PLINK)); 1940 if (cmd == I_LINK) { 1941 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL); 1942 } else { 1943 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST); 1944 } 1945 1946 link_rempassthru(passq); 1947 1948 mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss); 1949 1950 /* 1951 * Mark the upper stream as having dependent links 1952 * so that strclose can clean it up. 1953 */ 1954 if (cmd == I_LINK) { 1955 mutex_enter(&stp->sd_lock); 1956 stp->sd_flag |= STRHASLINKS; 1957 mutex_exit(&stp->sd_lock); 1958 } 1959 /* 1960 * Wake up any other processes that may have been 1961 * waiting on the lower stream. These will all 1962 * error out. 1963 */ 1964 mutex_enter(&stpdown->sd_lock); 1965 /* The passthru module is removed so we may release STRPLUMB */ 1966 stpdown->sd_flag &= ~STRPLUMB; 1967 cv_broadcast(&rq->q_wait); 1968 cv_broadcast(&_WR(rq)->q_wait); 1969 cv_broadcast(&stpdown->sd_monitor); 1970 mutex_exit(&stpdown->sd_lock); 1971 mutex_exit(&muxifier); 1972 *rvalp = linkp->li_lblk.l_index; 1973 netstack_rele(ss->ss_netstack); 1974 return (0); 1975 } 1976 1977 int 1978 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink) 1979 { 1980 int ret; 1981 struct file *fpdown; 1982 1983 fpdown = getf(arg); 1984 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink); 1985 if (fpdown != NULL) 1986 releasef(arg); 1987 return (ret); 1988 } 1989 1990 /* 1991 * Unlink a multiplexor link. Stp is the controlling stream for the 1992 * link, and linkp points to the link's entry in the linkinfo list. 1993 * The muxifier lock must be held on entry and is dropped on exit. 1994 * 1995 * NOTE : Currently it is assumed that mux would process all the messages 1996 * sitting on it's queue before ACKing the UNLINK. It is the responsibility 1997 * of the mux to handle all the messages that arrive before UNLINK. 1998 * If the mux has to send down messages on its lower stream before 1999 * ACKing I_UNLINK, then it *should* know to handle messages even 2000 * after the UNLINK is acked (actually it should be able to handle till we 2001 * re-block the read side of the pass queue here). If the mux does not 2002 * open up the lower stream, any messages that arrive during UNLINK 2003 * will be put in the stream head. In the case of lower stream opening 2004 * up, some messages might land in the stream head depending on when 2005 * the message arrived and when the read side of the pass queue was 2006 * re-blocked. 2007 */ 2008 int 2009 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp, 2010 str_stack_t *ss) 2011 { 2012 struct strioctl strioc; 2013 struct stdata *stpdown; 2014 queue_t *rq, *wrq; 2015 queue_t *passq; 2016 syncq_t *passyncq; 2017 int error = 0; 2018 file_t *fpdown; 2019 2020 ASSERT(MUTEX_HELD(&muxifier)); 2021 2022 stpdown = linkp->li_fpdown->f_vnode->v_stream; 2023 2024 /* 2025 * See the comment in mlink() concerning STRPLUMB/STPLEX flags. 2026 */ 2027 mutex_enter(&stpdown->sd_lock); 2028 stpdown->sd_flag |= STRPLUMB; 2029 mutex_exit(&stpdown->sd_lock); 2030 2031 /* 2032 * Add passthru queue below lower mux. This will block 2033 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 2034 */ 2035 passq = link_addpassthru(stpdown); 2036 2037 if ((flag & LINKTYPEMASK) == LINKNORMAL) 2038 strioc.ic_cmd = I_UNLINK; 2039 else 2040 strioc.ic_cmd = I_PUNLINK; 2041 strioc.ic_timout = INFTIM; 2042 strioc.ic_len = sizeof (struct linkblk); 2043 strioc.ic_dp = (char *)&linkp->li_lblk; 2044 2045 error = strdoioctl(stp, &strioc, FNATIVE, 2046 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 2047 2048 /* 2049 * If there was an error and this is not called via strclose, 2050 * return to the user. Otherwise, pretend there was no error 2051 * and close the link. 2052 */ 2053 if (error) { 2054 if (flag & LINKCLOSE) { 2055 cmn_err(CE_WARN, "KERNEL: munlink: could not perform " 2056 "unlink ioctl, closing anyway (%d)\n", error); 2057 } else { 2058 link_rempassthru(passq); 2059 mutex_enter(&stpdown->sd_lock); 2060 stpdown->sd_flag &= ~STRPLUMB; 2061 cv_broadcast(&stpdown->sd_monitor); 2062 mutex_exit(&stpdown->sd_lock); 2063 mutex_exit(&muxifier); 2064 return (error); 2065 } 2066 } 2067 2068 mux_rmvedge(stp, linkp->li_lblk.l_index, ss); 2069 fpdown = linkp->li_fpdown; 2070 lbfree(linkp); 2071 2072 /* 2073 * We go ahead and drop muxifier here--it's a nasty global lock that 2074 * can slow others down. It's okay to since attempts to mlink() this 2075 * stream will be stopped because STPLEX is still set in the stdata 2076 * structure, and munlink() is stopped because mux_rmvedge() and 2077 * lbfree() have removed it from mux_nodes[] and linkinfo_list, 2078 * respectively. Note that we defer the closef() of fpdown until 2079 * after we drop muxifier since strclose() can call munlinkall(). 2080 */ 2081 mutex_exit(&muxifier); 2082 2083 wrq = stpdown->sd_wrq; 2084 rq = _RD(wrq); 2085 2086 /* 2087 * Get rid of outstanding service procedure runs, before we make 2088 * it a stream head, since a stream head doesn't have any service 2089 * procedure. 2090 */ 2091 disable_svc(rq); 2092 wait_svc(rq); 2093 2094 /* 2095 * Since we don't disable the syncq for QPERMOD, we wait for whatever 2096 * is queued up to be finished. mux should take care that nothing is 2097 * send down to this queue. We should do it now as we're going to block 2098 * passyncq if it was unblocked. 2099 */ 2100 if (wrq->q_flag & QPERMOD) { 2101 syncq_t *sq = wrq->q_syncq; 2102 2103 mutex_enter(SQLOCK(sq)); 2104 while (wrq->q_sqflags & Q_SQQUEUED) { 2105 sq->sq_flags |= SQ_WANTWAKEUP; 2106 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2107 } 2108 mutex_exit(SQLOCK(sq)); 2109 } 2110 passyncq = passq->q_syncq; 2111 if (!(passyncq->sq_flags & SQ_BLOCKED)) { 2112 2113 syncq_t *sq, *outer; 2114 2115 /* 2116 * Messages could be flowing from underneath. We will 2117 * block the read side of the passq. This would be 2118 * sufficient for QPAIR and QPERQ muxes to ensure 2119 * that no data is flowing up into this queue 2120 * and hence no thread active in this instance of 2121 * lower mux. But for QPERMOD and QMTOUTPERIM there 2122 * could be messages on the inner and outer/inner 2123 * syncqs respectively. We will wait for them to drain. 2124 * Because passq is blocked messages end up in the syncq 2125 * And qfill_syncq could possibly end up setting QFULL 2126 * which will access the rq->q_flag. Hence, we have to 2127 * acquire the QLOCK in setq. 2128 * 2129 * XXX Messages can also flow from top into this 2130 * queue though the unlink is over (Ex. some instance 2131 * in putnext() called from top that has still not 2132 * accessed this queue. And also putq(lowerq) ?). 2133 * Solution : How about blocking the l_qtop queue ? 2134 * Do we really care about such pure D_MP muxes ? 2135 */ 2136 2137 blocksq(passyncq, SQ_BLOCKED, 0); 2138 2139 sq = rq->q_syncq; 2140 if ((outer = sq->sq_outer) != NULL) { 2141 2142 /* 2143 * We have to just wait for the outer sq_count 2144 * drop to zero. As this does not prevent new 2145 * messages to enter the outer perimeter, this 2146 * is subject to starvation. 2147 * 2148 * NOTE :Because of blocksq above, messages could 2149 * be in the inner syncq only because of some 2150 * thread holding the outer perimeter exclusively. 2151 * Hence it would be sufficient to wait for the 2152 * exclusive holder of the outer perimeter to drain 2153 * the inner and outer syncqs. But we will not depend 2154 * on this feature and hence check the inner syncqs 2155 * separately. 2156 */ 2157 wait_syncq(outer); 2158 } 2159 2160 2161 /* 2162 * There could be messages destined for 2163 * this queue. Let the exclusive holder 2164 * drain it. 2165 */ 2166 2167 wait_syncq(sq); 2168 ASSERT((rq->q_flag & QPERMOD) || 2169 ((rq->q_syncq->sq_head == NULL) && 2170 (_WR(rq)->q_syncq->sq_head == NULL))); 2171 } 2172 2173 /* 2174 * We haven't taken care of QPERMOD case yet. QPERMOD is a special 2175 * case as we don't disable its syncq or remove it off the syncq 2176 * service list. 2177 */ 2178 if (rq->q_flag & QPERMOD) { 2179 syncq_t *sq = rq->q_syncq; 2180 2181 mutex_enter(SQLOCK(sq)); 2182 while (rq->q_sqflags & Q_SQQUEUED) { 2183 sq->sq_flags |= SQ_WANTWAKEUP; 2184 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2185 } 2186 mutex_exit(SQLOCK(sq)); 2187 } 2188 2189 /* 2190 * flush_syncq changes states only when there are some messages to 2191 * free, i.e. when it returns non-zero value to return. 2192 */ 2193 ASSERT(flush_syncq(rq->q_syncq, rq) == 0); 2194 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0); 2195 2196 /* 2197 * Nobody else should know about this queue now. 2198 * If the mux did not process the messages before 2199 * acking the I_UNLINK, free them now. 2200 */ 2201 2202 flushq(rq, FLUSHALL); 2203 flushq(_WR(rq), FLUSHALL); 2204 2205 /* 2206 * Convert the mux lower queue into a stream head queue. 2207 * Turn off STPLEX before we turn on the stream by removing the passq. 2208 */ 2209 rq->q_ptr = wrq->q_ptr = stpdown; 2210 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE); 2211 2212 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 2213 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 2214 2215 enable_svc(rq); 2216 2217 /* 2218 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still 2219 * needs to be set to prevent reopen() of the stream - such reopen may 2220 * try to call non-existent pass queue open routine and panic. 2221 */ 2222 mutex_enter(&stpdown->sd_lock); 2223 stpdown->sd_flag &= ~STPLEX; 2224 mutex_exit(&stpdown->sd_lock); 2225 2226 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) || 2227 ((flag & LINKTYPEMASK) == LINKPERSIST)); 2228 2229 /* clean up the layered driver linkages */ 2230 if ((flag & LINKTYPEMASK) == LINKNORMAL) { 2231 ldi_munlink_fp(stp, fpdown, LINKNORMAL); 2232 } else { 2233 ldi_munlink_fp(stp, fpdown, LINKPERSIST); 2234 } 2235 2236 link_rempassthru(passq); 2237 2238 /* 2239 * Now all plumbing changes are finished and STRPLUMB is no 2240 * longer needed. 2241 */ 2242 mutex_enter(&stpdown->sd_lock); 2243 stpdown->sd_flag &= ~STRPLUMB; 2244 cv_broadcast(&stpdown->sd_monitor); 2245 mutex_exit(&stpdown->sd_lock); 2246 2247 (void) closef(fpdown); 2248 return (0); 2249 } 2250 2251 /* 2252 * Unlink all multiplexor links for which stp is the controlling stream. 2253 * Return 0, or a non-zero errno on failure. 2254 */ 2255 int 2256 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss) 2257 { 2258 linkinfo_t *linkp; 2259 int error = 0; 2260 2261 mutex_enter(&muxifier); 2262 while (linkp = findlinks(stp, 0, flag, ss)) { 2263 /* 2264 * munlink() releases the muxifier lock. 2265 */ 2266 if (error = munlink(stp, linkp, flag, crp, rvalp, ss)) 2267 return (error); 2268 mutex_enter(&muxifier); 2269 } 2270 mutex_exit(&muxifier); 2271 return (0); 2272 } 2273 2274 /* 2275 * A multiplexor link has been made. Add an 2276 * edge to the directed graph. 2277 */ 2278 void 2279 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss) 2280 { 2281 struct mux_node *np; 2282 struct mux_edge *ep; 2283 major_t upmaj; 2284 major_t lomaj; 2285 2286 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2287 lomaj = getmajor(lostp->sd_vnode->v_rdev); 2288 np = &ss->ss_mux_nodes[upmaj]; 2289 if (np->mn_outp) { 2290 ep = np->mn_outp; 2291 while (ep->me_nextp) 2292 ep = ep->me_nextp; 2293 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2294 ep = ep->me_nextp; 2295 } else { 2296 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2297 ep = np->mn_outp; 2298 } 2299 ep->me_nextp = NULL; 2300 ep->me_muxid = muxid; 2301 /* 2302 * Save the dev_t for the purposes of str_stack_shutdown. 2303 * str_stack_shutdown assumes that the device allows reopen, since 2304 * this dev_t is the one after any cloning by xx_open(). 2305 * Would prefer finding the dev_t from before any cloning, 2306 * but specfs doesn't retain that. 2307 */ 2308 ep->me_dev = upstp->sd_vnode->v_rdev; 2309 if (lostp->sd_vnode->v_type == VFIFO) 2310 ep->me_nodep = NULL; 2311 else 2312 ep->me_nodep = &ss->ss_mux_nodes[lomaj]; 2313 } 2314 2315 /* 2316 * A multiplexor link has been removed. Remove the 2317 * edge in the directed graph. 2318 */ 2319 void 2320 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss) 2321 { 2322 struct mux_node *np; 2323 struct mux_edge *ep; 2324 struct mux_edge *pep = NULL; 2325 major_t upmaj; 2326 2327 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2328 np = &ss->ss_mux_nodes[upmaj]; 2329 ASSERT(np->mn_outp != NULL); 2330 ep = np->mn_outp; 2331 while (ep) { 2332 if (ep->me_muxid == muxid) { 2333 if (pep) 2334 pep->me_nextp = ep->me_nextp; 2335 else 2336 np->mn_outp = ep->me_nextp; 2337 kmem_free(ep, sizeof (struct mux_edge)); 2338 return; 2339 } 2340 pep = ep; 2341 ep = ep->me_nextp; 2342 } 2343 ASSERT(0); /* should not reach here */ 2344 } 2345 2346 /* 2347 * Translate the device flags (from conf.h) to the corresponding 2348 * qflag and sq_flag (type) values. 2349 */ 2350 int 2351 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp, 2352 uint32_t *sqtypep) 2353 { 2354 uint32_t qflag = 0; 2355 uint32_t sqtype = 0; 2356 2357 if (devflag & _D_OLD) 2358 goto bad; 2359 2360 /* Inner perimeter presence and scope */ 2361 switch (devflag & D_MTINNER_MASK) { 2362 case D_MP: 2363 qflag |= QMTSAFE; 2364 sqtype |= SQ_CI; 2365 break; 2366 case D_MTPERQ|D_MP: 2367 qflag |= QPERQ; 2368 break; 2369 case D_MTQPAIR|D_MP: 2370 qflag |= QPAIR; 2371 break; 2372 case D_MTPERMOD|D_MP: 2373 qflag |= QPERMOD; 2374 break; 2375 default: 2376 goto bad; 2377 } 2378 2379 /* Outer perimeter */ 2380 if (devflag & D_MTOUTPERIM) { 2381 switch (devflag & D_MTINNER_MASK) { 2382 case D_MP: 2383 case D_MTPERQ|D_MP: 2384 case D_MTQPAIR|D_MP: 2385 break; 2386 default: 2387 goto bad; 2388 } 2389 qflag |= QMTOUTPERIM; 2390 } 2391 2392 /* Inner perimeter modifiers */ 2393 if (devflag & D_MTINNER_MOD) { 2394 switch (devflag & D_MTINNER_MASK) { 2395 case D_MP: 2396 goto bad; 2397 default: 2398 break; 2399 } 2400 if (devflag & D_MTPUTSHARED) 2401 sqtype |= SQ_CIPUT; 2402 if (devflag & _D_MTOCSHARED) { 2403 /* 2404 * The code in putnext assumes that it has the 2405 * highest concurrency by not checking sq_count. 2406 * Thus _D_MTOCSHARED can only be supported when 2407 * D_MTPUTSHARED is set. 2408 */ 2409 if (!(devflag & D_MTPUTSHARED)) 2410 goto bad; 2411 sqtype |= SQ_CIOC; 2412 } 2413 if (devflag & _D_MTCBSHARED) { 2414 /* 2415 * The code in putnext assumes that it has the 2416 * highest concurrency by not checking sq_count. 2417 * Thus _D_MTCBSHARED can only be supported when 2418 * D_MTPUTSHARED is set. 2419 */ 2420 if (!(devflag & D_MTPUTSHARED)) 2421 goto bad; 2422 sqtype |= SQ_CICB; 2423 } 2424 if (devflag & _D_MTSVCSHARED) { 2425 /* 2426 * The code in putnext assumes that it has the 2427 * highest concurrency by not checking sq_count. 2428 * Thus _D_MTSVCSHARED can only be supported when 2429 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is 2430 * supported only for QPERMOD. 2431 */ 2432 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD)) 2433 goto bad; 2434 sqtype |= SQ_CISVC; 2435 } 2436 } 2437 2438 /* Default outer perimeter concurrency */ 2439 sqtype |= SQ_CO; 2440 2441 /* Outer perimeter modifiers */ 2442 if (devflag & D_MTOCEXCL) { 2443 if (!(devflag & D_MTOUTPERIM)) { 2444 /* No outer perimeter */ 2445 goto bad; 2446 } 2447 sqtype &= ~SQ_COOC; 2448 } 2449 2450 /* Synchronous Streams extended qinit structure */ 2451 if (devflag & D_SYNCSTR) 2452 qflag |= QSYNCSTR; 2453 2454 /* 2455 * Private flag used by a transport module to indicate 2456 * to sockfs that it supports direct-access mode without 2457 * having to go through STREAMS. 2458 */ 2459 if (devflag & _D_DIRECT) { 2460 /* Reject unless the module is fully-MT (no perimeter) */ 2461 if ((qflag & QMT_TYPEMASK) != QMTSAFE) 2462 goto bad; 2463 qflag |= _QDIRECT; 2464 } 2465 2466 *qflagp = qflag; 2467 *sqtypep = sqtype; 2468 return (0); 2469 2470 bad: 2471 cmn_err(CE_WARN, 2472 "stropen: bad MT flags (0x%x) in driver '%s'", 2473 (int)(qflag & D_MTSAFETY_MASK), 2474 stp->st_rdinit->qi_minfo->mi_idname); 2475 2476 return (EINVAL); 2477 } 2478 2479 /* 2480 * Set the interface values for a pair of queues (qinit structure, 2481 * packet sizes, water marks). 2482 * setq assumes that the caller does not have a claim (entersq or claimq) 2483 * on the queue. 2484 */ 2485 void 2486 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit, 2487 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed) 2488 { 2489 queue_t *wq; 2490 syncq_t *sq, *outer; 2491 2492 ASSERT(rq->q_flag & QREADR); 2493 ASSERT((qflag & QMT_TYPEMASK) != 0); 2494 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 2495 2496 wq = _WR(rq); 2497 rq->q_qinfo = rinit; 2498 rq->q_hiwat = rinit->qi_minfo->mi_hiwat; 2499 rq->q_lowat = rinit->qi_minfo->mi_lowat; 2500 rq->q_minpsz = rinit->qi_minfo->mi_minpsz; 2501 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz; 2502 wq->q_qinfo = winit; 2503 wq->q_hiwat = winit->qi_minfo->mi_hiwat; 2504 wq->q_lowat = winit->qi_minfo->mi_lowat; 2505 wq->q_minpsz = winit->qi_minfo->mi_minpsz; 2506 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz; 2507 2508 /* Remove old syncqs */ 2509 sq = rq->q_syncq; 2510 outer = sq->sq_outer; 2511 if (outer != NULL) { 2512 ASSERT(wq->q_syncq->sq_outer == outer); 2513 outer_remove(outer, rq->q_syncq); 2514 if (wq->q_syncq != rq->q_syncq) 2515 outer_remove(outer, wq->q_syncq); 2516 } 2517 ASSERT(sq->sq_outer == NULL); 2518 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2519 2520 if (sq != SQ(rq)) { 2521 if (!(rq->q_flag & QPERMOD)) 2522 free_syncq(sq); 2523 if (wq->q_syncq == rq->q_syncq) 2524 wq->q_syncq = NULL; 2525 rq->q_syncq = NULL; 2526 } 2527 if (wq->q_syncq != NULL && wq->q_syncq != sq && 2528 wq->q_syncq != SQ(rq)) { 2529 free_syncq(wq->q_syncq); 2530 wq->q_syncq = NULL; 2531 } 2532 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL && 2533 rq->q_syncq->sq_tail == NULL)); 2534 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL && 2535 wq->q_syncq->sq_tail == NULL)); 2536 2537 if (!(rq->q_flag & QPERMOD) && 2538 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) { 2539 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2540 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl, 2541 rq->q_syncq->sq_nciputctrl, 0); 2542 ASSERT(ciputctrl_cache != NULL); 2543 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl); 2544 rq->q_syncq->sq_ciputctrl = NULL; 2545 rq->q_syncq->sq_nciputctrl = 0; 2546 } 2547 2548 if (!(wq->q_flag & QPERMOD) && 2549 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) { 2550 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2551 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl, 2552 wq->q_syncq->sq_nciputctrl, 0); 2553 ASSERT(ciputctrl_cache != NULL); 2554 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl); 2555 wq->q_syncq->sq_ciputctrl = NULL; 2556 wq->q_syncq->sq_nciputctrl = 0; 2557 } 2558 2559 sq = SQ(rq); 2560 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 2561 ASSERT(sq->sq_outer == NULL); 2562 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2563 2564 /* 2565 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS 2566 * bits in sq_flag based on the sqtype. 2567 */ 2568 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); 2569 2570 rq->q_syncq = wq->q_syncq = sq; 2571 sq->sq_type = sqtype; 2572 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); 2573 2574 /* 2575 * We are making sq_svcflags zero, 2576 * resetting SQ_DISABLED in case it was set by 2577 * wait_svc() in the munlink path. 2578 * 2579 */ 2580 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); 2581 sq->sq_svcflags = 0; 2582 2583 /* 2584 * We need to acquire the lock here for the mlink and munlink case, 2585 * where canputnext, backenable, etc can access the q_flag. 2586 */ 2587 if (lock_needed) { 2588 mutex_enter(QLOCK(rq)); 2589 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2590 mutex_exit(QLOCK(rq)); 2591 mutex_enter(QLOCK(wq)); 2592 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2593 mutex_exit(QLOCK(wq)); 2594 } else { 2595 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2596 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2597 } 2598 2599 if (qflag & QPERQ) { 2600 /* Allocate a separate syncq for the write side */ 2601 sq = new_syncq(); 2602 sq->sq_type = rq->q_syncq->sq_type; 2603 sq->sq_flags = rq->q_syncq->sq_flags; 2604 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2605 sq->sq_oprev == NULL); 2606 wq->q_syncq = sq; 2607 } 2608 if (qflag & QPERMOD) { 2609 sq = dmp->dm_sq; 2610 2611 /* 2612 * Assert that we do have an inner perimeter syncq and that it 2613 * does not have an outer perimeter associated with it. 2614 */ 2615 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2616 sq->sq_oprev == NULL); 2617 rq->q_syncq = wq->q_syncq = sq; 2618 } 2619 if (qflag & QMTOUTPERIM) { 2620 outer = dmp->dm_sq; 2621 2622 ASSERT(outer->sq_outer == NULL); 2623 outer_insert(outer, rq->q_syncq); 2624 if (wq->q_syncq != rq->q_syncq) 2625 outer_insert(outer, wq->q_syncq); 2626 } 2627 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2628 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2629 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2630 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2631 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK)); 2632 2633 /* 2634 * Initialize struio() types. 2635 */ 2636 rq->q_struiot = 2637 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE; 2638 wq->q_struiot = 2639 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE; 2640 } 2641 2642 perdm_t * 2643 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype) 2644 { 2645 syncq_t *sq; 2646 perdm_t **pp; 2647 perdm_t *p; 2648 perdm_t *dmp; 2649 2650 ASSERT(str != NULL); 2651 ASSERT(qflag & (QPERMOD | QMTOUTPERIM)); 2652 2653 rw_enter(&perdm_rwlock, RW_READER); 2654 for (p = perdm_list; p != NULL; p = p->dm_next) { 2655 if (p->dm_str == str) { /* found one */ 2656 atomic_add_32(&(p->dm_ref), 1); 2657 rw_exit(&perdm_rwlock); 2658 return (p); 2659 } 2660 } 2661 rw_exit(&perdm_rwlock); 2662 2663 sq = new_syncq(); 2664 if (qflag & QPERMOD) { 2665 sq->sq_type = sqtype | SQ_PERMOD; 2666 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; 2667 } else { 2668 ASSERT(qflag & QMTOUTPERIM); 2669 sq->sq_onext = sq->sq_oprev = sq; 2670 } 2671 2672 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP); 2673 dmp->dm_sq = sq; 2674 dmp->dm_str = str; 2675 dmp->dm_ref = 1; 2676 dmp->dm_next = NULL; 2677 2678 rw_enter(&perdm_rwlock, RW_WRITER); 2679 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) { 2680 if (p->dm_str == str) { /* already present */ 2681 p->dm_ref++; 2682 rw_exit(&perdm_rwlock); 2683 free_syncq(sq); 2684 kmem_free(dmp, sizeof (perdm_t)); 2685 return (p); 2686 } 2687 } 2688 2689 *pp = dmp; 2690 rw_exit(&perdm_rwlock); 2691 return (dmp); 2692 } 2693 2694 void 2695 rele_dm(perdm_t *dmp) 2696 { 2697 perdm_t **pp; 2698 perdm_t *p; 2699 2700 rw_enter(&perdm_rwlock, RW_WRITER); 2701 ASSERT(dmp->dm_ref > 0); 2702 2703 if (--dmp->dm_ref > 0) { 2704 rw_exit(&perdm_rwlock); 2705 return; 2706 } 2707 2708 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) 2709 if (p == dmp) 2710 break; 2711 ASSERT(p == dmp); 2712 *pp = p->dm_next; 2713 rw_exit(&perdm_rwlock); 2714 2715 /* 2716 * Wait for any background processing that relies on the 2717 * syncq to complete before it is freed. 2718 */ 2719 wait_sq_svc(p->dm_sq); 2720 free_syncq(p->dm_sq); 2721 kmem_free(p, sizeof (perdm_t)); 2722 } 2723 2724 /* 2725 * Make a protocol message given control and data buffers. 2726 * n.b., this can block; be careful of what locks you hold when calling it. 2727 * 2728 * If sd_maxblk is less than *iosize this routine can fail part way through 2729 * (due to an allocation failure). In this case on return *iosize will contain 2730 * the amount that was consumed. Otherwise *iosize will not be modified 2731 * i.e. it will contain the amount that was consumed. 2732 */ 2733 int 2734 strmakemsg( 2735 struct strbuf *mctl, 2736 ssize_t *iosize, 2737 struct uio *uiop, 2738 stdata_t *stp, 2739 int32_t flag, 2740 mblk_t **mpp) 2741 { 2742 mblk_t *mpctl = NULL; 2743 mblk_t *mpdata = NULL; 2744 int error; 2745 2746 ASSERT(uiop != NULL); 2747 2748 *mpp = NULL; 2749 /* Create control part, if any */ 2750 if ((mctl != NULL) && (mctl->len >= 0)) { 2751 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl); 2752 if (error) 2753 return (error); 2754 } 2755 /* Create data part, if any */ 2756 if (*iosize >= 0) { 2757 error = strmakedata(iosize, uiop, stp, flag, &mpdata); 2758 if (error) { 2759 freemsg(mpctl); 2760 return (error); 2761 } 2762 } 2763 if (mpctl != NULL) { 2764 if (mpdata != NULL) 2765 linkb(mpctl, mpdata); 2766 *mpp = mpctl; 2767 } else { 2768 *mpp = mpdata; 2769 } 2770 return (0); 2771 } 2772 2773 /* 2774 * Make the control part of a protocol message given a control buffer. 2775 * n.b., this can block; be careful of what locks you hold when calling it. 2776 */ 2777 int 2778 strmakectl( 2779 struct strbuf *mctl, 2780 int32_t flag, 2781 int32_t fflag, 2782 mblk_t **mpp) 2783 { 2784 mblk_t *bp = NULL; 2785 unsigned char msgtype; 2786 int error = 0; 2787 cred_t *cr = CRED(); 2788 2789 /* We do not support interrupt threads using the stream head to send */ 2790 ASSERT(cr != NULL); 2791 2792 *mpp = NULL; 2793 /* 2794 * Create control part of message, if any. 2795 */ 2796 if ((mctl != NULL) && (mctl->len >= 0)) { 2797 caddr_t base; 2798 int ctlcount; 2799 int allocsz; 2800 2801 if (flag & RS_HIPRI) 2802 msgtype = M_PCPROTO; 2803 else 2804 msgtype = M_PROTO; 2805 2806 ctlcount = mctl->len; 2807 base = mctl->buf; 2808 2809 /* 2810 * Give modules a better chance to reuse M_PROTO/M_PCPROTO 2811 * blocks by increasing the size to something more usable. 2812 */ 2813 allocsz = MAX(ctlcount, 64); 2814 2815 /* 2816 * Range checking has already been done; simply try 2817 * to allocate a message block for the ctl part. 2818 */ 2819 while ((bp = allocb_cred(allocsz, cr, 2820 curproc->p_pid)) == NULL) { 2821 if (fflag & (FNDELAY|FNONBLOCK)) 2822 return (EAGAIN); 2823 if (error = strwaitbuf(allocsz, BPRI_MED)) 2824 return (error); 2825 } 2826 2827 bp->b_datap->db_type = msgtype; 2828 if (copyin(base, bp->b_wptr, ctlcount)) { 2829 freeb(bp); 2830 return (EFAULT); 2831 } 2832 bp->b_wptr += ctlcount; 2833 } 2834 *mpp = bp; 2835 return (0); 2836 } 2837 2838 /* 2839 * Make a protocol message given data buffers. 2840 * n.b., this can block; be careful of what locks you hold when calling it. 2841 * 2842 * If sd_maxblk is less than *iosize this routine can fail part way through 2843 * (due to an allocation failure). In this case on return *iosize will contain 2844 * the amount that was consumed. Otherwise *iosize will not be modified 2845 * i.e. it will contain the amount that was consumed. 2846 */ 2847 int 2848 strmakedata( 2849 ssize_t *iosize, 2850 struct uio *uiop, 2851 stdata_t *stp, 2852 int32_t flag, 2853 mblk_t **mpp) 2854 { 2855 mblk_t *mp = NULL; 2856 mblk_t *bp; 2857 int wroff = (int)stp->sd_wroff; 2858 int tail_len = (int)stp->sd_tail; 2859 int extra = wroff + tail_len; 2860 int error = 0; 2861 ssize_t maxblk; 2862 ssize_t count = *iosize; 2863 cred_t *cr; 2864 2865 *mpp = NULL; 2866 if (count < 0) 2867 return (0); 2868 2869 /* We do not support interrupt threads using the stream head to send */ 2870 cr = CRED(); 2871 ASSERT(cr != NULL); 2872 2873 maxblk = stp->sd_maxblk; 2874 if (maxblk == INFPSZ) 2875 maxblk = count; 2876 2877 /* 2878 * Create data part of message, if any. 2879 */ 2880 do { 2881 ssize_t size; 2882 dblk_t *dp; 2883 2884 ASSERT(uiop); 2885 2886 size = MIN(count, maxblk); 2887 2888 while ((bp = allocb_cred(size + extra, cr, 2889 curproc->p_pid)) == NULL) { 2890 error = EAGAIN; 2891 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 2892 (error = strwaitbuf(size + extra, BPRI_MED)) != 0) { 2893 if (count == *iosize) { 2894 freemsg(mp); 2895 return (error); 2896 } else { 2897 *iosize -= count; 2898 *mpp = mp; 2899 return (0); 2900 } 2901 } 2902 } 2903 dp = bp->b_datap; 2904 dp->db_cpid = curproc->p_pid; 2905 ASSERT(wroff <= dp->db_lim - bp->b_wptr); 2906 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff; 2907 2908 if (flag & STRUIO_POSTPONE) { 2909 /* 2910 * Setup the stream uio portion of the 2911 * dblk for subsequent use by struioget(). 2912 */ 2913 dp->db_struioflag = STRUIO_SPEC; 2914 dp->db_cksumstart = 0; 2915 dp->db_cksumstuff = 0; 2916 dp->db_cksumend = size; 2917 *(long long *)dp->db_struioun.data = 0ll; 2918 bp->b_wptr += size; 2919 } else { 2920 if (stp->sd_copyflag & STRCOPYCACHED) 2921 uiop->uio_extflg |= UIO_COPY_CACHED; 2922 2923 if (size != 0) { 2924 error = uiomove(bp->b_wptr, size, UIO_WRITE, 2925 uiop); 2926 if (error != 0) { 2927 freeb(bp); 2928 freemsg(mp); 2929 return (error); 2930 } 2931 } 2932 bp->b_wptr += size; 2933 2934 if (stp->sd_wputdatafunc != NULL) { 2935 mblk_t *newbp; 2936 2937 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode, 2938 bp, NULL, NULL, NULL, NULL); 2939 if (newbp == NULL) { 2940 freeb(bp); 2941 freemsg(mp); 2942 return (ECOMM); 2943 } 2944 bp = newbp; 2945 } 2946 } 2947 2948 count -= size; 2949 2950 if (mp == NULL) 2951 mp = bp; 2952 else 2953 linkb(mp, bp); 2954 } while (count > 0); 2955 2956 *mpp = mp; 2957 return (0); 2958 } 2959 2960 /* 2961 * Wait for a buffer to become available. Return non-zero errno 2962 * if not able to wait, 0 if buffer is probably there. 2963 */ 2964 int 2965 strwaitbuf(size_t size, int pri) 2966 { 2967 bufcall_id_t id; 2968 2969 mutex_enter(&bcall_monitor); 2970 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast, 2971 &ttoproc(curthread)->p_flag_cv)) == 0) { 2972 mutex_exit(&bcall_monitor); 2973 return (ENOSR); 2974 } 2975 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) { 2976 unbufcall(id); 2977 mutex_exit(&bcall_monitor); 2978 return (EINTR); 2979 } 2980 unbufcall(id); 2981 mutex_exit(&bcall_monitor); 2982 return (0); 2983 } 2984 2985 /* 2986 * This function waits for a read or write event to happen on a stream. 2987 * fmode can specify FNDELAY and/or FNONBLOCK. 2988 * The timeout is in ms with -1 meaning infinite. 2989 * The flag values work as follows: 2990 * READWAIT Check for read side errors, send M_READ 2991 * GETWAIT Check for read side errors, no M_READ 2992 * WRITEWAIT Check for write side errors. 2993 * NOINTR Do not return error if nonblocking or timeout. 2994 * STR_NOERROR Ignore all errors except STPLEX. 2995 * STR_NOSIG Ignore/hold signals during the duration of the call. 2996 * STR_PEEK Pass through the strgeterr(). 2997 */ 2998 int 2999 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout, 3000 int *done) 3001 { 3002 int slpflg, errs; 3003 int error; 3004 kcondvar_t *sleepon; 3005 mblk_t *mp; 3006 ssize_t *rd_count; 3007 clock_t rval; 3008 3009 ASSERT(MUTEX_HELD(&stp->sd_lock)); 3010 if ((flag & READWAIT) || (flag & GETWAIT)) { 3011 slpflg = RSLEEP; 3012 sleepon = &_RD(stp->sd_wrq)->q_wait; 3013 errs = STRDERR|STPLEX; 3014 } else { 3015 slpflg = WSLEEP; 3016 sleepon = &stp->sd_wrq->q_wait; 3017 errs = STWRERR|STRHUP|STPLEX; 3018 } 3019 if (flag & STR_NOERROR) 3020 errs = STPLEX; 3021 3022 if (stp->sd_wakeq & slpflg) { 3023 /* 3024 * A strwakeq() is pending, no need to sleep. 3025 */ 3026 stp->sd_wakeq &= ~slpflg; 3027 *done = 0; 3028 return (0); 3029 } 3030 3031 if (stp->sd_flag & errs) { 3032 /* 3033 * Check for errors before going to sleep since the 3034 * caller might not have checked this while holding 3035 * sd_lock. 3036 */ 3037 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3038 if (error != 0) { 3039 *done = 1; 3040 return (error); 3041 } 3042 } 3043 3044 /* 3045 * If any module downstream has requested read notification 3046 * by setting SNDMREAD flag using M_SETOPTS, send a message 3047 * down stream. 3048 */ 3049 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) { 3050 mutex_exit(&stp->sd_lock); 3051 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED, 3052 (flag & STR_NOSIG), &error))) { 3053 mutex_enter(&stp->sd_lock); 3054 *done = 1; 3055 return (error); 3056 } 3057 mp->b_datap->db_type = M_READ; 3058 rd_count = (ssize_t *)mp->b_wptr; 3059 *rd_count = count; 3060 mp->b_wptr += sizeof (ssize_t); 3061 /* 3062 * Send the number of bytes requested by the 3063 * read as the argument to M_READ. 3064 */ 3065 stream_willservice(stp); 3066 putnext(stp->sd_wrq, mp); 3067 stream_runservice(stp); 3068 mutex_enter(&stp->sd_lock); 3069 3070 /* 3071 * If any data arrived due to inline processing 3072 * of putnext(), don't sleep. 3073 */ 3074 if (_RD(stp->sd_wrq)->q_first != NULL) { 3075 *done = 0; 3076 return (0); 3077 } 3078 } 3079 3080 if (fmode & (FNDELAY|FNONBLOCK)) { 3081 if (!(flag & NOINTR)) 3082 error = EAGAIN; 3083 else 3084 error = 0; 3085 *done = 1; 3086 return (error); 3087 } 3088 3089 stp->sd_flag |= slpflg; 3090 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2, 3091 "strwaitq sleeps (2):%p, %X, %lX, %X, %p", 3092 stp, flag, count, fmode, done); 3093 3094 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG); 3095 if (rval > 0) { 3096 /* EMPTY */ 3097 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2, 3098 "strwaitq awakes(2):%X, %X, %X, %X, %X", 3099 stp, flag, count, fmode, done); 3100 } else if (rval == 0) { 3101 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2, 3102 "strwaitq interrupt #2:%p, %X, %lX, %X, %p", 3103 stp, flag, count, fmode, done); 3104 stp->sd_flag &= ~slpflg; 3105 cv_broadcast(sleepon); 3106 if (!(flag & NOINTR)) 3107 error = EINTR; 3108 else 3109 error = 0; 3110 *done = 1; 3111 return (error); 3112 } else { 3113 /* timeout */ 3114 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME, 3115 "strwaitq timeout:%p, %X, %lX, %X, %p", 3116 stp, flag, count, fmode, done); 3117 *done = 1; 3118 if (!(flag & NOINTR)) 3119 return (ETIME); 3120 else 3121 return (0); 3122 } 3123 /* 3124 * If the caller implements delayed errors (i.e. queued after data) 3125 * we can not check for errors here since data as well as an 3126 * error might have arrived at the stream head. We return to 3127 * have the caller check the read queue before checking for errors. 3128 */ 3129 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) { 3130 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3131 if (error != 0) { 3132 *done = 1; 3133 return (error); 3134 } 3135 } 3136 *done = 0; 3137 return (0); 3138 } 3139 3140 /* 3141 * Perform job control discipline access checks. 3142 * Return 0 for success and the errno for failure. 3143 */ 3144 3145 #define cantsend(p, t, sig) \ 3146 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig)) 3147 3148 int 3149 straccess(struct stdata *stp, enum jcaccess mode) 3150 { 3151 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */ 3152 kthread_t *t = curthread; 3153 proc_t *p = ttoproc(t); 3154 sess_t *sp; 3155 3156 ASSERT(mutex_owned(&stp->sd_lock)); 3157 3158 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO) 3159 return (0); 3160 3161 mutex_enter(&p->p_lock); /* protects p_pgidp */ 3162 3163 for (;;) { 3164 mutex_enter(&p->p_splock); /* protects p->p_sessp */ 3165 sp = p->p_sessp; 3166 mutex_enter(&sp->s_lock); /* protects sp->* */ 3167 3168 /* 3169 * If this is not the calling process's controlling terminal 3170 * or if the calling process is already in the foreground 3171 * then allow access. 3172 */ 3173 if (sp->s_dev != stp->sd_vnode->v_rdev || 3174 p->p_pgidp == stp->sd_pgidp) { 3175 mutex_exit(&sp->s_lock); 3176 mutex_exit(&p->p_splock); 3177 mutex_exit(&p->p_lock); 3178 return (0); 3179 } 3180 3181 /* 3182 * Check to see if controlling terminal has been deallocated. 3183 */ 3184 if (sp->s_vp == NULL) { 3185 if (!cantsend(p, t, SIGHUP)) 3186 sigtoproc(p, t, SIGHUP); 3187 mutex_exit(&sp->s_lock); 3188 mutex_exit(&p->p_splock); 3189 mutex_exit(&p->p_lock); 3190 return (EIO); 3191 } 3192 3193 mutex_exit(&sp->s_lock); 3194 mutex_exit(&p->p_splock); 3195 3196 if (mode == JCGETP) { 3197 mutex_exit(&p->p_lock); 3198 return (0); 3199 } 3200 3201 if (mode == JCREAD) { 3202 if (p->p_detached || cantsend(p, t, SIGTTIN)) { 3203 mutex_exit(&p->p_lock); 3204 return (EIO); 3205 } 3206 mutex_exit(&p->p_lock); 3207 mutex_exit(&stp->sd_lock); 3208 pgsignal(p->p_pgidp, SIGTTIN); 3209 mutex_enter(&stp->sd_lock); 3210 mutex_enter(&p->p_lock); 3211 } else { /* mode == JCWRITE or JCSETP */ 3212 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) || 3213 cantsend(p, t, SIGTTOU)) { 3214 mutex_exit(&p->p_lock); 3215 return (0); 3216 } 3217 if (p->p_detached) { 3218 mutex_exit(&p->p_lock); 3219 return (EIO); 3220 } 3221 mutex_exit(&p->p_lock); 3222 mutex_exit(&stp->sd_lock); 3223 pgsignal(p->p_pgidp, SIGTTOU); 3224 mutex_enter(&stp->sd_lock); 3225 mutex_enter(&p->p_lock); 3226 } 3227 3228 /* 3229 * We call cv_wait_sig_swap() to cause the appropriate 3230 * action for the jobcontrol signal to take place. 3231 * If the signal is being caught, we will take the 3232 * EINTR error return. Otherwise, the default action 3233 * of causing the process to stop will take place. 3234 * In this case, we rely on the periodic cv_broadcast() on 3235 * &lbolt_cv to wake us up to loop around and test again. 3236 * We can't get here if the signal is ignored or 3237 * if the current thread is blocking the signal. 3238 */ 3239 mutex_exit(&stp->sd_lock); 3240 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) { 3241 mutex_exit(&p->p_lock); 3242 mutex_enter(&stp->sd_lock); 3243 return (EINTR); 3244 } 3245 mutex_exit(&p->p_lock); 3246 mutex_enter(&stp->sd_lock); 3247 mutex_enter(&p->p_lock); 3248 } 3249 } 3250 3251 /* 3252 * Return size of message of block type (bp->b_datap->db_type) 3253 */ 3254 size_t 3255 xmsgsize(mblk_t *bp) 3256 { 3257 unsigned char type; 3258 size_t count = 0; 3259 3260 type = bp->b_datap->db_type; 3261 3262 for (; bp; bp = bp->b_cont) { 3263 if (type != bp->b_datap->db_type) 3264 break; 3265 ASSERT(bp->b_wptr >= bp->b_rptr); 3266 count += bp->b_wptr - bp->b_rptr; 3267 } 3268 return (count); 3269 } 3270 3271 /* 3272 * Allocate a stream head. 3273 */ 3274 struct stdata * 3275 shalloc(queue_t *qp) 3276 { 3277 stdata_t *stp; 3278 3279 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP); 3280 3281 stp->sd_wrq = _WR(qp); 3282 stp->sd_strtab = NULL; 3283 stp->sd_iocid = 0; 3284 stp->sd_mate = NULL; 3285 stp->sd_freezer = NULL; 3286 stp->sd_refcnt = 0; 3287 stp->sd_wakeq = 0; 3288 stp->sd_anchor = 0; 3289 stp->sd_struiowrq = NULL; 3290 stp->sd_struiordq = NULL; 3291 stp->sd_struiodnak = 0; 3292 stp->sd_struionak = NULL; 3293 stp->sd_t_audit_data = NULL; 3294 stp->sd_rput_opt = 0; 3295 stp->sd_wput_opt = 0; 3296 stp->sd_read_opt = 0; 3297 stp->sd_rprotofunc = strrput_proto; 3298 stp->sd_rmiscfunc = strrput_misc; 3299 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL; 3300 stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL; 3301 stp->sd_ciputctrl = NULL; 3302 stp->sd_nciputctrl = 0; 3303 stp->sd_qhead = NULL; 3304 stp->sd_qtail = NULL; 3305 stp->sd_servid = NULL; 3306 stp->sd_nqueues = 0; 3307 stp->sd_svcflags = 0; 3308 stp->sd_copyflag = 0; 3309 3310 return (stp); 3311 } 3312 3313 /* 3314 * Free a stream head. 3315 */ 3316 void 3317 shfree(stdata_t *stp) 3318 { 3319 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock)); 3320 3321 stp->sd_wrq = NULL; 3322 3323 mutex_enter(&stp->sd_qlock); 3324 while (stp->sd_svcflags & STRS_SCHEDULED) { 3325 STRSTAT(strwaits); 3326 cv_wait(&stp->sd_qcv, &stp->sd_qlock); 3327 } 3328 mutex_exit(&stp->sd_qlock); 3329 3330 if (stp->sd_ciputctrl != NULL) { 3331 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1); 3332 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl, 3333 stp->sd_nciputctrl, 0); 3334 ASSERT(ciputctrl_cache != NULL); 3335 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl); 3336 stp->sd_ciputctrl = NULL; 3337 stp->sd_nciputctrl = 0; 3338 } 3339 ASSERT(stp->sd_qhead == NULL); 3340 ASSERT(stp->sd_qtail == NULL); 3341 ASSERT(stp->sd_nqueues == 0); 3342 kmem_cache_free(stream_head_cache, stp); 3343 } 3344 3345 /* 3346 * Allocate a pair of queues and a syncq for the pair 3347 */ 3348 queue_t * 3349 allocq(void) 3350 { 3351 queinfo_t *qip; 3352 queue_t *qp, *wqp; 3353 syncq_t *sq; 3354 3355 qip = kmem_cache_alloc(queue_cache, KM_SLEEP); 3356 3357 qp = &qip->qu_rqueue; 3358 wqp = &qip->qu_wqueue; 3359 sq = &qip->qu_syncq; 3360 3361 qp->q_last = NULL; 3362 qp->q_next = NULL; 3363 qp->q_ptr = NULL; 3364 qp->q_flag = QUSE | QREADR; 3365 qp->q_bandp = NULL; 3366 qp->q_stream = NULL; 3367 qp->q_syncq = sq; 3368 qp->q_nband = 0; 3369 qp->q_nfsrv = NULL; 3370 qp->q_draining = 0; 3371 qp->q_syncqmsgs = 0; 3372 qp->q_spri = 0; 3373 qp->q_qtstamp = 0; 3374 qp->q_sqtstamp = 0; 3375 qp->q_fp = NULL; 3376 3377 wqp->q_last = NULL; 3378 wqp->q_next = NULL; 3379 wqp->q_ptr = NULL; 3380 wqp->q_flag = QUSE; 3381 wqp->q_bandp = NULL; 3382 wqp->q_stream = NULL; 3383 wqp->q_syncq = sq; 3384 wqp->q_nband = 0; 3385 wqp->q_nfsrv = NULL; 3386 wqp->q_draining = 0; 3387 wqp->q_syncqmsgs = 0; 3388 wqp->q_qtstamp = 0; 3389 wqp->q_sqtstamp = 0; 3390 wqp->q_spri = 0; 3391 3392 sq->sq_count = 0; 3393 sq->sq_rmqcount = 0; 3394 sq->sq_flags = 0; 3395 sq->sq_type = 0; 3396 sq->sq_callbflags = 0; 3397 sq->sq_cancelid = 0; 3398 sq->sq_ciputctrl = NULL; 3399 sq->sq_nciputctrl = 0; 3400 sq->sq_needexcl = 0; 3401 sq->sq_svcflags = 0; 3402 3403 return (qp); 3404 } 3405 3406 /* 3407 * Free a pair of queues and the "attached" syncq. 3408 * Discard any messages left on the syncq(s), remove the syncq(s) from the 3409 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq. 3410 */ 3411 void 3412 freeq(queue_t *qp) 3413 { 3414 qband_t *qbp, *nqbp; 3415 syncq_t *sq, *outer; 3416 queue_t *wqp = _WR(qp); 3417 3418 ASSERT(qp->q_flag & QREADR); 3419 3420 /* 3421 * If a previously dispatched taskq job is scheduled to run 3422 * sync_service() or a service routine is scheduled for the 3423 * queues about to be freed, wait here until all service is 3424 * done on the queue and all associated queues and syncqs. 3425 */ 3426 wait_svc(qp); 3427 3428 (void) flush_syncq(qp->q_syncq, qp); 3429 (void) flush_syncq(wqp->q_syncq, wqp); 3430 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0); 3431 3432 /* 3433 * Flush the queues before q_next is set to NULL This is needed 3434 * in order to backenable any downstream queue before we go away. 3435 * Note: we are already removed from the stream so that the 3436 * backenabling will not cause any messages to be delivered to our 3437 * put procedures. 3438 */ 3439 flushq(qp, FLUSHALL); 3440 flushq(wqp, FLUSHALL); 3441 3442 /* Tidy up - removeq only does a half-remove from stream */ 3443 qp->q_next = wqp->q_next = NULL; 3444 ASSERT(!(qp->q_flag & QENAB)); 3445 ASSERT(!(wqp->q_flag & QENAB)); 3446 3447 outer = qp->q_syncq->sq_outer; 3448 if (outer != NULL) { 3449 outer_remove(outer, qp->q_syncq); 3450 if (wqp->q_syncq != qp->q_syncq) 3451 outer_remove(outer, wqp->q_syncq); 3452 } 3453 /* 3454 * Free any syncqs that are outside what allocq returned. 3455 */ 3456 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD)) 3457 free_syncq(qp->q_syncq); 3458 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp)) 3459 free_syncq(wqp->q_syncq); 3460 3461 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3462 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3463 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 3464 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp))); 3465 sq = SQ(qp); 3466 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 3467 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 3468 ASSERT(sq->sq_outer == NULL); 3469 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 3470 ASSERT(sq->sq_callbpend == NULL); 3471 ASSERT(sq->sq_needexcl == 0); 3472 3473 if (sq->sq_ciputctrl != NULL) { 3474 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 3475 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 3476 sq->sq_nciputctrl, 0); 3477 ASSERT(ciputctrl_cache != NULL); 3478 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 3479 sq->sq_ciputctrl = NULL; 3480 sq->sq_nciputctrl = 0; 3481 } 3482 3483 ASSERT(qp->q_first == NULL && wqp->q_first == NULL); 3484 ASSERT(qp->q_count == 0 && wqp->q_count == 0); 3485 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0); 3486 3487 qp->q_flag &= ~QUSE; 3488 wqp->q_flag &= ~QUSE; 3489 3490 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */ 3491 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */ 3492 3493 qbp = qp->q_bandp; 3494 while (qbp) { 3495 nqbp = qbp->qb_next; 3496 freeband(qbp); 3497 qbp = nqbp; 3498 } 3499 qbp = wqp->q_bandp; 3500 while (qbp) { 3501 nqbp = qbp->qb_next; 3502 freeband(qbp); 3503 qbp = nqbp; 3504 } 3505 kmem_cache_free(queue_cache, qp); 3506 } 3507 3508 /* 3509 * Allocate a qband structure. 3510 */ 3511 qband_t * 3512 allocband(void) 3513 { 3514 qband_t *qbp; 3515 3516 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP); 3517 if (qbp == NULL) 3518 return (NULL); 3519 3520 qbp->qb_next = NULL; 3521 qbp->qb_count = 0; 3522 qbp->qb_mblkcnt = 0; 3523 qbp->qb_first = NULL; 3524 qbp->qb_last = NULL; 3525 qbp->qb_flag = 0; 3526 3527 return (qbp); 3528 } 3529 3530 /* 3531 * Free a qband structure. 3532 */ 3533 void 3534 freeband(qband_t *qbp) 3535 { 3536 kmem_cache_free(qband_cache, qbp); 3537 } 3538 3539 /* 3540 * Just like putnextctl(9F), except that allocb_wait() is used. 3541 * 3542 * Consolidation Private, and of course only callable from the stream head or 3543 * routines that may block. 3544 */ 3545 int 3546 putnextctl_wait(queue_t *q, int type) 3547 { 3548 mblk_t *bp; 3549 int error; 3550 3551 if ((datamsg(type) && (type != M_DELAY)) || 3552 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL) 3553 return (0); 3554 3555 bp->b_datap->db_type = (unsigned char)type; 3556 putnext(q, bp); 3557 return (1); 3558 } 3559 3560 /* 3561 * Run any possible bufcalls. 3562 */ 3563 void 3564 runbufcalls(void) 3565 { 3566 strbufcall_t *bcp; 3567 3568 mutex_enter(&bcall_monitor); 3569 mutex_enter(&strbcall_lock); 3570 3571 if (strbcalls.bc_head) { 3572 size_t count; 3573 int nevent; 3574 3575 /* 3576 * count how many events are on the list 3577 * now so we can check to avoid looping 3578 * in low memory situations 3579 */ 3580 nevent = 0; 3581 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next) 3582 nevent++; 3583 3584 /* 3585 * get estimate of available memory from kmem_avail(). 3586 * awake all bufcall functions waiting for 3587 * memory whose request could be satisfied 3588 * by 'count' memory and let 'em fight for it. 3589 */ 3590 count = kmem_avail(); 3591 while ((bcp = strbcalls.bc_head) != NULL && nevent) { 3592 STRSTAT(bufcalls); 3593 --nevent; 3594 if (bcp->bc_size <= count) { 3595 bcp->bc_executor = curthread; 3596 mutex_exit(&strbcall_lock); 3597 (*bcp->bc_func)(bcp->bc_arg); 3598 mutex_enter(&strbcall_lock); 3599 bcp->bc_executor = NULL; 3600 cv_broadcast(&bcall_cv); 3601 strbcalls.bc_head = bcp->bc_next; 3602 kmem_free(bcp, sizeof (strbufcall_t)); 3603 } else { 3604 /* 3605 * too big, try again later - note 3606 * that nevent was decremented above 3607 * so we won't retry this one on this 3608 * iteration of the loop 3609 */ 3610 if (bcp->bc_next != NULL) { 3611 strbcalls.bc_head = bcp->bc_next; 3612 bcp->bc_next = NULL; 3613 strbcalls.bc_tail->bc_next = bcp; 3614 strbcalls.bc_tail = bcp; 3615 } 3616 } 3617 } 3618 if (strbcalls.bc_head == NULL) 3619 strbcalls.bc_tail = NULL; 3620 } 3621 3622 mutex_exit(&strbcall_lock); 3623 mutex_exit(&bcall_monitor); 3624 } 3625 3626 3627 /* 3628 * Actually run queue's service routine. 3629 */ 3630 static void 3631 runservice(queue_t *q) 3632 { 3633 qband_t *qbp; 3634 3635 ASSERT(q->q_qinfo->qi_srvp); 3636 again: 3637 entersq(q->q_syncq, SQ_SVC); 3638 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START, 3639 "runservice starts:%p", q); 3640 3641 if (!(q->q_flag & QWCLOSE)) 3642 (*q->q_qinfo->qi_srvp)(q); 3643 3644 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END, 3645 "runservice ends:(%p)", q); 3646 3647 leavesq(q->q_syncq, SQ_SVC); 3648 3649 mutex_enter(QLOCK(q)); 3650 if (q->q_flag & QENAB) { 3651 q->q_flag &= ~QENAB; 3652 mutex_exit(QLOCK(q)); 3653 goto again; 3654 } 3655 q->q_flag &= ~QINSERVICE; 3656 q->q_flag &= ~QBACK; 3657 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next) 3658 qbp->qb_flag &= ~QB_BACK; 3659 /* 3660 * Wakeup thread waiting for the service procedure 3661 * to be run (strclose and qdetach). 3662 */ 3663 cv_broadcast(&q->q_wait); 3664 3665 mutex_exit(QLOCK(q)); 3666 } 3667 3668 /* 3669 * Background processing of bufcalls. 3670 */ 3671 void 3672 streams_bufcall_service(void) 3673 { 3674 callb_cpr_t cprinfo; 3675 3676 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr, 3677 "streams_bufcall_service"); 3678 3679 mutex_enter(&strbcall_lock); 3680 3681 for (;;) { 3682 if (strbcalls.bc_head != NULL && kmem_avail() > 0) { 3683 mutex_exit(&strbcall_lock); 3684 runbufcalls(); 3685 mutex_enter(&strbcall_lock); 3686 } 3687 if (strbcalls.bc_head != NULL) { 3688 STRSTAT(bcwaits); 3689 /* Wait for memory to become available */ 3690 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3691 (void) cv_reltimedwait(&memavail_cv, &strbcall_lock, 3692 SEC_TO_TICK(60), TR_CLOCK_TICK); 3693 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3694 } 3695 3696 /* Wait for new work to arrive */ 3697 if (strbcalls.bc_head == NULL) { 3698 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3699 cv_wait(&strbcall_cv, &strbcall_lock); 3700 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3701 } 3702 } 3703 } 3704 3705 /* 3706 * Background processing of streams background tasks which failed 3707 * taskq_dispatch. 3708 */ 3709 static void 3710 streams_qbkgrnd_service(void) 3711 { 3712 callb_cpr_t cprinfo; 3713 queue_t *q; 3714 3715 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3716 "streams_bkgrnd_service"); 3717 3718 mutex_enter(&service_queue); 3719 3720 for (;;) { 3721 /* 3722 * Wait for work to arrive. 3723 */ 3724 while ((freebs_list == NULL) && (qhead == NULL)) { 3725 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3726 cv_wait(&services_to_run, &service_queue); 3727 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3728 } 3729 /* 3730 * Handle all pending freebs requests to free memory. 3731 */ 3732 while (freebs_list != NULL) { 3733 mblk_t *mp = freebs_list; 3734 freebs_list = mp->b_next; 3735 mutex_exit(&service_queue); 3736 mblk_free(mp); 3737 mutex_enter(&service_queue); 3738 } 3739 /* 3740 * Run pending queues. 3741 */ 3742 while (qhead != NULL) { 3743 DQ(q, qhead, qtail, q_link); 3744 ASSERT(q != NULL); 3745 mutex_exit(&service_queue); 3746 queue_service(q); 3747 mutex_enter(&service_queue); 3748 } 3749 ASSERT(qhead == NULL && qtail == NULL); 3750 } 3751 } 3752 3753 /* 3754 * Background processing of streams background tasks which failed 3755 * taskq_dispatch. 3756 */ 3757 static void 3758 streams_sqbkgrnd_service(void) 3759 { 3760 callb_cpr_t cprinfo; 3761 syncq_t *sq; 3762 3763 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3764 "streams_sqbkgrnd_service"); 3765 3766 mutex_enter(&service_queue); 3767 3768 for (;;) { 3769 /* 3770 * Wait for work to arrive. 3771 */ 3772 while (sqhead == NULL) { 3773 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3774 cv_wait(&syncqs_to_run, &service_queue); 3775 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3776 } 3777 3778 /* 3779 * Run pending syncqs. 3780 */ 3781 while (sqhead != NULL) { 3782 DQ(sq, sqhead, sqtail, sq_next); 3783 ASSERT(sq != NULL); 3784 ASSERT(sq->sq_svcflags & SQ_BGTHREAD); 3785 mutex_exit(&service_queue); 3786 syncq_service(sq); 3787 mutex_enter(&service_queue); 3788 } 3789 } 3790 } 3791 3792 /* 3793 * Disable the syncq and wait for background syncq processing to complete. 3794 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the 3795 * list. 3796 */ 3797 void 3798 wait_sq_svc(syncq_t *sq) 3799 { 3800 mutex_enter(SQLOCK(sq)); 3801 sq->sq_svcflags |= SQ_DISABLED; 3802 if (sq->sq_svcflags & SQ_BGTHREAD) { 3803 syncq_t *sq_chase; 3804 syncq_t *sq_curr; 3805 int removed; 3806 3807 ASSERT(sq->sq_servcount == 1); 3808 mutex_enter(&service_queue); 3809 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); 3810 mutex_exit(&service_queue); 3811 if (removed) { 3812 sq->sq_svcflags &= ~SQ_BGTHREAD; 3813 sq->sq_servcount = 0; 3814 STRSTAT(sqremoved); 3815 goto done; 3816 } 3817 } 3818 while (sq->sq_servcount != 0) { 3819 sq->sq_flags |= SQ_WANTWAKEUP; 3820 cv_wait(&sq->sq_wait, SQLOCK(sq)); 3821 } 3822 done: 3823 mutex_exit(SQLOCK(sq)); 3824 } 3825 3826 /* 3827 * Put a syncq on the list of syncq's to be serviced by the sqthread. 3828 * Add the argument to the end of the sqhead list and set the flag 3829 * indicating this syncq has been enabled. If it has already been 3830 * enabled, don't do anything. 3831 * This routine assumes that SQLOCK is held. 3832 * NOTE that the lock order is to have the SQLOCK first, 3833 * so if the service_syncq lock is held, we need to release it 3834 * before acquiring the SQLOCK (mostly relevant for the background 3835 * thread, and this seems to be common among the STREAMS global locks). 3836 * Note that the sq_svcflags are protected by the SQLOCK. 3837 */ 3838 void 3839 sqenable(syncq_t *sq) 3840 { 3841 /* 3842 * This is probably not important except for where I believe it 3843 * is being called. At that point, it should be held (and it 3844 * is a pain to release it just for this routine, so don't do 3845 * it). 3846 */ 3847 ASSERT(MUTEX_HELD(SQLOCK(sq))); 3848 3849 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); 3850 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); 3851 3852 /* 3853 * Do not put on list if background thread is scheduled or 3854 * syncq is disabled. 3855 */ 3856 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) 3857 return; 3858 3859 /* 3860 * Check whether we should enable sq at all. 3861 * Non PERMOD syncqs may be drained by at most one thread. 3862 * PERMOD syncqs may be drained by several threads but we limit the 3863 * total amount to the lesser of 3864 * Number of queues on the squeue and 3865 * Number of CPUs. 3866 */ 3867 if (sq->sq_servcount != 0) { 3868 if (((sq->sq_type & SQ_PERMOD) == 0) || 3869 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { 3870 STRSTAT(sqtoomany); 3871 return; 3872 } 3873 } 3874 3875 sq->sq_tstamp = ddi_get_lbolt(); 3876 STRSTAT(sqenables); 3877 3878 /* Attempt a taskq dispatch */ 3879 sq->sq_servid = (void *)taskq_dispatch(streams_taskq, 3880 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); 3881 if (sq->sq_servid != NULL) { 3882 sq->sq_servcount++; 3883 return; 3884 } 3885 3886 /* 3887 * This taskq dispatch failed, but a previous one may have succeeded. 3888 * Don't try to schedule on the background thread whilst there is 3889 * outstanding taskq processing. 3890 */ 3891 if (sq->sq_servcount != 0) 3892 return; 3893 3894 /* 3895 * System is low on resources and can't perform a non-sleeping 3896 * dispatch. Schedule the syncq for a background thread and mark the 3897 * syncq to avoid any further taskq dispatch attempts. 3898 */ 3899 mutex_enter(&service_queue); 3900 STRSTAT(taskqfails); 3901 ENQUEUE(sq, sqhead, sqtail, sq_next); 3902 sq->sq_svcflags |= SQ_BGTHREAD; 3903 sq->sq_servcount = 1; 3904 cv_signal(&syncqs_to_run); 3905 mutex_exit(&service_queue); 3906 } 3907 3908 /* 3909 * Note: fifo_close() depends on the mblk_t on the queue being freed 3910 * asynchronously. The asynchronous freeing of messages breaks the 3911 * recursive call chain of fifo_close() while there are I_SENDFD type of 3912 * messages referring to other file pointers on the queue. Then when 3913 * closing pipes it can avoid stack overflow in case of daisy-chained 3914 * pipes, and also avoid deadlock in case of fifonode_t pairs (which 3915 * share the same fifolock_t). 3916 * 3917 * No need to kpreempt_disable to access cpu_seqid. If we migrate and 3918 * the esb queue does not match the new CPU, that is OK. 3919 */ 3920 void 3921 freebs_enqueue(mblk_t *mp, dblk_t *dbp) 3922 { 3923 int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q; 3924 esb_queue_t *eqp; 3925 3926 ASSERT(dbp->db_mblk == mp); 3927 ASSERT(qindex < esbq_nelem); 3928 3929 eqp = system_esbq_array; 3930 if (eqp != NULL) { 3931 eqp += qindex; 3932 } else { 3933 mutex_enter(&esbq_lock); 3934 if (kmem_ready && system_esbq_array == NULL) 3935 system_esbq_array = (esb_queue_t *)kmem_zalloc( 3936 esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP); 3937 mutex_exit(&esbq_lock); 3938 eqp = system_esbq_array; 3939 if (eqp != NULL) 3940 eqp += qindex; 3941 else 3942 eqp = &system_esbq; 3943 } 3944 3945 /* 3946 * Check data sanity. The dblock should have non-empty free function. 3947 * It is better to panic here then later when the dblock is freed 3948 * asynchronously when the context is lost. 3949 */ 3950 if (dbp->db_frtnp->free_func == NULL) { 3951 panic("freebs_enqueue: dblock %p has a NULL free callback", 3952 (void *)dbp); 3953 } 3954 3955 mutex_enter(&eqp->eq_lock); 3956 /* queue the new mblk on the esballoc queue */ 3957 if (eqp->eq_head == NULL) { 3958 eqp->eq_head = eqp->eq_tail = mp; 3959 } else { 3960 eqp->eq_tail->b_next = mp; 3961 eqp->eq_tail = mp; 3962 } 3963 eqp->eq_len++; 3964 3965 /* If we're the first thread to reach the threshold, process */ 3966 if (eqp->eq_len >= esbq_max_qlen && 3967 !(eqp->eq_flags & ESBQ_PROCESSING)) 3968 esballoc_process_queue(eqp); 3969 3970 esballoc_set_timer(eqp, esbq_timeout); 3971 mutex_exit(&eqp->eq_lock); 3972 } 3973 3974 static void 3975 esballoc_process_queue(esb_queue_t *eqp) 3976 { 3977 mblk_t *mp; 3978 3979 ASSERT(MUTEX_HELD(&eqp->eq_lock)); 3980 3981 eqp->eq_flags |= ESBQ_PROCESSING; 3982 3983 do { 3984 /* 3985 * Detach the message chain for processing. 3986 */ 3987 mp = eqp->eq_head; 3988 eqp->eq_tail->b_next = NULL; 3989 eqp->eq_head = eqp->eq_tail = NULL; 3990 eqp->eq_len = 0; 3991 mutex_exit(&eqp->eq_lock); 3992 3993 /* 3994 * Process the message chain. 3995 */ 3996 esballoc_enqueue_mblk(mp); 3997 mutex_enter(&eqp->eq_lock); 3998 } while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0)); 3999 4000 eqp->eq_flags &= ~ESBQ_PROCESSING; 4001 } 4002 4003 /* 4004 * taskq callback routine to free esballoced mblk's 4005 */ 4006 static void 4007 esballoc_mblk_free(mblk_t *mp) 4008 { 4009 mblk_t *nextmp; 4010 4011 for (; mp != NULL; mp = nextmp) { 4012 nextmp = mp->b_next; 4013 mp->b_next = NULL; 4014 mblk_free(mp); 4015 } 4016 } 4017 4018 static void 4019 esballoc_enqueue_mblk(mblk_t *mp) 4020 { 4021 4022 if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp, 4023 TQ_NOSLEEP) == NULL) { 4024 mblk_t *first_mp = mp; 4025 /* 4026 * System is low on resources and can't perform a non-sleeping 4027 * dispatch. Schedule for a background thread. 4028 */ 4029 mutex_enter(&service_queue); 4030 STRSTAT(taskqfails); 4031 4032 while (mp->b_next != NULL) 4033 mp = mp->b_next; 4034 4035 mp->b_next = freebs_list; 4036 freebs_list = first_mp; 4037 cv_signal(&services_to_run); 4038 mutex_exit(&service_queue); 4039 } 4040 } 4041 4042 static void 4043 esballoc_timer(void *arg) 4044 { 4045 esb_queue_t *eqp = arg; 4046 4047 mutex_enter(&eqp->eq_lock); 4048 eqp->eq_flags &= ~ESBQ_TIMER; 4049 4050 if (!(eqp->eq_flags & ESBQ_PROCESSING) && 4051 eqp->eq_len > 0) 4052 esballoc_process_queue(eqp); 4053 4054 esballoc_set_timer(eqp, esbq_timeout); 4055 mutex_exit(&eqp->eq_lock); 4056 } 4057 4058 static void 4059 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout) 4060 { 4061 ASSERT(MUTEX_HELD(&eqp->eq_lock)); 4062 4063 if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) { 4064 (void) timeout(esballoc_timer, eqp, eq_timeout); 4065 eqp->eq_flags |= ESBQ_TIMER; 4066 } 4067 } 4068 4069 /* 4070 * Setup esbq array length based upon NCPU scaled by CPUs per 4071 * queue. Use static system_esbq until kmem_ready and we can 4072 * create an array in freebs_enqueue(). 4073 */ 4074 void 4075 esballoc_queue_init(void) 4076 { 4077 esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1); 4078 esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q; 4079 esbq_nelem = howmany(NCPU, esbq_cpus_per_q); 4080 system_esbq.eq_len = 0; 4081 system_esbq.eq_head = system_esbq.eq_tail = NULL; 4082 system_esbq.eq_flags = 0; 4083 } 4084 4085 /* 4086 * Set the QBACK or QB_BACK flag in the given queue for 4087 * the given priority band. 4088 */ 4089 void 4090 setqback(queue_t *q, unsigned char pri) 4091 { 4092 int i; 4093 qband_t *qbp; 4094 qband_t **qbpp; 4095 4096 ASSERT(MUTEX_HELD(QLOCK(q))); 4097 if (pri != 0) { 4098 if (pri > q->q_nband) { 4099 qbpp = &q->q_bandp; 4100 while (*qbpp) 4101 qbpp = &(*qbpp)->qb_next; 4102 while (pri > q->q_nband) { 4103 if ((*qbpp = allocband()) == NULL) { 4104 cmn_err(CE_WARN, 4105 "setqback: can't allocate qband\n"); 4106 return; 4107 } 4108 (*qbpp)->qb_hiwat = q->q_hiwat; 4109 (*qbpp)->qb_lowat = q->q_lowat; 4110 q->q_nband++; 4111 qbpp = &(*qbpp)->qb_next; 4112 } 4113 } 4114 qbp = q->q_bandp; 4115 i = pri; 4116 while (--i) 4117 qbp = qbp->qb_next; 4118 qbp->qb_flag |= QB_BACK; 4119 } else { 4120 q->q_flag |= QBACK; 4121 } 4122 } 4123 4124 int 4125 strcopyin(void *from, void *to, size_t len, int copyflag) 4126 { 4127 if (copyflag & U_TO_K) { 4128 ASSERT((copyflag & K_TO_K) == 0); 4129 if (copyin(from, to, len)) 4130 return (EFAULT); 4131 } else { 4132 ASSERT(copyflag & K_TO_K); 4133 bcopy(from, to, len); 4134 } 4135 return (0); 4136 } 4137 4138 int 4139 strcopyout(void *from, void *to, size_t len, int copyflag) 4140 { 4141 if (copyflag & U_TO_K) { 4142 if (copyout(from, to, len)) 4143 return (EFAULT); 4144 } else { 4145 ASSERT(copyflag & K_TO_K); 4146 bcopy(from, to, len); 4147 } 4148 return (0); 4149 } 4150 4151 /* 4152 * strsignal_nolock() posts a signal to the process(es) at the stream head. 4153 * It assumes that the stream head lock is already held, whereas strsignal() 4154 * acquires the lock first. This routine was created because a few callers 4155 * release the stream head lock before calling only to re-acquire it after 4156 * it returns. 4157 */ 4158 void 4159 strsignal_nolock(stdata_t *stp, int sig, uchar_t band) 4160 { 4161 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4162 switch (sig) { 4163 case SIGPOLL: 4164 if (stp->sd_sigflags & S_MSG) 4165 strsendsig(stp->sd_siglist, S_MSG, band, 0); 4166 break; 4167 default: 4168 if (stp->sd_pgidp) 4169 pgsignal(stp->sd_pgidp, sig); 4170 break; 4171 } 4172 } 4173 4174 void 4175 strsignal(stdata_t *stp, int sig, int32_t band) 4176 { 4177 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG, 4178 "strsignal:%p, %X, %X", stp, sig, band); 4179 4180 mutex_enter(&stp->sd_lock); 4181 switch (sig) { 4182 case SIGPOLL: 4183 if (stp->sd_sigflags & S_MSG) 4184 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 4185 break; 4186 4187 default: 4188 if (stp->sd_pgidp) { 4189 pgsignal(stp->sd_pgidp, sig); 4190 } 4191 break; 4192 } 4193 mutex_exit(&stp->sd_lock); 4194 } 4195 4196 void 4197 strhup(stdata_t *stp) 4198 { 4199 ASSERT(mutex_owned(&stp->sd_lock)); 4200 pollwakeup(&stp->sd_pollist, POLLHUP); 4201 if (stp->sd_sigflags & S_HANGUP) 4202 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0); 4203 } 4204 4205 /* 4206 * Backenable the first queue upstream from `q' with a service procedure. 4207 */ 4208 void 4209 backenable(queue_t *q, uchar_t pri) 4210 { 4211 queue_t *nq; 4212 4213 /* 4214 * Our presence might not prevent other modules in our own 4215 * stream from popping/pushing since the caller of getq might not 4216 * have a claim on the queue (some drivers do a getq on somebody 4217 * else's queue - they know that the queue itself is not going away 4218 * but the framework has to guarantee q_next in that stream). 4219 */ 4220 claimstr(q); 4221 4222 /* Find nearest back queue with service proc */ 4223 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) { 4224 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq)); 4225 } 4226 4227 if (nq) { 4228 kthread_t *freezer; 4229 /* 4230 * backenable can be called either with no locks held 4231 * or with the stream frozen (the latter occurs when a module 4232 * calls rmvq with the stream frozen). If the stream is frozen 4233 * by the caller the caller will hold all qlocks in the stream. 4234 * Note that a frozen stream doesn't freeze a mated stream, 4235 * so we explicitly check for that. 4236 */ 4237 freezer = STREAM(q)->sd_freezer; 4238 if (freezer != curthread || STREAM(q) != STREAM(nq)) { 4239 mutex_enter(QLOCK(nq)); 4240 } 4241 #ifdef DEBUG 4242 else { 4243 ASSERT(frozenstr(q)); 4244 ASSERT(MUTEX_HELD(QLOCK(q))); 4245 ASSERT(MUTEX_HELD(QLOCK(nq))); 4246 } 4247 #endif 4248 setqback(nq, pri); 4249 qenable_locked(nq); 4250 if (freezer != curthread || STREAM(q) != STREAM(nq)) 4251 mutex_exit(QLOCK(nq)); 4252 } 4253 releasestr(q); 4254 } 4255 4256 /* 4257 * Return the appropriate errno when one of flags_to_check is set 4258 * in sd_flags. Uses the exported error routines if they are set. 4259 * Will return 0 if non error is set (or if the exported error routines 4260 * do not return an error). 4261 * 4262 * If there is both a read and write error to check, we prefer the read error. 4263 * Also, give preference to recorded errno's over the error functions. 4264 * The flags that are handled are: 4265 * STPLEX return EINVAL 4266 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST) 4267 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST) 4268 * STRHUP return sd_werror 4269 * 4270 * If the caller indicates that the operation is a peek, a nonpersistent error 4271 * is not cleared. 4272 */ 4273 int 4274 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek) 4275 { 4276 int32_t sd_flag = stp->sd_flag & flags_to_check; 4277 int error = 0; 4278 4279 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4280 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0); 4281 if (sd_flag & STPLEX) 4282 error = EINVAL; 4283 else if (sd_flag & STRDERR) { 4284 error = stp->sd_rerror; 4285 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) { 4286 /* 4287 * Read errors are non-persistent i.e. discarded once 4288 * returned to a non-peeking caller, 4289 */ 4290 stp->sd_rerror = 0; 4291 stp->sd_flag &= ~STRDERR; 4292 } 4293 if (error == 0 && stp->sd_rderrfunc != NULL) { 4294 int clearerr = 0; 4295 4296 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek, 4297 &clearerr); 4298 if (clearerr) { 4299 stp->sd_flag &= ~STRDERR; 4300 stp->sd_rderrfunc = NULL; 4301 } 4302 } 4303 } else if (sd_flag & STWRERR) { 4304 error = stp->sd_werror; 4305 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) { 4306 /* 4307 * Write errors are non-persistent i.e. discarded once 4308 * returned to a non-peeking caller, 4309 */ 4310 stp->sd_werror = 0; 4311 stp->sd_flag &= ~STWRERR; 4312 } 4313 if (error == 0 && stp->sd_wrerrfunc != NULL) { 4314 int clearerr = 0; 4315 4316 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek, 4317 &clearerr); 4318 if (clearerr) { 4319 stp->sd_flag &= ~STWRERR; 4320 stp->sd_wrerrfunc = NULL; 4321 } 4322 } 4323 } else if (sd_flag & STRHUP) { 4324 /* sd_werror set when STRHUP */ 4325 error = stp->sd_werror; 4326 } 4327 return (error); 4328 } 4329 4330 4331 /* 4332 * Single-thread open/close/push/pop 4333 * for twisted streams also 4334 */ 4335 int 4336 strstartplumb(stdata_t *stp, int flag, int cmd) 4337 { 4338 int waited = 1; 4339 int error = 0; 4340 4341 if (STRMATED(stp)) { 4342 struct stdata *stmatep = stp->sd_mate; 4343 4344 STRLOCKMATES(stp); 4345 while (waited) { 4346 waited = 0; 4347 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4348 if ((cmd == I_POP) && 4349 (flag & (FNDELAY|FNONBLOCK))) { 4350 STRUNLOCKMATES(stp); 4351 return (EAGAIN); 4352 } 4353 waited = 1; 4354 mutex_exit(&stp->sd_lock); 4355 if (!cv_wait_sig(&stmatep->sd_monitor, 4356 &stmatep->sd_lock)) { 4357 mutex_exit(&stmatep->sd_lock); 4358 return (EINTR); 4359 } 4360 mutex_exit(&stmatep->sd_lock); 4361 STRLOCKMATES(stp); 4362 } 4363 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4364 if ((cmd == I_POP) && 4365 (flag & (FNDELAY|FNONBLOCK))) { 4366 STRUNLOCKMATES(stp); 4367 return (EAGAIN); 4368 } 4369 waited = 1; 4370 mutex_exit(&stmatep->sd_lock); 4371 if (!cv_wait_sig(&stp->sd_monitor, 4372 &stp->sd_lock)) { 4373 mutex_exit(&stp->sd_lock); 4374 return (EINTR); 4375 } 4376 mutex_exit(&stp->sd_lock); 4377 STRLOCKMATES(stp); 4378 } 4379 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4380 error = strgeterr(stp, 4381 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4382 if (error != 0) { 4383 STRUNLOCKMATES(stp); 4384 return (error); 4385 } 4386 } 4387 } 4388 stp->sd_flag |= STRPLUMB; 4389 STRUNLOCKMATES(stp); 4390 } else { 4391 mutex_enter(&stp->sd_lock); 4392 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4393 if (((cmd == I_POP) || (cmd == _I_REMOVE)) && 4394 (flag & (FNDELAY|FNONBLOCK))) { 4395 mutex_exit(&stp->sd_lock); 4396 return (EAGAIN); 4397 } 4398 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) { 4399 mutex_exit(&stp->sd_lock); 4400 return (EINTR); 4401 } 4402 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4403 error = strgeterr(stp, 4404 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4405 if (error != 0) { 4406 mutex_exit(&stp->sd_lock); 4407 return (error); 4408 } 4409 } 4410 } 4411 stp->sd_flag |= STRPLUMB; 4412 mutex_exit(&stp->sd_lock); 4413 } 4414 return (0); 4415 } 4416 4417 /* 4418 * Complete the plumbing operation associated with stream `stp'. 4419 */ 4420 void 4421 strendplumb(stdata_t *stp) 4422 { 4423 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4424 ASSERT(stp->sd_flag & STRPLUMB); 4425 stp->sd_flag &= ~STRPLUMB; 4426 cv_broadcast(&stp->sd_monitor); 4427 } 4428 4429 /* 4430 * This describes how the STREAMS framework handles synchronization 4431 * during open/push and close/pop. 4432 * The key interfaces for open and close are qprocson and qprocsoff, 4433 * respectively. While the close case in general is harder both open 4434 * have close have significant similarities. 4435 * 4436 * During close the STREAMS framework has to both ensure that there 4437 * are no stale references to the queue pair (and syncq) that 4438 * are being closed and also provide the guarantees that are documented 4439 * in qprocsoff(9F). 4440 * If there are stale references to the queue that is closing it can 4441 * result in kernel memory corruption or kernel panics. 4442 * 4443 * Note that is it up to the module/driver to ensure that it itself 4444 * does not have any stale references to the closing queues once its close 4445 * routine returns. This includes: 4446 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines 4447 * associated with the queues. For timeout and bufcall callbacks the 4448 * module/driver also has to ensure (or wait for) any callbacks that 4449 * are in progress. 4450 * - If the module/driver is using esballoc it has to ensure that any 4451 * esballoc free functions do not refer to a queue that has closed. 4452 * (Note that in general the close routine can not wait for the esballoc'ed 4453 * messages to be freed since that can cause a deadlock.) 4454 * - Cancelling any interrupts that refer to the closing queues and 4455 * also ensuring that there are no interrupts in progress that will 4456 * refer to the closing queues once the close routine returns. 4457 * - For multiplexors removing any driver global state that refers to 4458 * the closing queue and also ensuring that there are no threads in 4459 * the multiplexor that has picked up a queue pointer but not yet 4460 * finished using it. 4461 * 4462 * In addition, a driver/module can only reference the q_next pointer 4463 * in its open, close, put, or service procedures or in a 4464 * qtimeout/qbufcall callback procedure executing "on" the correct 4465 * stream. Thus it can not reference the q_next pointer in an interrupt 4466 * routine or a timeout, bufcall or esballoc callback routine. Likewise 4467 * it can not reference q_next of a different queue e.g. in a mux that 4468 * passes messages from one queues put/service procedure to another queue. 4469 * In all the cases when the driver/module can not access the q_next 4470 * field it must use the *next* versions e.g. canputnext instead of 4471 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...). 4472 * 4473 * 4474 * Assuming that the driver/module conforms to the above constraints 4475 * the STREAMS framework has to avoid stale references to q_next for all 4476 * the framework internal cases which include (but are not limited to): 4477 * - Threads in canput/canputnext/backenable and elsewhere that are 4478 * walking q_next. 4479 * - Messages on a syncq that have a reference to the queue through b_queue. 4480 * - Messages on an outer perimeter (syncq) that have a reference to the 4481 * queue through b_queue. 4482 * - Threads that use q_nfsrv (e.g. canput) to find a queue. 4483 * Note that only canput and bcanput use q_nfsrv without any locking. 4484 * 4485 * The STREAMS framework providing the qprocsoff(9F) guarantees means that 4486 * after qprocsoff returns, the framework has to ensure that no threads can 4487 * enter the put or service routines for the closing read or write-side queue. 4488 * In addition to preventing "direct" entry into the put procedures 4489 * the framework also has to prevent messages being drained from 4490 * the syncq or the outer perimeter. 4491 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only 4492 * mechanism to prevent qwriter(PERIM_OUTER) from running after 4493 * qprocsoff has returned. 4494 * Note that if a module/driver uses put(9F) on one of its own queues 4495 * it is up to the module/driver to ensure that the put() doesn't 4496 * get called when the queue is closing. 4497 * 4498 * 4499 * The framework aspects of the above "contract" is implemented by 4500 * qprocsoff, removeq, and strlock: 4501 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from 4502 * entering the service procedures. 4503 * - strlock acquires the sd_lock and sd_reflock to prevent putnext, 4504 * canputnext, backenable etc from dereferencing the q_next that will 4505 * soon change. 4506 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext 4507 * or other q_next walker that uses claimstr/releasestr to finish. 4508 * - optionally for every syncq in the stream strlock acquires all the 4509 * sq_lock's and waits for all sq_counts to drop to a value that indicates 4510 * that no thread executes in the put or service procedures and that no 4511 * thread is draining into the module/driver. This ensures that no 4512 * open, close, put, service, or qtimeout/qbufcall callback procedure is 4513 * currently executing hence no such thread can end up with the old stale 4514 * q_next value and no canput/backenable can have the old stale 4515 * q_nfsrv/q_next. 4516 * - qdetach (wait_svc) makes sure that any scheduled or running threads 4517 * have either finished or observed the QWCLOSE flag and gone away. 4518 */ 4519 4520 4521 /* 4522 * Get all the locks necessary to change q_next. 4523 * 4524 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the 4525 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that 4526 * the only threads inside the syncq are threads currently calling removeq(). 4527 * Since threads calling removeq() are in the process of removing their queues 4528 * from the stream, we do not need to worry about them accessing a stale q_next 4529 * pointer and thus we do not need to wait for them to exit (in fact, waiting 4530 * for them can cause deadlock). 4531 * 4532 * This routine is subject to starvation since it does not set any flag to 4533 * prevent threads from entering a module in the stream (i.e. sq_count can 4534 * increase on some syncq while it is waiting on some other syncq). 4535 * 4536 * Assumes that only one thread attempts to call strlock for a given 4537 * stream. If this is not the case the two threads would deadlock. 4538 * This assumption is guaranteed since strlock is only called by insertq 4539 * and removeq and streams plumbing changes are single-threaded for 4540 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags. 4541 * 4542 * For pipes, it is not difficult to atomically designate a pair of streams 4543 * to be mated. Once mated atomically by the framework the twisted pair remain 4544 * configured that way until dismantled atomically by the framework. 4545 * When plumbing takes place on a twisted stream it is necessary to ensure that 4546 * this operation is done exclusively on the twisted stream since two such 4547 * operations, each initiated on different ends of the pipe will deadlock 4548 * waiting for each other to complete. 4549 * 4550 * On entry, no locks should be held. 4551 * The locks acquired and held by strlock depends on a few factors. 4552 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired 4553 * and held on exit and all sq_count are at an acceptable level. 4554 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with 4555 * sd_refcnt being zero. 4556 */ 4557 4558 static void 4559 strlock(struct stdata *stp, sqlist_t *sqlist) 4560 { 4561 syncql_t *sql, *sql2; 4562 retry: 4563 /* 4564 * Wait for any claimstr to go away. 4565 */ 4566 if (STRMATED(stp)) { 4567 struct stdata *stp1, *stp2; 4568 4569 STRLOCKMATES(stp); 4570 /* 4571 * Note that the selection of locking order is not 4572 * important, just that they are always acquired in 4573 * the same order. To assure this, we choose this 4574 * order based on the value of the pointer, and since 4575 * the pointer will not change for the life of this 4576 * pair, we will always grab the locks in the same 4577 * order (and hence, prevent deadlocks). 4578 */ 4579 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) { 4580 stp1 = stp; 4581 stp2 = stp->sd_mate; 4582 } else { 4583 stp2 = stp; 4584 stp1 = stp->sd_mate; 4585 } 4586 mutex_enter(&stp1->sd_reflock); 4587 if (stp1->sd_refcnt > 0) { 4588 STRUNLOCKMATES(stp); 4589 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock); 4590 mutex_exit(&stp1->sd_reflock); 4591 goto retry; 4592 } 4593 mutex_enter(&stp2->sd_reflock); 4594 if (stp2->sd_refcnt > 0) { 4595 STRUNLOCKMATES(stp); 4596 mutex_exit(&stp1->sd_reflock); 4597 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock); 4598 mutex_exit(&stp2->sd_reflock); 4599 goto retry; 4600 } 4601 STREAM_PUTLOCKS_ENTER(stp1); 4602 STREAM_PUTLOCKS_ENTER(stp2); 4603 } else { 4604 mutex_enter(&stp->sd_lock); 4605 mutex_enter(&stp->sd_reflock); 4606 while (stp->sd_refcnt > 0) { 4607 mutex_exit(&stp->sd_lock); 4608 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock); 4609 if (mutex_tryenter(&stp->sd_lock) == 0) { 4610 mutex_exit(&stp->sd_reflock); 4611 mutex_enter(&stp->sd_lock); 4612 mutex_enter(&stp->sd_reflock); 4613 } 4614 } 4615 STREAM_PUTLOCKS_ENTER(stp); 4616 } 4617 4618 if (sqlist == NULL) 4619 return; 4620 4621 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4622 syncq_t *sq = sql->sql_sq; 4623 uint16_t count; 4624 4625 mutex_enter(SQLOCK(sq)); 4626 count = sq->sq_count; 4627 ASSERT(sq->sq_rmqcount <= count); 4628 SQ_PUTLOCKS_ENTER(sq); 4629 SUM_SQ_PUTCOUNTS(sq, count); 4630 if (count == sq->sq_rmqcount) 4631 continue; 4632 4633 /* Failed - drop all locks that we have acquired so far */ 4634 if (STRMATED(stp)) { 4635 STREAM_PUTLOCKS_EXIT(stp); 4636 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4637 STRUNLOCKMATES(stp); 4638 mutex_exit(&stp->sd_reflock); 4639 mutex_exit(&stp->sd_mate->sd_reflock); 4640 } else { 4641 STREAM_PUTLOCKS_EXIT(stp); 4642 mutex_exit(&stp->sd_lock); 4643 mutex_exit(&stp->sd_reflock); 4644 } 4645 for (sql2 = sqlist->sqlist_head; sql2 != sql; 4646 sql2 = sql2->sql_next) { 4647 SQ_PUTLOCKS_EXIT(sql2->sql_sq); 4648 mutex_exit(SQLOCK(sql2->sql_sq)); 4649 } 4650 4651 /* 4652 * The wait loop below may starve when there are many threads 4653 * claiming the syncq. This is especially a problem with permod 4654 * syncqs (IP). To lessen the impact of the problem we increment 4655 * sq_needexcl and clear fastbits so that putnexts will slow 4656 * down and call sqenable instead of draining right away. 4657 */ 4658 sq->sq_needexcl++; 4659 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4660 while (count > sq->sq_rmqcount) { 4661 sq->sq_flags |= SQ_WANTWAKEUP; 4662 SQ_PUTLOCKS_EXIT(sq); 4663 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4664 count = sq->sq_count; 4665 SQ_PUTLOCKS_ENTER(sq); 4666 SUM_SQ_PUTCOUNTS(sq, count); 4667 } 4668 sq->sq_needexcl--; 4669 if (sq->sq_needexcl == 0) 4670 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4671 SQ_PUTLOCKS_EXIT(sq); 4672 ASSERT(count == sq->sq_rmqcount); 4673 mutex_exit(SQLOCK(sq)); 4674 goto retry; 4675 } 4676 } 4677 4678 /* 4679 * Drop all the locks that strlock acquired. 4680 */ 4681 static void 4682 strunlock(struct stdata *stp, sqlist_t *sqlist) 4683 { 4684 syncql_t *sql; 4685 4686 if (STRMATED(stp)) { 4687 STREAM_PUTLOCKS_EXIT(stp); 4688 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4689 STRUNLOCKMATES(stp); 4690 mutex_exit(&stp->sd_reflock); 4691 mutex_exit(&stp->sd_mate->sd_reflock); 4692 } else { 4693 STREAM_PUTLOCKS_EXIT(stp); 4694 mutex_exit(&stp->sd_lock); 4695 mutex_exit(&stp->sd_reflock); 4696 } 4697 4698 if (sqlist == NULL) 4699 return; 4700 4701 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4702 SQ_PUTLOCKS_EXIT(sql->sql_sq); 4703 mutex_exit(SQLOCK(sql->sql_sq)); 4704 } 4705 } 4706 4707 /* 4708 * When the module has service procedure, we need check if the next 4709 * module which has service procedure is in flow control to trigger 4710 * the backenable. 4711 */ 4712 static void 4713 backenable_insertedq(queue_t *q) 4714 { 4715 qband_t *qbp; 4716 4717 claimstr(q); 4718 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) { 4719 if (q->q_next->q_nfsrv->q_flag & QWANTW) 4720 backenable(q, 0); 4721 4722 qbp = q->q_next->q_nfsrv->q_bandp; 4723 for (; qbp != NULL; qbp = qbp->qb_next) 4724 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL) 4725 backenable(q, qbp->qb_first->b_band); 4726 } 4727 releasestr(q); 4728 } 4729 4730 /* 4731 * Given two read queues, insert a new single one after another. 4732 * 4733 * This routine acquires all the necessary locks in order to change 4734 * q_next and related pointer using strlock(). 4735 * It depends on the stream head ensuring that there are no concurrent 4736 * insertq or removeq on the same stream. The stream head ensures this 4737 * using the flags STWOPEN, STRCLOSE, and STRPLUMB. 4738 * 4739 * Note that no syncq locks are held during the q_next change. This is 4740 * applied to all streams since, unlike removeq, there is no problem of stale 4741 * pointers when adding a module to the stream. Thus drivers/modules that do a 4742 * canput(rq->q_next) would never get a closed/freed queue pointer even if we 4743 * applied this optimization to all streams. 4744 */ 4745 void 4746 insertq(struct stdata *stp, queue_t *new) 4747 { 4748 queue_t *after; 4749 queue_t *wafter; 4750 queue_t *wnew = _WR(new); 4751 boolean_t have_fifo = B_FALSE; 4752 4753 if (new->q_flag & _QINSERTING) { 4754 ASSERT(stp->sd_vnode->v_type != VFIFO); 4755 after = new->q_next; 4756 wafter = _WR(new->q_next); 4757 } else { 4758 after = _RD(stp->sd_wrq); 4759 wafter = stp->sd_wrq; 4760 } 4761 4762 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ, 4763 "insertq:%p, %p", after, new); 4764 ASSERT(after->q_flag & QREADR); 4765 ASSERT(new->q_flag & QREADR); 4766 4767 strlock(stp, NULL); 4768 4769 /* Do we have a FIFO? */ 4770 if (wafter->q_next == after) { 4771 have_fifo = B_TRUE; 4772 wnew->q_next = new; 4773 } else { 4774 wnew->q_next = wafter->q_next; 4775 } 4776 new->q_next = after; 4777 4778 set_nfsrv_ptr(new, wnew, after, wafter); 4779 /* 4780 * set_nfsrv_ptr() needs to know if this is an insertion or not, 4781 * so only reset this flag after calling it. 4782 */ 4783 new->q_flag &= ~_QINSERTING; 4784 4785 if (have_fifo) { 4786 wafter->q_next = wnew; 4787 } else { 4788 if (wafter->q_next) 4789 _OTHERQ(wafter->q_next)->q_next = new; 4790 wafter->q_next = wnew; 4791 } 4792 4793 set_qend(new); 4794 /* The QEND flag might have to be updated for the upstream guy */ 4795 set_qend(after); 4796 4797 ASSERT(_SAMESTR(new) == O_SAMESTR(new)); 4798 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew)); 4799 ASSERT(_SAMESTR(after) == O_SAMESTR(after)); 4800 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter)); 4801 strsetuio(stp); 4802 4803 /* 4804 * If this was a module insertion, bump the push count. 4805 */ 4806 if (!(new->q_flag & QISDRV)) 4807 stp->sd_pushcnt++; 4808 4809 strunlock(stp, NULL); 4810 4811 /* check if the write Q needs backenable */ 4812 backenable_insertedq(wnew); 4813 4814 /* check if the read Q needs backenable */ 4815 backenable_insertedq(new); 4816 } 4817 4818 /* 4819 * Given a read queue, unlink it from any neighbors. 4820 * 4821 * This routine acquires all the necessary locks in order to 4822 * change q_next and related pointers and also guard against 4823 * stale references (e.g. through q_next) to the queue that 4824 * is being removed. It also plays part of the role in ensuring 4825 * that the module's/driver's put procedure doesn't get called 4826 * after qprocsoff returns. 4827 * 4828 * Removeq depends on the stream head ensuring that there are 4829 * no concurrent insertq or removeq on the same stream. The 4830 * stream head ensures this using the flags STWOPEN, STRCLOSE and 4831 * STRPLUMB. 4832 * 4833 * The set of locks needed to remove the queue is different in 4834 * different cases: 4835 * 4836 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after 4837 * waiting for the syncq reference count to drop to 0 indicating that no 4838 * non-close threads are present anywhere in the stream. This ensures that any 4839 * module/driver can reference q_next in its open, close, put, or service 4840 * procedures. 4841 * 4842 * The sq_rmqcount counter tracks the number of threads inside removeq(). 4843 * strlock() ensures that there is either no threads executing inside perimeter 4844 * or there is only a thread calling qprocsoff(). 4845 * 4846 * strlock() compares the value of sq_count with the number of threads inside 4847 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup 4848 * any threads waiting in strlock() when the sq_rmqcount increases. 4849 */ 4850 4851 void 4852 removeq(queue_t *qp) 4853 { 4854 queue_t *wqp = _WR(qp); 4855 struct stdata *stp = STREAM(qp); 4856 sqlist_t *sqlist = NULL; 4857 boolean_t isdriver; 4858 int moved; 4859 syncq_t *sq = qp->q_syncq; 4860 syncq_t *wsq = wqp->q_syncq; 4861 4862 ASSERT(stp); 4863 4864 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ, 4865 "removeq:%p %p", qp, wqp); 4866 ASSERT(qp->q_flag&QREADR); 4867 4868 /* 4869 * For queues using Synchronous streams, we must wait for all threads in 4870 * rwnext() to drain out before proceeding. 4871 */ 4872 if (qp->q_flag & QSYNCSTR) { 4873 /* First, we need wakeup any threads blocked in rwnext() */ 4874 mutex_enter(SQLOCK(sq)); 4875 if (sq->sq_flags & SQ_WANTWAKEUP) { 4876 sq->sq_flags &= ~SQ_WANTWAKEUP; 4877 cv_broadcast(&sq->sq_wait); 4878 } 4879 mutex_exit(SQLOCK(sq)); 4880 4881 if (wsq != sq) { 4882 mutex_enter(SQLOCK(wsq)); 4883 if (wsq->sq_flags & SQ_WANTWAKEUP) { 4884 wsq->sq_flags &= ~SQ_WANTWAKEUP; 4885 cv_broadcast(&wsq->sq_wait); 4886 } 4887 mutex_exit(SQLOCK(wsq)); 4888 } 4889 4890 mutex_enter(QLOCK(qp)); 4891 while (qp->q_rwcnt > 0) { 4892 qp->q_flag |= QWANTRMQSYNC; 4893 cv_wait(&qp->q_wait, QLOCK(qp)); 4894 } 4895 mutex_exit(QLOCK(qp)); 4896 4897 mutex_enter(QLOCK(wqp)); 4898 while (wqp->q_rwcnt > 0) { 4899 wqp->q_flag |= QWANTRMQSYNC; 4900 cv_wait(&wqp->q_wait, QLOCK(wqp)); 4901 } 4902 mutex_exit(QLOCK(wqp)); 4903 } 4904 4905 mutex_enter(SQLOCK(sq)); 4906 sq->sq_rmqcount++; 4907 if (sq->sq_flags & SQ_WANTWAKEUP) { 4908 sq->sq_flags &= ~SQ_WANTWAKEUP; 4909 cv_broadcast(&sq->sq_wait); 4910 } 4911 mutex_exit(SQLOCK(sq)); 4912 4913 isdriver = (qp->q_flag & QISDRV); 4914 4915 sqlist = sqlist_build(qp, stp, STRMATED(stp)); 4916 strlock(stp, sqlist); 4917 4918 reset_nfsrv_ptr(qp, wqp); 4919 4920 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp); 4921 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp); 4922 /* Do we have a FIFO? */ 4923 if (wqp->q_next == qp) { 4924 stp->sd_wrq->q_next = _RD(stp->sd_wrq); 4925 } else { 4926 if (wqp->q_next) 4927 backq(qp)->q_next = qp->q_next; 4928 if (qp->q_next) 4929 backq(wqp)->q_next = wqp->q_next; 4930 } 4931 4932 /* The QEND flag might have to be updated for the upstream guy */ 4933 if (qp->q_next) 4934 set_qend(qp->q_next); 4935 4936 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq)); 4937 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq))); 4938 4939 /* 4940 * Move any messages destined for the put procedures to the next 4941 * syncq in line. Otherwise free them. 4942 */ 4943 moved = 0; 4944 /* 4945 * Quick check to see whether there are any messages or events. 4946 */ 4947 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS)) 4948 moved += propagate_syncq(qp); 4949 if (wqp->q_syncqmsgs != 0 || 4950 (wqp->q_syncq->sq_flags & SQ_EVENTS)) 4951 moved += propagate_syncq(wqp); 4952 4953 strsetuio(stp); 4954 4955 /* 4956 * If this was a module removal, decrement the push count. 4957 */ 4958 if (!isdriver) 4959 stp->sd_pushcnt--; 4960 4961 strunlock(stp, sqlist); 4962 sqlist_free(sqlist); 4963 4964 /* 4965 * Make sure any messages that were propagated are drained. 4966 * Also clear any QFULL bit caused by messages that were propagated. 4967 */ 4968 4969 if (qp->q_next != NULL) { 4970 clr_qfull(qp); 4971 /* 4972 * For the driver calling qprocsoff, propagate_syncq 4973 * frees all the messages instead of putting it in 4974 * the stream head 4975 */ 4976 if (!isdriver && (moved > 0)) 4977 emptysq(qp->q_next->q_syncq); 4978 } 4979 if (wqp->q_next != NULL) { 4980 clr_qfull(wqp); 4981 /* 4982 * We come here for any pop of a module except for the 4983 * case of driver being removed. We don't call emptysq 4984 * if we did not move any messages. This will avoid holding 4985 * PERMOD syncq locks in emptysq 4986 */ 4987 if (moved > 0) 4988 emptysq(wqp->q_next->q_syncq); 4989 } 4990 4991 mutex_enter(SQLOCK(sq)); 4992 sq->sq_rmqcount--; 4993 mutex_exit(SQLOCK(sq)); 4994 } 4995 4996 /* 4997 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or 4998 * SQ_WRITER) on a syncq. 4999 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the 5000 * sync queue and waits until sq_count reaches maxcnt. 5001 * 5002 * If maxcnt is -1 there's no need to grab sq_putlocks since the caller 5003 * does not care about putnext threads that are in the middle of calling put 5004 * entry points. 5005 * 5006 * This routine is used for both inner and outer syncqs. 5007 */ 5008 static void 5009 blocksq(syncq_t *sq, ushort_t flag, int maxcnt) 5010 { 5011 uint16_t count = 0; 5012 5013 mutex_enter(SQLOCK(sq)); 5014 /* 5015 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset. 5016 * SQ_FROZEN will be set if there is a frozen stream that has a 5017 * queue which also refers to this "shared" syncq. 5018 * SQ_BLOCKED will be set if there is "off" queue which also 5019 * refers to this "shared" syncq. 5020 */ 5021 if (maxcnt != -1) { 5022 count = sq->sq_count; 5023 SQ_PUTLOCKS_ENTER(sq); 5024 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5025 SUM_SQ_PUTCOUNTS(sq, count); 5026 } 5027 sq->sq_needexcl++; 5028 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5029 5030 while ((sq->sq_flags & flag) || 5031 (maxcnt != -1 && count > (unsigned)maxcnt)) { 5032 sq->sq_flags |= SQ_WANTWAKEUP; 5033 if (maxcnt != -1) { 5034 SQ_PUTLOCKS_EXIT(sq); 5035 } 5036 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5037 if (maxcnt != -1) { 5038 count = sq->sq_count; 5039 SQ_PUTLOCKS_ENTER(sq); 5040 SUM_SQ_PUTCOUNTS(sq, count); 5041 } 5042 } 5043 sq->sq_needexcl--; 5044 sq->sq_flags |= flag; 5045 ASSERT(maxcnt == -1 || count == maxcnt); 5046 if (maxcnt != -1) { 5047 if (sq->sq_needexcl == 0) { 5048 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5049 } 5050 SQ_PUTLOCKS_EXIT(sq); 5051 } else if (sq->sq_needexcl == 0) { 5052 SQ_PUTCOUNT_SETFAST(sq); 5053 } 5054 5055 mutex_exit(SQLOCK(sq)); 5056 } 5057 5058 /* 5059 * Reset a flag that was set with blocksq. 5060 * 5061 * Can not use this routine to reset SQ_WRITER. 5062 * 5063 * If "isouter" is set then the syncq is assumed to be an outer perimeter 5064 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread 5065 * to handle the queued qwriter operations. 5066 * 5067 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5068 * sq_putlocks are used. 5069 */ 5070 static void 5071 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) 5072 { 5073 uint16_t flags; 5074 5075 mutex_enter(SQLOCK(sq)); 5076 ASSERT(resetflag != SQ_WRITER); 5077 ASSERT(sq->sq_flags & resetflag); 5078 flags = sq->sq_flags & ~resetflag; 5079 sq->sq_flags = flags; 5080 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) { 5081 if (flags & SQ_WANTWAKEUP) { 5082 flags &= ~SQ_WANTWAKEUP; 5083 cv_broadcast(&sq->sq_wait); 5084 } 5085 sq->sq_flags = flags; 5086 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5087 if (!isouter) { 5088 /* drain_syncq drops SQLOCK */ 5089 drain_syncq(sq); 5090 return; 5091 } 5092 } 5093 } 5094 mutex_exit(SQLOCK(sq)); 5095 } 5096 5097 /* 5098 * Reset a flag that was set with blocksq. 5099 * Does not drain the syncq. Use emptysq() for that. 5100 * Returns 1 if SQ_QUEUED is set. Otherwise 0. 5101 * 5102 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5103 * sq_putlocks are used. 5104 */ 5105 static int 5106 dropsq(syncq_t *sq, uint16_t resetflag) 5107 { 5108 uint16_t flags; 5109 5110 mutex_enter(SQLOCK(sq)); 5111 ASSERT(sq->sq_flags & resetflag); 5112 flags = sq->sq_flags & ~resetflag; 5113 if (flags & SQ_WANTWAKEUP) { 5114 flags &= ~SQ_WANTWAKEUP; 5115 cv_broadcast(&sq->sq_wait); 5116 } 5117 sq->sq_flags = flags; 5118 mutex_exit(SQLOCK(sq)); 5119 if (flags & SQ_QUEUED) 5120 return (1); 5121 return (0); 5122 } 5123 5124 /* 5125 * Empty all the messages on a syncq. 5126 * 5127 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5128 * sq_putlocks are used. 5129 */ 5130 static void 5131 emptysq(syncq_t *sq) 5132 { 5133 uint16_t flags; 5134 5135 mutex_enter(SQLOCK(sq)); 5136 flags = sq->sq_flags; 5137 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5138 /* 5139 * To prevent potential recursive invocation of drain_syncq we 5140 * do not call drain_syncq if count is non-zero. 5141 */ 5142 if (sq->sq_count == 0) { 5143 /* drain_syncq() drops SQLOCK */ 5144 drain_syncq(sq); 5145 return; 5146 } else 5147 sqenable(sq); 5148 } 5149 mutex_exit(SQLOCK(sq)); 5150 } 5151 5152 /* 5153 * Ordered insert while removing duplicates. 5154 */ 5155 static void 5156 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp) 5157 { 5158 syncql_t *sqlp, **prev_sqlpp, *new_sqlp; 5159 5160 prev_sqlpp = &sqlist->sqlist_head; 5161 while ((sqlp = *prev_sqlpp) != NULL) { 5162 if (sqlp->sql_sq >= sqp) { 5163 if (sqlp->sql_sq == sqp) /* duplicate */ 5164 return; 5165 break; 5166 } 5167 prev_sqlpp = &sqlp->sql_next; 5168 } 5169 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++]; 5170 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size); 5171 new_sqlp->sql_next = sqlp; 5172 new_sqlp->sql_sq = sqp; 5173 *prev_sqlpp = new_sqlp; 5174 } 5175 5176 /* 5177 * Walk the write side queues until we hit either the driver 5178 * or a twist in the stream (_SAMESTR will return false in both 5179 * these cases) then turn around and walk the read side queues 5180 * back up to the stream head. 5181 */ 5182 static void 5183 sqlist_insertall(sqlist_t *sqlist, queue_t *q) 5184 { 5185 while (q != NULL) { 5186 sqlist_insert(sqlist, q->q_syncq); 5187 5188 if (_SAMESTR(q)) 5189 q = q->q_next; 5190 else if (!(q->q_flag & QREADR)) 5191 q = _RD(q); 5192 else 5193 q = NULL; 5194 } 5195 } 5196 5197 /* 5198 * Allocate and build a list of all syncqs in a stream and the syncq(s) 5199 * associated with the "q" parameter. The resulting list is sorted in a 5200 * canonical order and is free of duplicates. 5201 * Assumes the passed queue is a _RD(q). 5202 */ 5203 static sqlist_t * 5204 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist) 5205 { 5206 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP); 5207 5208 /* 5209 * start with the current queue/qpair 5210 */ 5211 ASSERT(q->q_flag & QREADR); 5212 5213 sqlist_insert(sqlist, q->q_syncq); 5214 sqlist_insert(sqlist, _WR(q)->q_syncq); 5215 5216 sqlist_insertall(sqlist, stp->sd_wrq); 5217 if (do_twist) 5218 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq); 5219 5220 return (sqlist); 5221 } 5222 5223 static sqlist_t * 5224 sqlist_alloc(struct stdata *stp, int kmflag) 5225 { 5226 size_t sqlist_size; 5227 sqlist_t *sqlist; 5228 5229 /* 5230 * Allocate 2 syncql_t's for each pushed module. Note that 5231 * the sqlist_t structure already has 4 syncql_t's built in: 5232 * 2 for the stream head, and 2 for the driver/other stream head. 5233 */ 5234 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt + 5235 sizeof (sqlist_t); 5236 if (STRMATED(stp)) 5237 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt; 5238 sqlist = kmem_alloc(sqlist_size, kmflag); 5239 5240 sqlist->sqlist_head = NULL; 5241 sqlist->sqlist_size = sqlist_size; 5242 sqlist->sqlist_index = 0; 5243 5244 return (sqlist); 5245 } 5246 5247 /* 5248 * Free the list created by sqlist_alloc() 5249 */ 5250 static void 5251 sqlist_free(sqlist_t *sqlist) 5252 { 5253 kmem_free(sqlist, sqlist->sqlist_size); 5254 } 5255 5256 /* 5257 * Prevent any new entries into any syncq in this stream. 5258 * Used by freezestr. 5259 */ 5260 void 5261 strblock(queue_t *q) 5262 { 5263 struct stdata *stp; 5264 syncql_t *sql; 5265 sqlist_t *sqlist; 5266 5267 q = _RD(q); 5268 5269 stp = STREAM(q); 5270 ASSERT(stp != NULL); 5271 5272 /* 5273 * Get a sorted list with all the duplicates removed containing 5274 * all the syncqs referenced by this stream. 5275 */ 5276 sqlist = sqlist_build(q, stp, B_FALSE); 5277 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5278 blocksq(sql->sql_sq, SQ_FROZEN, -1); 5279 sqlist_free(sqlist); 5280 } 5281 5282 /* 5283 * Release the block on new entries into this stream 5284 */ 5285 void 5286 strunblock(queue_t *q) 5287 { 5288 struct stdata *stp; 5289 syncql_t *sql; 5290 sqlist_t *sqlist; 5291 int drain_needed; 5292 5293 q = _RD(q); 5294 5295 /* 5296 * Get a sorted list with all the duplicates removed containing 5297 * all the syncqs referenced by this stream. 5298 * Have to drop the SQ_FROZEN flag on all the syncqs before 5299 * starting to drain them; otherwise the draining might 5300 * cause a freezestr in some module on the stream (which 5301 * would deadlock). 5302 */ 5303 stp = STREAM(q); 5304 ASSERT(stp != NULL); 5305 sqlist = sqlist_build(q, stp, B_FALSE); 5306 drain_needed = 0; 5307 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5308 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN); 5309 if (drain_needed) { 5310 for (sql = sqlist->sqlist_head; sql != NULL; 5311 sql = sql->sql_next) 5312 emptysq(sql->sql_sq); 5313 } 5314 sqlist_free(sqlist); 5315 } 5316 5317 #ifdef DEBUG 5318 static int 5319 qprocsareon(queue_t *rq) 5320 { 5321 if (rq->q_next == NULL) 5322 return (0); 5323 return (_WR(rq->q_next)->q_next == _WR(rq)); 5324 } 5325 5326 int 5327 qclaimed(queue_t *q) 5328 { 5329 uint_t count; 5330 5331 count = q->q_syncq->sq_count; 5332 SUM_SQ_PUTCOUNTS(q->q_syncq, count); 5333 return (count != 0); 5334 } 5335 5336 /* 5337 * Check if anyone has frozen this stream with freezestr 5338 */ 5339 int 5340 frozenstr(queue_t *q) 5341 { 5342 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0); 5343 } 5344 #endif /* DEBUG */ 5345 5346 /* 5347 * Enter a queue. 5348 * Obsoleted interface. Should not be used. 5349 */ 5350 void 5351 enterq(queue_t *q) 5352 { 5353 entersq(q->q_syncq, SQ_CALLBACK); 5354 } 5355 5356 void 5357 leaveq(queue_t *q) 5358 { 5359 leavesq(q->q_syncq, SQ_CALLBACK); 5360 } 5361 5362 /* 5363 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits 5364 * to check. 5365 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter 5366 * calls and the running of open, close and service procedures. 5367 * 5368 * If c_inner bit is set no need to grab sq_putlocks since we don't care 5369 * if other threads have entered or are entering put entry point. 5370 * 5371 * If c_inner bit is set it might have been possible to use 5372 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize 5373 * open/close path for IP) but since the count may need to be decremented in 5374 * qwait() we wouldn't know which counter to decrement. Currently counter is 5375 * selected by current cpu_seqid and current CPU can change at any moment. XXX 5376 * in the future we might use curthread id bits to select the counter and this 5377 * would stay constant across routine calls. 5378 */ 5379 void 5380 entersq(syncq_t *sq, int entrypoint) 5381 { 5382 uint16_t count = 0; 5383 uint16_t flags; 5384 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 5385 uint16_t type; 5386 uint_t c_inner = entrypoint & SQ_CI; 5387 uint_t c_outer = entrypoint & SQ_CO; 5388 5389 /* 5390 * Increment ref count to keep closes out of this queue. 5391 */ 5392 ASSERT(sq); 5393 ASSERT(c_inner && c_outer); 5394 mutex_enter(SQLOCK(sq)); 5395 flags = sq->sq_flags; 5396 type = sq->sq_type; 5397 if (!(type & c_inner)) { 5398 /* Make sure all putcounts now use slowlock. */ 5399 count = sq->sq_count; 5400 SQ_PUTLOCKS_ENTER(sq); 5401 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5402 SUM_SQ_PUTCOUNTS(sq, count); 5403 sq->sq_needexcl++; 5404 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5405 waitflags |= SQ_MESSAGES; 5406 } 5407 /* 5408 * Wait until we can enter the inner perimeter. 5409 * If we want exclusive access we wait until sq_count is 0. 5410 * We have to do this before entering the outer perimeter in order 5411 * to preserve put/close message ordering. 5412 */ 5413 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) { 5414 sq->sq_flags = flags | SQ_WANTWAKEUP; 5415 if (!(type & c_inner)) { 5416 SQ_PUTLOCKS_EXIT(sq); 5417 } 5418 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5419 if (!(type & c_inner)) { 5420 count = sq->sq_count; 5421 SQ_PUTLOCKS_ENTER(sq); 5422 SUM_SQ_PUTCOUNTS(sq, count); 5423 } 5424 flags = sq->sq_flags; 5425 } 5426 5427 if (!(type & c_inner)) { 5428 ASSERT(sq->sq_needexcl > 0); 5429 sq->sq_needexcl--; 5430 if (sq->sq_needexcl == 0) { 5431 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5432 } 5433 } 5434 5435 /* Check if we need to enter the outer perimeter */ 5436 if (!(type & c_outer)) { 5437 /* 5438 * We have to enter the outer perimeter exclusively before 5439 * we can increment sq_count to avoid deadlock. This implies 5440 * that we have to re-check sq_flags and sq_count. 5441 * 5442 * is it possible to have c_inner set when c_outer is not set? 5443 */ 5444 if (!(type & c_inner)) { 5445 SQ_PUTLOCKS_EXIT(sq); 5446 } 5447 mutex_exit(SQLOCK(sq)); 5448 outer_enter(sq->sq_outer, SQ_GOAWAY); 5449 mutex_enter(SQLOCK(sq)); 5450 flags = sq->sq_flags; 5451 /* 5452 * there should be no need to recheck sq_putcounts 5453 * because outer_enter() has already waited for them to clear 5454 * after setting SQ_WRITER. 5455 */ 5456 count = sq->sq_count; 5457 #ifdef DEBUG 5458 /* 5459 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead 5460 * of doing an ASSERT internally. Others should do 5461 * something like 5462 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0); 5463 * without the need to #ifdef DEBUG it. 5464 */ 5465 SUMCHECK_SQ_PUTCOUNTS(sq, 0); 5466 #endif 5467 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) || 5468 (!(type & c_inner) && count != 0)) { 5469 sq->sq_flags = flags | SQ_WANTWAKEUP; 5470 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5471 count = sq->sq_count; 5472 flags = sq->sq_flags; 5473 } 5474 } 5475 5476 sq->sq_count++; 5477 ASSERT(sq->sq_count != 0); /* Wraparound */ 5478 if (!(type & c_inner)) { 5479 /* Exclusive entry */ 5480 ASSERT(sq->sq_count == 1); 5481 sq->sq_flags |= SQ_EXCL; 5482 if (type & c_outer) { 5483 SQ_PUTLOCKS_EXIT(sq); 5484 } 5485 } 5486 mutex_exit(SQLOCK(sq)); 5487 } 5488 5489 /* 5490 * Leave a syncq. Announce to framework that closes may proceed. 5491 * c_inner and c_outer specify which concurrency bits to check. 5492 * 5493 * Must never be called from driver or module put entry point. 5494 * 5495 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5496 * sq_putlocks are used. 5497 */ 5498 void 5499 leavesq(syncq_t *sq, int entrypoint) 5500 { 5501 uint16_t flags; 5502 uint16_t type; 5503 uint_t c_outer = entrypoint & SQ_CO; 5504 #ifdef DEBUG 5505 uint_t c_inner = entrypoint & SQ_CI; 5506 #endif 5507 5508 /* 5509 * Decrement ref count, drain the syncq if possible, and wake up 5510 * any waiting close. 5511 */ 5512 ASSERT(sq); 5513 ASSERT(c_inner && c_outer); 5514 mutex_enter(SQLOCK(sq)); 5515 flags = sq->sq_flags; 5516 type = sq->sq_type; 5517 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) { 5518 5519 if (flags & SQ_WANTWAKEUP) { 5520 flags &= ~SQ_WANTWAKEUP; 5521 cv_broadcast(&sq->sq_wait); 5522 } 5523 if (flags & SQ_WANTEXWAKEUP) { 5524 flags &= ~SQ_WANTEXWAKEUP; 5525 cv_broadcast(&sq->sq_exitwait); 5526 } 5527 5528 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) { 5529 /* 5530 * The syncq needs to be drained. "Exit" the syncq 5531 * before calling drain_syncq. 5532 */ 5533 ASSERT(sq->sq_count != 0); 5534 sq->sq_count--; 5535 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5536 sq->sq_flags = flags & ~SQ_EXCL; 5537 drain_syncq(sq); 5538 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 5539 /* Check if we need to exit the outer perimeter */ 5540 /* XXX will this ever be true? */ 5541 if (!(type & c_outer)) 5542 outer_exit(sq->sq_outer); 5543 return; 5544 } 5545 } 5546 ASSERT(sq->sq_count != 0); 5547 sq->sq_count--; 5548 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5549 sq->sq_flags = flags & ~SQ_EXCL; 5550 mutex_exit(SQLOCK(sq)); 5551 5552 /* Check if we need to exit the outer perimeter */ 5553 if (!(sq->sq_type & c_outer)) 5554 outer_exit(sq->sq_outer); 5555 } 5556 5557 /* 5558 * Prevent q_next from changing in this stream by incrementing sq_count. 5559 * 5560 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5561 * sq_putlocks are used. 5562 */ 5563 void 5564 claimq(queue_t *qp) 5565 { 5566 syncq_t *sq = qp->q_syncq; 5567 5568 mutex_enter(SQLOCK(sq)); 5569 sq->sq_count++; 5570 ASSERT(sq->sq_count != 0); /* Wraparound */ 5571 mutex_exit(SQLOCK(sq)); 5572 } 5573 5574 /* 5575 * Undo claimq. 5576 * 5577 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5578 * sq_putlocks are used. 5579 */ 5580 void 5581 releaseq(queue_t *qp) 5582 { 5583 syncq_t *sq = qp->q_syncq; 5584 uint16_t flags; 5585 5586 mutex_enter(SQLOCK(sq)); 5587 ASSERT(sq->sq_count > 0); 5588 sq->sq_count--; 5589 5590 flags = sq->sq_flags; 5591 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) { 5592 if (flags & SQ_WANTWAKEUP) { 5593 flags &= ~SQ_WANTWAKEUP; 5594 cv_broadcast(&sq->sq_wait); 5595 } 5596 sq->sq_flags = flags; 5597 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5598 /* 5599 * To prevent potential recursive invocation of 5600 * drain_syncq we do not call drain_syncq if count is 5601 * non-zero. 5602 */ 5603 if (sq->sq_count == 0) { 5604 drain_syncq(sq); 5605 return; 5606 } else 5607 sqenable(sq); 5608 } 5609 } 5610 mutex_exit(SQLOCK(sq)); 5611 } 5612 5613 /* 5614 * Prevent q_next from changing in this stream by incrementing sd_refcnt. 5615 */ 5616 void 5617 claimstr(queue_t *qp) 5618 { 5619 struct stdata *stp = STREAM(qp); 5620 5621 mutex_enter(&stp->sd_reflock); 5622 stp->sd_refcnt++; 5623 ASSERT(stp->sd_refcnt != 0); /* Wraparound */ 5624 mutex_exit(&stp->sd_reflock); 5625 } 5626 5627 /* 5628 * Undo claimstr. 5629 */ 5630 void 5631 releasestr(queue_t *qp) 5632 { 5633 struct stdata *stp = STREAM(qp); 5634 5635 mutex_enter(&stp->sd_reflock); 5636 ASSERT(stp->sd_refcnt != 0); 5637 if (--stp->sd_refcnt == 0) 5638 cv_broadcast(&stp->sd_refmonitor); 5639 mutex_exit(&stp->sd_reflock); 5640 } 5641 5642 static syncq_t * 5643 new_syncq(void) 5644 { 5645 return (kmem_cache_alloc(syncq_cache, KM_SLEEP)); 5646 } 5647 5648 static void 5649 free_syncq(syncq_t *sq) 5650 { 5651 ASSERT(sq->sq_head == NULL); 5652 ASSERT(sq->sq_outer == NULL); 5653 ASSERT(sq->sq_callbpend == NULL); 5654 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || 5655 (sq->sq_onext == sq && sq->sq_oprev == sq)); 5656 5657 if (sq->sq_ciputctrl != NULL) { 5658 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 5659 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 5660 sq->sq_nciputctrl, 0); 5661 ASSERT(ciputctrl_cache != NULL); 5662 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 5663 } 5664 5665 sq->sq_tail = NULL; 5666 sq->sq_evhead = NULL; 5667 sq->sq_evtail = NULL; 5668 sq->sq_ciputctrl = NULL; 5669 sq->sq_nciputctrl = 0; 5670 sq->sq_count = 0; 5671 sq->sq_rmqcount = 0; 5672 sq->sq_callbflags = 0; 5673 sq->sq_cancelid = 0; 5674 sq->sq_next = NULL; 5675 sq->sq_needexcl = 0; 5676 sq->sq_svcflags = 0; 5677 sq->sq_nqueues = 0; 5678 sq->sq_pri = 0; 5679 sq->sq_onext = NULL; 5680 sq->sq_oprev = NULL; 5681 sq->sq_flags = 0; 5682 sq->sq_type = 0; 5683 sq->sq_servcount = 0; 5684 5685 kmem_cache_free(syncq_cache, sq); 5686 } 5687 5688 /* Outer perimeter code */ 5689 5690 /* 5691 * The outer syncq uses the fields and flags in the syncq slightly 5692 * differently from the inner syncqs. 5693 * sq_count Incremented when there are pending or running 5694 * writers at the outer perimeter to prevent the set of 5695 * inner syncqs that belong to the outer perimeter from 5696 * changing. 5697 * sq_head/tail List of deferred qwriter(OUTER) operations. 5698 * 5699 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while 5700 * inner syncqs are added to or removed from the 5701 * outer perimeter. 5702 * SQ_QUEUED sq_head/tail has messages or events queued. 5703 * 5704 * SQ_WRITER A thread is currently traversing all the inner syncqs 5705 * setting the SQ_WRITER flag. 5706 */ 5707 5708 /* 5709 * Get write access at the outer perimeter. 5710 * Note that read access is done by entersq, putnext, and put by simply 5711 * incrementing sq_count in the inner syncq. 5712 * 5713 * Waits until "flags" is no longer set in the outer to prevent multiple 5714 * threads from having write access at the same time. SQ_WRITER has to be part 5715 * of "flags". 5716 * 5717 * Increases sq_count on the outer syncq to keep away outer_insert/remove 5718 * until the outer_exit is finished. 5719 * 5720 * outer_enter is vulnerable to starvation since it does not prevent new 5721 * threads from entering the inner syncqs while it is waiting for sq_count to 5722 * go to zero. 5723 */ 5724 void 5725 outer_enter(syncq_t *outer, uint16_t flags) 5726 { 5727 syncq_t *sq; 5728 int wait_needed; 5729 uint16_t count; 5730 5731 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5732 outer->sq_oprev != NULL); 5733 ASSERT(flags & SQ_WRITER); 5734 5735 retry: 5736 mutex_enter(SQLOCK(outer)); 5737 while (outer->sq_flags & flags) { 5738 outer->sq_flags |= SQ_WANTWAKEUP; 5739 cv_wait(&outer->sq_wait, SQLOCK(outer)); 5740 } 5741 5742 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5743 outer->sq_flags |= SQ_WRITER; 5744 outer->sq_count++; 5745 ASSERT(outer->sq_count != 0); /* wraparound */ 5746 wait_needed = 0; 5747 /* 5748 * Set SQ_WRITER on all the inner syncqs while holding 5749 * the SQLOCK on the outer syncq. This ensures that the changing 5750 * of SQ_WRITER is atomic under the outer SQLOCK. 5751 */ 5752 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5753 mutex_enter(SQLOCK(sq)); 5754 count = sq->sq_count; 5755 SQ_PUTLOCKS_ENTER(sq); 5756 sq->sq_flags |= SQ_WRITER; 5757 SUM_SQ_PUTCOUNTS(sq, count); 5758 if (count != 0) 5759 wait_needed = 1; 5760 SQ_PUTLOCKS_EXIT(sq); 5761 mutex_exit(SQLOCK(sq)); 5762 } 5763 mutex_exit(SQLOCK(outer)); 5764 5765 /* 5766 * Get everybody out of the syncqs sequentially. 5767 * Note that we don't actually need to acquire the PUTLOCKS, since 5768 * we have already cleared the fastbit, and set QWRITER. By 5769 * definition, the count can not increase since putnext will 5770 * take the slowlock path (and the purpose of acquiring the 5771 * putlocks was to make sure it didn't increase while we were 5772 * waiting). 5773 * 5774 * Note that we still acquire the PUTLOCKS to be safe. 5775 */ 5776 if (wait_needed) { 5777 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5778 mutex_enter(SQLOCK(sq)); 5779 count = sq->sq_count; 5780 SQ_PUTLOCKS_ENTER(sq); 5781 SUM_SQ_PUTCOUNTS(sq, count); 5782 while (count != 0) { 5783 sq->sq_flags |= SQ_WANTWAKEUP; 5784 SQ_PUTLOCKS_EXIT(sq); 5785 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5786 count = sq->sq_count; 5787 SQ_PUTLOCKS_ENTER(sq); 5788 SUM_SQ_PUTCOUNTS(sq, count); 5789 } 5790 SQ_PUTLOCKS_EXIT(sq); 5791 mutex_exit(SQLOCK(sq)); 5792 } 5793 /* 5794 * Verify that none of the flags got set while we 5795 * were waiting for the sq_counts to drop. 5796 * If this happens we exit and retry entering the 5797 * outer perimeter. 5798 */ 5799 mutex_enter(SQLOCK(outer)); 5800 if (outer->sq_flags & (flags & ~SQ_WRITER)) { 5801 mutex_exit(SQLOCK(outer)); 5802 outer_exit(outer); 5803 goto retry; 5804 } 5805 mutex_exit(SQLOCK(outer)); 5806 } 5807 } 5808 5809 /* 5810 * Drop the write access at the outer perimeter. 5811 * Read access is dropped implicitly (by putnext, put, and leavesq) by 5812 * decrementing sq_count. 5813 */ 5814 void 5815 outer_exit(syncq_t *outer) 5816 { 5817 syncq_t *sq; 5818 int drain_needed; 5819 uint16_t flags; 5820 5821 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5822 outer->sq_oprev != NULL); 5823 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer))); 5824 5825 /* 5826 * Atomically (from the perspective of threads calling become_writer) 5827 * drop the write access at the outer perimeter by holding 5828 * SQLOCK(outer) across all the dropsq calls and the resetting of 5829 * SQ_WRITER. 5830 * This defines a locking order between the outer perimeter 5831 * SQLOCK and the inner perimeter SQLOCKs. 5832 */ 5833 mutex_enter(SQLOCK(outer)); 5834 flags = outer->sq_flags; 5835 ASSERT(outer->sq_flags & SQ_WRITER); 5836 if (flags & SQ_QUEUED) { 5837 write_now(outer); 5838 flags = outer->sq_flags; 5839 } 5840 5841 /* 5842 * sq_onext is stable since sq_count has not yet been decreased. 5843 * Reset the SQ_WRITER flags in all syncqs. 5844 * After dropping SQ_WRITER on the outer syncq we empty all the 5845 * inner syncqs. 5846 */ 5847 drain_needed = 0; 5848 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5849 drain_needed += dropsq(sq, SQ_WRITER); 5850 ASSERT(!(outer->sq_flags & SQ_QUEUED)); 5851 flags &= ~SQ_WRITER; 5852 if (drain_needed) { 5853 outer->sq_flags = flags; 5854 mutex_exit(SQLOCK(outer)); 5855 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5856 emptysq(sq); 5857 mutex_enter(SQLOCK(outer)); 5858 flags = outer->sq_flags; 5859 } 5860 if (flags & SQ_WANTWAKEUP) { 5861 flags &= ~SQ_WANTWAKEUP; 5862 cv_broadcast(&outer->sq_wait); 5863 } 5864 outer->sq_flags = flags; 5865 ASSERT(outer->sq_count > 0); 5866 outer->sq_count--; 5867 mutex_exit(SQLOCK(outer)); 5868 } 5869 5870 /* 5871 * Add another syncq to an outer perimeter. 5872 * Block out all other access to the outer perimeter while it is being 5873 * changed using blocksq. 5874 * Assumes that the caller has *not* done an outer_enter. 5875 * 5876 * Vulnerable to starvation in blocksq. 5877 */ 5878 static void 5879 outer_insert(syncq_t *outer, syncq_t *sq) 5880 { 5881 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5882 outer->sq_oprev != NULL); 5883 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 5884 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ 5885 5886 /* Get exclusive access to the outer perimeter list */ 5887 blocksq(outer, SQ_BLOCKED, 0); 5888 ASSERT(outer->sq_flags & SQ_BLOCKED); 5889 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5890 5891 mutex_enter(SQLOCK(sq)); 5892 sq->sq_outer = outer; 5893 outer->sq_onext->sq_oprev = sq; 5894 sq->sq_onext = outer->sq_onext; 5895 outer->sq_onext = sq; 5896 sq->sq_oprev = outer; 5897 mutex_exit(SQLOCK(sq)); 5898 unblocksq(outer, SQ_BLOCKED, 1); 5899 } 5900 5901 /* 5902 * Remove a syncq from an outer perimeter. 5903 * Block out all other access to the outer perimeter while it is being 5904 * changed using blocksq. 5905 * Assumes that the caller has *not* done an outer_enter. 5906 * 5907 * Vulnerable to starvation in blocksq. 5908 */ 5909 static void 5910 outer_remove(syncq_t *outer, syncq_t *sq) 5911 { 5912 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5913 outer->sq_oprev != NULL); 5914 ASSERT(sq->sq_outer == outer); 5915 5916 /* Get exclusive access to the outer perimeter list */ 5917 blocksq(outer, SQ_BLOCKED, 0); 5918 ASSERT(outer->sq_flags & SQ_BLOCKED); 5919 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5920 5921 mutex_enter(SQLOCK(sq)); 5922 sq->sq_outer = NULL; 5923 sq->sq_onext->sq_oprev = sq->sq_oprev; 5924 sq->sq_oprev->sq_onext = sq->sq_onext; 5925 sq->sq_oprev = sq->sq_onext = NULL; 5926 mutex_exit(SQLOCK(sq)); 5927 unblocksq(outer, SQ_BLOCKED, 1); 5928 } 5929 5930 /* 5931 * Queue a deferred qwriter(OUTER) callback for this outer perimeter. 5932 * If this is the first callback for this outer perimeter then add 5933 * this outer perimeter to the list of outer perimeters that 5934 * the qwriter_outer_thread will process. 5935 * 5936 * Increments sq_count in the outer syncq to prevent the membership 5937 * of the outer perimeter (in terms of inner syncqs) to change while 5938 * the callback is pending. 5939 */ 5940 static void 5941 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp) 5942 { 5943 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5944 5945 mp->b_prev = (mblk_t *)func; 5946 mp->b_queue = q; 5947 mp->b_next = NULL; 5948 outer->sq_count++; /* Decremented when dequeued */ 5949 ASSERT(outer->sq_count != 0); /* Wraparound */ 5950 if (outer->sq_evhead == NULL) { 5951 /* First message. */ 5952 outer->sq_evhead = outer->sq_evtail = mp; 5953 outer->sq_flags |= SQ_EVENTS; 5954 mutex_exit(SQLOCK(outer)); 5955 STRSTAT(qwr_outer); 5956 (void) taskq_dispatch(streams_taskq, 5957 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP); 5958 } else { 5959 ASSERT(outer->sq_flags & SQ_EVENTS); 5960 outer->sq_evtail->b_next = mp; 5961 outer->sq_evtail = mp; 5962 mutex_exit(SQLOCK(outer)); 5963 } 5964 } 5965 5966 /* 5967 * Try and upgrade to write access at the outer perimeter. If this can 5968 * not be done without blocking then queue the callback to be done 5969 * by the qwriter_outer_thread. 5970 * 5971 * This routine can only be called from put or service procedures plus 5972 * asynchronous callback routines that have properly entered the queue (with 5973 * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq 5974 * associated with q. 5975 */ 5976 void 5977 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)()) 5978 { 5979 syncq_t *osq, *sq, *outer; 5980 int failed; 5981 uint16_t flags; 5982 5983 osq = q->q_syncq; 5984 outer = osq->sq_outer; 5985 if (outer == NULL) 5986 panic("qwriter(PERIM_OUTER): no outer perimeter"); 5987 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5988 outer->sq_oprev != NULL); 5989 5990 mutex_enter(SQLOCK(outer)); 5991 flags = outer->sq_flags; 5992 /* 5993 * If some thread is traversing sq_next, or if we are blocked by 5994 * outer_insert or outer_remove, or if the we already have queued 5995 * callbacks, then queue this callback for later processing. 5996 * 5997 * Also queue the qwriter for an interrupt thread in order 5998 * to reduce the time spent running at high IPL. 5999 * to identify there are events. 6000 */ 6001 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) { 6002 /* 6003 * Queue the become_writer request. 6004 * The queueing is atomic under SQLOCK(outer) in order 6005 * to synchronize with outer_exit. 6006 * queue_writer will drop the outer SQLOCK 6007 */ 6008 if (flags & SQ_BLOCKED) { 6009 /* Must set SQ_WRITER on inner perimeter */ 6010 mutex_enter(SQLOCK(osq)); 6011 osq->sq_flags |= SQ_WRITER; 6012 mutex_exit(SQLOCK(osq)); 6013 } else { 6014 if (!(flags & SQ_WRITER)) { 6015 /* 6016 * The outer could have been SQ_BLOCKED thus 6017 * SQ_WRITER might not be set on the inner. 6018 */ 6019 mutex_enter(SQLOCK(osq)); 6020 osq->sq_flags |= SQ_WRITER; 6021 mutex_exit(SQLOCK(osq)); 6022 } 6023 ASSERT(osq->sq_flags & SQ_WRITER); 6024 } 6025 queue_writer(outer, func, q, mp); 6026 return; 6027 } 6028 /* 6029 * We are half-way to exclusive access to the outer perimeter. 6030 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove 6031 * while the inner syncqs are traversed. 6032 */ 6033 outer->sq_count++; 6034 ASSERT(outer->sq_count != 0); /* wraparound */ 6035 flags |= SQ_WRITER; 6036 /* 6037 * Check if we can run the function immediately. Mark all 6038 * syncqs with the writer flag to prevent new entries into 6039 * put and service procedures. 6040 * 6041 * Set SQ_WRITER on all the inner syncqs while holding 6042 * the SQLOCK on the outer syncq. This ensures that the changing 6043 * of SQ_WRITER is atomic under the outer SQLOCK. 6044 */ 6045 failed = 0; 6046 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 6047 uint16_t count; 6048 uint_t maxcnt = (sq == osq) ? 1 : 0; 6049 6050 mutex_enter(SQLOCK(sq)); 6051 count = sq->sq_count; 6052 SQ_PUTLOCKS_ENTER(sq); 6053 SUM_SQ_PUTCOUNTS(sq, count); 6054 if (sq->sq_count > maxcnt) 6055 failed = 1; 6056 sq->sq_flags |= SQ_WRITER; 6057 SQ_PUTLOCKS_EXIT(sq); 6058 mutex_exit(SQLOCK(sq)); 6059 } 6060 if (failed) { 6061 /* 6062 * Some other thread has a read claim on the outer perimeter. 6063 * Queue the callback for deferred processing. 6064 * 6065 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER 6066 * so that other qwriter(OUTER) calls will queue their 6067 * callbacks as well. queue_writer increments sq_count so we 6068 * decrement to compensate for the our increment. 6069 * 6070 * Dropping SQ_WRITER enables the writer thread to work 6071 * on this outer perimeter. 6072 */ 6073 outer->sq_flags = flags; 6074 queue_writer(outer, func, q, mp); 6075 /* queue_writer dropper the lock */ 6076 mutex_enter(SQLOCK(outer)); 6077 ASSERT(outer->sq_count > 0); 6078 outer->sq_count--; 6079 ASSERT(outer->sq_flags & SQ_WRITER); 6080 flags = outer->sq_flags; 6081 flags &= ~SQ_WRITER; 6082 if (flags & SQ_WANTWAKEUP) { 6083 flags &= ~SQ_WANTWAKEUP; 6084 cv_broadcast(&outer->sq_wait); 6085 } 6086 outer->sq_flags = flags; 6087 mutex_exit(SQLOCK(outer)); 6088 return; 6089 } else { 6090 outer->sq_flags = flags; 6091 mutex_exit(SQLOCK(outer)); 6092 } 6093 6094 /* Can run it immediately */ 6095 (*func)(q, mp); 6096 6097 outer_exit(outer); 6098 } 6099 6100 /* 6101 * Dequeue all writer callbacks from the outer perimeter and run them. 6102 */ 6103 static void 6104 write_now(syncq_t *outer) 6105 { 6106 mblk_t *mp; 6107 queue_t *q; 6108 void (*func)(); 6109 6110 ASSERT(MUTEX_HELD(SQLOCK(outer))); 6111 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 6112 outer->sq_oprev != NULL); 6113 while ((mp = outer->sq_evhead) != NULL) { 6114 /* 6115 * queues cannot be placed on the queuelist on the outer 6116 * perimeter. 6117 */ 6118 ASSERT(!(outer->sq_flags & SQ_MESSAGES)); 6119 ASSERT((outer->sq_flags & SQ_EVENTS)); 6120 6121 outer->sq_evhead = mp->b_next; 6122 if (outer->sq_evhead == NULL) { 6123 outer->sq_evtail = NULL; 6124 outer->sq_flags &= ~SQ_EVENTS; 6125 } 6126 ASSERT(outer->sq_count != 0); 6127 outer->sq_count--; /* Incremented when enqueued. */ 6128 mutex_exit(SQLOCK(outer)); 6129 /* 6130 * Drop the message if the queue is closing. 6131 * Make sure that the queue is "claimed" when the callback 6132 * is run in order to satisfy various ASSERTs. 6133 */ 6134 q = mp->b_queue; 6135 func = (void (*)())mp->b_prev; 6136 ASSERT(func != NULL); 6137 mp->b_next = mp->b_prev = NULL; 6138 if (q->q_flag & QWCLOSE) { 6139 freemsg(mp); 6140 } else { 6141 claimq(q); 6142 (*func)(q, mp); 6143 releaseq(q); 6144 } 6145 mutex_enter(SQLOCK(outer)); 6146 } 6147 ASSERT(MUTEX_HELD(SQLOCK(outer))); 6148 } 6149 6150 /* 6151 * The list of messages on the inner syncq is effectively hashed 6152 * by destination queue. These destination queues are doubly 6153 * linked lists (hopefully) in priority order. Messages are then 6154 * put on the queue referenced by the q_sqhead/q_sqtail elements. 6155 * Additional messages are linked together by the b_next/b_prev 6156 * elements in the mblk, with (similar to putq()) the first message 6157 * having a NULL b_prev and the last message having a NULL b_next. 6158 * 6159 * Events, such as qwriter callbacks, are put onto a list in FIFO 6160 * order referenced by sq_evhead, and sq_evtail. This is a singly 6161 * linked list, and messages here MUST be processed in the order queued. 6162 */ 6163 6164 /* 6165 * Run the events on the syncq event list (sq_evhead). 6166 * Assumes there is only one claim on the syncq, it is 6167 * already exclusive (SQ_EXCL set), and the SQLOCK held. 6168 * Messages here are processed in order, with the SQ_EXCL bit 6169 * held all the way through till the last message is processed. 6170 */ 6171 void 6172 sq_run_events(syncq_t *sq) 6173 { 6174 mblk_t *bp; 6175 queue_t *qp; 6176 uint16_t flags = sq->sq_flags; 6177 void (*func)(); 6178 6179 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6180 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6181 sq->sq_oprev == NULL) || 6182 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6183 sq->sq_oprev != NULL)); 6184 6185 ASSERT(flags & SQ_EXCL); 6186 ASSERT(sq->sq_count == 1); 6187 6188 /* 6189 * We need to process all of the events on this list. It 6190 * is possible that new events will be added while we are 6191 * away processing a callback, so on every loop, we start 6192 * back at the beginning of the list. 6193 */ 6194 /* 6195 * We have to reaccess sq_evhead since there is a 6196 * possibility of a new entry while we were running 6197 * the callback. 6198 */ 6199 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { 6200 ASSERT(bp->b_queue->q_syncq == sq); 6201 ASSERT(sq->sq_flags & SQ_EVENTS); 6202 6203 qp = bp->b_queue; 6204 func = (void (*)())bp->b_prev; 6205 ASSERT(func != NULL); 6206 6207 /* 6208 * Messages from the event queue must be taken off in 6209 * FIFO order. 6210 */ 6211 ASSERT(sq->sq_evhead == bp); 6212 sq->sq_evhead = bp->b_next; 6213 6214 if (bp->b_next == NULL) { 6215 /* Deleting last */ 6216 ASSERT(sq->sq_evtail == bp); 6217 sq->sq_evtail = NULL; 6218 sq->sq_flags &= ~SQ_EVENTS; 6219 } 6220 bp->b_prev = bp->b_next = NULL; 6221 ASSERT(bp->b_datap->db_ref != 0); 6222 6223 mutex_exit(SQLOCK(sq)); 6224 6225 (*func)(qp, bp); 6226 6227 mutex_enter(SQLOCK(sq)); 6228 /* 6229 * re-read the flags, since they could have changed. 6230 */ 6231 flags = sq->sq_flags; 6232 ASSERT(flags & SQ_EXCL); 6233 } 6234 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); 6235 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6236 6237 if (flags & SQ_WANTWAKEUP) { 6238 flags &= ~SQ_WANTWAKEUP; 6239 cv_broadcast(&sq->sq_wait); 6240 } 6241 if (flags & SQ_WANTEXWAKEUP) { 6242 flags &= ~SQ_WANTEXWAKEUP; 6243 cv_broadcast(&sq->sq_exitwait); 6244 } 6245 sq->sq_flags = flags; 6246 } 6247 6248 /* 6249 * Put messages on the event list. 6250 * If we can go exclusive now, do so and process the event list, otherwise 6251 * let the last claim service this list (or wake the sqthread). 6252 * This procedure assumes SQLOCK is held. To run the event list, it 6253 * must be called with no claims. 6254 */ 6255 static void 6256 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) 6257 { 6258 uint16_t count; 6259 6260 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6261 ASSERT(func != NULL); 6262 6263 /* 6264 * This is a callback. Add it to the list of callbacks 6265 * and see about upgrading. 6266 */ 6267 mp->b_prev = (mblk_t *)func; 6268 mp->b_queue = q; 6269 mp->b_next = NULL; 6270 if (sq->sq_evhead == NULL) { 6271 sq->sq_evhead = sq->sq_evtail = mp; 6272 sq->sq_flags |= SQ_EVENTS; 6273 } else { 6274 ASSERT(sq->sq_evtail != NULL); 6275 ASSERT(sq->sq_evtail->b_next == NULL); 6276 ASSERT(sq->sq_flags & SQ_EVENTS); 6277 sq->sq_evtail->b_next = mp; 6278 sq->sq_evtail = mp; 6279 } 6280 /* 6281 * We have set SQ_EVENTS, so threads will have to 6282 * unwind out of the perimeter, and new entries will 6283 * not grab a putlock. But we still need to know 6284 * how many threads have already made a claim to the 6285 * syncq, so grab the putlocks, and sum the counts. 6286 * If there are no claims on the syncq, we can upgrade 6287 * to exclusive, and run the event list. 6288 * NOTE: We hold the SQLOCK, so we can just grab the 6289 * putlocks. 6290 */ 6291 count = sq->sq_count; 6292 SQ_PUTLOCKS_ENTER(sq); 6293 SUM_SQ_PUTCOUNTS(sq, count); 6294 /* 6295 * We have no claim, so we need to check if there 6296 * are no others, then we can upgrade. 6297 */ 6298 /* 6299 * There are currently no claims on 6300 * the syncq by this thread (at least on this entry). The thread who has 6301 * the claim should drain syncq. 6302 */ 6303 if (count > 0) { 6304 /* 6305 * Can't upgrade - other threads inside. 6306 */ 6307 SQ_PUTLOCKS_EXIT(sq); 6308 mutex_exit(SQLOCK(sq)); 6309 return; 6310 } 6311 /* 6312 * Need to set SQ_EXCL and make a claim on the syncq. 6313 */ 6314 ASSERT((sq->sq_flags & SQ_EXCL) == 0); 6315 sq->sq_flags |= SQ_EXCL; 6316 ASSERT(sq->sq_count == 0); 6317 sq->sq_count++; 6318 SQ_PUTLOCKS_EXIT(sq); 6319 6320 /* Process the events list */ 6321 sq_run_events(sq); 6322 6323 /* 6324 * Release our claim... 6325 */ 6326 sq->sq_count--; 6327 6328 /* 6329 * And release SQ_EXCL. 6330 * We don't need to acquire the putlocks to release 6331 * SQ_EXCL, since we are exclusive, and hold the SQLOCK. 6332 */ 6333 sq->sq_flags &= ~SQ_EXCL; 6334 6335 /* 6336 * sq_run_events should have released SQ_EXCL 6337 */ 6338 ASSERT(!(sq->sq_flags & SQ_EXCL)); 6339 6340 /* 6341 * If anything happened while we were running the 6342 * events (or was there before), we need to process 6343 * them now. We shouldn't be exclusive sine we 6344 * released the perimeter above (plus, we asserted 6345 * for it). 6346 */ 6347 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) 6348 drain_syncq(sq); 6349 else 6350 mutex_exit(SQLOCK(sq)); 6351 } 6352 6353 /* 6354 * Perform delayed processing. The caller has to make sure that it is safe 6355 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are 6356 * set). 6357 * 6358 * Assume that the caller has NO claims on the syncq. However, a claim 6359 * on the syncq does not indicate that a thread is draining the syncq. 6360 * There may be more claims on the syncq than there are threads draining 6361 * (i.e. #_threads_draining <= sq_count) 6362 * 6363 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set 6364 * in order to preserve qwriter(OUTER) ordering constraints. 6365 * 6366 * sq_putcount only needs to be checked when dispatching the queued 6367 * writer call for CIPUT sync queue, but this is handled in sq_run_events. 6368 */ 6369 void 6370 drain_syncq(syncq_t *sq) 6371 { 6372 queue_t *qp; 6373 uint16_t count; 6374 uint16_t type = sq->sq_type; 6375 uint16_t flags = sq->sq_flags; 6376 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; 6377 6378 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6379 "drain_syncq start:%p", sq); 6380 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6381 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6382 sq->sq_oprev == NULL) || 6383 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6384 sq->sq_oprev != NULL)); 6385 6386 /* 6387 * Drop SQ_SERVICE flag. 6388 */ 6389 if (bg_service) 6390 sq->sq_svcflags &= ~SQ_SERVICE; 6391 6392 /* 6393 * If SQ_EXCL is set, someone else is processing this syncq - let him 6394 * finish the job. 6395 */ 6396 if (flags & SQ_EXCL) { 6397 if (bg_service) { 6398 ASSERT(sq->sq_servcount != 0); 6399 sq->sq_servcount--; 6400 } 6401 mutex_exit(SQLOCK(sq)); 6402 return; 6403 } 6404 6405 /* 6406 * This routine can be called by a background thread if 6407 * it was scheduled by a hi-priority thread. SO, if there are 6408 * NOT messages queued, return (remember, we have the SQLOCK, 6409 * and it cannot change until we release it). Wakeup any waiters also. 6410 */ 6411 if (!(flags & SQ_QUEUED)) { 6412 if (flags & SQ_WANTWAKEUP) { 6413 flags &= ~SQ_WANTWAKEUP; 6414 cv_broadcast(&sq->sq_wait); 6415 } 6416 if (flags & SQ_WANTEXWAKEUP) { 6417 flags &= ~SQ_WANTEXWAKEUP; 6418 cv_broadcast(&sq->sq_exitwait); 6419 } 6420 sq->sq_flags = flags; 6421 if (bg_service) { 6422 ASSERT(sq->sq_servcount != 0); 6423 sq->sq_servcount--; 6424 } 6425 mutex_exit(SQLOCK(sq)); 6426 return; 6427 } 6428 6429 /* 6430 * If this is not a concurrent put perimeter, we need to 6431 * become exclusive to drain. Also, if not CIPUT, we would 6432 * not have acquired a putlock, so we don't need to check 6433 * the putcounts. If not entering with a claim, we test 6434 * for sq_count == 0. 6435 */ 6436 type = sq->sq_type; 6437 if (!(type & SQ_CIPUT)) { 6438 if (sq->sq_count > 1) { 6439 if (bg_service) { 6440 ASSERT(sq->sq_servcount != 0); 6441 sq->sq_servcount--; 6442 } 6443 mutex_exit(SQLOCK(sq)); 6444 return; 6445 } 6446 sq->sq_flags |= SQ_EXCL; 6447 } 6448 6449 /* 6450 * This is where we make a claim to the syncq. 6451 * This can either be done by incrementing a putlock, or 6452 * the sq_count. But since we already have the SQLOCK 6453 * here, we just bump the sq_count. 6454 * 6455 * Note that after we make a claim, we need to let the code 6456 * fall through to the end of this routine to clean itself 6457 * up. A return in the while loop will put the syncq in a 6458 * very bad state. 6459 */ 6460 sq->sq_count++; 6461 ASSERT(sq->sq_count != 0); /* wraparound */ 6462 6463 while ((flags = sq->sq_flags) & SQ_QUEUED) { 6464 /* 6465 * If we are told to stayaway or went exclusive, 6466 * we are done. 6467 */ 6468 if (flags & (SQ_STAYAWAY)) { 6469 break; 6470 } 6471 6472 /* 6473 * If there are events to run, do so. 6474 * We have one claim to the syncq, so if there are 6475 * more than one, other threads are running. 6476 */ 6477 if (sq->sq_evhead != NULL) { 6478 ASSERT(sq->sq_flags & SQ_EVENTS); 6479 6480 count = sq->sq_count; 6481 SQ_PUTLOCKS_ENTER(sq); 6482 SUM_SQ_PUTCOUNTS(sq, count); 6483 if (count > 1) { 6484 SQ_PUTLOCKS_EXIT(sq); 6485 /* Can't upgrade - other threads inside */ 6486 break; 6487 } 6488 ASSERT((flags & SQ_EXCL) == 0); 6489 sq->sq_flags = flags | SQ_EXCL; 6490 SQ_PUTLOCKS_EXIT(sq); 6491 /* 6492 * we have the only claim, run the events, 6493 * sq_run_events will clear the SQ_EXCL flag. 6494 */ 6495 sq_run_events(sq); 6496 6497 /* 6498 * If this is a CIPUT perimeter, we need 6499 * to drop the SQ_EXCL flag so we can properly 6500 * continue draining the syncq. 6501 */ 6502 if (type & SQ_CIPUT) { 6503 ASSERT(sq->sq_flags & SQ_EXCL); 6504 sq->sq_flags &= ~SQ_EXCL; 6505 } 6506 6507 /* 6508 * And go back to the beginning just in case 6509 * anything changed while we were away. 6510 */ 6511 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); 6512 continue; 6513 } 6514 6515 ASSERT(sq->sq_evhead == NULL); 6516 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6517 6518 /* 6519 * Find the queue that is not draining. 6520 * 6521 * q_draining is protected by QLOCK which we do not hold. 6522 * But if it was set, then a thread was draining, and if it gets 6523 * cleared, then it was because the thread has successfully 6524 * drained the syncq, or a GOAWAY state occurred. For the GOAWAY 6525 * state to happen, a thread needs the SQLOCK which we hold, and 6526 * if there was such a flag, we would have already seen it. 6527 */ 6528 6529 for (qp = sq->sq_head; 6530 qp != NULL && (qp->q_draining || 6531 (qp->q_sqflags & Q_SQDRAINING)); 6532 qp = qp->q_sqnext) 6533 ; 6534 6535 if (qp == NULL) 6536 break; 6537 6538 /* 6539 * We have a queue to work on, and we hold the 6540 * SQLOCK and one claim, call qdrain_syncq. 6541 * This means we need to release the SQLOCK and 6542 * acquire the QLOCK (OK since we have a claim). 6543 * Note that qdrain_syncq will actually dequeue 6544 * this queue from the sq_head list when it is 6545 * convinced all the work is done and release 6546 * the QLOCK before returning. 6547 */ 6548 qp->q_sqflags |= Q_SQDRAINING; 6549 mutex_exit(SQLOCK(sq)); 6550 mutex_enter(QLOCK(qp)); 6551 qdrain_syncq(sq, qp); 6552 mutex_enter(SQLOCK(sq)); 6553 6554 /* The queue is drained */ 6555 ASSERT(qp->q_sqflags & Q_SQDRAINING); 6556 qp->q_sqflags &= ~Q_SQDRAINING; 6557 /* 6558 * NOTE: After this point qp should not be used since it may be 6559 * closed. 6560 */ 6561 } 6562 6563 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6564 flags = sq->sq_flags; 6565 6566 /* 6567 * sq->sq_head cannot change because we hold the 6568 * sqlock. However, a thread CAN decide that it is no longer 6569 * going to drain that queue. However, this should be due to 6570 * a GOAWAY state, and we should see that here. 6571 * 6572 * This loop is not very efficient. One solution may be adding a second 6573 * pointer to the "draining" queue, but it is difficult to do when 6574 * queues are inserted in the middle due to priority ordering. Another 6575 * possibility is to yank the queue out of the sq list and put it onto 6576 * the "draining list" and then put it back if it can't be drained. 6577 */ 6578 6579 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || 6580 (type & SQ_CI) || sq->sq_head->q_draining); 6581 6582 /* Drop SQ_EXCL for non-CIPUT perimeters */ 6583 if (!(type & SQ_CIPUT)) 6584 flags &= ~SQ_EXCL; 6585 ASSERT((flags & SQ_EXCL) == 0); 6586 6587 /* Wake up any waiters. */ 6588 if (flags & SQ_WANTWAKEUP) { 6589 flags &= ~SQ_WANTWAKEUP; 6590 cv_broadcast(&sq->sq_wait); 6591 } 6592 if (flags & SQ_WANTEXWAKEUP) { 6593 flags &= ~SQ_WANTEXWAKEUP; 6594 cv_broadcast(&sq->sq_exitwait); 6595 } 6596 sq->sq_flags = flags; 6597 6598 ASSERT(sq->sq_count != 0); 6599 /* Release our claim. */ 6600 sq->sq_count--; 6601 6602 if (bg_service) { 6603 ASSERT(sq->sq_servcount != 0); 6604 sq->sq_servcount--; 6605 } 6606 6607 mutex_exit(SQLOCK(sq)); 6608 6609 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6610 "drain_syncq end:%p", sq); 6611 } 6612 6613 6614 /* 6615 * 6616 * qdrain_syncq can be called (currently) from only one of two places: 6617 * drain_syncq 6618 * putnext (or some variation of it). 6619 * and eventually 6620 * qwait(_sig) 6621 * 6622 * If called from drain_syncq, we found it in the list of queues needing 6623 * service, so there is work to be done (or it wouldn't be in the list). 6624 * 6625 * If called from some putnext variation, it was because the 6626 * perimeter is open, but messages are blocking a putnext and 6627 * there is not a thread working on it. Now a thread could start 6628 * working on it while we are getting ready to do so ourself, but 6629 * the thread would set the q_draining flag, and we can spin out. 6630 * 6631 * As for qwait(_sig), I think I shall let it continue to call 6632 * drain_syncq directly (after all, it will get here eventually). 6633 * 6634 * qdrain_syncq has to terminate when: 6635 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering 6636 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering 6637 * 6638 * ASSUMES: 6639 * One claim 6640 * QLOCK held 6641 * SQLOCK not held 6642 * Will release QLOCK before returning 6643 */ 6644 void 6645 qdrain_syncq(syncq_t *sq, queue_t *q) 6646 { 6647 mblk_t *bp; 6648 #ifdef DEBUG 6649 uint16_t count; 6650 #endif 6651 6652 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6653 "drain_syncq start:%p", sq); 6654 ASSERT(q->q_syncq == sq); 6655 ASSERT(MUTEX_HELD(QLOCK(q))); 6656 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6657 /* 6658 * For non-CIPUT perimeters, we should be called with the exclusive bit 6659 * set already. For CIPUT perimeters, we will be doing a concurrent 6660 * drain, so it better not be set. 6661 */ 6662 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); 6663 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); 6664 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); 6665 /* 6666 * All outer pointers are set, or none of them are 6667 */ 6668 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6669 sq->sq_oprev == NULL) || 6670 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6671 sq->sq_oprev != NULL)); 6672 #ifdef DEBUG 6673 count = sq->sq_count; 6674 /* 6675 * This is OK without the putlocks, because we have one 6676 * claim either from the sq_count, or a putcount. We could 6677 * get an erroneous value from other counts, but ours won't 6678 * change, so one way or another, we will have at least a 6679 * value of one. 6680 */ 6681 SUM_SQ_PUTCOUNTS(sq, count); 6682 ASSERT(count >= 1); 6683 #endif /* DEBUG */ 6684 6685 /* 6686 * The first thing to do is find out if a thread is already draining 6687 * this queue. If so, we are done, just return. 6688 */ 6689 if (q->q_draining) { 6690 mutex_exit(QLOCK(q)); 6691 return; 6692 } 6693 6694 /* 6695 * If the perimeter is exclusive, there is nothing we can do right now, 6696 * go away. Note that there is nothing to prevent this case from 6697 * changing right after this check, but the spin-out will catch it. 6698 */ 6699 6700 /* Tell other threads that we are draining this queue */ 6701 q->q_draining = 1; /* Protected by QLOCK */ 6702 6703 /* 6704 * If there is nothing to do, clear QFULL as necessary. This caters for 6705 * the case where an empty queue was enqueued onto the syncq. 6706 */ 6707 if (q->q_sqhead == NULL) { 6708 ASSERT(q->q_syncqmsgs == 0); 6709 mutex_exit(QLOCK(q)); 6710 clr_qfull(q); 6711 mutex_enter(QLOCK(q)); 6712 } 6713 6714 /* 6715 * Note that q_sqhead must be re-checked here in case another message 6716 * was enqueued whilst QLOCK was dropped during the call to clr_qfull. 6717 */ 6718 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) { 6719 /* 6720 * Because we can enter this routine just because a putnext is 6721 * blocked, we need to spin out if the perimeter wants to go 6722 * exclusive as well as just blocked. We need to spin out also 6723 * if events are queued on the syncq. 6724 * Don't check for SQ_EXCL, because non-CIPUT perimeters would 6725 * set it, and it can't become exclusive while we hold a claim. 6726 */ 6727 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { 6728 break; 6729 } 6730 6731 #ifdef DEBUG 6732 /* 6733 * Since we are in qdrain_syncq, we already know the queue, 6734 * but for sanity, we want to check this against the qp that 6735 * was passed in by bp->b_queue. 6736 */ 6737 6738 ASSERT(bp->b_queue == q); 6739 ASSERT(bp->b_queue->q_syncq == sq); 6740 bp->b_queue = NULL; 6741 6742 /* 6743 * We would have the following check in the DEBUG code: 6744 * 6745 * if (bp->b_prev != NULL) { 6746 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp); 6747 * } 6748 * 6749 * This can't be done, however, since IP modifies qinfo 6750 * structure at run-time (switching between IPv4 qinfo and IPv6 6751 * qinfo), invalidating the check. 6752 * So the assignment to func is left here, but the ASSERT itself 6753 * is removed until the whole issue is resolved. 6754 */ 6755 #endif 6756 ASSERT(q->q_sqhead == bp); 6757 q->q_sqhead = bp->b_next; 6758 bp->b_prev = bp->b_next = NULL; 6759 ASSERT(q->q_syncqmsgs > 0); 6760 mutex_exit(QLOCK(q)); 6761 6762 ASSERT(bp->b_datap->db_ref != 0); 6763 6764 (void) (*q->q_qinfo->qi_putp)(q, bp); 6765 6766 mutex_enter(QLOCK(q)); 6767 6768 /* 6769 * q_syncqmsgs should only be decremented after executing the 6770 * put procedure to avoid message re-ordering. This is due to an 6771 * optimisation in putnext() which can call the put procedure 6772 * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED 6773 * being set). 6774 * 6775 * We also need to clear QFULL in the next service procedure 6776 * queue if this is the last message destined for that queue. 6777 * 6778 * It would make better sense to have some sort of tunable for 6779 * the low water mark, but these semantics are not yet defined. 6780 * So, alas, we use a constant. 6781 */ 6782 if (--q->q_syncqmsgs == 0) { 6783 mutex_exit(QLOCK(q)); 6784 clr_qfull(q); 6785 mutex_enter(QLOCK(q)); 6786 } 6787 6788 /* 6789 * Always clear SQ_EXCL when CIPUT in order to handle 6790 * qwriter(INNER). The putp() can call qwriter and get exclusive 6791 * access IFF this is the only claim. So, we need to test for 6792 * this possibility, acquire the mutex and clear the bit. 6793 */ 6794 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { 6795 mutex_enter(SQLOCK(sq)); 6796 sq->sq_flags &= ~SQ_EXCL; 6797 mutex_exit(SQLOCK(sq)); 6798 } 6799 } 6800 6801 /* 6802 * We should either have no messages on this queue, or we were told to 6803 * goaway by a waiter (which we will wake up at the end of this 6804 * function). 6805 */ 6806 ASSERT((q->q_sqhead == NULL) || 6807 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); 6808 6809 ASSERT(MUTEX_HELD(QLOCK(q))); 6810 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6811 6812 /* Remove the q from the syncq list if all the messages are drained. */ 6813 if (q->q_sqhead == NULL) { 6814 ASSERT(q->q_syncqmsgs == 0); 6815 mutex_enter(SQLOCK(sq)); 6816 if (q->q_sqflags & Q_SQQUEUED) 6817 SQRM_Q(sq, q); 6818 mutex_exit(SQLOCK(sq)); 6819 /* 6820 * Since the queue is removed from the list, reset its priority. 6821 */ 6822 q->q_spri = 0; 6823 } 6824 6825 /* 6826 * Remember, the q_draining flag is used to let another thread know 6827 * that there is a thread currently draining the messages for a queue. 6828 * Since we are now done with this queue (even if there may be messages 6829 * still there), we need to clear this flag so some thread will work on 6830 * it if needed. 6831 */ 6832 ASSERT(q->q_draining); 6833 q->q_draining = 0; 6834 6835 /* Called with a claim, so OK to drop all locks. */ 6836 mutex_exit(QLOCK(q)); 6837 6838 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6839 "drain_syncq end:%p", sq); 6840 } 6841 /* END OF QDRAIN_SYNCQ */ 6842 6843 6844 /* 6845 * This is the mate to qdrain_syncq, except that it is putting the message onto 6846 * the queue instead of draining. Since the message is destined for the queue 6847 * that is selected, there is no need to identify the function because the 6848 * message is intended for the put routine for the queue. For debug kernels, 6849 * this routine will do it anyway just in case. 6850 * 6851 * After the message is enqueued on the syncq, it calls putnext_tail() 6852 * which will schedule a background thread to actually process the message. 6853 * 6854 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and 6855 * SQLOCK(sq) and QLOCK(q) are not held. 6856 */ 6857 void 6858 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) 6859 { 6860 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6861 ASSERT(MUTEX_NOT_HELD(QLOCK(q))); 6862 ASSERT(sq->sq_count > 0); 6863 ASSERT(q->q_syncq == sq); 6864 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6865 sq->sq_oprev == NULL) || 6866 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6867 sq->sq_oprev != NULL)); 6868 6869 mutex_enter(QLOCK(q)); 6870 6871 #ifdef DEBUG 6872 /* 6873 * This is used for debug in the qfill_syncq/qdrain_syncq case 6874 * to trace the queue that the message is intended for. Note 6875 * that the original use was to identify the queue and function 6876 * to call on the drain. In the new syncq, we have the context 6877 * of the queue that we are draining, so call it's putproc and 6878 * don't rely on the saved values. But for debug this is still 6879 * useful information. 6880 */ 6881 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp; 6882 mp->b_queue = q; 6883 mp->b_next = NULL; 6884 #endif 6885 ASSERT(q->q_syncq == sq); 6886 /* 6887 * Enqueue the message on the list. 6888 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to 6889 * protect it. So it's ok to acquire SQLOCK after SQPUT_MP(). 6890 */ 6891 SQPUT_MP(q, mp); 6892 mutex_enter(SQLOCK(sq)); 6893 6894 /* 6895 * And queue on syncq for scheduling, if not already queued. 6896 * Note that we need the SQLOCK for this, and for testing flags 6897 * at the end to see if we will drain. So grab it now, and 6898 * release it before we call qdrain_syncq or return. 6899 */ 6900 if (!(q->q_sqflags & Q_SQQUEUED)) { 6901 q->q_spri = curthread->t_pri; 6902 SQPUT_Q(sq, q); 6903 } 6904 #ifdef DEBUG 6905 else { 6906 /* 6907 * All of these conditions MUST be true! 6908 */ 6909 ASSERT(sq->sq_tail != NULL); 6910 if (sq->sq_tail == sq->sq_head) { 6911 ASSERT((q->q_sqprev == NULL) && 6912 (q->q_sqnext == NULL)); 6913 } else { 6914 ASSERT((q->q_sqprev != NULL) || 6915 (q->q_sqnext != NULL)); 6916 } 6917 ASSERT(sq->sq_flags & SQ_QUEUED); 6918 ASSERT(q->q_syncqmsgs != 0); 6919 ASSERT(q->q_sqflags & Q_SQQUEUED); 6920 } 6921 #endif 6922 mutex_exit(QLOCK(q)); 6923 /* 6924 * SQLOCK is still held, so sq_count can be safely decremented. 6925 */ 6926 sq->sq_count--; 6927 6928 putnext_tail(sq, q, 0); 6929 /* Should not reference sq or q after this point. */ 6930 } 6931 6932 /* End of qfill_syncq */ 6933 6934 /* 6935 * Remove all messages from a syncq (if qp is NULL) or remove all messages 6936 * that would be put into qp by drain_syncq. 6937 * Used when deleting the syncq (qp == NULL) or when detaching 6938 * a queue (qp != NULL). 6939 * Return non-zero if one or more messages were freed. 6940 * 6941 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 6942 * sq_putlocks are used. 6943 * 6944 * NOTE: This function assumes that it is called from the close() context and 6945 * that all the queues in the syncq are going away. For this reason it doesn't 6946 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is 6947 * currently valid, but it is useful to rethink this function to behave properly 6948 * in other cases. 6949 */ 6950 int 6951 flush_syncq(syncq_t *sq, queue_t *qp) 6952 { 6953 mblk_t *bp, *mp_head, *mp_next, *mp_prev; 6954 queue_t *q; 6955 int ret = 0; 6956 6957 mutex_enter(SQLOCK(sq)); 6958 6959 /* 6960 * Before we leave, we need to make sure there are no 6961 * events listed for this queue. All events for this queue 6962 * will just be freed. 6963 */ 6964 if (qp != NULL && sq->sq_evhead != NULL) { 6965 ASSERT(sq->sq_flags & SQ_EVENTS); 6966 6967 mp_prev = NULL; 6968 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { 6969 mp_next = bp->b_next; 6970 if (bp->b_queue == qp) { 6971 /* Delete this message */ 6972 if (mp_prev != NULL) { 6973 mp_prev->b_next = mp_next; 6974 /* 6975 * Update sq_evtail if the last element 6976 * is removed. 6977 */ 6978 if (bp == sq->sq_evtail) { 6979 ASSERT(mp_next == NULL); 6980 sq->sq_evtail = mp_prev; 6981 } 6982 } else 6983 sq->sq_evhead = mp_next; 6984 if (sq->sq_evhead == NULL) 6985 sq->sq_flags &= ~SQ_EVENTS; 6986 bp->b_prev = bp->b_next = NULL; 6987 freemsg(bp); 6988 ret++; 6989 } else { 6990 mp_prev = bp; 6991 } 6992 } 6993 } 6994 6995 /* 6996 * Walk sq_head and: 6997 * - match qp if qp is set, remove it's messages 6998 * - all if qp is not set 6999 */ 7000 q = sq->sq_head; 7001 while (q != NULL) { 7002 ASSERT(q->q_syncq == sq); 7003 if ((qp == NULL) || (qp == q)) { 7004 /* 7005 * Yank the messages as a list off the queue 7006 */ 7007 mp_head = q->q_sqhead; 7008 /* 7009 * We do not have QLOCK(q) here (which is safe due to 7010 * assumptions mentioned above). To obtain the lock we 7011 * need to release SQLOCK which may allow lots of things 7012 * to change upon us. This place requires more analysis. 7013 */ 7014 q->q_sqhead = q->q_sqtail = NULL; 7015 ASSERT(mp_head->b_queue && 7016 mp_head->b_queue->q_syncq == sq); 7017 7018 /* 7019 * Free each of the messages. 7020 */ 7021 for (bp = mp_head; bp != NULL; bp = mp_next) { 7022 mp_next = bp->b_next; 7023 bp->b_prev = bp->b_next = NULL; 7024 freemsg(bp); 7025 ret++; 7026 } 7027 /* 7028 * Now remove the queue from the syncq. 7029 */ 7030 ASSERT(q->q_sqflags & Q_SQQUEUED); 7031 SQRM_Q(sq, q); 7032 q->q_spri = 0; 7033 q->q_syncqmsgs = 0; 7034 7035 /* 7036 * If qp was specified, we are done with it and are 7037 * going to drop SQLOCK(sq) and return. We wakeup syncq 7038 * waiters while we still have the SQLOCK. 7039 */ 7040 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { 7041 sq->sq_flags &= ~SQ_WANTWAKEUP; 7042 cv_broadcast(&sq->sq_wait); 7043 } 7044 /* Drop SQLOCK across clr_qfull */ 7045 mutex_exit(SQLOCK(sq)); 7046 7047 /* 7048 * We avoid doing the test that drain_syncq does and 7049 * unconditionally clear qfull for every flushed 7050 * message. Since flush_syncq is only called during 7051 * close this should not be a problem. 7052 */ 7053 clr_qfull(q); 7054 if (qp != NULL) { 7055 return (ret); 7056 } else { 7057 mutex_enter(SQLOCK(sq)); 7058 /* 7059 * The head was removed by SQRM_Q above. 7060 * reread the new head and flush it. 7061 */ 7062 q = sq->sq_head; 7063 } 7064 } else { 7065 q = q->q_sqnext; 7066 } 7067 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7068 } 7069 7070 if (sq->sq_flags & SQ_WANTWAKEUP) { 7071 sq->sq_flags &= ~SQ_WANTWAKEUP; 7072 cv_broadcast(&sq->sq_wait); 7073 } 7074 7075 mutex_exit(SQLOCK(sq)); 7076 return (ret); 7077 } 7078 7079 /* 7080 * Propagate all messages from a syncq to the next syncq that are associated 7081 * with the specified queue. If the queue is attached to a driver or if the 7082 * messages have been added due to a qwriter(PERIM_INNER), free the messages. 7083 * 7084 * Assumes that the stream is strlock()'ed. We don't come here if there 7085 * are no messages to propagate. 7086 * 7087 * NOTE : If the queue is attached to a driver, all the messages are freed 7088 * as there is no point in propagating the messages from the driver syncq 7089 * to the closing stream head which will in turn get freed later. 7090 */ 7091 static int 7092 propagate_syncq(queue_t *qp) 7093 { 7094 mblk_t *bp, *head, *tail, *prev, *next; 7095 syncq_t *sq; 7096 queue_t *nqp; 7097 syncq_t *nsq; 7098 boolean_t isdriver; 7099 int moved = 0; 7100 uint16_t flags; 7101 pri_t priority = curthread->t_pri; 7102 #ifdef DEBUG 7103 void (*func)(); 7104 #endif 7105 7106 sq = qp->q_syncq; 7107 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7108 /* debug macro */ 7109 SQ_PUTLOCKS_HELD(sq); 7110 /* 7111 * As entersq() does not increment the sq_count for 7112 * the write side, check sq_count for non-QPERQ 7113 * perimeters alone. 7114 */ 7115 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); 7116 7117 /* 7118 * propagate_syncq() can be called because of either messages on the 7119 * queue syncq or because on events on the queue syncq. Do actual 7120 * message propagations if there are any messages. 7121 */ 7122 if (qp->q_syncqmsgs) { 7123 isdriver = (qp->q_flag & QISDRV); 7124 7125 if (!isdriver) { 7126 nqp = qp->q_next; 7127 nsq = nqp->q_syncq; 7128 ASSERT(MUTEX_HELD(SQLOCK(nsq))); 7129 /* debug macro */ 7130 SQ_PUTLOCKS_HELD(nsq); 7131 #ifdef DEBUG 7132 func = (void (*)())nqp->q_qinfo->qi_putp; 7133 #endif 7134 } 7135 7136 SQRM_Q(sq, qp); 7137 priority = MAX(qp->q_spri, priority); 7138 qp->q_spri = 0; 7139 head = qp->q_sqhead; 7140 tail = qp->q_sqtail; 7141 qp->q_sqhead = qp->q_sqtail = NULL; 7142 qp->q_syncqmsgs = 0; 7143 7144 /* 7145 * Walk the list of messages, and free them if this is a driver, 7146 * otherwise reset the b_prev and b_queue value to the new putp. 7147 * Afterward, we will just add the head to the end of the next 7148 * syncq, and point the tail to the end of this one. 7149 */ 7150 7151 for (bp = head; bp != NULL; bp = next) { 7152 next = bp->b_next; 7153 if (isdriver) { 7154 bp->b_prev = bp->b_next = NULL; 7155 freemsg(bp); 7156 continue; 7157 } 7158 /* Change the q values for this message */ 7159 bp->b_queue = nqp; 7160 #ifdef DEBUG 7161 bp->b_prev = (mblk_t *)func; 7162 #endif 7163 moved++; 7164 } 7165 /* 7166 * Attach list of messages to the end of the new queue (if there 7167 * is a list of messages). 7168 */ 7169 7170 if (!isdriver && head != NULL) { 7171 ASSERT(tail != NULL); 7172 if (nqp->q_sqhead == NULL) { 7173 nqp->q_sqhead = head; 7174 } else { 7175 ASSERT(nqp->q_sqtail != NULL); 7176 nqp->q_sqtail->b_next = head; 7177 } 7178 nqp->q_sqtail = tail; 7179 /* 7180 * When messages are moved from high priority queue to 7181 * another queue, the destination queue priority is 7182 * upgraded. 7183 */ 7184 7185 if (priority > nqp->q_spri) 7186 nqp->q_spri = priority; 7187 7188 SQPUT_Q(nsq, nqp); 7189 7190 nqp->q_syncqmsgs += moved; 7191 ASSERT(nqp->q_syncqmsgs != 0); 7192 } 7193 } 7194 7195 /* 7196 * Before we leave, we need to make sure there are no 7197 * events listed for this queue. All events for this queue 7198 * will just be freed. 7199 */ 7200 if (sq->sq_evhead != NULL) { 7201 ASSERT(sq->sq_flags & SQ_EVENTS); 7202 prev = NULL; 7203 for (bp = sq->sq_evhead; bp != NULL; bp = next) { 7204 next = bp->b_next; 7205 if (bp->b_queue == qp) { 7206 /* Delete this message */ 7207 if (prev != NULL) { 7208 prev->b_next = next; 7209 /* 7210 * Update sq_evtail if the last element 7211 * is removed. 7212 */ 7213 if (bp == sq->sq_evtail) { 7214 ASSERT(next == NULL); 7215 sq->sq_evtail = prev; 7216 } 7217 } else 7218 sq->sq_evhead = next; 7219 if (sq->sq_evhead == NULL) 7220 sq->sq_flags &= ~SQ_EVENTS; 7221 bp->b_prev = bp->b_next = NULL; 7222 freemsg(bp); 7223 } else { 7224 prev = bp; 7225 } 7226 } 7227 } 7228 7229 flags = sq->sq_flags; 7230 7231 /* Wake up any waiter before leaving. */ 7232 if (flags & SQ_WANTWAKEUP) { 7233 flags &= ~SQ_WANTWAKEUP; 7234 cv_broadcast(&sq->sq_wait); 7235 } 7236 sq->sq_flags = flags; 7237 7238 return (moved); 7239 } 7240 7241 /* 7242 * Try and upgrade to exclusive access at the inner perimeter. If this can 7243 * not be done without blocking then request will be queued on the syncq 7244 * and drain_syncq will run it later. 7245 * 7246 * This routine can only be called from put or service procedures plus 7247 * asynchronous callback routines that have properly entered the queue (with 7248 * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq 7249 * associated with q. 7250 */ 7251 void 7252 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)()) 7253 { 7254 syncq_t *sq = q->q_syncq; 7255 uint16_t count; 7256 7257 mutex_enter(SQLOCK(sq)); 7258 count = sq->sq_count; 7259 SQ_PUTLOCKS_ENTER(sq); 7260 SUM_SQ_PUTCOUNTS(sq, count); 7261 ASSERT(count >= 1); 7262 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); 7263 7264 if (count == 1) { 7265 /* 7266 * Can upgrade. This case also handles nested qwriter calls 7267 * (when the qwriter callback function calls qwriter). In that 7268 * case SQ_EXCL is already set. 7269 */ 7270 sq->sq_flags |= SQ_EXCL; 7271 SQ_PUTLOCKS_EXIT(sq); 7272 mutex_exit(SQLOCK(sq)); 7273 (*func)(q, mp); 7274 /* 7275 * Assumes that leavesq, putnext, and drain_syncq will reset 7276 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on 7277 * until putnext, leavesq, or drain_syncq drops it. 7278 * That way we handle nested qwriter(INNER) without dropping 7279 * SQ_EXCL until the outermost qwriter callback routine is 7280 * done. 7281 */ 7282 return; 7283 } 7284 SQ_PUTLOCKS_EXIT(sq); 7285 sqfill_events(sq, q, mp, func); 7286 } 7287 7288 /* 7289 * Synchronous callback support functions 7290 */ 7291 7292 /* 7293 * Allocate a callback parameter structure. 7294 * Assumes that caller initializes the flags and the id. 7295 * Acquires SQLOCK(sq) if non-NULL is returned. 7296 */ 7297 callbparams_t * 7298 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) 7299 { 7300 callbparams_t *cbp; 7301 size_t size = sizeof (callbparams_t); 7302 7303 cbp = kmem_alloc(size, kmflags & ~KM_PANIC); 7304 7305 /* 7306 * Only try tryhard allocation if the caller is ready to panic. 7307 * Otherwise just fail. 7308 */ 7309 if (cbp == NULL) { 7310 if (kmflags & KM_PANIC) 7311 cbp = kmem_alloc_tryhard(sizeof (callbparams_t), 7312 &size, kmflags); 7313 else 7314 return (NULL); 7315 } 7316 7317 ASSERT(size >= sizeof (callbparams_t)); 7318 cbp->cbp_size = size; 7319 cbp->cbp_sq = sq; 7320 cbp->cbp_func = func; 7321 cbp->cbp_arg = arg; 7322 mutex_enter(SQLOCK(sq)); 7323 cbp->cbp_next = sq->sq_callbpend; 7324 sq->sq_callbpend = cbp; 7325 return (cbp); 7326 } 7327 7328 void 7329 callbparams_free(syncq_t *sq, callbparams_t *cbp) 7330 { 7331 callbparams_t **pp, *p; 7332 7333 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7334 7335 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7336 if (p == cbp) { 7337 *pp = p->cbp_next; 7338 kmem_free(p, p->cbp_size); 7339 return; 7340 } 7341 } 7342 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7343 "callbparams_free: not found\n")); 7344 } 7345 7346 void 7347 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) 7348 { 7349 callbparams_t **pp, *p; 7350 7351 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7352 7353 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7354 if (p->cbp_id == id && p->cbp_flags == flag) { 7355 *pp = p->cbp_next; 7356 kmem_free(p, p->cbp_size); 7357 return; 7358 } 7359 } 7360 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7361 "callbparams_free_id: not found\n")); 7362 } 7363 7364 /* 7365 * Callback wrapper function used by once-only callbacks that can be 7366 * cancelled (qtimeout and qbufcall) 7367 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be 7368 * cancelled by the qun* functions. 7369 */ 7370 void 7371 qcallbwrapper(void *arg) 7372 { 7373 callbparams_t *cbp = arg; 7374 syncq_t *sq; 7375 uint16_t count = 0; 7376 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 7377 uint16_t type; 7378 7379 sq = cbp->cbp_sq; 7380 mutex_enter(SQLOCK(sq)); 7381 type = sq->sq_type; 7382 if (!(type & SQ_CICB)) { 7383 count = sq->sq_count; 7384 SQ_PUTLOCKS_ENTER(sq); 7385 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 7386 SUM_SQ_PUTCOUNTS(sq, count); 7387 sq->sq_needexcl++; 7388 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 7389 waitflags |= SQ_MESSAGES; 7390 } 7391 /* Can not handle exclusive entry at outer perimeter */ 7392 ASSERT(type & SQ_COCB); 7393 7394 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { 7395 if ((sq->sq_callbflags & cbp->cbp_flags) && 7396 (sq->sq_cancelid == cbp->cbp_id)) { 7397 /* timeout has been cancelled */ 7398 sq->sq_callbflags |= SQ_CALLB_BYPASSED; 7399 callbparams_free(sq, cbp); 7400 if (!(type & SQ_CICB)) { 7401 ASSERT(sq->sq_needexcl > 0); 7402 sq->sq_needexcl--; 7403 if (sq->sq_needexcl == 0) { 7404 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7405 } 7406 SQ_PUTLOCKS_EXIT(sq); 7407 } 7408 mutex_exit(SQLOCK(sq)); 7409 return; 7410 } 7411 sq->sq_flags |= SQ_WANTWAKEUP; 7412 if (!(type & SQ_CICB)) { 7413 SQ_PUTLOCKS_EXIT(sq); 7414 } 7415 cv_wait(&sq->sq_wait, SQLOCK(sq)); 7416 if (!(type & SQ_CICB)) { 7417 count = sq->sq_count; 7418 SQ_PUTLOCKS_ENTER(sq); 7419 SUM_SQ_PUTCOUNTS(sq, count); 7420 } 7421 } 7422 7423 sq->sq_count++; 7424 ASSERT(sq->sq_count != 0); /* Wraparound */ 7425 if (!(type & SQ_CICB)) { 7426 ASSERT(count == 0); 7427 sq->sq_flags |= SQ_EXCL; 7428 ASSERT(sq->sq_needexcl > 0); 7429 sq->sq_needexcl--; 7430 if (sq->sq_needexcl == 0) { 7431 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7432 } 7433 SQ_PUTLOCKS_EXIT(sq); 7434 } 7435 7436 mutex_exit(SQLOCK(sq)); 7437 7438 cbp->cbp_func(cbp->cbp_arg); 7439 7440 /* 7441 * We drop the lock only for leavesq to re-acquire it. 7442 * Possible optimization is inline of leavesq. 7443 */ 7444 mutex_enter(SQLOCK(sq)); 7445 callbparams_free(sq, cbp); 7446 mutex_exit(SQLOCK(sq)); 7447 leavesq(sq, SQ_CALLBACK); 7448 } 7449 7450 /* 7451 * No need to grab sq_putlocks here. See comment in strsubr.h that 7452 * explains when sq_putlocks are used. 7453 * 7454 * sq_count (or one of the sq_putcounts) has already been 7455 * decremented by the caller, and if SQ_QUEUED, we need to call 7456 * drain_syncq (the global syncq drain). 7457 * If putnext_tail is called with the SQ_EXCL bit set, we are in 7458 * one of two states, non-CIPUT perimeter, and we need to clear 7459 * it, or we went exclusive in the put procedure. In any case, 7460 * we want to clear the bit now, and it is probably easier to do 7461 * this at the beginning of this function (remember, we hold 7462 * the SQLOCK). Lastly, if there are other messages queued 7463 * on the syncq (and not for our destination), enable the syncq 7464 * for background work. 7465 */ 7466 7467 /* ARGSUSED */ 7468 void 7469 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) 7470 { 7471 uint16_t flags = sq->sq_flags; 7472 7473 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7474 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 7475 7476 /* Clear SQ_EXCL if set in passflags */ 7477 if (passflags & SQ_EXCL) { 7478 flags &= ~SQ_EXCL; 7479 } 7480 if (flags & SQ_WANTWAKEUP) { 7481 flags &= ~SQ_WANTWAKEUP; 7482 cv_broadcast(&sq->sq_wait); 7483 } 7484 if (flags & SQ_WANTEXWAKEUP) { 7485 flags &= ~SQ_WANTEXWAKEUP; 7486 cv_broadcast(&sq->sq_exitwait); 7487 } 7488 sq->sq_flags = flags; 7489 7490 /* 7491 * We have cleared SQ_EXCL if we were asked to, and started 7492 * the wakeup process for waiters. If there are no writers 7493 * then we need to drain the syncq if we were told to, or 7494 * enable the background thread to do it. 7495 */ 7496 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) { 7497 if ((passflags & SQ_QUEUED) || 7498 (sq->sq_svcflags & SQ_DISABLED)) { 7499 /* drain_syncq will take care of events in the list */ 7500 drain_syncq(sq); 7501 return; 7502 } else if (flags & SQ_QUEUED) { 7503 sqenable(sq); 7504 } 7505 } 7506 /* Drop the SQLOCK on exit */ 7507 mutex_exit(SQLOCK(sq)); 7508 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END, 7509 "putnext_end:(%p, %p, %p) done", NULL, qp, sq); 7510 } 7511 7512 void 7513 set_qend(queue_t *q) 7514 { 7515 mutex_enter(QLOCK(q)); 7516 if (!O_SAMESTR(q)) 7517 q->q_flag |= QEND; 7518 else 7519 q->q_flag &= ~QEND; 7520 mutex_exit(QLOCK(q)); 7521 q = _OTHERQ(q); 7522 mutex_enter(QLOCK(q)); 7523 if (!O_SAMESTR(q)) 7524 q->q_flag |= QEND; 7525 else 7526 q->q_flag &= ~QEND; 7527 mutex_exit(QLOCK(q)); 7528 } 7529 7530 /* 7531 * Set QFULL in next service procedure queue (that cares) if not already 7532 * set and if there are already more messages on the syncq than 7533 * sq_max_size. If sq_max_size is 0, no flow control will be asserted on 7534 * any syncq. 7535 * 7536 * The fq here is the next queue with a service procedure. This is where 7537 * we would fail canputnext, so this is where we need to set QFULL. 7538 * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag. 7539 * 7540 * We already have QLOCK at this point. To avoid cross-locks with 7541 * freezestr() which grabs all QLOCKs and with strlock() which grabs both 7542 * SQLOCK and sd_reflock, we need to drop respective locks first. 7543 */ 7544 void 7545 set_qfull(queue_t *q) 7546 { 7547 queue_t *fq = NULL; 7548 7549 ASSERT(MUTEX_HELD(QLOCK(q))); 7550 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) && 7551 (q->q_syncqmsgs > sq_max_size)) { 7552 if ((fq = q->q_nfsrv) == q) { 7553 fq->q_flag |= QFULL; 7554 } else { 7555 mutex_exit(QLOCK(q)); 7556 mutex_enter(QLOCK(fq)); 7557 fq->q_flag |= QFULL; 7558 mutex_exit(QLOCK(fq)); 7559 mutex_enter(QLOCK(q)); 7560 } 7561 } 7562 } 7563 7564 void 7565 clr_qfull(queue_t *q) 7566 { 7567 queue_t *oq = q; 7568 7569 q = q->q_nfsrv; 7570 /* Fast check if there is any work to do before getting the lock. */ 7571 if ((q->q_flag & (QFULL|QWANTW)) == 0) { 7572 return; 7573 } 7574 7575 /* 7576 * Do not reset QFULL (and backenable) if the q_count is the reason 7577 * for QFULL being set. 7578 */ 7579 mutex_enter(QLOCK(q)); 7580 /* 7581 * If queue is empty i.e q_mblkcnt is zero, queue can not be full. 7582 * Hence clear the QFULL. 7583 * If both q_count and q_mblkcnt are less than the hiwat mark, 7584 * clear the QFULL. 7585 */ 7586 if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) && 7587 (q->q_mblkcnt < q->q_hiwat))) { 7588 q->q_flag &= ~QFULL; 7589 /* 7590 * A little more confusing, how about this way: 7591 * if someone wants to write, 7592 * AND 7593 * both counts are less than the lowat mark 7594 * OR 7595 * the lowat mark is zero 7596 * THEN 7597 * backenable 7598 */ 7599 if ((q->q_flag & QWANTW) && 7600 (((q->q_count < q->q_lowat) && 7601 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) { 7602 q->q_flag &= ~QWANTW; 7603 mutex_exit(QLOCK(q)); 7604 backenable(oq, 0); 7605 } else 7606 mutex_exit(QLOCK(q)); 7607 } else 7608 mutex_exit(QLOCK(q)); 7609 } 7610 7611 /* 7612 * Set the forward service procedure pointer. 7613 * 7614 * Called at insert-time to cache a queue's next forward service procedure in 7615 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted 7616 * has a service procedure then q_nfsrv points to itself. If the queue to be 7617 * inserted does not have a service procedure, then q_nfsrv points to the next 7618 * queue forward that has a service procedure. If the queue is at the logical 7619 * end of the stream (driver for write side, stream head for the read side) 7620 * and does not have a service procedure, then q_nfsrv also points to itself. 7621 */ 7622 void 7623 set_nfsrv_ptr( 7624 queue_t *rnew, /* read queue pointer to new module */ 7625 queue_t *wnew, /* write queue pointer to new module */ 7626 queue_t *prev_rq, /* read queue pointer to the module above */ 7627 queue_t *prev_wq) /* write queue pointer to the module above */ 7628 { 7629 queue_t *qp; 7630 7631 if (prev_wq->q_next == NULL) { 7632 /* 7633 * Insert the driver, initialize the driver and stream head. 7634 * In this case, prev_rq/prev_wq should be the stream head. 7635 * _I_INSERT does not allow inserting a driver. Make sure 7636 * that it is not an insertion. 7637 */ 7638 ASSERT(!(rnew->q_flag & _QINSERTING)); 7639 wnew->q_nfsrv = wnew; 7640 if (rnew->q_qinfo->qi_srvp) 7641 rnew->q_nfsrv = rnew; 7642 else 7643 rnew->q_nfsrv = prev_rq; 7644 prev_rq->q_nfsrv = prev_rq; 7645 prev_wq->q_nfsrv = prev_wq; 7646 } else { 7647 /* 7648 * set up read side q_nfsrv pointer. This MUST be done 7649 * before setting the write side, because the setting of 7650 * the write side for a fifo may depend on it. 7651 * 7652 * Suppose we have a fifo that only has pipemod pushed. 7653 * pipemod has no read or write service procedures, so 7654 * nfsrv for both pipemod queues points to prev_rq (the 7655 * stream read head). Now push bufmod (which has only a 7656 * read service procedure). Doing the write side first, 7657 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which 7658 * is WRONG; the next queue forward from wnew with a 7659 * service procedure will be rnew, not the stream read head. 7660 * Since the downstream queue (which in the case of a fifo 7661 * is the read queue rnew) can affect upstream queues, it 7662 * needs to be done first. Setting up the read side first 7663 * sets nfsrv for both pipemod queues to rnew and then 7664 * when the write side is set up, wnew-q_nfsrv will also 7665 * point to rnew. 7666 */ 7667 if (rnew->q_qinfo->qi_srvp) { 7668 /* 7669 * use _OTHERQ() because, if this is a pipe, next 7670 * module may have been pushed from other end and 7671 * q_next could be a read queue. 7672 */ 7673 qp = _OTHERQ(prev_wq->q_next); 7674 while (qp && qp->q_nfsrv != qp) { 7675 qp->q_nfsrv = rnew; 7676 qp = backq(qp); 7677 } 7678 rnew->q_nfsrv = rnew; 7679 } else 7680 rnew->q_nfsrv = prev_rq->q_nfsrv; 7681 7682 /* set up write side q_nfsrv pointer */ 7683 if (wnew->q_qinfo->qi_srvp) { 7684 wnew->q_nfsrv = wnew; 7685 7686 /* 7687 * For insertion, need to update nfsrv of the modules 7688 * above which do not have a service routine. 7689 */ 7690 if (rnew->q_flag & _QINSERTING) { 7691 for (qp = prev_wq; 7692 qp != NULL && qp->q_nfsrv != qp; 7693 qp = backq(qp)) { 7694 qp->q_nfsrv = wnew->q_nfsrv; 7695 } 7696 } 7697 } else { 7698 if (prev_wq->q_next == prev_rq) 7699 /* 7700 * Since prev_wq/prev_rq are the middle of a 7701 * fifo, wnew/rnew will also be the middle of 7702 * a fifo and wnew's nfsrv is same as rnew's. 7703 */ 7704 wnew->q_nfsrv = rnew->q_nfsrv; 7705 else 7706 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv; 7707 } 7708 } 7709 } 7710 7711 /* 7712 * Reset the forward service procedure pointer; called at remove-time. 7713 */ 7714 void 7715 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp) 7716 { 7717 queue_t *tmp_qp; 7718 7719 /* Reset the write side q_nfsrv pointer for _I_REMOVE */ 7720 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) { 7721 for (tmp_qp = backq(wqp); 7722 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp; 7723 tmp_qp = backq(tmp_qp)) { 7724 tmp_qp->q_nfsrv = wqp->q_nfsrv; 7725 } 7726 } 7727 7728 /* reset the read side q_nfsrv pointer */ 7729 if (rqp->q_qinfo->qi_srvp) { 7730 if (wqp->q_next) { /* non-driver case */ 7731 tmp_qp = _OTHERQ(wqp->q_next); 7732 while (tmp_qp && tmp_qp->q_nfsrv == rqp) { 7733 /* Note that rqp->q_next cannot be NULL */ 7734 ASSERT(rqp->q_next != NULL); 7735 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv; 7736 tmp_qp = backq(tmp_qp); 7737 } 7738 } 7739 } 7740 } 7741 7742 /* 7743 * This routine should be called after all stream geometry changes to update 7744 * the stream head cached struio() rd/wr queue pointers. Note must be called 7745 * with the streamlock()ed. 7746 * 7747 * Note: only enables Synchronous STREAMS for a side of a Stream which has 7748 * an explicit synchronous barrier module queue. That is, a queue that 7749 * has specified a struio() type. 7750 */ 7751 static void 7752 strsetuio(stdata_t *stp) 7753 { 7754 queue_t *wrq; 7755 7756 if (stp->sd_flag & STPLEX) { 7757 /* 7758 * Not streamhead, but a mux, so no Synchronous STREAMS. 7759 */ 7760 stp->sd_struiowrq = NULL; 7761 stp->sd_struiordq = NULL; 7762 return; 7763 } 7764 /* 7765 * Scan the write queue(s) while synchronous 7766 * until we find a qinfo uio type specified. 7767 */ 7768 wrq = stp->sd_wrq->q_next; 7769 while (wrq) { 7770 if (wrq->q_struiot == STRUIOT_NONE) { 7771 wrq = 0; 7772 break; 7773 } 7774 if (wrq->q_struiot != STRUIOT_DONTCARE) 7775 break; 7776 if (! _SAMESTR(wrq)) { 7777 wrq = 0; 7778 break; 7779 } 7780 wrq = wrq->q_next; 7781 } 7782 stp->sd_struiowrq = wrq; 7783 /* 7784 * Scan the read queue(s) while synchronous 7785 * until we find a qinfo uio type specified. 7786 */ 7787 wrq = stp->sd_wrq->q_next; 7788 while (wrq) { 7789 if (_RD(wrq)->q_struiot == STRUIOT_NONE) { 7790 wrq = 0; 7791 break; 7792 } 7793 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE) 7794 break; 7795 if (! _SAMESTR(wrq)) { 7796 wrq = 0; 7797 break; 7798 } 7799 wrq = wrq->q_next; 7800 } 7801 stp->sd_struiordq = wrq ? _RD(wrq) : 0; 7802 } 7803 7804 /* 7805 * pass_wput, unblocks the passthru queues, so that 7806 * messages can arrive at muxs lower read queue, before 7807 * I_LINK/I_UNLINK is acked/nacked. 7808 */ 7809 static void 7810 pass_wput(queue_t *q, mblk_t *mp) 7811 { 7812 syncq_t *sq; 7813 7814 sq = _RD(q)->q_syncq; 7815 if (sq->sq_flags & SQ_BLOCKED) 7816 unblocksq(sq, SQ_BLOCKED, 0); 7817 putnext(q, mp); 7818 } 7819 7820 /* 7821 * Set up queues for the link/unlink. 7822 * Create a new queue and block it and then insert it 7823 * below the stream head on the lower stream. 7824 * This prevents any messages from arriving during the setq 7825 * as well as while the mux is processing the LINK/I_UNLINK. 7826 * The blocked passq is unblocked once the LINK/I_UNLINK has 7827 * been acked or nacked or if a message is generated and sent 7828 * down muxs write put procedure. 7829 * See pass_wput(). 7830 * 7831 * After the new queue is inserted, all messages coming from below are 7832 * blocked. The call to strlock will ensure that all activity in the stream head 7833 * read queue syncq is stopped (sq_count drops to zero). 7834 */ 7835 static queue_t * 7836 link_addpassthru(stdata_t *stpdown) 7837 { 7838 queue_t *passq; 7839 sqlist_t sqlist; 7840 7841 passq = allocq(); 7842 STREAM(passq) = STREAM(_WR(passq)) = stpdown; 7843 /* setq might sleep in allocator - avoid holding locks. */ 7844 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ, 7845 SQ_CI|SQ_CO, B_FALSE); 7846 claimq(passq); 7847 blocksq(passq->q_syncq, SQ_BLOCKED, 1); 7848 insertq(STREAM(passq), passq); 7849 7850 /* 7851 * Use strlock() to wait for the stream head sq_count to drop to zero 7852 * since we are going to change q_ptr in the stream head. Note that 7853 * insertq() doesn't wait for any syncq counts to drop to zero. 7854 */ 7855 sqlist.sqlist_head = NULL; 7856 sqlist.sqlist_index = 0; 7857 sqlist.sqlist_size = sizeof (sqlist_t); 7858 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq); 7859 strlock(stpdown, &sqlist); 7860 strunlock(stpdown, &sqlist); 7861 7862 releaseq(passq); 7863 return (passq); 7864 } 7865 7866 /* 7867 * Let messages flow up into the mux by removing 7868 * the passq. 7869 */ 7870 static void 7871 link_rempassthru(queue_t *passq) 7872 { 7873 claimq(passq); 7874 removeq(passq); 7875 releaseq(passq); 7876 freeq(passq); 7877 } 7878 7879 /* 7880 * Wait for the condition variable pointed to by `cvp' to be signaled, 7881 * or for `tim' milliseconds to elapse, whichever comes first. If `tim' 7882 * is negative, then there is no time limit. If `nosigs' is non-zero, 7883 * then the wait will be non-interruptible. 7884 * 7885 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout. 7886 */ 7887 clock_t 7888 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs) 7889 { 7890 clock_t ret; 7891 7892 if (tim < 0) { 7893 if (nosigs) { 7894 cv_wait(cvp, mp); 7895 ret = 1; 7896 } else { 7897 ret = cv_wait_sig(cvp, mp); 7898 } 7899 } else if (tim > 0) { 7900 /* 7901 * convert milliseconds to clock ticks 7902 */ 7903 if (nosigs) { 7904 ret = cv_reltimedwait(cvp, mp, 7905 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK); 7906 } else { 7907 ret = cv_reltimedwait_sig(cvp, mp, 7908 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK); 7909 } 7910 } else { 7911 ret = -1; 7912 } 7913 return (ret); 7914 } 7915 7916 /* 7917 * Wait until the stream head can determine if it is at the mark but 7918 * don't wait forever to prevent a race condition between the "mark" state 7919 * in the stream head and any mark state in the caller/user of this routine. 7920 * 7921 * This is used by sockets and for a socket it would be incorrect 7922 * to return a failure for SIOCATMARK when there is no data in the receive 7923 * queue and the marked urgent data is traveling up the stream. 7924 * 7925 * This routine waits until the mark is known by waiting for one of these 7926 * three events: 7927 * The stream head read queue becoming non-empty (including an EOF). 7928 * The STRATMARK flag being set (due to a MSGMARKNEXT message). 7929 * The STRNOTATMARK flag being set (which indicates that the transport 7930 * has sent a MSGNOTMARKNEXT message to indicate that it is not at 7931 * the mark). 7932 * 7933 * The routine returns 1 if the stream is at the mark; 0 if it can 7934 * be determined that the stream is not at the mark. 7935 * If the wait times out and it can't determine 7936 * whether or not the stream might be at the mark the routine will return -1. 7937 * 7938 * Note: This routine should only be used when a mark is pending i.e., 7939 * in the socket case the SIGURG has been posted. 7940 * Note2: This can not wakeup just because synchronous streams indicate 7941 * that data is available since it is not possible to use the synchronous 7942 * streams interfaces to determine the b_flag value for the data queued below 7943 * the stream head. 7944 */ 7945 int 7946 strwaitmark(vnode_t *vp) 7947 { 7948 struct stdata *stp = vp->v_stream; 7949 queue_t *rq = _RD(stp->sd_wrq); 7950 int mark; 7951 7952 mutex_enter(&stp->sd_lock); 7953 while (rq->q_first == NULL && 7954 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) { 7955 stp->sd_flag |= RSLEEP; 7956 7957 /* Wait for 100 milliseconds for any state change. */ 7958 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) { 7959 mutex_exit(&stp->sd_lock); 7960 return (-1); 7961 } 7962 } 7963 if (stp->sd_flag & STRATMARK) 7964 mark = 1; 7965 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK)) 7966 mark = 1; 7967 else 7968 mark = 0; 7969 7970 mutex_exit(&stp->sd_lock); 7971 return (mark); 7972 } 7973 7974 /* 7975 * Set a read side error. If persist is set change the socket error 7976 * to persistent. If errfunc is set install the function as the exported 7977 * error handler. 7978 */ 7979 void 7980 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 7981 { 7982 struct stdata *stp = vp->v_stream; 7983 7984 mutex_enter(&stp->sd_lock); 7985 stp->sd_rerror = error; 7986 if (error == 0 && errfunc == NULL) 7987 stp->sd_flag &= ~STRDERR; 7988 else 7989 stp->sd_flag |= STRDERR; 7990 if (persist) { 7991 stp->sd_flag &= ~STRDERRNONPERSIST; 7992 } else { 7993 stp->sd_flag |= STRDERRNONPERSIST; 7994 } 7995 stp->sd_rderrfunc = errfunc; 7996 if (error != 0 || errfunc != NULL) { 7997 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 7998 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 7999 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 8000 8001 mutex_exit(&stp->sd_lock); 8002 pollwakeup(&stp->sd_pollist, POLLERR); 8003 mutex_enter(&stp->sd_lock); 8004 8005 if (stp->sd_sigflags & S_ERROR) 8006 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 8007 } 8008 mutex_exit(&stp->sd_lock); 8009 } 8010 8011 /* 8012 * Set a write side error. If persist is set change the socket error 8013 * to persistent. 8014 */ 8015 void 8016 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 8017 { 8018 struct stdata *stp = vp->v_stream; 8019 8020 mutex_enter(&stp->sd_lock); 8021 stp->sd_werror = error; 8022 if (error == 0 && errfunc == NULL) 8023 stp->sd_flag &= ~STWRERR; 8024 else 8025 stp->sd_flag |= STWRERR; 8026 if (persist) { 8027 stp->sd_flag &= ~STWRERRNONPERSIST; 8028 } else { 8029 stp->sd_flag |= STWRERRNONPERSIST; 8030 } 8031 stp->sd_wrerrfunc = errfunc; 8032 if (error != 0 || errfunc != NULL) { 8033 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 8034 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 8035 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 8036 8037 mutex_exit(&stp->sd_lock); 8038 pollwakeup(&stp->sd_pollist, POLLERR); 8039 mutex_enter(&stp->sd_lock); 8040 8041 if (stp->sd_sigflags & S_ERROR) 8042 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 8043 } 8044 mutex_exit(&stp->sd_lock); 8045 } 8046 8047 /* 8048 * Make the stream return 0 (EOF) when all data has been read. 8049 * No effect on write side. 8050 */ 8051 void 8052 strseteof(vnode_t *vp, int eof) 8053 { 8054 struct stdata *stp = vp->v_stream; 8055 8056 mutex_enter(&stp->sd_lock); 8057 if (!eof) { 8058 stp->sd_flag &= ~STREOF; 8059 mutex_exit(&stp->sd_lock); 8060 return; 8061 } 8062 stp->sd_flag |= STREOF; 8063 if (stp->sd_flag & RSLEEP) { 8064 stp->sd_flag &= ~RSLEEP; 8065 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); 8066 } 8067 8068 mutex_exit(&stp->sd_lock); 8069 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM); 8070 mutex_enter(&stp->sd_lock); 8071 8072 if (stp->sd_sigflags & (S_INPUT|S_RDNORM)) 8073 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0); 8074 mutex_exit(&stp->sd_lock); 8075 } 8076 8077 void 8078 strflushrq(vnode_t *vp, int flag) 8079 { 8080 struct stdata *stp = vp->v_stream; 8081 8082 mutex_enter(&stp->sd_lock); 8083 flushq(_RD(stp->sd_wrq), flag); 8084 mutex_exit(&stp->sd_lock); 8085 } 8086 8087 void 8088 strsetrputhooks(vnode_t *vp, uint_t flags, 8089 msgfunc_t protofunc, msgfunc_t miscfunc) 8090 { 8091 struct stdata *stp = vp->v_stream; 8092 8093 mutex_enter(&stp->sd_lock); 8094 8095 if (protofunc == NULL) 8096 stp->sd_rprotofunc = strrput_proto; 8097 else 8098 stp->sd_rprotofunc = protofunc; 8099 8100 if (miscfunc == NULL) 8101 stp->sd_rmiscfunc = strrput_misc; 8102 else 8103 stp->sd_rmiscfunc = miscfunc; 8104 8105 if (flags & SH_CONSOL_DATA) 8106 stp->sd_rput_opt |= SR_CONSOL_DATA; 8107 else 8108 stp->sd_rput_opt &= ~SR_CONSOL_DATA; 8109 8110 if (flags & SH_SIGALLDATA) 8111 stp->sd_rput_opt |= SR_SIGALLDATA; 8112 else 8113 stp->sd_rput_opt &= ~SR_SIGALLDATA; 8114 8115 if (flags & SH_IGN_ZEROLEN) 8116 stp->sd_rput_opt |= SR_IGN_ZEROLEN; 8117 else 8118 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN; 8119 8120 mutex_exit(&stp->sd_lock); 8121 } 8122 8123 void 8124 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime) 8125 { 8126 struct stdata *stp = vp->v_stream; 8127 8128 mutex_enter(&stp->sd_lock); 8129 stp->sd_closetime = closetime; 8130 8131 if (flags & SH_SIGPIPE) 8132 stp->sd_wput_opt |= SW_SIGPIPE; 8133 else 8134 stp->sd_wput_opt &= ~SW_SIGPIPE; 8135 if (flags & SH_RECHECK_ERR) 8136 stp->sd_wput_opt |= SW_RECHECK_ERR; 8137 else 8138 stp->sd_wput_opt &= ~SW_RECHECK_ERR; 8139 8140 mutex_exit(&stp->sd_lock); 8141 } 8142 8143 void 8144 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc) 8145 { 8146 struct stdata *stp = vp->v_stream; 8147 8148 mutex_enter(&stp->sd_lock); 8149 8150 stp->sd_rputdatafunc = rdatafunc; 8151 stp->sd_wputdatafunc = wdatafunc; 8152 8153 mutex_exit(&stp->sd_lock); 8154 } 8155 8156 /* Used within framework when the queue is already locked */ 8157 void 8158 qenable_locked(queue_t *q) 8159 { 8160 stdata_t *stp = STREAM(q); 8161 8162 ASSERT(MUTEX_HELD(QLOCK(q))); 8163 8164 if (!q->q_qinfo->qi_srvp) 8165 return; 8166 8167 /* 8168 * Do not place on run queue if already enabled or closing. 8169 */ 8170 if (q->q_flag & (QWCLOSE|QENAB)) 8171 return; 8172 8173 /* 8174 * mark queue enabled and place on run list if it is not already being 8175 * serviced. If it is serviced, the runservice() function will detect 8176 * that QENAB is set and call service procedure before clearing 8177 * QINSERVICE flag. 8178 */ 8179 q->q_flag |= QENAB; 8180 if (q->q_flag & QINSERVICE) 8181 return; 8182 8183 /* Record the time of qenable */ 8184 q->q_qtstamp = ddi_get_lbolt(); 8185 8186 /* 8187 * Put the queue in the stp list and schedule it for background 8188 * processing if it is not already scheduled or if stream head does not 8189 * intent to process it in the foreground later by setting 8190 * STRS_WILLSERVICE flag. 8191 */ 8192 mutex_enter(&stp->sd_qlock); 8193 /* 8194 * If there are already something on the list, stp flags should show 8195 * intention to drain it. 8196 */ 8197 IMPLY(STREAM_NEEDSERVICE(stp), 8198 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))); 8199 8200 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link); 8201 stp->sd_nqueues++; 8202 8203 /* 8204 * If no one will drain this stream we are the first producer and 8205 * need to schedule it for background thread. 8206 */ 8207 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) { 8208 /* 8209 * No one will service this stream later, so we have to 8210 * schedule it now. 8211 */ 8212 STRSTAT(stenables); 8213 stp->sd_svcflags |= STRS_SCHEDULED; 8214 stp->sd_servid = (void *)taskq_dispatch(streams_taskq, 8215 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE); 8216 8217 if (stp->sd_servid == NULL) { 8218 /* 8219 * Task queue failed so fail over to the backup 8220 * servicing thread. 8221 */ 8222 STRSTAT(taskqfails); 8223 /* 8224 * It is safe to clear STRS_SCHEDULED flag because it 8225 * was set by this thread above. 8226 */ 8227 stp->sd_svcflags &= ~STRS_SCHEDULED; 8228 8229 /* 8230 * Failover scheduling is protected by service_queue 8231 * lock. 8232 */ 8233 mutex_enter(&service_queue); 8234 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q)); 8235 ASSERT(q->q_link == NULL); 8236 /* 8237 * Append the queue to qhead/qtail list. 8238 */ 8239 if (qhead == NULL) 8240 qhead = q; 8241 else 8242 qtail->q_link = q; 8243 qtail = q; 8244 /* 8245 * Clear stp queue list. 8246 */ 8247 stp->sd_qhead = stp->sd_qtail = NULL; 8248 stp->sd_nqueues = 0; 8249 /* 8250 * Wakeup background queue processing thread. 8251 */ 8252 cv_signal(&services_to_run); 8253 mutex_exit(&service_queue); 8254 } 8255 } 8256 mutex_exit(&stp->sd_qlock); 8257 } 8258 8259 static void 8260 queue_service(queue_t *q) 8261 { 8262 /* 8263 * The queue in the list should have 8264 * QENAB flag set and should not have 8265 * QINSERVICE flag set. QINSERVICE is 8266 * set when the queue is dequeued and 8267 * qenable_locked doesn't enqueue a 8268 * queue with QINSERVICE set. 8269 */ 8270 8271 ASSERT(!(q->q_flag & QINSERVICE)); 8272 ASSERT((q->q_flag & QENAB)); 8273 mutex_enter(QLOCK(q)); 8274 q->q_flag &= ~QENAB; 8275 q->q_flag |= QINSERVICE; 8276 mutex_exit(QLOCK(q)); 8277 runservice(q); 8278 } 8279 8280 static void 8281 syncq_service(syncq_t *sq) 8282 { 8283 STRSTAT(syncqservice); 8284 mutex_enter(SQLOCK(sq)); 8285 ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); 8286 ASSERT(sq->sq_servcount != 0); 8287 ASSERT(sq->sq_next == NULL); 8288 8289 /* if we came here from the background thread, clear the flag */ 8290 if (sq->sq_svcflags & SQ_BGTHREAD) 8291 sq->sq_svcflags &= ~SQ_BGTHREAD; 8292 8293 /* let drain_syncq know that it's being called in the background */ 8294 sq->sq_svcflags |= SQ_SERVICE; 8295 drain_syncq(sq); 8296 } 8297 8298 static void 8299 qwriter_outer_service(syncq_t *outer) 8300 { 8301 /* 8302 * Note that SQ_WRITER is used on the outer perimeter 8303 * to signal that a qwriter(OUTER) is either investigating 8304 * running or that it is actually running a function. 8305 */ 8306 outer_enter(outer, SQ_BLOCKED|SQ_WRITER); 8307 8308 /* 8309 * All inner syncq are empty and have SQ_WRITER set 8310 * to block entering the outer perimeter. 8311 * 8312 * We do not need to explicitly call write_now since 8313 * outer_exit does it for us. 8314 */ 8315 outer_exit(outer); 8316 } 8317 8318 static void 8319 mblk_free(mblk_t *mp) 8320 { 8321 dblk_t *dbp = mp->b_datap; 8322 frtn_t *frp = dbp->db_frtnp; 8323 8324 mp->b_next = NULL; 8325 if (dbp->db_fthdr != NULL) 8326 str_ftfree(dbp); 8327 8328 ASSERT(dbp->db_fthdr == NULL); 8329 frp->free_func(frp->free_arg); 8330 ASSERT(dbp->db_mblk == mp); 8331 8332 if (dbp->db_credp != NULL) { 8333 crfree(dbp->db_credp); 8334 dbp->db_credp = NULL; 8335 } 8336 dbp->db_cpid = -1; 8337 dbp->db_struioflag = 0; 8338 dbp->db_struioun.cksum.flags = 0; 8339 8340 kmem_cache_free(dbp->db_cache, dbp); 8341 } 8342 8343 /* 8344 * Background processing of the stream queue list. 8345 */ 8346 static void 8347 stream_service(stdata_t *stp) 8348 { 8349 queue_t *q; 8350 8351 mutex_enter(&stp->sd_qlock); 8352 8353 STR_SERVICE(stp, q); 8354 8355 stp->sd_svcflags &= ~STRS_SCHEDULED; 8356 stp->sd_servid = NULL; 8357 cv_signal(&stp->sd_qcv); 8358 mutex_exit(&stp->sd_qlock); 8359 } 8360 8361 /* 8362 * Foreground processing of the stream queue list. 8363 */ 8364 void 8365 stream_runservice(stdata_t *stp) 8366 { 8367 queue_t *q; 8368 8369 mutex_enter(&stp->sd_qlock); 8370 STRSTAT(rservice); 8371 /* 8372 * We are going to drain this stream queue list, so qenable_locked will 8373 * not schedule it until we finish. 8374 */ 8375 stp->sd_svcflags |= STRS_WILLSERVICE; 8376 8377 STR_SERVICE(stp, q); 8378 8379 stp->sd_svcflags &= ~STRS_WILLSERVICE; 8380 mutex_exit(&stp->sd_qlock); 8381 /* 8382 * Help backup background thread to drain the qhead/qtail list. 8383 */ 8384 while (qhead != NULL) { 8385 STRSTAT(qhelps); 8386 mutex_enter(&service_queue); 8387 DQ(q, qhead, qtail, q_link); 8388 mutex_exit(&service_queue); 8389 if (q != NULL) 8390 queue_service(q); 8391 } 8392 } 8393 8394 void 8395 stream_willservice(stdata_t *stp) 8396 { 8397 mutex_enter(&stp->sd_qlock); 8398 stp->sd_svcflags |= STRS_WILLSERVICE; 8399 mutex_exit(&stp->sd_qlock); 8400 } 8401 8402 /* 8403 * Replace the cred currently in the mblk with a different one. 8404 * Also update db_cpid. 8405 */ 8406 void 8407 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid) 8408 { 8409 dblk_t *dbp = mp->b_datap; 8410 cred_t *ocr = dbp->db_credp; 8411 8412 ASSERT(cr != NULL); 8413 8414 if (cr != ocr) { 8415 crhold(dbp->db_credp = cr); 8416 if (ocr != NULL) 8417 crfree(ocr); 8418 } 8419 /* Don't overwrite with NOPID */ 8420 if (cpid != NOPID) 8421 dbp->db_cpid = cpid; 8422 } 8423 8424 /* 8425 * If the src message has a cred, then replace the cred currently in the mblk 8426 * with it. 8427 * Also update db_cpid. 8428 */ 8429 void 8430 mblk_copycred(mblk_t *mp, const mblk_t *src) 8431 { 8432 dblk_t *dbp = mp->b_datap; 8433 cred_t *cr, *ocr; 8434 pid_t cpid; 8435 8436 cr = msg_getcred(src, &cpid); 8437 if (cr == NULL) 8438 return; 8439 8440 ocr = dbp->db_credp; 8441 if (cr != ocr) { 8442 crhold(dbp->db_credp = cr); 8443 if (ocr != NULL) 8444 crfree(ocr); 8445 } 8446 /* Don't overwrite with NOPID */ 8447 if (cpid != NOPID) 8448 dbp->db_cpid = cpid; 8449 } 8450 8451 int 8452 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8453 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value, 8454 uint32_t flags, int km_flags) 8455 { 8456 int rc = 0; 8457 8458 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8459 if (mp->b_datap->db_type == M_DATA) { 8460 /* Associate values for M_DATA type */ 8461 DB_CKSUMSTART(mp) = (intptr_t)start; 8462 DB_CKSUMSTUFF(mp) = (intptr_t)stuff; 8463 DB_CKSUMEND(mp) = (intptr_t)end; 8464 DB_CKSUMFLAGS(mp) = flags; 8465 DB_CKSUM16(mp) = (uint16_t)value; 8466 8467 } else { 8468 pattrinfo_t pa_info; 8469 8470 ASSERT(mmd != NULL); 8471 8472 pa_info.type = PATTR_HCKSUM; 8473 pa_info.len = sizeof (pattr_hcksum_t); 8474 8475 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) { 8476 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf; 8477 8478 hck->hcksum_start_offset = start; 8479 hck->hcksum_stuff_offset = stuff; 8480 hck->hcksum_end_offset = end; 8481 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value; 8482 hck->hcksum_flags = flags; 8483 } else { 8484 rc = -1; 8485 } 8486 } 8487 return (rc); 8488 } 8489 8490 void 8491 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8492 uint32_t *start, uint32_t *stuff, uint32_t *end, 8493 uint32_t *value, uint32_t *flags) 8494 { 8495 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8496 if (mp->b_datap->db_type == M_DATA) { 8497 if (flags != NULL) { 8498 *flags = DB_CKSUMFLAGS(mp) & HCK_FLAGS; 8499 if ((*flags & (HCK_PARTIALCKSUM | 8500 HCK_FULLCKSUM)) != 0) { 8501 if (value != NULL) 8502 *value = (uint32_t)DB_CKSUM16(mp); 8503 if ((*flags & HCK_PARTIALCKSUM) != 0) { 8504 if (start != NULL) 8505 *start = 8506 (uint32_t)DB_CKSUMSTART(mp); 8507 if (stuff != NULL) 8508 *stuff = 8509 (uint32_t)DB_CKSUMSTUFF(mp); 8510 if (end != NULL) 8511 *end = 8512 (uint32_t)DB_CKSUMEND(mp); 8513 } 8514 } 8515 } 8516 } else { 8517 pattrinfo_t hck_attr = {PATTR_HCKSUM}; 8518 8519 ASSERT(mmd != NULL); 8520 8521 /* get hardware checksum attribute */ 8522 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) { 8523 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf; 8524 8525 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t)); 8526 if (flags != NULL) 8527 *flags = hck->hcksum_flags; 8528 if (start != NULL) 8529 *start = hck->hcksum_start_offset; 8530 if (stuff != NULL) 8531 *stuff = hck->hcksum_stuff_offset; 8532 if (end != NULL) 8533 *end = hck->hcksum_end_offset; 8534 if (value != NULL) 8535 *value = (uint32_t) 8536 hck->hcksum_cksum_val.inet_cksum; 8537 } 8538 } 8539 } 8540 8541 void 8542 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags) 8543 { 8544 ASSERT(DB_TYPE(mp) == M_DATA); 8545 ASSERT((flags & ~HW_LSO_FLAGS) == 0); 8546 8547 /* Set the flags */ 8548 DB_LSOFLAGS(mp) |= flags; 8549 DB_LSOMSS(mp) = mss; 8550 } 8551 8552 void 8553 lso_info_cleanup(mblk_t *mp) 8554 { 8555 ASSERT(DB_TYPE(mp) == M_DATA); 8556 8557 /* Clear the flags */ 8558 DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS; 8559 DB_LSOMSS(mp) = 0; 8560 } 8561 8562 /* 8563 * Checksum buffer *bp for len bytes with psum partial checksum, 8564 * or 0 if none, and return the 16 bit partial checksum. 8565 */ 8566 unsigned 8567 bcksum(uchar_t *bp, int len, unsigned int psum) 8568 { 8569 int odd = len & 1; 8570 extern unsigned int ip_ocsum(); 8571 8572 if (((intptr_t)bp & 1) == 0 && !odd) { 8573 /* 8574 * Bp is 16 bit aligned and len is multiple of 16 bit word. 8575 */ 8576 return (ip_ocsum((ushort_t *)bp, len >> 1, psum)); 8577 } 8578 if (((intptr_t)bp & 1) != 0) { 8579 /* 8580 * Bp isn't 16 bit aligned. 8581 */ 8582 unsigned int tsum; 8583 8584 #ifdef _LITTLE_ENDIAN 8585 psum += *bp; 8586 #else 8587 psum += *bp << 8; 8588 #endif 8589 len--; 8590 bp++; 8591 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0); 8592 psum += (tsum << 8) & 0xffff | (tsum >> 8); 8593 if (len & 1) { 8594 bp += len - 1; 8595 #ifdef _LITTLE_ENDIAN 8596 psum += *bp << 8; 8597 #else 8598 psum += *bp; 8599 #endif 8600 } 8601 } else { 8602 /* 8603 * Bp is 16 bit aligned. 8604 */ 8605 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum); 8606 if (odd) { 8607 bp += len - 1; 8608 #ifdef _LITTLE_ENDIAN 8609 psum += *bp; 8610 #else 8611 psum += *bp << 8; 8612 #endif 8613 } 8614 } 8615 /* 8616 * Normalize psum to 16 bits before returning the new partial 8617 * checksum. The max psum value before normalization is 0x3FDFE. 8618 */ 8619 return ((psum >> 16) + (psum & 0xFFFF)); 8620 } 8621 8622 boolean_t 8623 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd) 8624 { 8625 boolean_t rc; 8626 8627 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8628 if (DB_TYPE(mp) == M_DATA) { 8629 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0); 8630 } else { 8631 pattrinfo_t zcopy_attr = {PATTR_ZCOPY}; 8632 8633 ASSERT(mmd != NULL); 8634 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL); 8635 } 8636 return (rc); 8637 } 8638 8639 void 8640 freemsgchain(mblk_t *mp) 8641 { 8642 mblk_t *next; 8643 8644 while (mp != NULL) { 8645 next = mp->b_next; 8646 mp->b_next = NULL; 8647 8648 freemsg(mp); 8649 mp = next; 8650 } 8651 } 8652 8653 mblk_t * 8654 copymsgchain(mblk_t *mp) 8655 { 8656 mblk_t *nmp = NULL; 8657 mblk_t **nmpp = &nmp; 8658 8659 for (; mp != NULL; mp = mp->b_next) { 8660 if ((*nmpp = copymsg(mp)) == NULL) { 8661 freemsgchain(nmp); 8662 return (NULL); 8663 } 8664 8665 nmpp = &((*nmpp)->b_next); 8666 } 8667 8668 return (nmp); 8669 } 8670 8671 /* NOTE: Do not add code after this point. */ 8672 #undef QLOCK 8673 8674 /* 8675 * Replacement for QLOCK macro for those that can't use it. 8676 */ 8677 kmutex_t * 8678 QLOCK(queue_t *q) 8679 { 8680 return (&(q)->q_lock); 8681 } 8682 8683 /* 8684 * Dummy runqueues/queuerun functions functions for backwards compatibility. 8685 */ 8686 #undef runqueues 8687 void 8688 runqueues(void) 8689 { 8690 } 8691 8692 #undef queuerun 8693 void 8694 queuerun(void) 8695 { 8696 } 8697 8698 /* 8699 * Initialize the STR stack instance, which tracks autopush and persistent 8700 * links. 8701 */ 8702 /* ARGSUSED */ 8703 static void * 8704 str_stack_init(netstackid_t stackid, netstack_t *ns) 8705 { 8706 str_stack_t *ss; 8707 int i; 8708 8709 ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP); 8710 ss->ss_netstack = ns; 8711 8712 /* 8713 * set up autopush 8714 */ 8715 sad_initspace(ss); 8716 8717 /* 8718 * set up mux_node structures. 8719 */ 8720 ss->ss_devcnt = devcnt; /* In case it should change before free */ 8721 ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) * 8722 ss->ss_devcnt), KM_SLEEP); 8723 for (i = 0; i < ss->ss_devcnt; i++) 8724 ss->ss_mux_nodes[i].mn_imaj = i; 8725 return (ss); 8726 } 8727 8728 /* 8729 * Note: run at zone shutdown and not destroy so that the PLINKs are 8730 * gone by the time other cleanup happens from the destroy callbacks. 8731 */ 8732 static void 8733 str_stack_shutdown(netstackid_t stackid, void *arg) 8734 { 8735 str_stack_t *ss = (str_stack_t *)arg; 8736 int i; 8737 cred_t *cr; 8738 8739 cr = zone_get_kcred(netstackid_to_zoneid(stackid)); 8740 ASSERT(cr != NULL); 8741 8742 /* Undo all the I_PLINKs for this zone */ 8743 for (i = 0; i < ss->ss_devcnt; i++) { 8744 struct mux_edge *ep; 8745 ldi_handle_t lh; 8746 ldi_ident_t li; 8747 int ret; 8748 int rval; 8749 dev_t rdev; 8750 8751 ep = ss->ss_mux_nodes[i].mn_outp; 8752 if (ep == NULL) 8753 continue; 8754 ret = ldi_ident_from_major((major_t)i, &li); 8755 if (ret != 0) { 8756 continue; 8757 } 8758 rdev = ep->me_dev; 8759 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE, 8760 cr, &lh, li); 8761 if (ret != 0) { 8762 ldi_ident_release(li); 8763 continue; 8764 } 8765 8766 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL, 8767 cr, &rval); 8768 if (ret) { 8769 (void) ldi_close(lh, FREAD|FWRITE, cr); 8770 ldi_ident_release(li); 8771 continue; 8772 } 8773 (void) ldi_close(lh, FREAD|FWRITE, cr); 8774 8775 /* Close layered handles */ 8776 ldi_ident_release(li); 8777 } 8778 crfree(cr); 8779 8780 sad_freespace(ss); 8781 8782 kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt); 8783 ss->ss_mux_nodes = NULL; 8784 } 8785 8786 /* 8787 * Free the structure; str_stack_shutdown did the other cleanup work. 8788 */ 8789 /* ARGSUSED */ 8790 static void 8791 str_stack_fini(netstackid_t stackid, void *arg) 8792 { 8793 str_stack_t *ss = (str_stack_t *)arg; 8794 8795 kmem_free(ss, sizeof (*ss)); 8796 } 8797