1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 22 /* All Rights Reserved */ 23 24 25 /* 26 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 27 * Use is subject to license terms. 28 * Copyright (c) 2016 by Delphix. All rights reserved. 29 * Copyright 2018 OmniOS Community Edition (OmniOSce) Association. 30 * Copyright 2018 Joyent, Inc. 31 */ 32 33 #include <sys/types.h> 34 #include <sys/sysmacros.h> 35 #include <sys/param.h> 36 #include <sys/errno.h> 37 #include <sys/signal.h> 38 #include <sys/proc.h> 39 #include <sys/conf.h> 40 #include <sys/cred.h> 41 #include <sys/user.h> 42 #include <sys/vnode.h> 43 #include <sys/file.h> 44 #include <sys/session.h> 45 #include <sys/stream.h> 46 #include <sys/strsubr.h> 47 #include <sys/stropts.h> 48 #include <sys/poll.h> 49 #include <sys/systm.h> 50 #include <sys/cpuvar.h> 51 #include <sys/uio.h> 52 #include <sys/cmn_err.h> 53 #include <sys/priocntl.h> 54 #include <sys/procset.h> 55 #include <sys/vmem.h> 56 #include <sys/bitmap.h> 57 #include <sys/kmem.h> 58 #include <sys/siginfo.h> 59 #include <sys/vtrace.h> 60 #include <sys/callb.h> 61 #include <sys/debug.h> 62 #include <sys/modctl.h> 63 #include <sys/vmsystm.h> 64 #include <vm/page.h> 65 #include <sys/atomic.h> 66 #include <sys/suntpi.h> 67 #include <sys/strlog.h> 68 #include <sys/promif.h> 69 #include <sys/project.h> 70 #include <sys/vm.h> 71 #include <sys/taskq.h> 72 #include <sys/sunddi.h> 73 #include <sys/sunldi_impl.h> 74 #include <sys/strsun.h> 75 #include <sys/isa_defs.h> 76 #include <sys/multidata.h> 77 #include <sys/pattr.h> 78 #include <sys/strft.h> 79 #include <sys/fs/snode.h> 80 #include <sys/zone.h> 81 #include <sys/open.h> 82 #include <sys/sunldi.h> 83 #include <sys/sad.h> 84 #include <sys/netstack.h> 85 86 #define O_SAMESTR(q) (((q)->q_next) && \ 87 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR))) 88 89 /* 90 * WARNING: 91 * The variables and routines in this file are private, belonging 92 * to the STREAMS subsystem. These should not be used by modules 93 * or drivers. Compatibility will not be guaranteed. 94 */ 95 96 /* 97 * Id value used to distinguish between different multiplexor links. 98 */ 99 static int32_t lnk_id = 0; 100 101 #define STREAMS_LOPRI MINCLSYSPRI 102 static pri_t streams_lopri = STREAMS_LOPRI; 103 104 #define STRSTAT(x) (str_statistics.x.value.ui64++) 105 typedef struct str_stat { 106 kstat_named_t sqenables; 107 kstat_named_t stenables; 108 kstat_named_t syncqservice; 109 kstat_named_t freebs; 110 kstat_named_t qwr_outer; 111 kstat_named_t rservice; 112 kstat_named_t strwaits; 113 kstat_named_t taskqfails; 114 kstat_named_t bufcalls; 115 kstat_named_t qhelps; 116 kstat_named_t qremoved; 117 kstat_named_t sqremoved; 118 kstat_named_t bcwaits; 119 kstat_named_t sqtoomany; 120 } str_stat_t; 121 122 static str_stat_t str_statistics = { 123 { "sqenables", KSTAT_DATA_UINT64 }, 124 { "stenables", KSTAT_DATA_UINT64 }, 125 { "syncqservice", KSTAT_DATA_UINT64 }, 126 { "freebs", KSTAT_DATA_UINT64 }, 127 { "qwr_outer", KSTAT_DATA_UINT64 }, 128 { "rservice", KSTAT_DATA_UINT64 }, 129 { "strwaits", KSTAT_DATA_UINT64 }, 130 { "taskqfails", KSTAT_DATA_UINT64 }, 131 { "bufcalls", KSTAT_DATA_UINT64 }, 132 { "qhelps", KSTAT_DATA_UINT64 }, 133 { "qremoved", KSTAT_DATA_UINT64 }, 134 { "sqremoved", KSTAT_DATA_UINT64 }, 135 { "bcwaits", KSTAT_DATA_UINT64 }, 136 { "sqtoomany", KSTAT_DATA_UINT64 }, 137 }; 138 139 static kstat_t *str_kstat; 140 141 /* 142 * qrunflag was used previously to control background scheduling of queues. It 143 * is not used anymore, but kept here in case some module still wants to access 144 * it via qready() and setqsched macros. 145 */ 146 char qrunflag; /* Unused */ 147 148 /* 149 * Most of the streams scheduling is done via task queues. Task queues may fail 150 * for non-sleep dispatches, so there are two backup threads servicing failed 151 * requests for queues and syncqs. Both of these threads also service failed 152 * dispatches freebs requests. Queues are put in the list specified by `qhead' 153 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs 154 * requests are put into `freebs_list' which has no tail pointer. All three 155 * lists are protected by a single `service_queue' lock and use 156 * `services_to_run' condition variable for signaling background threads. Use of 157 * a single lock should not be a problem because it is only used under heavy 158 * loads when task queues start to fail and at that time it may be a good idea 159 * to throttle scheduling requests. 160 * 161 * NOTE: queues and syncqs should be scheduled by two separate threads because 162 * queue servicing may be blocked waiting for a syncq which may be also 163 * scheduled for background execution. This may create a deadlock when only one 164 * thread is used for both. 165 */ 166 167 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */ 168 169 static kmutex_t service_queue; /* protects all of servicing vars */ 170 static kcondvar_t services_to_run; /* wake up background service thread */ 171 static kcondvar_t syncqs_to_run; /* wake up background service thread */ 172 173 /* 174 * List of queues scheduled for background processing due to lack of resources 175 * in the task queues. Protected by service_queue lock; 176 */ 177 static struct queue *qhead; 178 static struct queue *qtail; 179 180 /* 181 * Same list for syncqs 182 */ 183 static syncq_t *sqhead; 184 static syncq_t *sqtail; 185 186 static mblk_t *freebs_list; /* list of buffers to free */ 187 188 /* 189 * Backup threads for servicing queues and syncqs 190 */ 191 kthread_t *streams_qbkgrnd_thread; 192 kthread_t *streams_sqbkgrnd_thread; 193 194 /* 195 * Bufcalls related variables. 196 */ 197 struct bclist strbcalls; /* list of waiting bufcalls */ 198 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */ 199 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */ 200 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */ 201 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */ 202 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */ 203 204 kmutex_t strresources; /* protects global resources */ 205 kmutex_t muxifier; /* single-threads multiplexor creation */ 206 207 static void *str_stack_init(netstackid_t stackid, netstack_t *ns); 208 static void str_stack_shutdown(netstackid_t stackid, void *arg); 209 static void str_stack_fini(netstackid_t stackid, void *arg); 210 211 /* 212 * run_queues is no longer used, but is kept in case some 3rd party 213 * module/driver decides to use it. 214 */ 215 int run_queues = 0; 216 217 /* 218 * sq_max_size is the depth of the syncq (in number of messages) before 219 * qfill_syncq() starts QFULL'ing destination queues. As its primary 220 * consumer - IP is no longer D_MTPERMOD, but there may be other 221 * modules/drivers depend on this syncq flow control, we prefer to 222 * choose a large number as the default value. For potential 223 * performance gain, this value is tunable in /etc/system. 224 */ 225 int sq_max_size = 10000; 226 227 /* 228 * The number of ciputctrl structures per syncq and stream we create when 229 * needed. 230 */ 231 int n_ciputctrl; 232 int max_n_ciputctrl = 16; 233 /* 234 * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache. 235 */ 236 int min_n_ciputctrl = 2; 237 238 /* 239 * Per-driver/module syncqs 240 * ======================== 241 * 242 * For drivers/modules that use PERMOD or outer syncqs we keep a list of 243 * perdm structures, new entries being added (and new syncqs allocated) when 244 * setq() encounters a module/driver with a streamtab that it hasn't seen 245 * before. 246 * The reason for this mechanism is that some modules and drivers share a 247 * common streamtab and it is necessary for those modules and drivers to also 248 * share a common PERMOD syncq. 249 * 250 * perdm_list --> dm_str == streamtab_1 251 * dm_sq == syncq_1 252 * dm_ref 253 * dm_next --> dm_str == streamtab_2 254 * dm_sq == syncq_2 255 * dm_ref 256 * dm_next --> ... NULL 257 * 258 * The dm_ref field is incremented for each new driver/module that takes 259 * a reference to the perdm structure and hence shares the syncq. 260 * References are held in the fmodsw_impl_t structure for each STREAMS module 261 * or the dev_impl array (indexed by device major number) for each driver. 262 * 263 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL 264 * ^ ^ ^ ^ 265 * | ______________/ | | 266 * | / | | 267 * dev_impl: ...|x|y|... module A module B 268 * 269 * When a module/driver is unloaded the reference count is decremented and, 270 * when it falls to zero, the perdm structure is removed from the list and 271 * the syncq is freed (see rele_dm()). 272 */ 273 perdm_t *perdm_list = NULL; 274 static krwlock_t perdm_rwlock; 275 cdevsw_impl_t *devimpl; 276 277 extern struct qinit strdata; 278 extern struct qinit stwdata; 279 280 static void runservice(queue_t *); 281 static void streams_bufcall_service(void); 282 static void streams_qbkgrnd_service(void); 283 static void streams_sqbkgrnd_service(void); 284 static syncq_t *new_syncq(void); 285 static void free_syncq(syncq_t *); 286 static void outer_insert(syncq_t *, syncq_t *); 287 static void outer_remove(syncq_t *, syncq_t *); 288 static void write_now(syncq_t *); 289 static void clr_qfull(queue_t *); 290 static void runbufcalls(void); 291 static void sqenable(syncq_t *); 292 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)()); 293 static void wait_q_syncq(queue_t *); 294 static void backenable_insertedq(queue_t *); 295 296 static void queue_service(queue_t *); 297 static void stream_service(stdata_t *); 298 static void syncq_service(syncq_t *); 299 static void qwriter_outer_service(syncq_t *); 300 static void mblk_free(mblk_t *); 301 #ifdef DEBUG 302 static int qprocsareon(queue_t *); 303 #endif 304 305 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *); 306 static void reset_nfsrv_ptr(queue_t *, queue_t *); 307 void set_qfull(queue_t *); 308 309 static void sq_run_events(syncq_t *); 310 static int propagate_syncq(queue_t *); 311 312 static void blocksq(syncq_t *, ushort_t, int); 313 static void unblocksq(syncq_t *, ushort_t, int); 314 static int dropsq(syncq_t *, uint16_t); 315 static void emptysq(syncq_t *); 316 static sqlist_t *sqlist_alloc(struct stdata *, int); 317 static void sqlist_free(sqlist_t *); 318 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t); 319 static void sqlist_insert(sqlist_t *, syncq_t *); 320 static void sqlist_insertall(sqlist_t *, queue_t *); 321 322 static void strsetuio(stdata_t *); 323 324 struct kmem_cache *stream_head_cache; 325 struct kmem_cache *queue_cache; 326 struct kmem_cache *syncq_cache; 327 struct kmem_cache *qband_cache; 328 struct kmem_cache *linkinfo_cache; 329 struct kmem_cache *ciputctrl_cache = NULL; 330 331 static linkinfo_t *linkinfo_list; 332 333 /* Global esballoc throttling queue */ 334 static esb_queue_t system_esbq; 335 336 /* Array of esballoc throttling queues, of length esbq_nelem */ 337 static esb_queue_t *volatile system_esbq_array; 338 static int esbq_nelem; 339 static kmutex_t esbq_lock; 340 static int esbq_log2_cpus_per_q = 0; 341 342 /* Scale the system_esbq length by setting number of CPUs per queue. */ 343 uint_t esbq_cpus_per_q = 1; 344 345 /* 346 * esballoc tunable parameters. 347 */ 348 int esbq_max_qlen = 0x16; /* throttled queue length */ 349 clock_t esbq_timeout = 0x8; /* timeout to process esb queue */ 350 351 /* 352 * Routines to handle esballoc queueing. 353 */ 354 static void esballoc_process_queue(esb_queue_t *); 355 static void esballoc_enqueue_mblk(mblk_t *); 356 static void esballoc_timer(void *); 357 static void esballoc_set_timer(esb_queue_t *, clock_t); 358 static void esballoc_mblk_free(mblk_t *); 359 360 /* 361 * Qinit structure and Module_info structures 362 * for passthru read and write queues 363 */ 364 365 static int pass_rput(queue_t *, mblk_t *); 366 static int pass_wput(queue_t *, mblk_t *); 367 static queue_t *link_addpassthru(stdata_t *); 368 static void link_rempassthru(queue_t *); 369 370 struct module_info passthru_info = { 371 0, 372 "passthru", 373 0, 374 INFPSZ, 375 STRHIGH, 376 STRLOW 377 }; 378 379 struct qinit passthru_rinit = { 380 pass_rput, 381 NULL, 382 NULL, 383 NULL, 384 NULL, 385 &passthru_info, 386 NULL 387 }; 388 389 struct qinit passthru_winit = { 390 pass_wput, 391 NULL, 392 NULL, 393 NULL, 394 NULL, 395 &passthru_info, 396 NULL 397 }; 398 399 /* 400 * Verify correctness of list head/tail pointers. 401 */ 402 #define LISTCHECK(head, tail, link) { \ 403 EQUIV(head, tail); \ 404 IMPLY(tail != NULL, tail->link == NULL); \ 405 } 406 407 /* 408 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail' 409 * using a `link' field. 410 */ 411 #define ENQUEUE(el, head, tail, link) { \ 412 ASSERT(el->link == NULL); \ 413 LISTCHECK(head, tail, link); \ 414 if (head == NULL) \ 415 head = el; \ 416 else \ 417 tail->link = el; \ 418 tail = el; \ 419 } 420 421 /* 422 * Dequeue the first element of the list denoted by `head' and `tail' pointers 423 * using a `link' field and put result into `el'. 424 */ 425 #define DQ(el, head, tail, link) { \ 426 LISTCHECK(head, tail, link); \ 427 el = head; \ 428 if (head != NULL) { \ 429 head = head->link; \ 430 if (head == NULL) \ 431 tail = NULL; \ 432 el->link = NULL; \ 433 } \ 434 } 435 436 /* 437 * Remove `el' from the list using `chase' and `curr' pointers and return result 438 * in `succeed'. 439 */ 440 #define RMQ(el, head, tail, link, chase, curr, succeed) { \ 441 LISTCHECK(head, tail, link); \ 442 chase = NULL; \ 443 succeed = 0; \ 444 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \ 445 chase = curr; \ 446 if (curr != NULL) { \ 447 succeed = 1; \ 448 ASSERT(curr == el); \ 449 if (chase != NULL) \ 450 chase->link = curr->link; \ 451 else \ 452 head = curr->link; \ 453 curr->link = NULL; \ 454 if (curr == tail) \ 455 tail = chase; \ 456 } \ 457 LISTCHECK(head, tail, link); \ 458 } 459 460 /* Handling of delayed messages on the inner syncq. */ 461 462 /* 463 * DEBUG versions should use function versions (to simplify tracing) and 464 * non-DEBUG kernels should use macro versions. 465 */ 466 467 /* 468 * Put a queue on the syncq list of queues. 469 * Assumes SQLOCK held. 470 */ 471 #define SQPUT_Q(sq, qp) \ 472 { \ 473 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 474 if (!(qp->q_sqflags & Q_SQQUEUED)) { \ 475 /* The queue should not be linked anywhere */ \ 476 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \ 477 /* Head and tail may only be NULL simultaneously */ \ 478 EQUIV(sq->sq_head, sq->sq_tail); \ 479 /* Queue may be only enqueued on its syncq */ \ 480 ASSERT(sq == qp->q_syncq); \ 481 /* Check the correctness of SQ_MESSAGES flag */ \ 482 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \ 483 /* Sanity check first/last elements of the list */ \ 484 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\ 485 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\ 486 /* \ 487 * Sanity check of priority field: empty queue should \ 488 * have zero priority \ 489 * and nqueues equal to zero. \ 490 */ \ 491 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \ 492 /* Sanity check of sq_nqueues field */ \ 493 EQUIV(sq->sq_head, sq->sq_nqueues); \ 494 if (sq->sq_head == NULL) { \ 495 sq->sq_head = sq->sq_tail = qp; \ 496 sq->sq_flags |= SQ_MESSAGES; \ 497 } else if (qp->q_spri == 0) { \ 498 qp->q_sqprev = sq->sq_tail; \ 499 sq->sq_tail->q_sqnext = qp; \ 500 sq->sq_tail = qp; \ 501 } else { \ 502 /* \ 503 * Put this queue in priority order: higher \ 504 * priority gets closer to the head. \ 505 */ \ 506 queue_t **qpp = &sq->sq_tail; \ 507 queue_t *qnext = NULL; \ 508 \ 509 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \ 510 qnext = *qpp; \ 511 qpp = &(*qpp)->q_sqprev; \ 512 } \ 513 qp->q_sqnext = qnext; \ 514 qp->q_sqprev = *qpp; \ 515 if (*qpp != NULL) { \ 516 (*qpp)->q_sqnext = qp; \ 517 } else { \ 518 sq->sq_head = qp; \ 519 sq->sq_pri = sq->sq_head->q_spri; \ 520 } \ 521 *qpp = qp; \ 522 } \ 523 qp->q_sqflags |= Q_SQQUEUED; \ 524 qp->q_sqtstamp = ddi_get_lbolt(); \ 525 sq->sq_nqueues++; \ 526 } \ 527 } 528 529 /* 530 * Remove a queue from the syncq list 531 * Assumes SQLOCK held. 532 */ 533 #define SQRM_Q(sq, qp) \ 534 { \ 535 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 536 ASSERT(qp->q_sqflags & Q_SQQUEUED); \ 537 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \ 538 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \ 539 /* Check that the queue is actually in the list */ \ 540 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \ 541 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \ 542 ASSERT(sq->sq_nqueues != 0); \ 543 if (qp->q_sqprev == NULL) { \ 544 /* First queue on list, make head q_sqnext */ \ 545 sq->sq_head = qp->q_sqnext; \ 546 } else { \ 547 /* Make prev->next == next */ \ 548 qp->q_sqprev->q_sqnext = qp->q_sqnext; \ 549 } \ 550 if (qp->q_sqnext == NULL) { \ 551 /* Last queue on list, make tail sqprev */ \ 552 sq->sq_tail = qp->q_sqprev; \ 553 } else { \ 554 /* Make next->prev == prev */ \ 555 qp->q_sqnext->q_sqprev = qp->q_sqprev; \ 556 } \ 557 /* clear out references on this queue */ \ 558 qp->q_sqprev = qp->q_sqnext = NULL; \ 559 qp->q_sqflags &= ~Q_SQQUEUED; \ 560 /* If there is nothing queued, clear SQ_MESSAGES */ \ 561 if (sq->sq_head != NULL) { \ 562 sq->sq_pri = sq->sq_head->q_spri; \ 563 } else { \ 564 sq->sq_flags &= ~SQ_MESSAGES; \ 565 sq->sq_pri = 0; \ 566 } \ 567 sq->sq_nqueues--; \ 568 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \ 569 (sq->sq_flags & SQ_QUEUED) == 0); \ 570 } 571 572 /* Hide the definition from the header file. */ 573 #ifdef SQPUT_MP 574 #undef SQPUT_MP 575 #endif 576 577 /* 578 * Put a message on the queue syncq. 579 * Assumes QLOCK held. 580 */ 581 #define SQPUT_MP(qp, mp) \ 582 { \ 583 ASSERT(MUTEX_HELD(QLOCK(qp))); \ 584 ASSERT(qp->q_sqhead == NULL || \ 585 (qp->q_sqtail != NULL && \ 586 qp->q_sqtail->b_next == NULL)); \ 587 qp->q_syncqmsgs++; \ 588 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \ 589 if (qp->q_sqhead == NULL) { \ 590 qp->q_sqhead = qp->q_sqtail = mp; \ 591 } else { \ 592 qp->q_sqtail->b_next = mp; \ 593 qp->q_sqtail = mp; \ 594 } \ 595 ASSERT(qp->q_syncqmsgs > 0); \ 596 set_qfull(qp); \ 597 } 598 599 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ 600 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 601 if ((sq)->sq_ciputctrl != NULL) { \ 602 int i; \ 603 int nlocks = (sq)->sq_nciputctrl; \ 604 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 605 ASSERT((sq)->sq_type & SQ_CIPUT); \ 606 for (i = 0; i <= nlocks; i++) { \ 607 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 608 cip[i].ciputctrl_count |= SQ_FASTPUT; \ 609 } \ 610 } \ 611 } 612 613 614 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ 615 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 616 if ((sq)->sq_ciputctrl != NULL) { \ 617 int i; \ 618 int nlocks = (sq)->sq_nciputctrl; \ 619 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 620 ASSERT((sq)->sq_type & SQ_CIPUT); \ 621 for (i = 0; i <= nlocks; i++) { \ 622 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 623 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \ 624 } \ 625 } \ 626 } 627 628 /* 629 * Run service procedures for all queues in the stream head. 630 */ 631 #define STR_SERVICE(stp, q) { \ 632 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \ 633 while (stp->sd_qhead != NULL) { \ 634 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \ 635 ASSERT(stp->sd_nqueues > 0); \ 636 stp->sd_nqueues--; \ 637 ASSERT(!(q->q_flag & QINSERVICE)); \ 638 mutex_exit(&stp->sd_qlock); \ 639 queue_service(q); \ 640 mutex_enter(&stp->sd_qlock); \ 641 } \ 642 ASSERT(stp->sd_nqueues == 0); \ 643 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \ 644 } 645 646 /* 647 * Constructor/destructor routines for the stream head cache 648 */ 649 /* ARGSUSED */ 650 static int 651 stream_head_constructor(void *buf, void *cdrarg, int kmflags) 652 { 653 stdata_t *stp = buf; 654 655 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL); 656 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL); 657 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL); 658 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL); 659 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL); 660 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL); 661 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL); 662 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL); 663 stp->sd_wrq = NULL; 664 665 return (0); 666 } 667 668 /* ARGSUSED */ 669 static void 670 stream_head_destructor(void *buf, void *cdrarg) 671 { 672 stdata_t *stp = buf; 673 674 mutex_destroy(&stp->sd_lock); 675 mutex_destroy(&stp->sd_reflock); 676 mutex_destroy(&stp->sd_qlock); 677 cv_destroy(&stp->sd_monitor); 678 cv_destroy(&stp->sd_iocmonitor); 679 cv_destroy(&stp->sd_refmonitor); 680 cv_destroy(&stp->sd_qcv); 681 cv_destroy(&stp->sd_zcopy_wait); 682 } 683 684 /* 685 * Constructor/destructor routines for the queue cache 686 */ 687 /* ARGSUSED */ 688 static int 689 queue_constructor(void *buf, void *cdrarg, int kmflags) 690 { 691 queinfo_t *qip = buf; 692 queue_t *qp = &qip->qu_rqueue; 693 queue_t *wqp = &qip->qu_wqueue; 694 syncq_t *sq = &qip->qu_syncq; 695 696 qp->q_first = NULL; 697 qp->q_link = NULL; 698 qp->q_count = 0; 699 qp->q_mblkcnt = 0; 700 qp->q_sqhead = NULL; 701 qp->q_sqtail = NULL; 702 qp->q_sqnext = NULL; 703 qp->q_sqprev = NULL; 704 qp->q_sqflags = 0; 705 qp->q_rwcnt = 0; 706 qp->q_spri = 0; 707 708 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL); 709 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL); 710 711 wqp->q_first = NULL; 712 wqp->q_link = NULL; 713 wqp->q_count = 0; 714 wqp->q_mblkcnt = 0; 715 wqp->q_sqhead = NULL; 716 wqp->q_sqtail = NULL; 717 wqp->q_sqnext = NULL; 718 wqp->q_sqprev = NULL; 719 wqp->q_sqflags = 0; 720 wqp->q_rwcnt = 0; 721 wqp->q_spri = 0; 722 723 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL); 724 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL); 725 726 sq->sq_head = NULL; 727 sq->sq_tail = NULL; 728 sq->sq_evhead = NULL; 729 sq->sq_evtail = NULL; 730 sq->sq_callbpend = NULL; 731 sq->sq_outer = NULL; 732 sq->sq_onext = NULL; 733 sq->sq_oprev = NULL; 734 sq->sq_next = NULL; 735 sq->sq_svcflags = 0; 736 sq->sq_servcount = 0; 737 sq->sq_needexcl = 0; 738 sq->sq_nqueues = 0; 739 sq->sq_pri = 0; 740 741 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 742 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 743 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 744 745 return (0); 746 } 747 748 /* ARGSUSED */ 749 static void 750 queue_destructor(void *buf, void *cdrarg) 751 { 752 queinfo_t *qip = buf; 753 queue_t *qp = &qip->qu_rqueue; 754 queue_t *wqp = &qip->qu_wqueue; 755 syncq_t *sq = &qip->qu_syncq; 756 757 ASSERT(qp->q_sqhead == NULL); 758 ASSERT(wqp->q_sqhead == NULL); 759 ASSERT(qp->q_sqnext == NULL); 760 ASSERT(wqp->q_sqnext == NULL); 761 ASSERT(qp->q_rwcnt == 0); 762 ASSERT(wqp->q_rwcnt == 0); 763 764 mutex_destroy(&qp->q_lock); 765 cv_destroy(&qp->q_wait); 766 767 mutex_destroy(&wqp->q_lock); 768 cv_destroy(&wqp->q_wait); 769 770 mutex_destroy(&sq->sq_lock); 771 cv_destroy(&sq->sq_wait); 772 cv_destroy(&sq->sq_exitwait); 773 } 774 775 /* 776 * Constructor/destructor routines for the syncq cache 777 */ 778 /* ARGSUSED */ 779 static int 780 syncq_constructor(void *buf, void *cdrarg, int kmflags) 781 { 782 syncq_t *sq = buf; 783 784 bzero(buf, sizeof (syncq_t)); 785 786 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 787 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 788 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 789 790 return (0); 791 } 792 793 /* ARGSUSED */ 794 static void 795 syncq_destructor(void *buf, void *cdrarg) 796 { 797 syncq_t *sq = buf; 798 799 ASSERT(sq->sq_head == NULL); 800 ASSERT(sq->sq_tail == NULL); 801 ASSERT(sq->sq_evhead == NULL); 802 ASSERT(sq->sq_evtail == NULL); 803 ASSERT(sq->sq_callbpend == NULL); 804 ASSERT(sq->sq_callbflags == 0); 805 ASSERT(sq->sq_outer == NULL); 806 ASSERT(sq->sq_onext == NULL); 807 ASSERT(sq->sq_oprev == NULL); 808 ASSERT(sq->sq_next == NULL); 809 ASSERT(sq->sq_needexcl == 0); 810 ASSERT(sq->sq_svcflags == 0); 811 ASSERT(sq->sq_servcount == 0); 812 ASSERT(sq->sq_nqueues == 0); 813 ASSERT(sq->sq_pri == 0); 814 ASSERT(sq->sq_count == 0); 815 ASSERT(sq->sq_rmqcount == 0); 816 ASSERT(sq->sq_cancelid == 0); 817 ASSERT(sq->sq_ciputctrl == NULL); 818 ASSERT(sq->sq_nciputctrl == 0); 819 ASSERT(sq->sq_type == 0); 820 ASSERT(sq->sq_flags == 0); 821 822 mutex_destroy(&sq->sq_lock); 823 cv_destroy(&sq->sq_wait); 824 cv_destroy(&sq->sq_exitwait); 825 } 826 827 /* ARGSUSED */ 828 static int 829 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags) 830 { 831 ciputctrl_t *cip = buf; 832 int i; 833 834 for (i = 0; i < n_ciputctrl; i++) { 835 cip[i].ciputctrl_count = SQ_FASTPUT; 836 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL); 837 } 838 839 return (0); 840 } 841 842 /* ARGSUSED */ 843 static void 844 ciputctrl_destructor(void *buf, void *cdrarg) 845 { 846 ciputctrl_t *cip = buf; 847 int i; 848 849 for (i = 0; i < n_ciputctrl; i++) { 850 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT); 851 mutex_destroy(&cip[i].ciputctrl_lock); 852 } 853 } 854 855 /* 856 * Init routine run from main at boot time. 857 */ 858 void 859 strinit(void) 860 { 861 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); 862 863 stream_head_cache = kmem_cache_create("stream_head_cache", 864 sizeof (stdata_t), 0, 865 stream_head_constructor, stream_head_destructor, NULL, 866 NULL, NULL, 0); 867 868 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0, 869 queue_constructor, queue_destructor, NULL, NULL, NULL, 0); 870 871 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0, 872 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0); 873 874 qband_cache = kmem_cache_create("qband_cache", 875 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 876 877 linkinfo_cache = kmem_cache_create("linkinfo_cache", 878 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 879 880 n_ciputctrl = ncpus; 881 n_ciputctrl = 1 << highbit(n_ciputctrl - 1); 882 ASSERT(n_ciputctrl >= 1); 883 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl); 884 if (n_ciputctrl >= min_n_ciputctrl) { 885 ciputctrl_cache = kmem_cache_create("ciputctrl_cache", 886 sizeof (ciputctrl_t) * n_ciputctrl, 887 sizeof (ciputctrl_t), ciputctrl_constructor, 888 ciputctrl_destructor, NULL, NULL, NULL, 0); 889 } 890 891 streams_taskq = system_taskq; 892 893 if (streams_taskq == NULL) 894 panic("strinit: no memory for streams taskq!"); 895 896 bc_bkgrnd_thread = thread_create(NULL, 0, 897 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri); 898 899 streams_qbkgrnd_thread = thread_create(NULL, 0, 900 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 901 902 streams_sqbkgrnd_thread = thread_create(NULL, 0, 903 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 904 905 /* 906 * Create STREAMS kstats. 907 */ 908 str_kstat = kstat_create("streams", 0, "strstat", 909 "net", KSTAT_TYPE_NAMED, 910 sizeof (str_statistics) / sizeof (kstat_named_t), 911 KSTAT_FLAG_VIRTUAL); 912 913 if (str_kstat != NULL) { 914 str_kstat->ks_data = &str_statistics; 915 kstat_install(str_kstat); 916 } 917 918 /* 919 * TPI support routine initialisation. 920 */ 921 tpi_init(); 922 923 /* 924 * Handle to have autopush and persistent link information per 925 * zone. 926 * Note: uses shutdown hook instead of destroy hook so that the 927 * persistent links can be torn down before the destroy hooks 928 * in the TCP/IP stack are called. 929 */ 930 netstack_register(NS_STR, str_stack_init, str_stack_shutdown, 931 str_stack_fini); 932 } 933 934 void 935 str_sendsig(vnode_t *vp, int event, uchar_t band, int error) 936 { 937 struct stdata *stp; 938 939 ASSERT(vp->v_stream); 940 stp = vp->v_stream; 941 /* Have to hold sd_lock to prevent siglist from changing */ 942 mutex_enter(&stp->sd_lock); 943 if (stp->sd_sigflags & event) 944 strsendsig(stp->sd_siglist, event, band, error); 945 mutex_exit(&stp->sd_lock); 946 } 947 948 /* 949 * Send the "sevent" set of signals to a process. 950 * This might send more than one signal if the process is registered 951 * for multiple events. The caller should pass in an sevent that only 952 * includes the events for which the process has registered. 953 */ 954 static void 955 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info, 956 uchar_t band, int error) 957 { 958 ASSERT(MUTEX_HELD(&proc->p_lock)); 959 960 info->si_band = 0; 961 info->si_errno = 0; 962 963 if (sevent & S_ERROR) { 964 sevent &= ~S_ERROR; 965 info->si_code = POLL_ERR; 966 info->si_errno = error; 967 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 968 "strsendsig:proc %p info %p", proc, info); 969 sigaddq(proc, NULL, info, KM_NOSLEEP); 970 info->si_errno = 0; 971 } 972 if (sevent & S_HANGUP) { 973 sevent &= ~S_HANGUP; 974 info->si_code = POLL_HUP; 975 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 976 "strsendsig:proc %p info %p", proc, info); 977 sigaddq(proc, NULL, info, KM_NOSLEEP); 978 } 979 if (sevent & S_HIPRI) { 980 sevent &= ~S_HIPRI; 981 info->si_code = POLL_PRI; 982 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 983 "strsendsig:proc %p info %p", proc, info); 984 sigaddq(proc, NULL, info, KM_NOSLEEP); 985 } 986 if (sevent & S_RDBAND) { 987 sevent &= ~S_RDBAND; 988 if (events & S_BANDURG) 989 sigtoproc(proc, NULL, SIGURG); 990 else 991 sigtoproc(proc, NULL, SIGPOLL); 992 } 993 if (sevent & S_WRBAND) { 994 sevent &= ~S_WRBAND; 995 sigtoproc(proc, NULL, SIGPOLL); 996 } 997 if (sevent & S_INPUT) { 998 sevent &= ~S_INPUT; 999 info->si_code = POLL_IN; 1000 info->si_band = band; 1001 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1002 "strsendsig:proc %p info %p", proc, info); 1003 sigaddq(proc, NULL, info, KM_NOSLEEP); 1004 info->si_band = 0; 1005 } 1006 if (sevent & S_OUTPUT) { 1007 sevent &= ~S_OUTPUT; 1008 info->si_code = POLL_OUT; 1009 info->si_band = band; 1010 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1011 "strsendsig:proc %p info %p", proc, info); 1012 sigaddq(proc, NULL, info, KM_NOSLEEP); 1013 info->si_band = 0; 1014 } 1015 if (sevent & S_MSG) { 1016 sevent &= ~S_MSG; 1017 info->si_code = POLL_MSG; 1018 info->si_band = band; 1019 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1020 "strsendsig:proc %p info %p", proc, info); 1021 sigaddq(proc, NULL, info, KM_NOSLEEP); 1022 info->si_band = 0; 1023 } 1024 if (sevent & S_RDNORM) { 1025 sevent &= ~S_RDNORM; 1026 sigtoproc(proc, NULL, SIGPOLL); 1027 } 1028 if (sevent != 0) { 1029 panic("strsendsig: unknown event(s) %x", sevent); 1030 } 1031 } 1032 1033 /* 1034 * Send SIGPOLL/SIGURG signal to all processes and process groups 1035 * registered on the given signal list that want a signal for at 1036 * least one of the specified events. 1037 * 1038 * Must be called with exclusive access to siglist (caller holding sd_lock). 1039 * 1040 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding 1041 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure 1042 * while it is in the siglist. 1043 * 1044 * For performance reasons (MP scalability) the code drops pidlock 1045 * when sending signals to a single process. 1046 * When sending to a process group the code holds 1047 * pidlock to prevent the membership in the process group from changing 1048 * while walking the p_pglink list. 1049 */ 1050 void 1051 strsendsig(strsig_t *siglist, int event, uchar_t band, int error) 1052 { 1053 strsig_t *ssp; 1054 k_siginfo_t info; 1055 struct pid *pidp; 1056 proc_t *proc; 1057 1058 info.si_signo = SIGPOLL; 1059 info.si_errno = 0; 1060 for (ssp = siglist; ssp; ssp = ssp->ss_next) { 1061 int sevent; 1062 1063 sevent = ssp->ss_events & event; 1064 if (sevent == 0) 1065 continue; 1066 1067 if ((pidp = ssp->ss_pidp) == NULL) { 1068 /* pid was released but still on event list */ 1069 continue; 1070 } 1071 1072 1073 if (ssp->ss_pid > 0) { 1074 /* 1075 * XXX This unfortunately still generates 1076 * a signal when a fd is closed but 1077 * the proc is active. 1078 */ 1079 ASSERT(ssp->ss_pid == pidp->pid_id); 1080 1081 mutex_enter(&pidlock); 1082 proc = prfind_zone(pidp->pid_id, ALL_ZONES); 1083 if (proc == NULL) { 1084 mutex_exit(&pidlock); 1085 continue; 1086 } 1087 mutex_enter(&proc->p_lock); 1088 mutex_exit(&pidlock); 1089 dosendsig(proc, ssp->ss_events, sevent, &info, 1090 band, error); 1091 mutex_exit(&proc->p_lock); 1092 } else { 1093 /* 1094 * Send to process group. Hold pidlock across 1095 * calls to dosendsig(). 1096 */ 1097 pid_t pgrp = -ssp->ss_pid; 1098 1099 mutex_enter(&pidlock); 1100 proc = pgfind_zone(pgrp, ALL_ZONES); 1101 while (proc != NULL) { 1102 mutex_enter(&proc->p_lock); 1103 dosendsig(proc, ssp->ss_events, sevent, 1104 &info, band, error); 1105 mutex_exit(&proc->p_lock); 1106 proc = proc->p_pglink; 1107 } 1108 mutex_exit(&pidlock); 1109 } 1110 } 1111 } 1112 1113 /* 1114 * Attach a stream device or module. 1115 * qp is a read queue; the new queue goes in so its next 1116 * read ptr is the argument, and the write queue corresponding 1117 * to the argument points to this queue. Return 0 on success, 1118 * or a non-zero errno on failure. 1119 */ 1120 int 1121 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp, 1122 boolean_t is_insert) 1123 { 1124 major_t major; 1125 cdevsw_impl_t *dp; 1126 struct streamtab *str; 1127 queue_t *rq; 1128 queue_t *wrq; 1129 uint32_t qflag; 1130 uint32_t sqtype; 1131 perdm_t *dmp; 1132 int error; 1133 int sflag; 1134 1135 rq = allocq(); 1136 wrq = _WR(rq); 1137 STREAM(rq) = STREAM(wrq) = STREAM(qp); 1138 1139 if (fp != NULL) { 1140 str = fp->f_str; 1141 qflag = fp->f_qflag; 1142 sqtype = fp->f_sqtype; 1143 dmp = fp->f_dmp; 1144 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 1145 sflag = MODOPEN; 1146 1147 /* 1148 * stash away a pointer to the module structure so we can 1149 * unref it in qdetach. 1150 */ 1151 rq->q_fp = fp; 1152 } else { 1153 ASSERT(!is_insert); 1154 1155 major = getmajor(*devp); 1156 dp = &devimpl[major]; 1157 1158 str = dp->d_str; 1159 ASSERT(str == STREAMSTAB(major)); 1160 1161 qflag = dp->d_qflag; 1162 ASSERT(qflag & QISDRV); 1163 sqtype = dp->d_sqtype; 1164 1165 /* create perdm_t if needed */ 1166 if (NEED_DM(dp->d_dmp, qflag)) 1167 dp->d_dmp = hold_dm(str, qflag, sqtype); 1168 1169 dmp = dp->d_dmp; 1170 sflag = 0; 1171 } 1172 1173 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS, 1174 "qattach:qflag == %X(%X)", qflag, *devp); 1175 1176 /* setq might sleep in allocator - avoid holding locks. */ 1177 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE); 1178 1179 /* 1180 * Before calling the module's open routine, set up the q_next 1181 * pointer for inserting a module in the middle of a stream. 1182 * 1183 * Note that we can always set _QINSERTING and set up q_next 1184 * pointer for both inserting and pushing a module. Then there 1185 * is no need for the is_insert parameter. In insertq(), called 1186 * by qprocson(), assume that q_next of the new module always points 1187 * to the correct queue and use it for insertion. Everything should 1188 * work out fine. But in the first release of _I_INSERT, we 1189 * distinguish between inserting and pushing to make sure that 1190 * pushing a module follows the same code path as before. 1191 */ 1192 if (is_insert) { 1193 rq->q_flag |= _QINSERTING; 1194 rq->q_next = qp; 1195 } 1196 1197 /* 1198 * If there is an outer perimeter get exclusive access during 1199 * the open procedure. Bump up the reference count on the queue. 1200 */ 1201 entersq(rq->q_syncq, SQ_OPENCLOSE); 1202 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp); 1203 if (error != 0) 1204 goto failed; 1205 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1206 ASSERT(qprocsareon(rq)); 1207 return (0); 1208 1209 failed: 1210 rq->q_flag &= ~_QINSERTING; 1211 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq) 1212 qprocsoff(rq); 1213 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1214 rq->q_next = wrq->q_next = NULL; 1215 qdetach(rq, 0, 0, crp, B_FALSE); 1216 return (error); 1217 } 1218 1219 /* 1220 * Handle second open of stream. For modules, set the 1221 * last argument to MODOPEN and do not pass any open flags. 1222 * Ignore dummydev since this is not the first open. 1223 */ 1224 int 1225 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp) 1226 { 1227 int error; 1228 dev_t dummydev; 1229 queue_t *wqp = _WR(qp); 1230 1231 ASSERT(qp->q_flag & QREADR); 1232 entersq(qp->q_syncq, SQ_OPENCLOSE); 1233 1234 dummydev = *devp; 1235 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev, 1236 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) { 1237 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1238 mutex_enter(&STREAM(qp)->sd_lock); 1239 qp->q_stream->sd_flag |= STREOPENFAIL; 1240 mutex_exit(&STREAM(qp)->sd_lock); 1241 return (error); 1242 } 1243 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1244 1245 /* 1246 * successful open should have done qprocson() 1247 */ 1248 ASSERT(qprocsareon(_RD(qp))); 1249 return (0); 1250 } 1251 1252 /* 1253 * Detach a stream module or device. 1254 * If clmode == 1 then the module or driver was opened and its 1255 * close routine must be called. If clmode == 0, the module 1256 * or driver was never opened or the open failed, and so its close 1257 * should not be called. 1258 */ 1259 void 1260 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove) 1261 { 1262 queue_t *wqp = _WR(qp); 1263 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB)); 1264 1265 if (STREAM_NEEDSERVICE(STREAM(qp))) 1266 stream_runservice(STREAM(qp)); 1267 1268 if (clmode) { 1269 /* 1270 * Make sure that all the messages on the write side syncq are 1271 * processed and nothing is left. Since we are closing, no new 1272 * messages may appear there. 1273 */ 1274 wait_q_syncq(wqp); 1275 1276 entersq(qp->q_syncq, SQ_OPENCLOSE); 1277 if (is_remove) { 1278 mutex_enter(QLOCK(qp)); 1279 qp->q_flag |= _QREMOVING; 1280 mutex_exit(QLOCK(qp)); 1281 } 1282 (*qp->q_qinfo->qi_qclose)(qp, flag, crp); 1283 /* 1284 * Check that qprocsoff() was actually called. 1285 */ 1286 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE)); 1287 1288 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1289 } else { 1290 disable_svc(qp); 1291 } 1292 1293 /* 1294 * Allow any threads blocked in entersq to proceed and discover 1295 * the QWCLOSE is set. 1296 * Note: This assumes that all users of entersq check QWCLOSE. 1297 * Currently runservice is the only entersq that can happen 1298 * after removeq has finished. 1299 * Removeq will have discarded all messages destined to the closing 1300 * pair of queues from the syncq. 1301 * NOTE: Calling a function inside an assert is unconventional. 1302 * However, it does not cause any problem since flush_syncq() does 1303 * not change any state except when it returns non-zero i.e. 1304 * when the assert will trigger. 1305 */ 1306 ASSERT(flush_syncq(qp->q_syncq, qp) == 0); 1307 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0); 1308 ASSERT((qp->q_flag & QPERMOD) || 1309 ((qp->q_syncq->sq_head == NULL) && 1310 (wqp->q_syncq->sq_head == NULL))); 1311 1312 /* release any fmodsw_impl_t structure held on behalf of the queue */ 1313 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV); 1314 if (qp->q_fp != NULL) 1315 fmodsw_rele(qp->q_fp); 1316 1317 /* freeq removes us from the outer perimeter if any */ 1318 freeq(qp); 1319 } 1320 1321 /* Prevent service procedures from being called */ 1322 void 1323 disable_svc(queue_t *qp) 1324 { 1325 queue_t *wqp = _WR(qp); 1326 1327 ASSERT(qp->q_flag & QREADR); 1328 mutex_enter(QLOCK(qp)); 1329 qp->q_flag |= QWCLOSE; 1330 mutex_exit(QLOCK(qp)); 1331 mutex_enter(QLOCK(wqp)); 1332 wqp->q_flag |= QWCLOSE; 1333 mutex_exit(QLOCK(wqp)); 1334 } 1335 1336 /* Allow service procedures to be called again */ 1337 void 1338 enable_svc(queue_t *qp) 1339 { 1340 queue_t *wqp = _WR(qp); 1341 1342 ASSERT(qp->q_flag & QREADR); 1343 mutex_enter(QLOCK(qp)); 1344 qp->q_flag &= ~QWCLOSE; 1345 mutex_exit(QLOCK(qp)); 1346 mutex_enter(QLOCK(wqp)); 1347 wqp->q_flag &= ~QWCLOSE; 1348 mutex_exit(QLOCK(wqp)); 1349 } 1350 1351 /* 1352 * Remove queue from qhead/qtail if it is enabled. 1353 * Only reset QENAB if the queue was removed from the runlist. 1354 * A queue goes through 3 stages: 1355 * It is on the service list and QENAB is set. 1356 * It is removed from the service list but QENAB is still set. 1357 * QENAB gets changed to QINSERVICE. 1358 * QINSERVICE is reset (when the service procedure is done) 1359 * Thus we can not reset QENAB unless we actually removed it from the service 1360 * queue. 1361 */ 1362 void 1363 remove_runlist(queue_t *qp) 1364 { 1365 if (qp->q_flag & QENAB && qhead != NULL) { 1366 queue_t *q_chase; 1367 queue_t *q_curr; 1368 int removed; 1369 1370 mutex_enter(&service_queue); 1371 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed); 1372 mutex_exit(&service_queue); 1373 if (removed) { 1374 STRSTAT(qremoved); 1375 qp->q_flag &= ~QENAB; 1376 } 1377 } 1378 } 1379 1380 1381 /* 1382 * Wait for any pending service processing to complete. 1383 * The removal of queues from the runlist is not atomic with the 1384 * clearing of the QENABLED flag and setting the INSERVICE flag. 1385 * consequently it is possible for remove_runlist in strclose 1386 * to not find the queue on the runlist but for it to be QENABLED 1387 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED 1388 * as well as INSERVICE. 1389 */ 1390 void 1391 wait_svc(queue_t *qp) 1392 { 1393 queue_t *wqp = _WR(qp); 1394 1395 ASSERT(qp->q_flag & QREADR); 1396 1397 /* 1398 * Try to remove queues from qhead/qtail list. 1399 */ 1400 if (qhead != NULL) { 1401 remove_runlist(qp); 1402 remove_runlist(wqp); 1403 } 1404 /* 1405 * Wait till the syncqs associated with the queue disappear from the 1406 * background processing list. 1407 * This only needs to be done for non-PERMOD perimeters since 1408 * for PERMOD perimeters the syncq may be shared and will only be freed 1409 * when the last module/driver is unloaded. 1410 * If for PERMOD perimeters queue was on the syncq list, removeq() 1411 * should call propagate_syncq() or drain_syncq() for it. Both of these 1412 * functions remove the queue from its syncq list, so sqthread will not 1413 * try to access the queue. 1414 */ 1415 if (!(qp->q_flag & QPERMOD)) { 1416 syncq_t *rsq = qp->q_syncq; 1417 syncq_t *wsq = wqp->q_syncq; 1418 1419 /* 1420 * Disable rsq and wsq and wait for any background processing of 1421 * syncq to complete. 1422 */ 1423 wait_sq_svc(rsq); 1424 if (wsq != rsq) 1425 wait_sq_svc(wsq); 1426 } 1427 1428 mutex_enter(QLOCK(qp)); 1429 while (qp->q_flag & (QINSERVICE|QENAB)) 1430 cv_wait(&qp->q_wait, QLOCK(qp)); 1431 mutex_exit(QLOCK(qp)); 1432 mutex_enter(QLOCK(wqp)); 1433 while (wqp->q_flag & (QINSERVICE|QENAB)) 1434 cv_wait(&wqp->q_wait, QLOCK(wqp)); 1435 mutex_exit(QLOCK(wqp)); 1436 } 1437 1438 /* 1439 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'. 1440 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may 1441 * also be set, and is passed through to allocb_cred_wait(). 1442 * 1443 * Returns errno on failure, zero on success. 1444 */ 1445 int 1446 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr) 1447 { 1448 mblk_t *tmp; 1449 ssize_t count; 1450 int error = 0; 1451 1452 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K || 1453 (flag & (U_TO_K | K_TO_K)) == K_TO_K); 1454 1455 if (bp->b_datap->db_type == M_IOCTL) { 1456 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1457 } else { 1458 ASSERT(bp->b_datap->db_type == M_COPYIN); 1459 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1460 } 1461 /* 1462 * strdoioctl validates ioc_count, so if this assert fails it 1463 * cannot be due to user error. 1464 */ 1465 ASSERT(count >= 0); 1466 1467 if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr, 1468 curproc->p_pid)) == NULL) { 1469 return (error); 1470 } 1471 error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K)); 1472 if (error != 0) { 1473 freeb(tmp); 1474 return (error); 1475 } 1476 DB_CPID(tmp) = curproc->p_pid; 1477 tmp->b_wptr += count; 1478 bp->b_cont = tmp; 1479 1480 return (0); 1481 } 1482 1483 /* 1484 * Copy ioctl data to user-land. Return non-zero errno on failure, 1485 * 0 for success. 1486 */ 1487 int 1488 getiocd(mblk_t *bp, char *arg, int copymode) 1489 { 1490 ssize_t count; 1491 size_t n; 1492 int error; 1493 1494 if (bp->b_datap->db_type == M_IOCACK) 1495 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1496 else { 1497 ASSERT(bp->b_datap->db_type == M_COPYOUT); 1498 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1499 } 1500 ASSERT(count >= 0); 1501 1502 for (bp = bp->b_cont; bp && count; 1503 count -= n, bp = bp->b_cont, arg += n) { 1504 n = MIN(count, bp->b_wptr - bp->b_rptr); 1505 error = strcopyout(bp->b_rptr, arg, n, copymode); 1506 if (error) 1507 return (error); 1508 } 1509 ASSERT(count == 0); 1510 return (0); 1511 } 1512 1513 /* 1514 * Allocate a linkinfo entry given the write queue of the 1515 * bottom module of the top stream and the write queue of the 1516 * stream head of the bottom stream. 1517 */ 1518 linkinfo_t * 1519 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown) 1520 { 1521 linkinfo_t *linkp; 1522 1523 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP); 1524 1525 linkp->li_lblk.l_qtop = qup; 1526 linkp->li_lblk.l_qbot = qdown; 1527 linkp->li_fpdown = fpdown; 1528 1529 mutex_enter(&strresources); 1530 linkp->li_next = linkinfo_list; 1531 linkp->li_prev = NULL; 1532 if (linkp->li_next) 1533 linkp->li_next->li_prev = linkp; 1534 linkinfo_list = linkp; 1535 linkp->li_lblk.l_index = ++lnk_id; 1536 ASSERT(lnk_id != 0); /* this should never wrap in practice */ 1537 mutex_exit(&strresources); 1538 1539 return (linkp); 1540 } 1541 1542 /* 1543 * Free a linkinfo entry. 1544 */ 1545 void 1546 lbfree(linkinfo_t *linkp) 1547 { 1548 mutex_enter(&strresources); 1549 if (linkp->li_next) 1550 linkp->li_next->li_prev = linkp->li_prev; 1551 if (linkp->li_prev) 1552 linkp->li_prev->li_next = linkp->li_next; 1553 else 1554 linkinfo_list = linkp->li_next; 1555 mutex_exit(&strresources); 1556 1557 kmem_cache_free(linkinfo_cache, linkp); 1558 } 1559 1560 /* 1561 * Check for a potential linking cycle. 1562 * Return 1 if a link will result in a cycle, 1563 * and 0 otherwise. 1564 */ 1565 int 1566 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss) 1567 { 1568 struct mux_node *np; 1569 struct mux_edge *ep; 1570 int i; 1571 major_t lomaj; 1572 major_t upmaj; 1573 /* 1574 * if the lower stream is a pipe/FIFO, return, since link 1575 * cycles can not happen on pipes/FIFOs 1576 */ 1577 if (lostp->sd_vnode->v_type == VFIFO) 1578 return (0); 1579 1580 for (i = 0; i < ss->ss_devcnt; i++) { 1581 np = &ss->ss_mux_nodes[i]; 1582 MUX_CLEAR(np); 1583 } 1584 lomaj = getmajor(lostp->sd_vnode->v_rdev); 1585 upmaj = getmajor(upstp->sd_vnode->v_rdev); 1586 np = &ss->ss_mux_nodes[lomaj]; 1587 for (;;) { 1588 if (!MUX_DIDVISIT(np)) { 1589 if (np->mn_imaj == upmaj) 1590 return (1); 1591 if (np->mn_outp == NULL) { 1592 MUX_VISIT(np); 1593 if (np->mn_originp == NULL) 1594 return (0); 1595 np = np->mn_originp; 1596 continue; 1597 } 1598 MUX_VISIT(np); 1599 np->mn_startp = np->mn_outp; 1600 } else { 1601 if (np->mn_startp == NULL) { 1602 if (np->mn_originp == NULL) 1603 return (0); 1604 else { 1605 np = np->mn_originp; 1606 continue; 1607 } 1608 } 1609 /* 1610 * If ep->me_nodep is a FIFO (me_nodep == NULL), 1611 * ignore the edge and move on. ep->me_nodep gets 1612 * set to NULL in mux_addedge() if it is a FIFO. 1613 * 1614 */ 1615 ep = np->mn_startp; 1616 np->mn_startp = ep->me_nextp; 1617 if (ep->me_nodep == NULL) 1618 continue; 1619 ep->me_nodep->mn_originp = np; 1620 np = ep->me_nodep; 1621 } 1622 } 1623 } 1624 1625 /* 1626 * Find linkinfo entry corresponding to the parameters. 1627 */ 1628 linkinfo_t * 1629 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss) 1630 { 1631 linkinfo_t *linkp; 1632 struct mux_edge *mep; 1633 struct mux_node *mnp; 1634 queue_t *qup; 1635 1636 mutex_enter(&strresources); 1637 if ((type & LINKTYPEMASK) == LINKNORMAL) { 1638 qup = getendq(stp->sd_wrq); 1639 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1640 if ((qup == linkp->li_lblk.l_qtop) && 1641 (!index || (index == linkp->li_lblk.l_index))) { 1642 mutex_exit(&strresources); 1643 return (linkp); 1644 } 1645 } 1646 } else { 1647 ASSERT((type & LINKTYPEMASK) == LINKPERSIST); 1648 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)]; 1649 mep = mnp->mn_outp; 1650 while (mep) { 1651 if ((index == 0) || (index == mep->me_muxid)) 1652 break; 1653 mep = mep->me_nextp; 1654 } 1655 if (!mep) { 1656 mutex_exit(&strresources); 1657 return (NULL); 1658 } 1659 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1660 if ((!linkp->li_lblk.l_qtop) && 1661 (mep->me_muxid == linkp->li_lblk.l_index)) { 1662 mutex_exit(&strresources); 1663 return (linkp); 1664 } 1665 } 1666 } 1667 mutex_exit(&strresources); 1668 return (NULL); 1669 } 1670 1671 /* 1672 * Given a queue ptr, follow the chain of q_next pointers until you reach the 1673 * last queue on the chain and return it. 1674 */ 1675 queue_t * 1676 getendq(queue_t *q) 1677 { 1678 ASSERT(q != NULL); 1679 while (_SAMESTR(q)) 1680 q = q->q_next; 1681 return (q); 1682 } 1683 1684 /* 1685 * Wait for the syncq count to drop to zero. 1686 * sq could be either outer or inner. 1687 */ 1688 1689 static void 1690 wait_syncq(syncq_t *sq) 1691 { 1692 uint16_t count; 1693 1694 mutex_enter(SQLOCK(sq)); 1695 count = sq->sq_count; 1696 SQ_PUTLOCKS_ENTER(sq); 1697 SUM_SQ_PUTCOUNTS(sq, count); 1698 while (count != 0) { 1699 sq->sq_flags |= SQ_WANTWAKEUP; 1700 SQ_PUTLOCKS_EXIT(sq); 1701 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1702 count = sq->sq_count; 1703 SQ_PUTLOCKS_ENTER(sq); 1704 SUM_SQ_PUTCOUNTS(sq, count); 1705 } 1706 SQ_PUTLOCKS_EXIT(sq); 1707 mutex_exit(SQLOCK(sq)); 1708 } 1709 1710 /* 1711 * Wait while there are any messages for the queue in its syncq. 1712 */ 1713 static void 1714 wait_q_syncq(queue_t *q) 1715 { 1716 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1717 syncq_t *sq = q->q_syncq; 1718 1719 mutex_enter(SQLOCK(sq)); 1720 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1721 sq->sq_flags |= SQ_WANTWAKEUP; 1722 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1723 } 1724 mutex_exit(SQLOCK(sq)); 1725 } 1726 } 1727 1728 1729 int 1730 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp, 1731 int lhlink) 1732 { 1733 struct stdata *stp; 1734 struct strioctl strioc; 1735 struct linkinfo *linkp; 1736 struct stdata *stpdown; 1737 struct streamtab *str; 1738 queue_t *passq; 1739 syncq_t *passyncq; 1740 queue_t *rq; 1741 cdevsw_impl_t *dp; 1742 uint32_t qflag; 1743 uint32_t sqtype; 1744 perdm_t *dmp; 1745 int error = 0; 1746 netstack_t *ns; 1747 str_stack_t *ss; 1748 1749 stp = vp->v_stream; 1750 TRACE_1(TR_FAC_STREAMS_FR, 1751 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp); 1752 /* 1753 * Test for invalid upper stream 1754 */ 1755 if (stp->sd_flag & STRHUP) { 1756 return (ENXIO); 1757 } 1758 if (vp->v_type == VFIFO) { 1759 return (EINVAL); 1760 } 1761 if (stp->sd_strtab == NULL) { 1762 return (EINVAL); 1763 } 1764 if (!stp->sd_strtab->st_muxwinit) { 1765 return (EINVAL); 1766 } 1767 if (fpdown == NULL) { 1768 return (EBADF); 1769 } 1770 ns = netstack_find_by_cred(crp); 1771 ASSERT(ns != NULL); 1772 ss = ns->netstack_str; 1773 ASSERT(ss != NULL); 1774 1775 if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) { 1776 netstack_rele(ss->ss_netstack); 1777 return (EINVAL); 1778 } 1779 mutex_enter(&muxifier); 1780 if (stp->sd_flag & STPLEX) { 1781 mutex_exit(&muxifier); 1782 netstack_rele(ss->ss_netstack); 1783 return (ENXIO); 1784 } 1785 1786 /* 1787 * Test for invalid lower stream. 1788 * The check for the v_type != VFIFO and having a major 1789 * number not >= devcnt is done to avoid problems with 1790 * adding mux_node entry past the end of mux_nodes[]. 1791 * For FIFO's we don't add an entry so this isn't a 1792 * problem. 1793 */ 1794 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) || 1795 (stpdown == stp) || (stpdown->sd_flag & 1796 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) || 1797 ((stpdown->sd_vnode->v_type != VFIFO) && 1798 (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) || 1799 linkcycle(stp, stpdown, ss)) { 1800 mutex_exit(&muxifier); 1801 netstack_rele(ss->ss_netstack); 1802 return (EINVAL); 1803 } 1804 TRACE_1(TR_FAC_STREAMS_FR, 1805 TR_STPDOWN, "stpdown:%p", stpdown); 1806 rq = getendq(stp->sd_wrq); 1807 if (cmd == I_PLINK) 1808 rq = NULL; 1809 1810 linkp = alloclink(rq, stpdown->sd_wrq, fpdown); 1811 1812 strioc.ic_cmd = cmd; 1813 strioc.ic_timout = INFTIM; 1814 strioc.ic_len = sizeof (struct linkblk); 1815 strioc.ic_dp = (char *)&linkp->li_lblk; 1816 1817 /* 1818 * STRPLUMB protects plumbing changes and should be set before 1819 * link_addpassthru()/link_rempassthru() are called, so it is set here 1820 * and cleared in the end of mlink when passthru queue is removed. 1821 * Setting of STRPLUMB prevents reopens of the stream while passthru 1822 * queue is in-place (it is not a proper module and doesn't have open 1823 * entry point). 1824 * 1825 * STPLEX prevents any threads from entering the stream from above. It 1826 * can't be set before the call to link_addpassthru() because putnext 1827 * from below may cause stream head I/O routines to be called and these 1828 * routines assert that STPLEX is not set. After link_addpassthru() 1829 * nothing may come from below since the pass queue syncq is blocked. 1830 * Note also that STPLEX should be cleared before the call to 1831 * link_rempassthru() since when messages start flowing to the stream 1832 * head (e.g. because of message propagation from the pass queue) stream 1833 * head I/O routines may be called with STPLEX flag set. 1834 * 1835 * When STPLEX is set, nothing may come into the stream from above and 1836 * it is safe to do a setq which will change stream head. So, the 1837 * correct sequence of actions is: 1838 * 1839 * 1) Set STRPLUMB 1840 * 2) Call link_addpassthru() 1841 * 3) Set STPLEX 1842 * 4) Call setq and update the stream state 1843 * 5) Clear STPLEX 1844 * 6) Call link_rempassthru() 1845 * 7) Clear STRPLUMB 1846 * 1847 * The same sequence applies to munlink() code. 1848 */ 1849 mutex_enter(&stpdown->sd_lock); 1850 stpdown->sd_flag |= STRPLUMB; 1851 mutex_exit(&stpdown->sd_lock); 1852 /* 1853 * Add passthru queue below lower mux. This will block 1854 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 1855 */ 1856 passq = link_addpassthru(stpdown); 1857 1858 mutex_enter(&stpdown->sd_lock); 1859 stpdown->sd_flag |= STPLEX; 1860 mutex_exit(&stpdown->sd_lock); 1861 1862 rq = _RD(stpdown->sd_wrq); 1863 /* 1864 * There may be messages in the streamhead's syncq due to messages 1865 * that arrived before link_addpassthru() was done. To avoid 1866 * background processing of the syncq happening simultaneous with 1867 * setq processing, we disable the streamhead syncq and wait until 1868 * existing background thread finishes working on it. 1869 */ 1870 wait_sq_svc(rq->q_syncq); 1871 passyncq = passq->q_syncq; 1872 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1873 blocksq(passyncq, SQ_BLOCKED, 0); 1874 1875 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 1876 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 1877 rq->q_ptr = _WR(rq)->q_ptr = NULL; 1878 1879 /* setq might sleep in allocator - avoid holding locks. */ 1880 /* Note: we are holding muxifier here. */ 1881 1882 str = stp->sd_strtab; 1883 dp = &devimpl[getmajor(vp->v_rdev)]; 1884 ASSERT(dp->d_str == str); 1885 1886 qflag = dp->d_qflag; 1887 sqtype = dp->d_sqtype; 1888 1889 /* create perdm_t if needed */ 1890 if (NEED_DM(dp->d_dmp, qflag)) 1891 dp->d_dmp = hold_dm(str, qflag, sqtype); 1892 1893 dmp = dp->d_dmp; 1894 1895 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype, 1896 B_TRUE); 1897 1898 /* 1899 * XXX Remove any "odd" messages from the queue. 1900 * Keep only M_DATA, M_PROTO, M_PCPROTO. 1901 */ 1902 error = strdoioctl(stp, &strioc, FNATIVE, 1903 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 1904 if (error != 0) 1905 goto cleanup; 1906 1907 mutex_enter(&fpdown->f_tlock); 1908 fpdown->f_count++; 1909 mutex_exit(&fpdown->f_tlock); 1910 1911 /* 1912 * if we've made it here the linkage is all set up so we should also 1913 * set up the layered driver linkages 1914 */ 1915 1916 ASSERT((cmd == I_LINK) || (cmd == I_PLINK)); 1917 if (cmd == I_LINK) { 1918 error = ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL); 1919 } else { 1920 error = ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST); 1921 } 1922 1923 if (error != 0) { 1924 mutex_enter(&fpdown->f_tlock); 1925 fpdown->f_count--; 1926 mutex_exit(&fpdown->f_tlock); 1927 goto cleanup; 1928 } 1929 1930 link_rempassthru(passq); 1931 1932 mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss); 1933 1934 /* 1935 * Mark the upper stream as having dependent links 1936 * so that strclose can clean it up. 1937 */ 1938 if (cmd == I_LINK) { 1939 mutex_enter(&stp->sd_lock); 1940 stp->sd_flag |= STRHASLINKS; 1941 mutex_exit(&stp->sd_lock); 1942 } 1943 /* 1944 * Wake up any other processes that may have been 1945 * waiting on the lower stream. These will all 1946 * error out. 1947 */ 1948 mutex_enter(&stpdown->sd_lock); 1949 /* The passthru module is removed so we may release STRPLUMB */ 1950 stpdown->sd_flag &= ~STRPLUMB; 1951 cv_broadcast(&rq->q_wait); 1952 cv_broadcast(&_WR(rq)->q_wait); 1953 cv_broadcast(&stpdown->sd_monitor); 1954 mutex_exit(&stpdown->sd_lock); 1955 mutex_exit(&muxifier); 1956 *rvalp = linkp->li_lblk.l_index; 1957 netstack_rele(ss->ss_netstack); 1958 return (0); 1959 1960 cleanup: 1961 lbfree(linkp); 1962 1963 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1964 blocksq(passyncq, SQ_BLOCKED, 0); 1965 /* 1966 * Restore the stream head queue and then remove 1967 * the passq. Turn off STPLEX before we turn on 1968 * the stream by removing the passq. 1969 */ 1970 rq->q_ptr = _WR(rq)->q_ptr = stpdown; 1971 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, 1972 B_TRUE); 1973 1974 mutex_enter(&stpdown->sd_lock); 1975 stpdown->sd_flag &= ~STPLEX; 1976 mutex_exit(&stpdown->sd_lock); 1977 1978 link_rempassthru(passq); 1979 1980 mutex_enter(&stpdown->sd_lock); 1981 stpdown->sd_flag &= ~STRPLUMB; 1982 /* Wakeup anyone waiting for STRPLUMB to clear. */ 1983 cv_broadcast(&stpdown->sd_monitor); 1984 mutex_exit(&stpdown->sd_lock); 1985 1986 mutex_exit(&muxifier); 1987 netstack_rele(ss->ss_netstack); 1988 return (error); 1989 } 1990 1991 int 1992 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink) 1993 { 1994 int ret; 1995 struct file *fpdown; 1996 1997 fpdown = getf(arg); 1998 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink); 1999 if (fpdown != NULL) 2000 releasef(arg); 2001 return (ret); 2002 } 2003 2004 /* 2005 * Unlink a multiplexor link. Stp is the controlling stream for the 2006 * link, and linkp points to the link's entry in the linkinfo list. 2007 * The muxifier lock must be held on entry and is dropped on exit. 2008 * 2009 * NOTE : Currently it is assumed that mux would process all the messages 2010 * sitting on it's queue before ACKing the UNLINK. It is the responsibility 2011 * of the mux to handle all the messages that arrive before UNLINK. 2012 * If the mux has to send down messages on its lower stream before 2013 * ACKing I_UNLINK, then it *should* know to handle messages even 2014 * after the UNLINK is acked (actually it should be able to handle till we 2015 * re-block the read side of the pass queue here). If the mux does not 2016 * open up the lower stream, any messages that arrive during UNLINK 2017 * will be put in the stream head. In the case of lower stream opening 2018 * up, some messages might land in the stream head depending on when 2019 * the message arrived and when the read side of the pass queue was 2020 * re-blocked. 2021 */ 2022 int 2023 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp, 2024 str_stack_t *ss) 2025 { 2026 struct strioctl strioc; 2027 struct stdata *stpdown; 2028 queue_t *rq, *wrq; 2029 queue_t *passq; 2030 syncq_t *passyncq; 2031 int error = 0; 2032 file_t *fpdown; 2033 2034 ASSERT(MUTEX_HELD(&muxifier)); 2035 2036 stpdown = linkp->li_fpdown->f_vnode->v_stream; 2037 2038 /* 2039 * See the comment in mlink() concerning STRPLUMB/STPLEX flags. 2040 */ 2041 mutex_enter(&stpdown->sd_lock); 2042 stpdown->sd_flag |= STRPLUMB; 2043 mutex_exit(&stpdown->sd_lock); 2044 2045 /* 2046 * Add passthru queue below lower mux. This will block 2047 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 2048 */ 2049 passq = link_addpassthru(stpdown); 2050 2051 if ((flag & LINKTYPEMASK) == LINKNORMAL) 2052 strioc.ic_cmd = I_UNLINK; 2053 else 2054 strioc.ic_cmd = I_PUNLINK; 2055 strioc.ic_timout = INFTIM; 2056 strioc.ic_len = sizeof (struct linkblk); 2057 strioc.ic_dp = (char *)&linkp->li_lblk; 2058 2059 error = strdoioctl(stp, &strioc, FNATIVE, 2060 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 2061 2062 /* 2063 * If there was an error and this is not called via strclose, 2064 * return to the user. Otherwise, pretend there was no error 2065 * and close the link. 2066 */ 2067 if (error) { 2068 if (flag & LINKCLOSE) { 2069 cmn_err(CE_WARN, "KERNEL: munlink: could not perform " 2070 "unlink ioctl, closing anyway (%d)\n", error); 2071 } else { 2072 link_rempassthru(passq); 2073 mutex_enter(&stpdown->sd_lock); 2074 stpdown->sd_flag &= ~STRPLUMB; 2075 cv_broadcast(&stpdown->sd_monitor); 2076 mutex_exit(&stpdown->sd_lock); 2077 mutex_exit(&muxifier); 2078 return (error); 2079 } 2080 } 2081 2082 mux_rmvedge(stp, linkp->li_lblk.l_index, ss); 2083 fpdown = linkp->li_fpdown; 2084 lbfree(linkp); 2085 2086 /* 2087 * We go ahead and drop muxifier here--it's a nasty global lock that 2088 * can slow others down. It's okay to since attempts to mlink() this 2089 * stream will be stopped because STPLEX is still set in the stdata 2090 * structure, and munlink() is stopped because mux_rmvedge() and 2091 * lbfree() have removed it from mux_nodes[] and linkinfo_list, 2092 * respectively. Note that we defer the closef() of fpdown until 2093 * after we drop muxifier since strclose() can call munlinkall(). 2094 */ 2095 mutex_exit(&muxifier); 2096 2097 wrq = stpdown->sd_wrq; 2098 rq = _RD(wrq); 2099 2100 /* 2101 * Get rid of outstanding service procedure runs, before we make 2102 * it a stream head, since a stream head doesn't have any service 2103 * procedure. 2104 */ 2105 disable_svc(rq); 2106 wait_svc(rq); 2107 2108 /* 2109 * Since we don't disable the syncq for QPERMOD, we wait for whatever 2110 * is queued up to be finished. mux should take care that nothing is 2111 * send down to this queue. We should do it now as we're going to block 2112 * passyncq if it was unblocked. 2113 */ 2114 if (wrq->q_flag & QPERMOD) { 2115 syncq_t *sq = wrq->q_syncq; 2116 2117 mutex_enter(SQLOCK(sq)); 2118 while (wrq->q_sqflags & Q_SQQUEUED) { 2119 sq->sq_flags |= SQ_WANTWAKEUP; 2120 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2121 } 2122 mutex_exit(SQLOCK(sq)); 2123 } 2124 passyncq = passq->q_syncq; 2125 if (!(passyncq->sq_flags & SQ_BLOCKED)) { 2126 2127 syncq_t *sq, *outer; 2128 2129 /* 2130 * Messages could be flowing from underneath. We will 2131 * block the read side of the passq. This would be 2132 * sufficient for QPAIR and QPERQ muxes to ensure 2133 * that no data is flowing up into this queue 2134 * and hence no thread active in this instance of 2135 * lower mux. But for QPERMOD and QMTOUTPERIM there 2136 * could be messages on the inner and outer/inner 2137 * syncqs respectively. We will wait for them to drain. 2138 * Because passq is blocked messages end up in the syncq 2139 * And qfill_syncq could possibly end up setting QFULL 2140 * which will access the rq->q_flag. Hence, we have to 2141 * acquire the QLOCK in setq. 2142 * 2143 * XXX Messages can also flow from top into this 2144 * queue though the unlink is over (Ex. some instance 2145 * in putnext() called from top that has still not 2146 * accessed this queue. And also putq(lowerq) ?). 2147 * Solution : How about blocking the l_qtop queue ? 2148 * Do we really care about such pure D_MP muxes ? 2149 */ 2150 2151 blocksq(passyncq, SQ_BLOCKED, 0); 2152 2153 sq = rq->q_syncq; 2154 if ((outer = sq->sq_outer) != NULL) { 2155 2156 /* 2157 * We have to just wait for the outer sq_count 2158 * drop to zero. As this does not prevent new 2159 * messages to enter the outer perimeter, this 2160 * is subject to starvation. 2161 * 2162 * NOTE :Because of blocksq above, messages could 2163 * be in the inner syncq only because of some 2164 * thread holding the outer perimeter exclusively. 2165 * Hence it would be sufficient to wait for the 2166 * exclusive holder of the outer perimeter to drain 2167 * the inner and outer syncqs. But we will not depend 2168 * on this feature and hence check the inner syncqs 2169 * separately. 2170 */ 2171 wait_syncq(outer); 2172 } 2173 2174 2175 /* 2176 * There could be messages destined for 2177 * this queue. Let the exclusive holder 2178 * drain it. 2179 */ 2180 2181 wait_syncq(sq); 2182 ASSERT((rq->q_flag & QPERMOD) || 2183 ((rq->q_syncq->sq_head == NULL) && 2184 (_WR(rq)->q_syncq->sq_head == NULL))); 2185 } 2186 2187 /* 2188 * We haven't taken care of QPERMOD case yet. QPERMOD is a special 2189 * case as we don't disable its syncq or remove it off the syncq 2190 * service list. 2191 */ 2192 if (rq->q_flag & QPERMOD) { 2193 syncq_t *sq = rq->q_syncq; 2194 2195 mutex_enter(SQLOCK(sq)); 2196 while (rq->q_sqflags & Q_SQQUEUED) { 2197 sq->sq_flags |= SQ_WANTWAKEUP; 2198 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2199 } 2200 mutex_exit(SQLOCK(sq)); 2201 } 2202 2203 /* 2204 * flush_syncq changes states only when there are some messages to 2205 * free, i.e. when it returns non-zero value to return. 2206 */ 2207 ASSERT(flush_syncq(rq->q_syncq, rq) == 0); 2208 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0); 2209 2210 /* 2211 * Nobody else should know about this queue now. 2212 * If the mux did not process the messages before 2213 * acking the I_UNLINK, free them now. 2214 */ 2215 2216 flushq(rq, FLUSHALL); 2217 flushq(_WR(rq), FLUSHALL); 2218 2219 /* 2220 * Convert the mux lower queue into a stream head queue. 2221 * Turn off STPLEX before we turn on the stream by removing the passq. 2222 */ 2223 rq->q_ptr = wrq->q_ptr = stpdown; 2224 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE); 2225 2226 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 2227 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 2228 2229 enable_svc(rq); 2230 2231 /* 2232 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still 2233 * needs to be set to prevent reopen() of the stream - such reopen may 2234 * try to call non-existent pass queue open routine and panic. 2235 */ 2236 mutex_enter(&stpdown->sd_lock); 2237 stpdown->sd_flag &= ~STPLEX; 2238 mutex_exit(&stpdown->sd_lock); 2239 2240 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) || 2241 ((flag & LINKTYPEMASK) == LINKPERSIST)); 2242 2243 /* clean up the layered driver linkages */ 2244 if ((flag & LINKTYPEMASK) == LINKNORMAL) { 2245 VERIFY0(ldi_munlink_fp(stp, fpdown, LINKNORMAL)); 2246 } else { 2247 VERIFY0(ldi_munlink_fp(stp, fpdown, LINKPERSIST)); 2248 } 2249 2250 link_rempassthru(passq); 2251 2252 /* 2253 * Now all plumbing changes are finished and STRPLUMB is no 2254 * longer needed. 2255 */ 2256 mutex_enter(&stpdown->sd_lock); 2257 stpdown->sd_flag &= ~STRPLUMB; 2258 cv_broadcast(&stpdown->sd_monitor); 2259 mutex_exit(&stpdown->sd_lock); 2260 2261 (void) closef(fpdown); 2262 return (0); 2263 } 2264 2265 /* 2266 * Unlink all multiplexor links for which stp is the controlling stream. 2267 * Return 0, or a non-zero errno on failure. 2268 */ 2269 int 2270 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss) 2271 { 2272 linkinfo_t *linkp; 2273 int error = 0; 2274 2275 mutex_enter(&muxifier); 2276 while (linkp = findlinks(stp, 0, flag, ss)) { 2277 /* 2278 * munlink() releases the muxifier lock. 2279 */ 2280 if (error = munlink(stp, linkp, flag, crp, rvalp, ss)) 2281 return (error); 2282 mutex_enter(&muxifier); 2283 } 2284 mutex_exit(&muxifier); 2285 return (0); 2286 } 2287 2288 /* 2289 * A multiplexor link has been made. Add an 2290 * edge to the directed graph. 2291 */ 2292 void 2293 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss) 2294 { 2295 struct mux_node *np; 2296 struct mux_edge *ep; 2297 major_t upmaj; 2298 major_t lomaj; 2299 2300 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2301 lomaj = getmajor(lostp->sd_vnode->v_rdev); 2302 np = &ss->ss_mux_nodes[upmaj]; 2303 if (np->mn_outp) { 2304 ep = np->mn_outp; 2305 while (ep->me_nextp) 2306 ep = ep->me_nextp; 2307 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2308 ep = ep->me_nextp; 2309 } else { 2310 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2311 ep = np->mn_outp; 2312 } 2313 ep->me_nextp = NULL; 2314 ep->me_muxid = muxid; 2315 /* 2316 * Save the dev_t for the purposes of str_stack_shutdown. 2317 * str_stack_shutdown assumes that the device allows reopen, since 2318 * this dev_t is the one after any cloning by xx_open(). 2319 * Would prefer finding the dev_t from before any cloning, 2320 * but specfs doesn't retain that. 2321 */ 2322 ep->me_dev = upstp->sd_vnode->v_rdev; 2323 if (lostp->sd_vnode->v_type == VFIFO) 2324 ep->me_nodep = NULL; 2325 else 2326 ep->me_nodep = &ss->ss_mux_nodes[lomaj]; 2327 } 2328 2329 /* 2330 * A multiplexor link has been removed. Remove the 2331 * edge in the directed graph. 2332 */ 2333 void 2334 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss) 2335 { 2336 struct mux_node *np; 2337 struct mux_edge *ep; 2338 struct mux_edge *pep = NULL; 2339 major_t upmaj; 2340 2341 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2342 np = &ss->ss_mux_nodes[upmaj]; 2343 ASSERT(np->mn_outp != NULL); 2344 ep = np->mn_outp; 2345 while (ep) { 2346 if (ep->me_muxid == muxid) { 2347 if (pep) 2348 pep->me_nextp = ep->me_nextp; 2349 else 2350 np->mn_outp = ep->me_nextp; 2351 kmem_free(ep, sizeof (struct mux_edge)); 2352 return; 2353 } 2354 pep = ep; 2355 ep = ep->me_nextp; 2356 } 2357 ASSERT(0); /* should not reach here */ 2358 } 2359 2360 /* 2361 * Translate the device flags (from conf.h) to the corresponding 2362 * qflag and sq_flag (type) values. 2363 */ 2364 int 2365 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp, 2366 uint32_t *sqtypep) 2367 { 2368 uint32_t qflag = 0; 2369 uint32_t sqtype = 0; 2370 2371 if (devflag & _D_OLD) 2372 goto bad; 2373 2374 /* Inner perimeter presence and scope */ 2375 switch (devflag & D_MTINNER_MASK) { 2376 case D_MP: 2377 qflag |= QMTSAFE; 2378 sqtype |= SQ_CI; 2379 break; 2380 case D_MTPERQ|D_MP: 2381 qflag |= QPERQ; 2382 break; 2383 case D_MTQPAIR|D_MP: 2384 qflag |= QPAIR; 2385 break; 2386 case D_MTPERMOD|D_MP: 2387 qflag |= QPERMOD; 2388 break; 2389 default: 2390 goto bad; 2391 } 2392 2393 /* Outer perimeter */ 2394 if (devflag & D_MTOUTPERIM) { 2395 switch (devflag & D_MTINNER_MASK) { 2396 case D_MP: 2397 case D_MTPERQ|D_MP: 2398 case D_MTQPAIR|D_MP: 2399 break; 2400 default: 2401 goto bad; 2402 } 2403 qflag |= QMTOUTPERIM; 2404 } 2405 2406 /* Inner perimeter modifiers */ 2407 if (devflag & D_MTINNER_MOD) { 2408 switch (devflag & D_MTINNER_MASK) { 2409 case D_MP: 2410 goto bad; 2411 default: 2412 break; 2413 } 2414 if (devflag & D_MTPUTSHARED) 2415 sqtype |= SQ_CIPUT; 2416 if (devflag & _D_MTOCSHARED) { 2417 /* 2418 * The code in putnext assumes that it has the 2419 * highest concurrency by not checking sq_count. 2420 * Thus _D_MTOCSHARED can only be supported when 2421 * D_MTPUTSHARED is set. 2422 */ 2423 if (!(devflag & D_MTPUTSHARED)) 2424 goto bad; 2425 sqtype |= SQ_CIOC; 2426 } 2427 if (devflag & _D_MTCBSHARED) { 2428 /* 2429 * The code in putnext assumes that it has the 2430 * highest concurrency by not checking sq_count. 2431 * Thus _D_MTCBSHARED can only be supported when 2432 * D_MTPUTSHARED is set. 2433 */ 2434 if (!(devflag & D_MTPUTSHARED)) 2435 goto bad; 2436 sqtype |= SQ_CICB; 2437 } 2438 if (devflag & _D_MTSVCSHARED) { 2439 /* 2440 * The code in putnext assumes that it has the 2441 * highest concurrency by not checking sq_count. 2442 * Thus _D_MTSVCSHARED can only be supported when 2443 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is 2444 * supported only for QPERMOD. 2445 */ 2446 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD)) 2447 goto bad; 2448 sqtype |= SQ_CISVC; 2449 } 2450 } 2451 2452 /* Default outer perimeter concurrency */ 2453 sqtype |= SQ_CO; 2454 2455 /* Outer perimeter modifiers */ 2456 if (devflag & D_MTOCEXCL) { 2457 if (!(devflag & D_MTOUTPERIM)) { 2458 /* No outer perimeter */ 2459 goto bad; 2460 } 2461 sqtype &= ~SQ_COOC; 2462 } 2463 2464 /* Synchronous Streams extended qinit structure */ 2465 if (devflag & D_SYNCSTR) 2466 qflag |= QSYNCSTR; 2467 2468 /* 2469 * Private flag used by a transport module to indicate 2470 * to sockfs that it supports direct-access mode without 2471 * having to go through STREAMS. 2472 */ 2473 if (devflag & _D_DIRECT) { 2474 /* Reject unless the module is fully-MT (no perimeter) */ 2475 if ((qflag & QMT_TYPEMASK) != QMTSAFE) 2476 goto bad; 2477 qflag |= _QDIRECT; 2478 } 2479 2480 /* 2481 * Private flag used to indicate that a streams module should only 2482 * be pushed once. The TTY streams modules have this flag since if 2483 * libc believes itself to be an xpg4 process then it will 2484 * automatically and unconditionally push them when a PTS device is 2485 * opened. If an application is not aware of this then without this 2486 * flag we would end up with duplicate modules. 2487 */ 2488 if (devflag & _D_SINGLE_INSTANCE) 2489 qflag |= _QSINGLE_INSTANCE; 2490 2491 *qflagp = qflag; 2492 *sqtypep = sqtype; 2493 return (0); 2494 2495 bad: 2496 cmn_err(CE_WARN, 2497 "stropen: bad MT flags (0x%x) in driver '%s'", 2498 (int)(qflag & D_MTSAFETY_MASK), 2499 stp->st_rdinit->qi_minfo->mi_idname); 2500 2501 return (EINVAL); 2502 } 2503 2504 /* 2505 * Set the interface values for a pair of queues (qinit structure, 2506 * packet sizes, water marks). 2507 * setq assumes that the caller does not have a claim (entersq or claimq) 2508 * on the queue. 2509 */ 2510 void 2511 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit, 2512 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed) 2513 { 2514 queue_t *wq; 2515 syncq_t *sq, *outer; 2516 2517 ASSERT(rq->q_flag & QREADR); 2518 ASSERT((qflag & QMT_TYPEMASK) != 0); 2519 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 2520 2521 wq = _WR(rq); 2522 rq->q_qinfo = rinit; 2523 rq->q_hiwat = rinit->qi_minfo->mi_hiwat; 2524 rq->q_lowat = rinit->qi_minfo->mi_lowat; 2525 rq->q_minpsz = rinit->qi_minfo->mi_minpsz; 2526 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz; 2527 wq->q_qinfo = winit; 2528 wq->q_hiwat = winit->qi_minfo->mi_hiwat; 2529 wq->q_lowat = winit->qi_minfo->mi_lowat; 2530 wq->q_minpsz = winit->qi_minfo->mi_minpsz; 2531 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz; 2532 2533 /* Remove old syncqs */ 2534 sq = rq->q_syncq; 2535 outer = sq->sq_outer; 2536 if (outer != NULL) { 2537 ASSERT(wq->q_syncq->sq_outer == outer); 2538 outer_remove(outer, rq->q_syncq); 2539 if (wq->q_syncq != rq->q_syncq) 2540 outer_remove(outer, wq->q_syncq); 2541 } 2542 ASSERT(sq->sq_outer == NULL); 2543 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2544 2545 if (sq != SQ(rq)) { 2546 if (!(rq->q_flag & QPERMOD)) 2547 free_syncq(sq); 2548 if (wq->q_syncq == rq->q_syncq) 2549 wq->q_syncq = NULL; 2550 rq->q_syncq = NULL; 2551 } 2552 if (wq->q_syncq != NULL && wq->q_syncq != sq && 2553 wq->q_syncq != SQ(rq)) { 2554 free_syncq(wq->q_syncq); 2555 wq->q_syncq = NULL; 2556 } 2557 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL && 2558 rq->q_syncq->sq_tail == NULL)); 2559 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL && 2560 wq->q_syncq->sq_tail == NULL)); 2561 2562 if (!(rq->q_flag & QPERMOD) && 2563 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) { 2564 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2565 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl, 2566 rq->q_syncq->sq_nciputctrl, 0); 2567 ASSERT(ciputctrl_cache != NULL); 2568 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl); 2569 rq->q_syncq->sq_ciputctrl = NULL; 2570 rq->q_syncq->sq_nciputctrl = 0; 2571 } 2572 2573 if (!(wq->q_flag & QPERMOD) && 2574 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) { 2575 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2576 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl, 2577 wq->q_syncq->sq_nciputctrl, 0); 2578 ASSERT(ciputctrl_cache != NULL); 2579 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl); 2580 wq->q_syncq->sq_ciputctrl = NULL; 2581 wq->q_syncq->sq_nciputctrl = 0; 2582 } 2583 2584 sq = SQ(rq); 2585 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 2586 ASSERT(sq->sq_outer == NULL); 2587 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2588 2589 /* 2590 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS 2591 * bits in sq_flag based on the sqtype. 2592 */ 2593 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); 2594 2595 rq->q_syncq = wq->q_syncq = sq; 2596 sq->sq_type = sqtype; 2597 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); 2598 2599 /* 2600 * We are making sq_svcflags zero, 2601 * resetting SQ_DISABLED in case it was set by 2602 * wait_svc() in the munlink path. 2603 * 2604 */ 2605 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); 2606 sq->sq_svcflags = 0; 2607 2608 /* 2609 * We need to acquire the lock here for the mlink and munlink case, 2610 * where canputnext, backenable, etc can access the q_flag. 2611 */ 2612 if (lock_needed) { 2613 mutex_enter(QLOCK(rq)); 2614 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2615 mutex_exit(QLOCK(rq)); 2616 mutex_enter(QLOCK(wq)); 2617 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2618 mutex_exit(QLOCK(wq)); 2619 } else { 2620 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2621 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2622 } 2623 2624 if (qflag & QPERQ) { 2625 /* Allocate a separate syncq for the write side */ 2626 sq = new_syncq(); 2627 sq->sq_type = rq->q_syncq->sq_type; 2628 sq->sq_flags = rq->q_syncq->sq_flags; 2629 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2630 sq->sq_oprev == NULL); 2631 wq->q_syncq = sq; 2632 } 2633 if (qflag & QPERMOD) { 2634 sq = dmp->dm_sq; 2635 2636 /* 2637 * Assert that we do have an inner perimeter syncq and that it 2638 * does not have an outer perimeter associated with it. 2639 */ 2640 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2641 sq->sq_oprev == NULL); 2642 rq->q_syncq = wq->q_syncq = sq; 2643 } 2644 if (qflag & QMTOUTPERIM) { 2645 outer = dmp->dm_sq; 2646 2647 ASSERT(outer->sq_outer == NULL); 2648 outer_insert(outer, rq->q_syncq); 2649 if (wq->q_syncq != rq->q_syncq) 2650 outer_insert(outer, wq->q_syncq); 2651 } 2652 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2653 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2654 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2655 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2656 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK)); 2657 2658 /* 2659 * Initialize struio() types. 2660 */ 2661 rq->q_struiot = 2662 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE; 2663 wq->q_struiot = 2664 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE; 2665 } 2666 2667 perdm_t * 2668 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype) 2669 { 2670 syncq_t *sq; 2671 perdm_t **pp; 2672 perdm_t *p; 2673 perdm_t *dmp; 2674 2675 ASSERT(str != NULL); 2676 ASSERT(qflag & (QPERMOD | QMTOUTPERIM)); 2677 2678 rw_enter(&perdm_rwlock, RW_READER); 2679 for (p = perdm_list; p != NULL; p = p->dm_next) { 2680 if (p->dm_str == str) { /* found one */ 2681 atomic_inc_32(&(p->dm_ref)); 2682 rw_exit(&perdm_rwlock); 2683 return (p); 2684 } 2685 } 2686 rw_exit(&perdm_rwlock); 2687 2688 sq = new_syncq(); 2689 if (qflag & QPERMOD) { 2690 sq->sq_type = sqtype | SQ_PERMOD; 2691 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; 2692 } else { 2693 ASSERT(qflag & QMTOUTPERIM); 2694 sq->sq_onext = sq->sq_oprev = sq; 2695 } 2696 2697 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP); 2698 dmp->dm_sq = sq; 2699 dmp->dm_str = str; 2700 dmp->dm_ref = 1; 2701 dmp->dm_next = NULL; 2702 2703 rw_enter(&perdm_rwlock, RW_WRITER); 2704 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) { 2705 if (p->dm_str == str) { /* already present */ 2706 p->dm_ref++; 2707 rw_exit(&perdm_rwlock); 2708 free_syncq(sq); 2709 kmem_free(dmp, sizeof (perdm_t)); 2710 return (p); 2711 } 2712 } 2713 2714 *pp = dmp; 2715 rw_exit(&perdm_rwlock); 2716 return (dmp); 2717 } 2718 2719 void 2720 rele_dm(perdm_t *dmp) 2721 { 2722 perdm_t **pp; 2723 perdm_t *p; 2724 2725 rw_enter(&perdm_rwlock, RW_WRITER); 2726 ASSERT(dmp->dm_ref > 0); 2727 2728 if (--dmp->dm_ref > 0) { 2729 rw_exit(&perdm_rwlock); 2730 return; 2731 } 2732 2733 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) 2734 if (p == dmp) 2735 break; 2736 ASSERT(p == dmp); 2737 *pp = p->dm_next; 2738 rw_exit(&perdm_rwlock); 2739 2740 /* 2741 * Wait for any background processing that relies on the 2742 * syncq to complete before it is freed. 2743 */ 2744 wait_sq_svc(p->dm_sq); 2745 free_syncq(p->dm_sq); 2746 kmem_free(p, sizeof (perdm_t)); 2747 } 2748 2749 /* 2750 * Make a protocol message given control and data buffers. 2751 * n.b., this can block; be careful of what locks you hold when calling it. 2752 * 2753 * If sd_maxblk is less than *iosize this routine can fail part way through 2754 * (due to an allocation failure). In this case on return *iosize will contain 2755 * the amount that was consumed. Otherwise *iosize will not be modified 2756 * i.e. it will contain the amount that was consumed. 2757 */ 2758 int 2759 strmakemsg( 2760 struct strbuf *mctl, 2761 ssize_t *iosize, 2762 struct uio *uiop, 2763 stdata_t *stp, 2764 int32_t flag, 2765 mblk_t **mpp) 2766 { 2767 mblk_t *mpctl = NULL; 2768 mblk_t *mpdata = NULL; 2769 int error; 2770 2771 ASSERT(uiop != NULL); 2772 2773 *mpp = NULL; 2774 /* Create control part, if any */ 2775 if ((mctl != NULL) && (mctl->len >= 0)) { 2776 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl); 2777 if (error) 2778 return (error); 2779 } 2780 /* Create data part, if any */ 2781 if (*iosize >= 0) { 2782 error = strmakedata(iosize, uiop, stp, flag, &mpdata); 2783 if (error) { 2784 freemsg(mpctl); 2785 return (error); 2786 } 2787 } 2788 if (mpctl != NULL) { 2789 if (mpdata != NULL) 2790 linkb(mpctl, mpdata); 2791 *mpp = mpctl; 2792 } else { 2793 *mpp = mpdata; 2794 } 2795 return (0); 2796 } 2797 2798 /* 2799 * Make the control part of a protocol message given a control buffer. 2800 * n.b., this can block; be careful of what locks you hold when calling it. 2801 */ 2802 int 2803 strmakectl( 2804 struct strbuf *mctl, 2805 int32_t flag, 2806 int32_t fflag, 2807 mblk_t **mpp) 2808 { 2809 mblk_t *bp = NULL; 2810 unsigned char msgtype; 2811 int error = 0; 2812 cred_t *cr = CRED(); 2813 2814 /* We do not support interrupt threads using the stream head to send */ 2815 ASSERT(cr != NULL); 2816 2817 *mpp = NULL; 2818 /* 2819 * Create control part of message, if any. 2820 */ 2821 if ((mctl != NULL) && (mctl->len >= 0)) { 2822 caddr_t base; 2823 int ctlcount; 2824 int allocsz; 2825 2826 if (flag & RS_HIPRI) 2827 msgtype = M_PCPROTO; 2828 else 2829 msgtype = M_PROTO; 2830 2831 ctlcount = mctl->len; 2832 base = mctl->buf; 2833 2834 /* 2835 * Give modules a better chance to reuse M_PROTO/M_PCPROTO 2836 * blocks by increasing the size to something more usable. 2837 */ 2838 allocsz = MAX(ctlcount, 64); 2839 2840 /* 2841 * Range checking has already been done; simply try 2842 * to allocate a message block for the ctl part. 2843 */ 2844 while ((bp = allocb_cred(allocsz, cr, 2845 curproc->p_pid)) == NULL) { 2846 if (fflag & (FNDELAY|FNONBLOCK)) 2847 return (EAGAIN); 2848 if (error = strwaitbuf(allocsz, BPRI_MED)) 2849 return (error); 2850 } 2851 2852 bp->b_datap->db_type = msgtype; 2853 if (copyin(base, bp->b_wptr, ctlcount)) { 2854 freeb(bp); 2855 return (EFAULT); 2856 } 2857 bp->b_wptr += ctlcount; 2858 } 2859 *mpp = bp; 2860 return (0); 2861 } 2862 2863 /* 2864 * Make a protocol message given data buffers. 2865 * n.b., this can block; be careful of what locks you hold when calling it. 2866 * 2867 * If sd_maxblk is less than *iosize this routine can fail part way through 2868 * (due to an allocation failure). In this case on return *iosize will contain 2869 * the amount that was consumed. Otherwise *iosize will not be modified 2870 * i.e. it will contain the amount that was consumed. 2871 */ 2872 int 2873 strmakedata( 2874 ssize_t *iosize, 2875 struct uio *uiop, 2876 stdata_t *stp, 2877 int32_t flag, 2878 mblk_t **mpp) 2879 { 2880 mblk_t *mp = NULL; 2881 mblk_t *bp; 2882 int wroff = (int)stp->sd_wroff; 2883 int tail_len = (int)stp->sd_tail; 2884 int extra = wroff + tail_len; 2885 int error = 0; 2886 ssize_t maxblk; 2887 ssize_t count = *iosize; 2888 cred_t *cr; 2889 2890 *mpp = NULL; 2891 if (count < 0) 2892 return (0); 2893 2894 /* We do not support interrupt threads using the stream head to send */ 2895 cr = CRED(); 2896 ASSERT(cr != NULL); 2897 2898 maxblk = stp->sd_maxblk; 2899 if (maxblk == INFPSZ) 2900 maxblk = count; 2901 2902 /* 2903 * Create data part of message, if any. 2904 */ 2905 do { 2906 ssize_t size; 2907 dblk_t *dp; 2908 2909 ASSERT(uiop); 2910 2911 size = MIN(count, maxblk); 2912 2913 while ((bp = allocb_cred(size + extra, cr, 2914 curproc->p_pid)) == NULL) { 2915 error = EAGAIN; 2916 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 2917 (error = strwaitbuf(size + extra, BPRI_MED)) != 0) { 2918 if (count == *iosize) { 2919 freemsg(mp); 2920 return (error); 2921 } else { 2922 *iosize -= count; 2923 *mpp = mp; 2924 return (0); 2925 } 2926 } 2927 } 2928 dp = bp->b_datap; 2929 dp->db_cpid = curproc->p_pid; 2930 ASSERT(wroff <= dp->db_lim - bp->b_wptr); 2931 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff; 2932 2933 if (flag & STRUIO_POSTPONE) { 2934 /* 2935 * Setup the stream uio portion of the 2936 * dblk for subsequent use by struioget(). 2937 */ 2938 dp->db_struioflag = STRUIO_SPEC; 2939 dp->db_cksumstart = 0; 2940 dp->db_cksumstuff = 0; 2941 dp->db_cksumend = size; 2942 *(long long *)dp->db_struioun.data = 0ll; 2943 bp->b_wptr += size; 2944 } else { 2945 if (stp->sd_copyflag & STRCOPYCACHED) 2946 uiop->uio_extflg |= UIO_COPY_CACHED; 2947 2948 if (size != 0) { 2949 error = uiomove(bp->b_wptr, size, UIO_WRITE, 2950 uiop); 2951 if (error != 0) { 2952 freeb(bp); 2953 freemsg(mp); 2954 return (error); 2955 } 2956 } 2957 bp->b_wptr += size; 2958 2959 if (stp->sd_wputdatafunc != NULL) { 2960 mblk_t *newbp; 2961 2962 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode, 2963 bp, NULL, NULL, NULL, NULL); 2964 if (newbp == NULL) { 2965 freeb(bp); 2966 freemsg(mp); 2967 return (ECOMM); 2968 } 2969 bp = newbp; 2970 } 2971 } 2972 2973 count -= size; 2974 2975 if (mp == NULL) 2976 mp = bp; 2977 else 2978 linkb(mp, bp); 2979 } while (count > 0); 2980 2981 *mpp = mp; 2982 return (0); 2983 } 2984 2985 /* 2986 * Wait for a buffer to become available. Return non-zero errno 2987 * if not able to wait, 0 if buffer is probably there. 2988 */ 2989 int 2990 strwaitbuf(size_t size, int pri) 2991 { 2992 bufcall_id_t id; 2993 2994 mutex_enter(&bcall_monitor); 2995 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast, 2996 &ttoproc(curthread)->p_flag_cv)) == 0) { 2997 mutex_exit(&bcall_monitor); 2998 return (ENOSR); 2999 } 3000 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) { 3001 unbufcall(id); 3002 mutex_exit(&bcall_monitor); 3003 return (EINTR); 3004 } 3005 unbufcall(id); 3006 mutex_exit(&bcall_monitor); 3007 return (0); 3008 } 3009 3010 /* 3011 * This function waits for a read or write event to happen on a stream. 3012 * fmode can specify FNDELAY and/or FNONBLOCK. 3013 * The timeout is in ms with -1 meaning infinite. 3014 * The flag values work as follows: 3015 * READWAIT Check for read side errors, send M_READ 3016 * GETWAIT Check for read side errors, no M_READ 3017 * WRITEWAIT Check for write side errors. 3018 * NOINTR Do not return error if nonblocking or timeout. 3019 * STR_NOERROR Ignore all errors except STPLEX. 3020 * STR_NOSIG Ignore/hold signals during the duration of the call. 3021 * STR_PEEK Pass through the strgeterr(). 3022 */ 3023 int 3024 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout, 3025 int *done) 3026 { 3027 int slpflg, errs; 3028 int error; 3029 kcondvar_t *sleepon; 3030 mblk_t *mp; 3031 ssize_t *rd_count; 3032 clock_t rval; 3033 3034 ASSERT(MUTEX_HELD(&stp->sd_lock)); 3035 if ((flag & READWAIT) || (flag & GETWAIT)) { 3036 slpflg = RSLEEP; 3037 sleepon = &_RD(stp->sd_wrq)->q_wait; 3038 errs = STRDERR|STPLEX; 3039 } else { 3040 slpflg = WSLEEP; 3041 sleepon = &stp->sd_wrq->q_wait; 3042 errs = STWRERR|STRHUP|STPLEX; 3043 } 3044 if (flag & STR_NOERROR) 3045 errs = STPLEX; 3046 3047 if (stp->sd_wakeq & slpflg) { 3048 /* 3049 * A strwakeq() is pending, no need to sleep. 3050 */ 3051 stp->sd_wakeq &= ~slpflg; 3052 *done = 0; 3053 return (0); 3054 } 3055 3056 if (stp->sd_flag & errs) { 3057 /* 3058 * Check for errors before going to sleep since the 3059 * caller might not have checked this while holding 3060 * sd_lock. 3061 */ 3062 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3063 if (error != 0) { 3064 *done = 1; 3065 return (error); 3066 } 3067 } 3068 3069 /* 3070 * If any module downstream has requested read notification 3071 * by setting SNDMREAD flag using M_SETOPTS, send a message 3072 * down stream. 3073 */ 3074 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) { 3075 mutex_exit(&stp->sd_lock); 3076 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED, 3077 (flag & STR_NOSIG), &error))) { 3078 mutex_enter(&stp->sd_lock); 3079 *done = 1; 3080 return (error); 3081 } 3082 mp->b_datap->db_type = M_READ; 3083 rd_count = (ssize_t *)mp->b_wptr; 3084 *rd_count = count; 3085 mp->b_wptr += sizeof (ssize_t); 3086 /* 3087 * Send the number of bytes requested by the 3088 * read as the argument to M_READ. 3089 */ 3090 stream_willservice(stp); 3091 putnext(stp->sd_wrq, mp); 3092 stream_runservice(stp); 3093 mutex_enter(&stp->sd_lock); 3094 3095 /* 3096 * If any data arrived due to inline processing 3097 * of putnext(), don't sleep. 3098 */ 3099 if (_RD(stp->sd_wrq)->q_first != NULL) { 3100 *done = 0; 3101 return (0); 3102 } 3103 } 3104 3105 if (fmode & (FNDELAY|FNONBLOCK)) { 3106 if (!(flag & NOINTR)) 3107 error = EAGAIN; 3108 else 3109 error = 0; 3110 *done = 1; 3111 return (error); 3112 } 3113 3114 stp->sd_flag |= slpflg; 3115 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2, 3116 "strwaitq sleeps (2):%p, %X, %lX, %X, %p", 3117 stp, flag, count, fmode, done); 3118 3119 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG); 3120 if (rval > 0) { 3121 /* EMPTY */ 3122 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2, 3123 "strwaitq awakes(2):%X, %X, %X, %X, %X", 3124 stp, flag, count, fmode, done); 3125 } else if (rval == 0) { 3126 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2, 3127 "strwaitq interrupt #2:%p, %X, %lX, %X, %p", 3128 stp, flag, count, fmode, done); 3129 stp->sd_flag &= ~slpflg; 3130 cv_broadcast(sleepon); 3131 if (!(flag & NOINTR)) 3132 error = EINTR; 3133 else 3134 error = 0; 3135 *done = 1; 3136 return (error); 3137 } else { 3138 /* timeout */ 3139 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME, 3140 "strwaitq timeout:%p, %X, %lX, %X, %p", 3141 stp, flag, count, fmode, done); 3142 *done = 1; 3143 if (!(flag & NOINTR)) 3144 return (ETIME); 3145 else 3146 return (0); 3147 } 3148 /* 3149 * If the caller implements delayed errors (i.e. queued after data) 3150 * we can not check for errors here since data as well as an 3151 * error might have arrived at the stream head. We return to 3152 * have the caller check the read queue before checking for errors. 3153 */ 3154 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) { 3155 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3156 if (error != 0) { 3157 *done = 1; 3158 return (error); 3159 } 3160 } 3161 *done = 0; 3162 return (0); 3163 } 3164 3165 /* 3166 * Perform job control discipline access checks. 3167 * Return 0 for success and the errno for failure. 3168 */ 3169 3170 #define cantsend(p, t, sig) \ 3171 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig)) 3172 3173 int 3174 straccess(struct stdata *stp, enum jcaccess mode) 3175 { 3176 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */ 3177 kthread_t *t = curthread; 3178 proc_t *p = ttoproc(t); 3179 sess_t *sp; 3180 3181 ASSERT(mutex_owned(&stp->sd_lock)); 3182 3183 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO) 3184 return (0); 3185 3186 mutex_enter(&p->p_lock); /* protects p_pgidp */ 3187 3188 for (;;) { 3189 mutex_enter(&p->p_splock); /* protects p->p_sessp */ 3190 sp = p->p_sessp; 3191 mutex_enter(&sp->s_lock); /* protects sp->* */ 3192 3193 /* 3194 * If this is not the calling process's controlling terminal 3195 * or if the calling process is already in the foreground 3196 * then allow access. 3197 */ 3198 if (sp->s_dev != stp->sd_vnode->v_rdev || 3199 p->p_pgidp == stp->sd_pgidp) { 3200 mutex_exit(&sp->s_lock); 3201 mutex_exit(&p->p_splock); 3202 mutex_exit(&p->p_lock); 3203 return (0); 3204 } 3205 3206 /* 3207 * Check to see if controlling terminal has been deallocated. 3208 */ 3209 if (sp->s_vp == NULL) { 3210 if (!cantsend(p, t, SIGHUP)) 3211 sigtoproc(p, t, SIGHUP); 3212 mutex_exit(&sp->s_lock); 3213 mutex_exit(&p->p_splock); 3214 mutex_exit(&p->p_lock); 3215 return (EIO); 3216 } 3217 3218 mutex_exit(&sp->s_lock); 3219 mutex_exit(&p->p_splock); 3220 3221 if (mode == JCGETP) { 3222 mutex_exit(&p->p_lock); 3223 return (0); 3224 } 3225 3226 if (mode == JCREAD) { 3227 if (p->p_detached || cantsend(p, t, SIGTTIN)) { 3228 mutex_exit(&p->p_lock); 3229 return (EIO); 3230 } 3231 mutex_exit(&p->p_lock); 3232 mutex_exit(&stp->sd_lock); 3233 pgsignal(p->p_pgidp, SIGTTIN); 3234 mutex_enter(&stp->sd_lock); 3235 mutex_enter(&p->p_lock); 3236 } else { /* mode == JCWRITE or JCSETP */ 3237 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) || 3238 cantsend(p, t, SIGTTOU)) { 3239 mutex_exit(&p->p_lock); 3240 return (0); 3241 } 3242 if (p->p_detached) { 3243 mutex_exit(&p->p_lock); 3244 return (EIO); 3245 } 3246 mutex_exit(&p->p_lock); 3247 mutex_exit(&stp->sd_lock); 3248 pgsignal(p->p_pgidp, SIGTTOU); 3249 mutex_enter(&stp->sd_lock); 3250 mutex_enter(&p->p_lock); 3251 } 3252 3253 /* 3254 * We call cv_wait_sig_swap() to cause the appropriate 3255 * action for the jobcontrol signal to take place. 3256 * If the signal is being caught, we will take the 3257 * EINTR error return. Otherwise, the default action 3258 * of causing the process to stop will take place. 3259 * In this case, we rely on the periodic cv_broadcast() on 3260 * &lbolt_cv to wake us up to loop around and test again. 3261 * We can't get here if the signal is ignored or 3262 * if the current thread is blocking the signal. 3263 */ 3264 mutex_exit(&stp->sd_lock); 3265 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) { 3266 mutex_exit(&p->p_lock); 3267 mutex_enter(&stp->sd_lock); 3268 return (EINTR); 3269 } 3270 mutex_exit(&p->p_lock); 3271 mutex_enter(&stp->sd_lock); 3272 mutex_enter(&p->p_lock); 3273 } 3274 } 3275 3276 /* 3277 * Return size of message of block type (bp->b_datap->db_type) 3278 */ 3279 size_t 3280 xmsgsize(mblk_t *bp) 3281 { 3282 unsigned char type; 3283 size_t count = 0; 3284 3285 type = bp->b_datap->db_type; 3286 3287 for (; bp; bp = bp->b_cont) { 3288 if (type != bp->b_datap->db_type) 3289 break; 3290 ASSERT(bp->b_wptr >= bp->b_rptr); 3291 count += bp->b_wptr - bp->b_rptr; 3292 } 3293 return (count); 3294 } 3295 3296 /* 3297 * Allocate a stream head. 3298 */ 3299 struct stdata * 3300 shalloc(queue_t *qp) 3301 { 3302 stdata_t *stp; 3303 3304 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP); 3305 3306 stp->sd_wrq = _WR(qp); 3307 stp->sd_strtab = NULL; 3308 stp->sd_iocid = 0; 3309 stp->sd_mate = NULL; 3310 stp->sd_freezer = NULL; 3311 stp->sd_refcnt = 0; 3312 stp->sd_wakeq = 0; 3313 stp->sd_anchor = 0; 3314 stp->sd_struiowrq = NULL; 3315 stp->sd_struiordq = NULL; 3316 stp->sd_struiodnak = 0; 3317 stp->sd_struionak = NULL; 3318 stp->sd_t_audit_data = NULL; 3319 stp->sd_rput_opt = 0; 3320 stp->sd_wput_opt = 0; 3321 stp->sd_read_opt = 0; 3322 stp->sd_rprotofunc = strrput_proto; 3323 stp->sd_rmiscfunc = strrput_misc; 3324 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL; 3325 stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL; 3326 stp->sd_ciputctrl = NULL; 3327 stp->sd_nciputctrl = 0; 3328 stp->sd_qhead = NULL; 3329 stp->sd_qtail = NULL; 3330 stp->sd_servid = NULL; 3331 stp->sd_nqueues = 0; 3332 stp->sd_svcflags = 0; 3333 stp->sd_copyflag = 0; 3334 3335 return (stp); 3336 } 3337 3338 /* 3339 * Free a stream head. 3340 */ 3341 void 3342 shfree(stdata_t *stp) 3343 { 3344 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock)); 3345 3346 stp->sd_wrq = NULL; 3347 3348 mutex_enter(&stp->sd_qlock); 3349 while (stp->sd_svcflags & STRS_SCHEDULED) { 3350 STRSTAT(strwaits); 3351 cv_wait(&stp->sd_qcv, &stp->sd_qlock); 3352 } 3353 mutex_exit(&stp->sd_qlock); 3354 3355 if (stp->sd_ciputctrl != NULL) { 3356 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1); 3357 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl, 3358 stp->sd_nciputctrl, 0); 3359 ASSERT(ciputctrl_cache != NULL); 3360 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl); 3361 stp->sd_ciputctrl = NULL; 3362 stp->sd_nciputctrl = 0; 3363 } 3364 ASSERT(stp->sd_qhead == NULL); 3365 ASSERT(stp->sd_qtail == NULL); 3366 ASSERT(stp->sd_nqueues == 0); 3367 kmem_cache_free(stream_head_cache, stp); 3368 } 3369 3370 /* 3371 * Allocate a pair of queues and a syncq for the pair 3372 */ 3373 queue_t * 3374 allocq(void) 3375 { 3376 queinfo_t *qip; 3377 queue_t *qp, *wqp; 3378 syncq_t *sq; 3379 3380 qip = kmem_cache_alloc(queue_cache, KM_SLEEP); 3381 3382 qp = &qip->qu_rqueue; 3383 wqp = &qip->qu_wqueue; 3384 sq = &qip->qu_syncq; 3385 3386 qp->q_last = NULL; 3387 qp->q_next = NULL; 3388 qp->q_ptr = NULL; 3389 qp->q_flag = QUSE | QREADR; 3390 qp->q_bandp = NULL; 3391 qp->q_stream = NULL; 3392 qp->q_syncq = sq; 3393 qp->q_nband = 0; 3394 qp->q_nfsrv = NULL; 3395 qp->q_draining = 0; 3396 qp->q_syncqmsgs = 0; 3397 qp->q_spri = 0; 3398 qp->q_qtstamp = 0; 3399 qp->q_sqtstamp = 0; 3400 qp->q_fp = NULL; 3401 3402 wqp->q_last = NULL; 3403 wqp->q_next = NULL; 3404 wqp->q_ptr = NULL; 3405 wqp->q_flag = QUSE; 3406 wqp->q_bandp = NULL; 3407 wqp->q_stream = NULL; 3408 wqp->q_syncq = sq; 3409 wqp->q_nband = 0; 3410 wqp->q_nfsrv = NULL; 3411 wqp->q_draining = 0; 3412 wqp->q_syncqmsgs = 0; 3413 wqp->q_qtstamp = 0; 3414 wqp->q_sqtstamp = 0; 3415 wqp->q_spri = 0; 3416 3417 sq->sq_count = 0; 3418 sq->sq_rmqcount = 0; 3419 sq->sq_flags = 0; 3420 sq->sq_type = 0; 3421 sq->sq_callbflags = 0; 3422 sq->sq_cancelid = 0; 3423 sq->sq_ciputctrl = NULL; 3424 sq->sq_nciputctrl = 0; 3425 sq->sq_needexcl = 0; 3426 sq->sq_svcflags = 0; 3427 3428 return (qp); 3429 } 3430 3431 /* 3432 * Free a pair of queues and the "attached" syncq. 3433 * Discard any messages left on the syncq(s), remove the syncq(s) from the 3434 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq. 3435 */ 3436 void 3437 freeq(queue_t *qp) 3438 { 3439 qband_t *qbp, *nqbp; 3440 syncq_t *sq, *outer; 3441 queue_t *wqp = _WR(qp); 3442 3443 ASSERT(qp->q_flag & QREADR); 3444 3445 /* 3446 * If a previously dispatched taskq job is scheduled to run 3447 * sync_service() or a service routine is scheduled for the 3448 * queues about to be freed, wait here until all service is 3449 * done on the queue and all associated queues and syncqs. 3450 */ 3451 wait_svc(qp); 3452 3453 (void) flush_syncq(qp->q_syncq, qp); 3454 (void) flush_syncq(wqp->q_syncq, wqp); 3455 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0); 3456 3457 /* 3458 * Flush the queues before q_next is set to NULL This is needed 3459 * in order to backenable any downstream queue before we go away. 3460 * Note: we are already removed from the stream so that the 3461 * backenabling will not cause any messages to be delivered to our 3462 * put procedures. 3463 */ 3464 flushq(qp, FLUSHALL); 3465 flushq(wqp, FLUSHALL); 3466 3467 /* Tidy up - removeq only does a half-remove from stream */ 3468 qp->q_next = wqp->q_next = NULL; 3469 ASSERT(!(qp->q_flag & QENAB)); 3470 ASSERT(!(wqp->q_flag & QENAB)); 3471 3472 outer = qp->q_syncq->sq_outer; 3473 if (outer != NULL) { 3474 outer_remove(outer, qp->q_syncq); 3475 if (wqp->q_syncq != qp->q_syncq) 3476 outer_remove(outer, wqp->q_syncq); 3477 } 3478 /* 3479 * Free any syncqs that are outside what allocq returned. 3480 */ 3481 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD)) 3482 free_syncq(qp->q_syncq); 3483 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp)) 3484 free_syncq(wqp->q_syncq); 3485 3486 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3487 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3488 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 3489 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp))); 3490 sq = SQ(qp); 3491 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 3492 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 3493 ASSERT(sq->sq_outer == NULL); 3494 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 3495 ASSERT(sq->sq_callbpend == NULL); 3496 ASSERT(sq->sq_needexcl == 0); 3497 3498 if (sq->sq_ciputctrl != NULL) { 3499 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 3500 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 3501 sq->sq_nciputctrl, 0); 3502 ASSERT(ciputctrl_cache != NULL); 3503 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 3504 sq->sq_ciputctrl = NULL; 3505 sq->sq_nciputctrl = 0; 3506 } 3507 3508 ASSERT(qp->q_first == NULL && wqp->q_first == NULL); 3509 ASSERT(qp->q_count == 0 && wqp->q_count == 0); 3510 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0); 3511 3512 qp->q_flag &= ~QUSE; 3513 wqp->q_flag &= ~QUSE; 3514 3515 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */ 3516 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */ 3517 3518 qbp = qp->q_bandp; 3519 while (qbp) { 3520 nqbp = qbp->qb_next; 3521 freeband(qbp); 3522 qbp = nqbp; 3523 } 3524 qbp = wqp->q_bandp; 3525 while (qbp) { 3526 nqbp = qbp->qb_next; 3527 freeband(qbp); 3528 qbp = nqbp; 3529 } 3530 kmem_cache_free(queue_cache, qp); 3531 } 3532 3533 /* 3534 * Allocate a qband structure. 3535 */ 3536 qband_t * 3537 allocband(void) 3538 { 3539 qband_t *qbp; 3540 3541 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP); 3542 if (qbp == NULL) 3543 return (NULL); 3544 3545 qbp->qb_next = NULL; 3546 qbp->qb_count = 0; 3547 qbp->qb_mblkcnt = 0; 3548 qbp->qb_first = NULL; 3549 qbp->qb_last = NULL; 3550 qbp->qb_flag = 0; 3551 3552 return (qbp); 3553 } 3554 3555 /* 3556 * Free a qband structure. 3557 */ 3558 void 3559 freeband(qband_t *qbp) 3560 { 3561 kmem_cache_free(qband_cache, qbp); 3562 } 3563 3564 /* 3565 * Just like putnextctl(9F), except that allocb_wait() is used. 3566 * 3567 * Consolidation Private, and of course only callable from the stream head or 3568 * routines that may block. 3569 */ 3570 int 3571 putnextctl_wait(queue_t *q, int type) 3572 { 3573 mblk_t *bp; 3574 int error; 3575 3576 if ((datamsg(type) && (type != M_DELAY)) || 3577 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL) 3578 return (0); 3579 3580 bp->b_datap->db_type = (unsigned char)type; 3581 putnext(q, bp); 3582 return (1); 3583 } 3584 3585 /* 3586 * Run any possible bufcalls. 3587 */ 3588 void 3589 runbufcalls(void) 3590 { 3591 strbufcall_t *bcp; 3592 3593 mutex_enter(&bcall_monitor); 3594 mutex_enter(&strbcall_lock); 3595 3596 if (strbcalls.bc_head) { 3597 size_t count; 3598 int nevent; 3599 3600 /* 3601 * count how many events are on the list 3602 * now so we can check to avoid looping 3603 * in low memory situations 3604 */ 3605 nevent = 0; 3606 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next) 3607 nevent++; 3608 3609 /* 3610 * get estimate of available memory from kmem_avail(). 3611 * awake all bufcall functions waiting for 3612 * memory whose request could be satisfied 3613 * by 'count' memory and let 'em fight for it. 3614 */ 3615 count = kmem_avail(); 3616 while ((bcp = strbcalls.bc_head) != NULL && nevent) { 3617 STRSTAT(bufcalls); 3618 --nevent; 3619 if (bcp->bc_size <= count) { 3620 bcp->bc_executor = curthread; 3621 mutex_exit(&strbcall_lock); 3622 (*bcp->bc_func)(bcp->bc_arg); 3623 mutex_enter(&strbcall_lock); 3624 bcp->bc_executor = NULL; 3625 cv_broadcast(&bcall_cv); 3626 strbcalls.bc_head = bcp->bc_next; 3627 kmem_free(bcp, sizeof (strbufcall_t)); 3628 } else { 3629 /* 3630 * too big, try again later - note 3631 * that nevent was decremented above 3632 * so we won't retry this one on this 3633 * iteration of the loop 3634 */ 3635 if (bcp->bc_next != NULL) { 3636 strbcalls.bc_head = bcp->bc_next; 3637 bcp->bc_next = NULL; 3638 strbcalls.bc_tail->bc_next = bcp; 3639 strbcalls.bc_tail = bcp; 3640 } 3641 } 3642 } 3643 if (strbcalls.bc_head == NULL) 3644 strbcalls.bc_tail = NULL; 3645 } 3646 3647 mutex_exit(&strbcall_lock); 3648 mutex_exit(&bcall_monitor); 3649 } 3650 3651 3652 /* 3653 * Actually run queue's service routine. 3654 */ 3655 static void 3656 runservice(queue_t *q) 3657 { 3658 qband_t *qbp; 3659 3660 ASSERT(q->q_qinfo->qi_srvp); 3661 again: 3662 entersq(q->q_syncq, SQ_SVC); 3663 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START, 3664 "runservice starts:%p", q); 3665 3666 if (!(q->q_flag & QWCLOSE)) 3667 (*q->q_qinfo->qi_srvp)(q); 3668 3669 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END, 3670 "runservice ends:(%p)", q); 3671 3672 leavesq(q->q_syncq, SQ_SVC); 3673 3674 mutex_enter(QLOCK(q)); 3675 if (q->q_flag & QENAB) { 3676 q->q_flag &= ~QENAB; 3677 mutex_exit(QLOCK(q)); 3678 goto again; 3679 } 3680 q->q_flag &= ~QINSERVICE; 3681 q->q_flag &= ~QBACK; 3682 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next) 3683 qbp->qb_flag &= ~QB_BACK; 3684 /* 3685 * Wakeup thread waiting for the service procedure 3686 * to be run (strclose and qdetach). 3687 */ 3688 cv_broadcast(&q->q_wait); 3689 3690 mutex_exit(QLOCK(q)); 3691 } 3692 3693 /* 3694 * Background processing of bufcalls. 3695 */ 3696 void 3697 streams_bufcall_service(void) 3698 { 3699 callb_cpr_t cprinfo; 3700 3701 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr, 3702 "streams_bufcall_service"); 3703 3704 mutex_enter(&strbcall_lock); 3705 3706 for (;;) { 3707 if (strbcalls.bc_head != NULL && kmem_avail() > 0) { 3708 mutex_exit(&strbcall_lock); 3709 runbufcalls(); 3710 mutex_enter(&strbcall_lock); 3711 } 3712 if (strbcalls.bc_head != NULL) { 3713 STRSTAT(bcwaits); 3714 /* Wait for memory to become available */ 3715 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3716 (void) cv_reltimedwait(&memavail_cv, &strbcall_lock, 3717 SEC_TO_TICK(60), TR_CLOCK_TICK); 3718 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3719 } 3720 3721 /* Wait for new work to arrive */ 3722 if (strbcalls.bc_head == NULL) { 3723 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3724 cv_wait(&strbcall_cv, &strbcall_lock); 3725 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3726 } 3727 } 3728 } 3729 3730 /* 3731 * Background processing of streams background tasks which failed 3732 * taskq_dispatch. 3733 */ 3734 static void 3735 streams_qbkgrnd_service(void) 3736 { 3737 callb_cpr_t cprinfo; 3738 queue_t *q; 3739 3740 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3741 "streams_bkgrnd_service"); 3742 3743 mutex_enter(&service_queue); 3744 3745 for (;;) { 3746 /* 3747 * Wait for work to arrive. 3748 */ 3749 while ((freebs_list == NULL) && (qhead == NULL)) { 3750 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3751 cv_wait(&services_to_run, &service_queue); 3752 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3753 } 3754 /* 3755 * Handle all pending freebs requests to free memory. 3756 */ 3757 while (freebs_list != NULL) { 3758 mblk_t *mp = freebs_list; 3759 freebs_list = mp->b_next; 3760 mutex_exit(&service_queue); 3761 mblk_free(mp); 3762 mutex_enter(&service_queue); 3763 } 3764 /* 3765 * Run pending queues. 3766 */ 3767 while (qhead != NULL) { 3768 DQ(q, qhead, qtail, q_link); 3769 ASSERT(q != NULL); 3770 mutex_exit(&service_queue); 3771 queue_service(q); 3772 mutex_enter(&service_queue); 3773 } 3774 ASSERT(qhead == NULL && qtail == NULL); 3775 } 3776 } 3777 3778 /* 3779 * Background processing of streams background tasks which failed 3780 * taskq_dispatch. 3781 */ 3782 static void 3783 streams_sqbkgrnd_service(void) 3784 { 3785 callb_cpr_t cprinfo; 3786 syncq_t *sq; 3787 3788 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3789 "streams_sqbkgrnd_service"); 3790 3791 mutex_enter(&service_queue); 3792 3793 for (;;) { 3794 /* 3795 * Wait for work to arrive. 3796 */ 3797 while (sqhead == NULL) { 3798 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3799 cv_wait(&syncqs_to_run, &service_queue); 3800 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3801 } 3802 3803 /* 3804 * Run pending syncqs. 3805 */ 3806 while (sqhead != NULL) { 3807 DQ(sq, sqhead, sqtail, sq_next); 3808 ASSERT(sq != NULL); 3809 ASSERT(sq->sq_svcflags & SQ_BGTHREAD); 3810 mutex_exit(&service_queue); 3811 syncq_service(sq); 3812 mutex_enter(&service_queue); 3813 } 3814 } 3815 } 3816 3817 /* 3818 * Disable the syncq and wait for background syncq processing to complete. 3819 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the 3820 * list. 3821 */ 3822 void 3823 wait_sq_svc(syncq_t *sq) 3824 { 3825 mutex_enter(SQLOCK(sq)); 3826 sq->sq_svcflags |= SQ_DISABLED; 3827 if (sq->sq_svcflags & SQ_BGTHREAD) { 3828 syncq_t *sq_chase; 3829 syncq_t *sq_curr; 3830 int removed; 3831 3832 ASSERT(sq->sq_servcount == 1); 3833 mutex_enter(&service_queue); 3834 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); 3835 mutex_exit(&service_queue); 3836 if (removed) { 3837 sq->sq_svcflags &= ~SQ_BGTHREAD; 3838 sq->sq_servcount = 0; 3839 STRSTAT(sqremoved); 3840 goto done; 3841 } 3842 } 3843 while (sq->sq_servcount != 0) { 3844 sq->sq_flags |= SQ_WANTWAKEUP; 3845 cv_wait(&sq->sq_wait, SQLOCK(sq)); 3846 } 3847 done: 3848 mutex_exit(SQLOCK(sq)); 3849 } 3850 3851 /* 3852 * Put a syncq on the list of syncq's to be serviced by the sqthread. 3853 * Add the argument to the end of the sqhead list and set the flag 3854 * indicating this syncq has been enabled. If it has already been 3855 * enabled, don't do anything. 3856 * This routine assumes that SQLOCK is held. 3857 * NOTE that the lock order is to have the SQLOCK first, 3858 * so if the service_syncq lock is held, we need to release it 3859 * before acquiring the SQLOCK (mostly relevant for the background 3860 * thread, and this seems to be common among the STREAMS global locks). 3861 * Note that the sq_svcflags are protected by the SQLOCK. 3862 */ 3863 void 3864 sqenable(syncq_t *sq) 3865 { 3866 /* 3867 * This is probably not important except for where I believe it 3868 * is being called. At that point, it should be held (and it 3869 * is a pain to release it just for this routine, so don't do 3870 * it). 3871 */ 3872 ASSERT(MUTEX_HELD(SQLOCK(sq))); 3873 3874 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); 3875 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); 3876 3877 /* 3878 * Do not put on list if background thread is scheduled or 3879 * syncq is disabled. 3880 */ 3881 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) 3882 return; 3883 3884 /* 3885 * Check whether we should enable sq at all. 3886 * Non PERMOD syncqs may be drained by at most one thread. 3887 * PERMOD syncqs may be drained by several threads but we limit the 3888 * total amount to the lesser of 3889 * Number of queues on the squeue and 3890 * Number of CPUs. 3891 */ 3892 if (sq->sq_servcount != 0) { 3893 if (((sq->sq_type & SQ_PERMOD) == 0) || 3894 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { 3895 STRSTAT(sqtoomany); 3896 return; 3897 } 3898 } 3899 3900 sq->sq_tstamp = ddi_get_lbolt(); 3901 STRSTAT(sqenables); 3902 3903 /* Attempt a taskq dispatch */ 3904 sq->sq_servid = (void *)taskq_dispatch(streams_taskq, 3905 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); 3906 if (sq->sq_servid != NULL) { 3907 sq->sq_servcount++; 3908 return; 3909 } 3910 3911 /* 3912 * This taskq dispatch failed, but a previous one may have succeeded. 3913 * Don't try to schedule on the background thread whilst there is 3914 * outstanding taskq processing. 3915 */ 3916 if (sq->sq_servcount != 0) 3917 return; 3918 3919 /* 3920 * System is low on resources and can't perform a non-sleeping 3921 * dispatch. Schedule the syncq for a background thread and mark the 3922 * syncq to avoid any further taskq dispatch attempts. 3923 */ 3924 mutex_enter(&service_queue); 3925 STRSTAT(taskqfails); 3926 ENQUEUE(sq, sqhead, sqtail, sq_next); 3927 sq->sq_svcflags |= SQ_BGTHREAD; 3928 sq->sq_servcount = 1; 3929 cv_signal(&syncqs_to_run); 3930 mutex_exit(&service_queue); 3931 } 3932 3933 /* 3934 * Note: fifo_close() depends on the mblk_t on the queue being freed 3935 * asynchronously. The asynchronous freeing of messages breaks the 3936 * recursive call chain of fifo_close() while there are I_SENDFD type of 3937 * messages referring to other file pointers on the queue. Then when 3938 * closing pipes it can avoid stack overflow in case of daisy-chained 3939 * pipes, and also avoid deadlock in case of fifonode_t pairs (which 3940 * share the same fifolock_t). 3941 * 3942 * No need to kpreempt_disable to access cpu_seqid. If we migrate and 3943 * the esb queue does not match the new CPU, that is OK. 3944 */ 3945 void 3946 freebs_enqueue(mblk_t *mp, dblk_t *dbp) 3947 { 3948 int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q; 3949 esb_queue_t *eqp; 3950 3951 ASSERT(dbp->db_mblk == mp); 3952 ASSERT(qindex < esbq_nelem); 3953 3954 eqp = system_esbq_array; 3955 if (eqp != NULL) { 3956 eqp += qindex; 3957 } else { 3958 mutex_enter(&esbq_lock); 3959 if (kmem_ready && system_esbq_array == NULL) 3960 system_esbq_array = (esb_queue_t *)kmem_zalloc( 3961 esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP); 3962 mutex_exit(&esbq_lock); 3963 eqp = system_esbq_array; 3964 if (eqp != NULL) 3965 eqp += qindex; 3966 else 3967 eqp = &system_esbq; 3968 } 3969 3970 /* 3971 * Check data sanity. The dblock should have non-empty free function. 3972 * It is better to panic here then later when the dblock is freed 3973 * asynchronously when the context is lost. 3974 */ 3975 if (dbp->db_frtnp->free_func == NULL) { 3976 panic("freebs_enqueue: dblock %p has a NULL free callback", 3977 (void *)dbp); 3978 } 3979 3980 mutex_enter(&eqp->eq_lock); 3981 /* queue the new mblk on the esballoc queue */ 3982 if (eqp->eq_head == NULL) { 3983 eqp->eq_head = eqp->eq_tail = mp; 3984 } else { 3985 eqp->eq_tail->b_next = mp; 3986 eqp->eq_tail = mp; 3987 } 3988 eqp->eq_len++; 3989 3990 /* If we're the first thread to reach the threshold, process */ 3991 if (eqp->eq_len >= esbq_max_qlen && 3992 !(eqp->eq_flags & ESBQ_PROCESSING)) 3993 esballoc_process_queue(eqp); 3994 3995 esballoc_set_timer(eqp, esbq_timeout); 3996 mutex_exit(&eqp->eq_lock); 3997 } 3998 3999 static void 4000 esballoc_process_queue(esb_queue_t *eqp) 4001 { 4002 mblk_t *mp; 4003 4004 ASSERT(MUTEX_HELD(&eqp->eq_lock)); 4005 4006 eqp->eq_flags |= ESBQ_PROCESSING; 4007 4008 do { 4009 /* 4010 * Detach the message chain for processing. 4011 */ 4012 mp = eqp->eq_head; 4013 eqp->eq_tail->b_next = NULL; 4014 eqp->eq_head = eqp->eq_tail = NULL; 4015 eqp->eq_len = 0; 4016 mutex_exit(&eqp->eq_lock); 4017 4018 /* 4019 * Process the message chain. 4020 */ 4021 esballoc_enqueue_mblk(mp); 4022 mutex_enter(&eqp->eq_lock); 4023 } while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0)); 4024 4025 eqp->eq_flags &= ~ESBQ_PROCESSING; 4026 } 4027 4028 /* 4029 * taskq callback routine to free esballoced mblk's 4030 */ 4031 static void 4032 esballoc_mblk_free(mblk_t *mp) 4033 { 4034 mblk_t *nextmp; 4035 4036 for (; mp != NULL; mp = nextmp) { 4037 nextmp = mp->b_next; 4038 mp->b_next = NULL; 4039 mblk_free(mp); 4040 } 4041 } 4042 4043 static void 4044 esballoc_enqueue_mblk(mblk_t *mp) 4045 { 4046 4047 if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp, 4048 TQ_NOSLEEP) == TASKQID_INVALID) { 4049 mblk_t *first_mp = mp; 4050 /* 4051 * System is low on resources and can't perform a non-sleeping 4052 * dispatch. Schedule for a background thread. 4053 */ 4054 mutex_enter(&service_queue); 4055 STRSTAT(taskqfails); 4056 4057 while (mp->b_next != NULL) 4058 mp = mp->b_next; 4059 4060 mp->b_next = freebs_list; 4061 freebs_list = first_mp; 4062 cv_signal(&services_to_run); 4063 mutex_exit(&service_queue); 4064 } 4065 } 4066 4067 static void 4068 esballoc_timer(void *arg) 4069 { 4070 esb_queue_t *eqp = arg; 4071 4072 mutex_enter(&eqp->eq_lock); 4073 eqp->eq_flags &= ~ESBQ_TIMER; 4074 4075 if (!(eqp->eq_flags & ESBQ_PROCESSING) && 4076 eqp->eq_len > 0) 4077 esballoc_process_queue(eqp); 4078 4079 esballoc_set_timer(eqp, esbq_timeout); 4080 mutex_exit(&eqp->eq_lock); 4081 } 4082 4083 static void 4084 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout) 4085 { 4086 ASSERT(MUTEX_HELD(&eqp->eq_lock)); 4087 4088 if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) { 4089 (void) timeout(esballoc_timer, eqp, eq_timeout); 4090 eqp->eq_flags |= ESBQ_TIMER; 4091 } 4092 } 4093 4094 /* 4095 * Setup esbq array length based upon NCPU scaled by CPUs per 4096 * queue. Use static system_esbq until kmem_ready and we can 4097 * create an array in freebs_enqueue(). 4098 */ 4099 void 4100 esballoc_queue_init(void) 4101 { 4102 esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1); 4103 esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q; 4104 esbq_nelem = howmany(NCPU, esbq_cpus_per_q); 4105 system_esbq.eq_len = 0; 4106 system_esbq.eq_head = system_esbq.eq_tail = NULL; 4107 system_esbq.eq_flags = 0; 4108 } 4109 4110 /* 4111 * Set the QBACK or QB_BACK flag in the given queue for 4112 * the given priority band. 4113 */ 4114 void 4115 setqback(queue_t *q, unsigned char pri) 4116 { 4117 int i; 4118 qband_t *qbp; 4119 qband_t **qbpp; 4120 4121 ASSERT(MUTEX_HELD(QLOCK(q))); 4122 if (pri != 0) { 4123 if (pri > q->q_nband) { 4124 qbpp = &q->q_bandp; 4125 while (*qbpp) 4126 qbpp = &(*qbpp)->qb_next; 4127 while (pri > q->q_nband) { 4128 if ((*qbpp = allocband()) == NULL) { 4129 cmn_err(CE_WARN, 4130 "setqback: can't allocate qband\n"); 4131 return; 4132 } 4133 (*qbpp)->qb_hiwat = q->q_hiwat; 4134 (*qbpp)->qb_lowat = q->q_lowat; 4135 q->q_nband++; 4136 qbpp = &(*qbpp)->qb_next; 4137 } 4138 } 4139 qbp = q->q_bandp; 4140 i = pri; 4141 while (--i) 4142 qbp = qbp->qb_next; 4143 qbp->qb_flag |= QB_BACK; 4144 } else { 4145 q->q_flag |= QBACK; 4146 } 4147 } 4148 4149 int 4150 strcopyin(void *from, void *to, size_t len, int copyflag) 4151 { 4152 if (copyflag & U_TO_K) { 4153 ASSERT((copyflag & K_TO_K) == 0); 4154 if (copyin(from, to, len)) 4155 return (EFAULT); 4156 } else { 4157 ASSERT(copyflag & K_TO_K); 4158 bcopy(from, to, len); 4159 } 4160 return (0); 4161 } 4162 4163 int 4164 strcopyout(void *from, void *to, size_t len, int copyflag) 4165 { 4166 if (copyflag & U_TO_K) { 4167 if (copyout(from, to, len)) 4168 return (EFAULT); 4169 } else { 4170 ASSERT(copyflag & K_TO_K); 4171 bcopy(from, to, len); 4172 } 4173 return (0); 4174 } 4175 4176 /* 4177 * strsignal_nolock() posts a signal to the process(es) at the stream head. 4178 * It assumes that the stream head lock is already held, whereas strsignal() 4179 * acquires the lock first. This routine was created because a few callers 4180 * release the stream head lock before calling only to re-acquire it after 4181 * it returns. 4182 */ 4183 void 4184 strsignal_nolock(stdata_t *stp, int sig, uchar_t band) 4185 { 4186 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4187 switch (sig) { 4188 case SIGPOLL: 4189 if (stp->sd_sigflags & S_MSG) 4190 strsendsig(stp->sd_siglist, S_MSG, band, 0); 4191 break; 4192 default: 4193 if (stp->sd_pgidp) 4194 pgsignal(stp->sd_pgidp, sig); 4195 break; 4196 } 4197 } 4198 4199 void 4200 strsignal(stdata_t *stp, int sig, int32_t band) 4201 { 4202 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG, 4203 "strsignal:%p, %X, %X", stp, sig, band); 4204 4205 mutex_enter(&stp->sd_lock); 4206 switch (sig) { 4207 case SIGPOLL: 4208 if (stp->sd_sigflags & S_MSG) 4209 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 4210 break; 4211 4212 default: 4213 if (stp->sd_pgidp) { 4214 pgsignal(stp->sd_pgidp, sig); 4215 } 4216 break; 4217 } 4218 mutex_exit(&stp->sd_lock); 4219 } 4220 4221 void 4222 strhup(stdata_t *stp) 4223 { 4224 ASSERT(mutex_owned(&stp->sd_lock)); 4225 pollwakeup(&stp->sd_pollist, POLLHUP); 4226 if (stp->sd_sigflags & S_HANGUP) 4227 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0); 4228 } 4229 4230 /* 4231 * Backenable the first queue upstream from `q' with a service procedure. 4232 */ 4233 void 4234 backenable(queue_t *q, uchar_t pri) 4235 { 4236 queue_t *nq; 4237 4238 /* 4239 * Our presence might not prevent other modules in our own 4240 * stream from popping/pushing since the caller of getq might not 4241 * have a claim on the queue (some drivers do a getq on somebody 4242 * else's queue - they know that the queue itself is not going away 4243 * but the framework has to guarantee q_next in that stream). 4244 */ 4245 claimstr(q); 4246 4247 /* Find nearest back queue with service proc */ 4248 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) { 4249 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq)); 4250 } 4251 4252 if (nq) { 4253 kthread_t *freezer; 4254 /* 4255 * backenable can be called either with no locks held 4256 * or with the stream frozen (the latter occurs when a module 4257 * calls rmvq with the stream frozen). If the stream is frozen 4258 * by the caller the caller will hold all qlocks in the stream. 4259 * Note that a frozen stream doesn't freeze a mated stream, 4260 * so we explicitly check for that. 4261 */ 4262 freezer = STREAM(q)->sd_freezer; 4263 if (freezer != curthread || STREAM(q) != STREAM(nq)) { 4264 mutex_enter(QLOCK(nq)); 4265 } 4266 #ifdef DEBUG 4267 else { 4268 ASSERT(frozenstr(q)); 4269 ASSERT(MUTEX_HELD(QLOCK(q))); 4270 ASSERT(MUTEX_HELD(QLOCK(nq))); 4271 } 4272 #endif 4273 setqback(nq, pri); 4274 qenable_locked(nq); 4275 if (freezer != curthread || STREAM(q) != STREAM(nq)) 4276 mutex_exit(QLOCK(nq)); 4277 } 4278 releasestr(q); 4279 } 4280 4281 /* 4282 * Return the appropriate errno when one of flags_to_check is set 4283 * in sd_flags. Uses the exported error routines if they are set. 4284 * Will return 0 if non error is set (or if the exported error routines 4285 * do not return an error). 4286 * 4287 * If there is both a read and write error to check, we prefer the read error. 4288 * Also, give preference to recorded errno's over the error functions. 4289 * The flags that are handled are: 4290 * STPLEX return EINVAL 4291 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST) 4292 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST) 4293 * STRHUP return sd_werror 4294 * 4295 * If the caller indicates that the operation is a peek, a nonpersistent error 4296 * is not cleared. 4297 */ 4298 int 4299 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek) 4300 { 4301 int32_t sd_flag = stp->sd_flag & flags_to_check; 4302 int error = 0; 4303 4304 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4305 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0); 4306 if (sd_flag & STPLEX) 4307 error = EINVAL; 4308 else if (sd_flag & STRDERR) { 4309 error = stp->sd_rerror; 4310 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) { 4311 /* 4312 * Read errors are non-persistent i.e. discarded once 4313 * returned to a non-peeking caller, 4314 */ 4315 stp->sd_rerror = 0; 4316 stp->sd_flag &= ~STRDERR; 4317 } 4318 if (error == 0 && stp->sd_rderrfunc != NULL) { 4319 int clearerr = 0; 4320 4321 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek, 4322 &clearerr); 4323 if (clearerr) { 4324 stp->sd_flag &= ~STRDERR; 4325 stp->sd_rderrfunc = NULL; 4326 } 4327 } 4328 } else if (sd_flag & STWRERR) { 4329 error = stp->sd_werror; 4330 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) { 4331 /* 4332 * Write errors are non-persistent i.e. discarded once 4333 * returned to a non-peeking caller, 4334 */ 4335 stp->sd_werror = 0; 4336 stp->sd_flag &= ~STWRERR; 4337 } 4338 if (error == 0 && stp->sd_wrerrfunc != NULL) { 4339 int clearerr = 0; 4340 4341 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek, 4342 &clearerr); 4343 if (clearerr) { 4344 stp->sd_flag &= ~STWRERR; 4345 stp->sd_wrerrfunc = NULL; 4346 } 4347 } 4348 } else if (sd_flag & STRHUP) { 4349 /* sd_werror set when STRHUP */ 4350 error = stp->sd_werror; 4351 } 4352 return (error); 4353 } 4354 4355 4356 /* 4357 * Single-thread open/close/push/pop 4358 * for twisted streams also 4359 */ 4360 int 4361 strstartplumb(stdata_t *stp, int flag, int cmd) 4362 { 4363 int waited = 1; 4364 int error = 0; 4365 4366 if (STRMATED(stp)) { 4367 struct stdata *stmatep = stp->sd_mate; 4368 4369 STRLOCKMATES(stp); 4370 while (waited) { 4371 waited = 0; 4372 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4373 if ((cmd == I_POP) && 4374 (flag & (FNDELAY|FNONBLOCK))) { 4375 STRUNLOCKMATES(stp); 4376 return (EAGAIN); 4377 } 4378 waited = 1; 4379 mutex_exit(&stp->sd_lock); 4380 if (!cv_wait_sig(&stmatep->sd_monitor, 4381 &stmatep->sd_lock)) { 4382 mutex_exit(&stmatep->sd_lock); 4383 return (EINTR); 4384 } 4385 mutex_exit(&stmatep->sd_lock); 4386 STRLOCKMATES(stp); 4387 } 4388 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4389 if ((cmd == I_POP) && 4390 (flag & (FNDELAY|FNONBLOCK))) { 4391 STRUNLOCKMATES(stp); 4392 return (EAGAIN); 4393 } 4394 waited = 1; 4395 mutex_exit(&stmatep->sd_lock); 4396 if (!cv_wait_sig(&stp->sd_monitor, 4397 &stp->sd_lock)) { 4398 mutex_exit(&stp->sd_lock); 4399 return (EINTR); 4400 } 4401 mutex_exit(&stp->sd_lock); 4402 STRLOCKMATES(stp); 4403 } 4404 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4405 error = strgeterr(stp, 4406 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4407 if (error != 0) { 4408 STRUNLOCKMATES(stp); 4409 return (error); 4410 } 4411 } 4412 } 4413 stp->sd_flag |= STRPLUMB; 4414 STRUNLOCKMATES(stp); 4415 } else { 4416 mutex_enter(&stp->sd_lock); 4417 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4418 if (((cmd == I_POP) || (cmd == _I_REMOVE)) && 4419 (flag & (FNDELAY|FNONBLOCK))) { 4420 mutex_exit(&stp->sd_lock); 4421 return (EAGAIN); 4422 } 4423 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) { 4424 mutex_exit(&stp->sd_lock); 4425 return (EINTR); 4426 } 4427 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4428 error = strgeterr(stp, 4429 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4430 if (error != 0) { 4431 mutex_exit(&stp->sd_lock); 4432 return (error); 4433 } 4434 } 4435 } 4436 stp->sd_flag |= STRPLUMB; 4437 mutex_exit(&stp->sd_lock); 4438 } 4439 return (0); 4440 } 4441 4442 /* 4443 * Complete the plumbing operation associated with stream `stp'. 4444 */ 4445 void 4446 strendplumb(stdata_t *stp) 4447 { 4448 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4449 ASSERT(stp->sd_flag & STRPLUMB); 4450 stp->sd_flag &= ~STRPLUMB; 4451 cv_broadcast(&stp->sd_monitor); 4452 } 4453 4454 /* 4455 * This describes how the STREAMS framework handles synchronization 4456 * during open/push and close/pop. 4457 * The key interfaces for open and close are qprocson and qprocsoff, 4458 * respectively. While the close case in general is harder both open 4459 * have close have significant similarities. 4460 * 4461 * During close the STREAMS framework has to both ensure that there 4462 * are no stale references to the queue pair (and syncq) that 4463 * are being closed and also provide the guarantees that are documented 4464 * in qprocsoff(9F). 4465 * If there are stale references to the queue that is closing it can 4466 * result in kernel memory corruption or kernel panics. 4467 * 4468 * Note that is it up to the module/driver to ensure that it itself 4469 * does not have any stale references to the closing queues once its close 4470 * routine returns. This includes: 4471 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines 4472 * associated with the queues. For timeout and bufcall callbacks the 4473 * module/driver also has to ensure (or wait for) any callbacks that 4474 * are in progress. 4475 * - If the module/driver is using esballoc it has to ensure that any 4476 * esballoc free functions do not refer to a queue that has closed. 4477 * (Note that in general the close routine can not wait for the esballoc'ed 4478 * messages to be freed since that can cause a deadlock.) 4479 * - Cancelling any interrupts that refer to the closing queues and 4480 * also ensuring that there are no interrupts in progress that will 4481 * refer to the closing queues once the close routine returns. 4482 * - For multiplexors removing any driver global state that refers to 4483 * the closing queue and also ensuring that there are no threads in 4484 * the multiplexor that has picked up a queue pointer but not yet 4485 * finished using it. 4486 * 4487 * In addition, a driver/module can only reference the q_next pointer 4488 * in its open, close, put, or service procedures or in a 4489 * qtimeout/qbufcall callback procedure executing "on" the correct 4490 * stream. Thus it can not reference the q_next pointer in an interrupt 4491 * routine or a timeout, bufcall or esballoc callback routine. Likewise 4492 * it can not reference q_next of a different queue e.g. in a mux that 4493 * passes messages from one queues put/service procedure to another queue. 4494 * In all the cases when the driver/module can not access the q_next 4495 * field it must use the *next* versions e.g. canputnext instead of 4496 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...). 4497 * 4498 * 4499 * Assuming that the driver/module conforms to the above constraints 4500 * the STREAMS framework has to avoid stale references to q_next for all 4501 * the framework internal cases which include (but are not limited to): 4502 * - Threads in canput/canputnext/backenable and elsewhere that are 4503 * walking q_next. 4504 * - Messages on a syncq that have a reference to the queue through b_queue. 4505 * - Messages on an outer perimeter (syncq) that have a reference to the 4506 * queue through b_queue. 4507 * - Threads that use q_nfsrv (e.g. canput) to find a queue. 4508 * Note that only canput and bcanput use q_nfsrv without any locking. 4509 * 4510 * The STREAMS framework providing the qprocsoff(9F) guarantees means that 4511 * after qprocsoff returns, the framework has to ensure that no threads can 4512 * enter the put or service routines for the closing read or write-side queue. 4513 * In addition to preventing "direct" entry into the put procedures 4514 * the framework also has to prevent messages being drained from 4515 * the syncq or the outer perimeter. 4516 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only 4517 * mechanism to prevent qwriter(PERIM_OUTER) from running after 4518 * qprocsoff has returned. 4519 * Note that if a module/driver uses put(9F) on one of its own queues 4520 * it is up to the module/driver to ensure that the put() doesn't 4521 * get called when the queue is closing. 4522 * 4523 * 4524 * The framework aspects of the above "contract" is implemented by 4525 * qprocsoff, removeq, and strlock: 4526 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from 4527 * entering the service procedures. 4528 * - strlock acquires the sd_lock and sd_reflock to prevent putnext, 4529 * canputnext, backenable etc from dereferencing the q_next that will 4530 * soon change. 4531 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext 4532 * or other q_next walker that uses claimstr/releasestr to finish. 4533 * - optionally for every syncq in the stream strlock acquires all the 4534 * sq_lock's and waits for all sq_counts to drop to a value that indicates 4535 * that no thread executes in the put or service procedures and that no 4536 * thread is draining into the module/driver. This ensures that no 4537 * open, close, put, service, or qtimeout/qbufcall callback procedure is 4538 * currently executing hence no such thread can end up with the old stale 4539 * q_next value and no canput/backenable can have the old stale 4540 * q_nfsrv/q_next. 4541 * - qdetach (wait_svc) makes sure that any scheduled or running threads 4542 * have either finished or observed the QWCLOSE flag and gone away. 4543 */ 4544 4545 4546 /* 4547 * Get all the locks necessary to change q_next. 4548 * 4549 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the 4550 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that 4551 * the only threads inside the syncq are threads currently calling removeq(). 4552 * Since threads calling removeq() are in the process of removing their queues 4553 * from the stream, we do not need to worry about them accessing a stale q_next 4554 * pointer and thus we do not need to wait for them to exit (in fact, waiting 4555 * for them can cause deadlock). 4556 * 4557 * This routine is subject to starvation since it does not set any flag to 4558 * prevent threads from entering a module in the stream (i.e. sq_count can 4559 * increase on some syncq while it is waiting on some other syncq). 4560 * 4561 * Assumes that only one thread attempts to call strlock for a given 4562 * stream. If this is not the case the two threads would deadlock. 4563 * This assumption is guaranteed since strlock is only called by insertq 4564 * and removeq and streams plumbing changes are single-threaded for 4565 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags. 4566 * 4567 * For pipes, it is not difficult to atomically designate a pair of streams 4568 * to be mated. Once mated atomically by the framework the twisted pair remain 4569 * configured that way until dismantled atomically by the framework. 4570 * When plumbing takes place on a twisted stream it is necessary to ensure that 4571 * this operation is done exclusively on the twisted stream since two such 4572 * operations, each initiated on different ends of the pipe will deadlock 4573 * waiting for each other to complete. 4574 * 4575 * On entry, no locks should be held. 4576 * The locks acquired and held by strlock depends on a few factors. 4577 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired 4578 * and held on exit and all sq_count are at an acceptable level. 4579 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with 4580 * sd_refcnt being zero. 4581 */ 4582 4583 static void 4584 strlock(struct stdata *stp, sqlist_t *sqlist) 4585 { 4586 syncql_t *sql, *sql2; 4587 retry: 4588 /* 4589 * Wait for any claimstr to go away. 4590 */ 4591 if (STRMATED(stp)) { 4592 struct stdata *stp1, *stp2; 4593 4594 STRLOCKMATES(stp); 4595 /* 4596 * Note that the selection of locking order is not 4597 * important, just that they are always acquired in 4598 * the same order. To assure this, we choose this 4599 * order based on the value of the pointer, and since 4600 * the pointer will not change for the life of this 4601 * pair, we will always grab the locks in the same 4602 * order (and hence, prevent deadlocks). 4603 */ 4604 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) { 4605 stp1 = stp; 4606 stp2 = stp->sd_mate; 4607 } else { 4608 stp2 = stp; 4609 stp1 = stp->sd_mate; 4610 } 4611 mutex_enter(&stp1->sd_reflock); 4612 if (stp1->sd_refcnt > 0) { 4613 STRUNLOCKMATES(stp); 4614 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock); 4615 mutex_exit(&stp1->sd_reflock); 4616 goto retry; 4617 } 4618 mutex_enter(&stp2->sd_reflock); 4619 if (stp2->sd_refcnt > 0) { 4620 STRUNLOCKMATES(stp); 4621 mutex_exit(&stp1->sd_reflock); 4622 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock); 4623 mutex_exit(&stp2->sd_reflock); 4624 goto retry; 4625 } 4626 STREAM_PUTLOCKS_ENTER(stp1); 4627 STREAM_PUTLOCKS_ENTER(stp2); 4628 } else { 4629 mutex_enter(&stp->sd_lock); 4630 mutex_enter(&stp->sd_reflock); 4631 while (stp->sd_refcnt > 0) { 4632 mutex_exit(&stp->sd_lock); 4633 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock); 4634 if (mutex_tryenter(&stp->sd_lock) == 0) { 4635 mutex_exit(&stp->sd_reflock); 4636 mutex_enter(&stp->sd_lock); 4637 mutex_enter(&stp->sd_reflock); 4638 } 4639 } 4640 STREAM_PUTLOCKS_ENTER(stp); 4641 } 4642 4643 if (sqlist == NULL) 4644 return; 4645 4646 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4647 syncq_t *sq = sql->sql_sq; 4648 uint16_t count; 4649 4650 mutex_enter(SQLOCK(sq)); 4651 count = sq->sq_count; 4652 ASSERT(sq->sq_rmqcount <= count); 4653 SQ_PUTLOCKS_ENTER(sq); 4654 SUM_SQ_PUTCOUNTS(sq, count); 4655 if (count == sq->sq_rmqcount) 4656 continue; 4657 4658 /* Failed - drop all locks that we have acquired so far */ 4659 if (STRMATED(stp)) { 4660 STREAM_PUTLOCKS_EXIT(stp); 4661 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4662 STRUNLOCKMATES(stp); 4663 mutex_exit(&stp->sd_reflock); 4664 mutex_exit(&stp->sd_mate->sd_reflock); 4665 } else { 4666 STREAM_PUTLOCKS_EXIT(stp); 4667 mutex_exit(&stp->sd_lock); 4668 mutex_exit(&stp->sd_reflock); 4669 } 4670 for (sql2 = sqlist->sqlist_head; sql2 != sql; 4671 sql2 = sql2->sql_next) { 4672 SQ_PUTLOCKS_EXIT(sql2->sql_sq); 4673 mutex_exit(SQLOCK(sql2->sql_sq)); 4674 } 4675 4676 /* 4677 * The wait loop below may starve when there are many threads 4678 * claiming the syncq. This is especially a problem with permod 4679 * syncqs (IP). To lessen the impact of the problem we increment 4680 * sq_needexcl and clear fastbits so that putnexts will slow 4681 * down and call sqenable instead of draining right away. 4682 */ 4683 sq->sq_needexcl++; 4684 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4685 while (count > sq->sq_rmqcount) { 4686 sq->sq_flags |= SQ_WANTWAKEUP; 4687 SQ_PUTLOCKS_EXIT(sq); 4688 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4689 count = sq->sq_count; 4690 SQ_PUTLOCKS_ENTER(sq); 4691 SUM_SQ_PUTCOUNTS(sq, count); 4692 } 4693 sq->sq_needexcl--; 4694 if (sq->sq_needexcl == 0) 4695 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4696 SQ_PUTLOCKS_EXIT(sq); 4697 ASSERT(count == sq->sq_rmqcount); 4698 mutex_exit(SQLOCK(sq)); 4699 goto retry; 4700 } 4701 } 4702 4703 /* 4704 * Drop all the locks that strlock acquired. 4705 */ 4706 static void 4707 strunlock(struct stdata *stp, sqlist_t *sqlist) 4708 { 4709 syncql_t *sql; 4710 4711 if (STRMATED(stp)) { 4712 STREAM_PUTLOCKS_EXIT(stp); 4713 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4714 STRUNLOCKMATES(stp); 4715 mutex_exit(&stp->sd_reflock); 4716 mutex_exit(&stp->sd_mate->sd_reflock); 4717 } else { 4718 STREAM_PUTLOCKS_EXIT(stp); 4719 mutex_exit(&stp->sd_lock); 4720 mutex_exit(&stp->sd_reflock); 4721 } 4722 4723 if (sqlist == NULL) 4724 return; 4725 4726 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4727 SQ_PUTLOCKS_EXIT(sql->sql_sq); 4728 mutex_exit(SQLOCK(sql->sql_sq)); 4729 } 4730 } 4731 4732 /* 4733 * When the module has service procedure, we need check if the next 4734 * module which has service procedure is in flow control to trigger 4735 * the backenable. 4736 */ 4737 static void 4738 backenable_insertedq(queue_t *q) 4739 { 4740 qband_t *qbp; 4741 4742 claimstr(q); 4743 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) { 4744 if (q->q_next->q_nfsrv->q_flag & QWANTW) 4745 backenable(q, 0); 4746 4747 qbp = q->q_next->q_nfsrv->q_bandp; 4748 for (; qbp != NULL; qbp = qbp->qb_next) 4749 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL) 4750 backenable(q, qbp->qb_first->b_band); 4751 } 4752 releasestr(q); 4753 } 4754 4755 /* 4756 * Given two read queues, insert a new single one after another. 4757 * 4758 * This routine acquires all the necessary locks in order to change 4759 * q_next and related pointer using strlock(). 4760 * It depends on the stream head ensuring that there are no concurrent 4761 * insertq or removeq on the same stream. The stream head ensures this 4762 * using the flags STWOPEN, STRCLOSE, and STRPLUMB. 4763 * 4764 * Note that no syncq locks are held during the q_next change. This is 4765 * applied to all streams since, unlike removeq, there is no problem of stale 4766 * pointers when adding a module to the stream. Thus drivers/modules that do a 4767 * canput(rq->q_next) would never get a closed/freed queue pointer even if we 4768 * applied this optimization to all streams. 4769 */ 4770 void 4771 insertq(struct stdata *stp, queue_t *new) 4772 { 4773 queue_t *after; 4774 queue_t *wafter; 4775 queue_t *wnew = _WR(new); 4776 boolean_t have_fifo = B_FALSE; 4777 4778 if (new->q_flag & _QINSERTING) { 4779 ASSERT(stp->sd_vnode->v_type != VFIFO); 4780 after = new->q_next; 4781 wafter = _WR(new->q_next); 4782 } else { 4783 after = _RD(stp->sd_wrq); 4784 wafter = stp->sd_wrq; 4785 } 4786 4787 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ, 4788 "insertq:%p, %p", after, new); 4789 ASSERT(after->q_flag & QREADR); 4790 ASSERT(new->q_flag & QREADR); 4791 4792 strlock(stp, NULL); 4793 4794 /* Do we have a FIFO? */ 4795 if (wafter->q_next == after) { 4796 have_fifo = B_TRUE; 4797 wnew->q_next = new; 4798 } else { 4799 wnew->q_next = wafter->q_next; 4800 } 4801 new->q_next = after; 4802 4803 set_nfsrv_ptr(new, wnew, after, wafter); 4804 /* 4805 * set_nfsrv_ptr() needs to know if this is an insertion or not, 4806 * so only reset this flag after calling it. 4807 */ 4808 new->q_flag &= ~_QINSERTING; 4809 4810 if (have_fifo) { 4811 wafter->q_next = wnew; 4812 } else { 4813 if (wafter->q_next) 4814 _OTHERQ(wafter->q_next)->q_next = new; 4815 wafter->q_next = wnew; 4816 } 4817 4818 set_qend(new); 4819 /* The QEND flag might have to be updated for the upstream guy */ 4820 set_qend(after); 4821 4822 ASSERT(_SAMESTR(new) == O_SAMESTR(new)); 4823 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew)); 4824 ASSERT(_SAMESTR(after) == O_SAMESTR(after)); 4825 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter)); 4826 strsetuio(stp); 4827 4828 /* 4829 * If this was a module insertion, bump the push count. 4830 */ 4831 if (!(new->q_flag & QISDRV)) 4832 stp->sd_pushcnt++; 4833 4834 strunlock(stp, NULL); 4835 4836 /* check if the write Q needs backenable */ 4837 backenable_insertedq(wnew); 4838 4839 /* check if the read Q needs backenable */ 4840 backenable_insertedq(new); 4841 } 4842 4843 /* 4844 * Given a read queue, unlink it from any neighbors. 4845 * 4846 * This routine acquires all the necessary locks in order to 4847 * change q_next and related pointers and also guard against 4848 * stale references (e.g. through q_next) to the queue that 4849 * is being removed. It also plays part of the role in ensuring 4850 * that the module's/driver's put procedure doesn't get called 4851 * after qprocsoff returns. 4852 * 4853 * Removeq depends on the stream head ensuring that there are 4854 * no concurrent insertq or removeq on the same stream. The 4855 * stream head ensures this using the flags STWOPEN, STRCLOSE and 4856 * STRPLUMB. 4857 * 4858 * The set of locks needed to remove the queue is different in 4859 * different cases: 4860 * 4861 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after 4862 * waiting for the syncq reference count to drop to 0 indicating that no 4863 * non-close threads are present anywhere in the stream. This ensures that any 4864 * module/driver can reference q_next in its open, close, put, or service 4865 * procedures. 4866 * 4867 * The sq_rmqcount counter tracks the number of threads inside removeq(). 4868 * strlock() ensures that there is either no threads executing inside perimeter 4869 * or there is only a thread calling qprocsoff(). 4870 * 4871 * strlock() compares the value of sq_count with the number of threads inside 4872 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup 4873 * any threads waiting in strlock() when the sq_rmqcount increases. 4874 */ 4875 4876 void 4877 removeq(queue_t *qp) 4878 { 4879 queue_t *wqp = _WR(qp); 4880 struct stdata *stp = STREAM(qp); 4881 sqlist_t *sqlist = NULL; 4882 boolean_t isdriver; 4883 int moved; 4884 syncq_t *sq = qp->q_syncq; 4885 syncq_t *wsq = wqp->q_syncq; 4886 4887 ASSERT(stp); 4888 4889 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ, 4890 "removeq:%p %p", qp, wqp); 4891 ASSERT(qp->q_flag&QREADR); 4892 4893 /* 4894 * For queues using Synchronous streams, we must wait for all threads in 4895 * rwnext() to drain out before proceeding. 4896 */ 4897 if (qp->q_flag & QSYNCSTR) { 4898 /* First, we need wakeup any threads blocked in rwnext() */ 4899 mutex_enter(SQLOCK(sq)); 4900 if (sq->sq_flags & SQ_WANTWAKEUP) { 4901 sq->sq_flags &= ~SQ_WANTWAKEUP; 4902 cv_broadcast(&sq->sq_wait); 4903 } 4904 mutex_exit(SQLOCK(sq)); 4905 4906 if (wsq != sq) { 4907 mutex_enter(SQLOCK(wsq)); 4908 if (wsq->sq_flags & SQ_WANTWAKEUP) { 4909 wsq->sq_flags &= ~SQ_WANTWAKEUP; 4910 cv_broadcast(&wsq->sq_wait); 4911 } 4912 mutex_exit(SQLOCK(wsq)); 4913 } 4914 4915 mutex_enter(QLOCK(qp)); 4916 while (qp->q_rwcnt > 0) { 4917 qp->q_flag |= QWANTRMQSYNC; 4918 cv_wait(&qp->q_wait, QLOCK(qp)); 4919 } 4920 mutex_exit(QLOCK(qp)); 4921 4922 mutex_enter(QLOCK(wqp)); 4923 while (wqp->q_rwcnt > 0) { 4924 wqp->q_flag |= QWANTRMQSYNC; 4925 cv_wait(&wqp->q_wait, QLOCK(wqp)); 4926 } 4927 mutex_exit(QLOCK(wqp)); 4928 } 4929 4930 mutex_enter(SQLOCK(sq)); 4931 sq->sq_rmqcount++; 4932 if (sq->sq_flags & SQ_WANTWAKEUP) { 4933 sq->sq_flags &= ~SQ_WANTWAKEUP; 4934 cv_broadcast(&sq->sq_wait); 4935 } 4936 mutex_exit(SQLOCK(sq)); 4937 4938 isdriver = (qp->q_flag & QISDRV); 4939 4940 sqlist = sqlist_build(qp, stp, STRMATED(stp)); 4941 strlock(stp, sqlist); 4942 4943 reset_nfsrv_ptr(qp, wqp); 4944 4945 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp); 4946 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp); 4947 /* Do we have a FIFO? */ 4948 if (wqp->q_next == qp) { 4949 stp->sd_wrq->q_next = _RD(stp->sd_wrq); 4950 } else { 4951 if (wqp->q_next) 4952 backq(qp)->q_next = qp->q_next; 4953 if (qp->q_next) 4954 backq(wqp)->q_next = wqp->q_next; 4955 } 4956 4957 /* The QEND flag might have to be updated for the upstream guy */ 4958 if (qp->q_next) 4959 set_qend(qp->q_next); 4960 4961 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq)); 4962 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq))); 4963 4964 /* 4965 * Move any messages destined for the put procedures to the next 4966 * syncq in line. Otherwise free them. 4967 */ 4968 moved = 0; 4969 /* 4970 * Quick check to see whether there are any messages or events. 4971 */ 4972 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS)) 4973 moved += propagate_syncq(qp); 4974 if (wqp->q_syncqmsgs != 0 || 4975 (wqp->q_syncq->sq_flags & SQ_EVENTS)) 4976 moved += propagate_syncq(wqp); 4977 4978 strsetuio(stp); 4979 4980 /* 4981 * If this was a module removal, decrement the push count. 4982 */ 4983 if (!isdriver) 4984 stp->sd_pushcnt--; 4985 4986 strunlock(stp, sqlist); 4987 sqlist_free(sqlist); 4988 4989 /* 4990 * Make sure any messages that were propagated are drained. 4991 * Also clear any QFULL bit caused by messages that were propagated. 4992 */ 4993 4994 if (qp->q_next != NULL) { 4995 clr_qfull(qp); 4996 /* 4997 * For the driver calling qprocsoff, propagate_syncq 4998 * frees all the messages instead of putting it in 4999 * the stream head 5000 */ 5001 if (!isdriver && (moved > 0)) 5002 emptysq(qp->q_next->q_syncq); 5003 } 5004 if (wqp->q_next != NULL) { 5005 clr_qfull(wqp); 5006 /* 5007 * We come here for any pop of a module except for the 5008 * case of driver being removed. We don't call emptysq 5009 * if we did not move any messages. This will avoid holding 5010 * PERMOD syncq locks in emptysq 5011 */ 5012 if (moved > 0) 5013 emptysq(wqp->q_next->q_syncq); 5014 } 5015 5016 mutex_enter(SQLOCK(sq)); 5017 sq->sq_rmqcount--; 5018 mutex_exit(SQLOCK(sq)); 5019 } 5020 5021 /* 5022 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or 5023 * SQ_WRITER) on a syncq. 5024 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the 5025 * sync queue and waits until sq_count reaches maxcnt. 5026 * 5027 * If maxcnt is -1 there's no need to grab sq_putlocks since the caller 5028 * does not care about putnext threads that are in the middle of calling put 5029 * entry points. 5030 * 5031 * This routine is used for both inner and outer syncqs. 5032 */ 5033 static void 5034 blocksq(syncq_t *sq, ushort_t flag, int maxcnt) 5035 { 5036 uint16_t count = 0; 5037 5038 mutex_enter(SQLOCK(sq)); 5039 /* 5040 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset. 5041 * SQ_FROZEN will be set if there is a frozen stream that has a 5042 * queue which also refers to this "shared" syncq. 5043 * SQ_BLOCKED will be set if there is "off" queue which also 5044 * refers to this "shared" syncq. 5045 */ 5046 if (maxcnt != -1) { 5047 count = sq->sq_count; 5048 SQ_PUTLOCKS_ENTER(sq); 5049 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5050 SUM_SQ_PUTCOUNTS(sq, count); 5051 } 5052 sq->sq_needexcl++; 5053 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5054 5055 while ((sq->sq_flags & flag) || 5056 (maxcnt != -1 && count > (unsigned)maxcnt)) { 5057 sq->sq_flags |= SQ_WANTWAKEUP; 5058 if (maxcnt != -1) { 5059 SQ_PUTLOCKS_EXIT(sq); 5060 } 5061 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5062 if (maxcnt != -1) { 5063 count = sq->sq_count; 5064 SQ_PUTLOCKS_ENTER(sq); 5065 SUM_SQ_PUTCOUNTS(sq, count); 5066 } 5067 } 5068 sq->sq_needexcl--; 5069 sq->sq_flags |= flag; 5070 ASSERT(maxcnt == -1 || count == maxcnt); 5071 if (maxcnt != -1) { 5072 if (sq->sq_needexcl == 0) { 5073 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5074 } 5075 SQ_PUTLOCKS_EXIT(sq); 5076 } else if (sq->sq_needexcl == 0) { 5077 SQ_PUTCOUNT_SETFAST(sq); 5078 } 5079 5080 mutex_exit(SQLOCK(sq)); 5081 } 5082 5083 /* 5084 * Reset a flag that was set with blocksq. 5085 * 5086 * Can not use this routine to reset SQ_WRITER. 5087 * 5088 * If "isouter" is set then the syncq is assumed to be an outer perimeter 5089 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread 5090 * to handle the queued qwriter operations. 5091 * 5092 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5093 * sq_putlocks are used. 5094 */ 5095 static void 5096 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) 5097 { 5098 uint16_t flags; 5099 5100 mutex_enter(SQLOCK(sq)); 5101 ASSERT(resetflag != SQ_WRITER); 5102 ASSERT(sq->sq_flags & resetflag); 5103 flags = sq->sq_flags & ~resetflag; 5104 sq->sq_flags = flags; 5105 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) { 5106 if (flags & SQ_WANTWAKEUP) { 5107 flags &= ~SQ_WANTWAKEUP; 5108 cv_broadcast(&sq->sq_wait); 5109 } 5110 sq->sq_flags = flags; 5111 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5112 if (!isouter) { 5113 /* drain_syncq drops SQLOCK */ 5114 drain_syncq(sq); 5115 return; 5116 } 5117 } 5118 } 5119 mutex_exit(SQLOCK(sq)); 5120 } 5121 5122 /* 5123 * Reset a flag that was set with blocksq. 5124 * Does not drain the syncq. Use emptysq() for that. 5125 * Returns 1 if SQ_QUEUED is set. Otherwise 0. 5126 * 5127 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5128 * sq_putlocks are used. 5129 */ 5130 static int 5131 dropsq(syncq_t *sq, uint16_t resetflag) 5132 { 5133 uint16_t flags; 5134 5135 mutex_enter(SQLOCK(sq)); 5136 ASSERT(sq->sq_flags & resetflag); 5137 flags = sq->sq_flags & ~resetflag; 5138 if (flags & SQ_WANTWAKEUP) { 5139 flags &= ~SQ_WANTWAKEUP; 5140 cv_broadcast(&sq->sq_wait); 5141 } 5142 sq->sq_flags = flags; 5143 mutex_exit(SQLOCK(sq)); 5144 if (flags & SQ_QUEUED) 5145 return (1); 5146 return (0); 5147 } 5148 5149 /* 5150 * Empty all the messages on a syncq. 5151 * 5152 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5153 * sq_putlocks are used. 5154 */ 5155 static void 5156 emptysq(syncq_t *sq) 5157 { 5158 uint16_t flags; 5159 5160 mutex_enter(SQLOCK(sq)); 5161 flags = sq->sq_flags; 5162 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5163 /* 5164 * To prevent potential recursive invocation of drain_syncq we 5165 * do not call drain_syncq if count is non-zero. 5166 */ 5167 if (sq->sq_count == 0) { 5168 /* drain_syncq() drops SQLOCK */ 5169 drain_syncq(sq); 5170 return; 5171 } else 5172 sqenable(sq); 5173 } 5174 mutex_exit(SQLOCK(sq)); 5175 } 5176 5177 /* 5178 * Ordered insert while removing duplicates. 5179 */ 5180 static void 5181 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp) 5182 { 5183 syncql_t *sqlp, **prev_sqlpp, *new_sqlp; 5184 5185 prev_sqlpp = &sqlist->sqlist_head; 5186 while ((sqlp = *prev_sqlpp) != NULL) { 5187 if (sqlp->sql_sq >= sqp) { 5188 if (sqlp->sql_sq == sqp) /* duplicate */ 5189 return; 5190 break; 5191 } 5192 prev_sqlpp = &sqlp->sql_next; 5193 } 5194 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++]; 5195 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size); 5196 new_sqlp->sql_next = sqlp; 5197 new_sqlp->sql_sq = sqp; 5198 *prev_sqlpp = new_sqlp; 5199 } 5200 5201 /* 5202 * Walk the write side queues until we hit either the driver 5203 * or a twist in the stream (_SAMESTR will return false in both 5204 * these cases) then turn around and walk the read side queues 5205 * back up to the stream head. 5206 */ 5207 static void 5208 sqlist_insertall(sqlist_t *sqlist, queue_t *q) 5209 { 5210 while (q != NULL) { 5211 sqlist_insert(sqlist, q->q_syncq); 5212 5213 if (_SAMESTR(q)) 5214 q = q->q_next; 5215 else if (!(q->q_flag & QREADR)) 5216 q = _RD(q); 5217 else 5218 q = NULL; 5219 } 5220 } 5221 5222 /* 5223 * Allocate and build a list of all syncqs in a stream and the syncq(s) 5224 * associated with the "q" parameter. The resulting list is sorted in a 5225 * canonical order and is free of duplicates. 5226 * Assumes the passed queue is a _RD(q). 5227 */ 5228 static sqlist_t * 5229 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist) 5230 { 5231 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP); 5232 5233 /* 5234 * start with the current queue/qpair 5235 */ 5236 ASSERT(q->q_flag & QREADR); 5237 5238 sqlist_insert(sqlist, q->q_syncq); 5239 sqlist_insert(sqlist, _WR(q)->q_syncq); 5240 5241 sqlist_insertall(sqlist, stp->sd_wrq); 5242 if (do_twist) 5243 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq); 5244 5245 return (sqlist); 5246 } 5247 5248 static sqlist_t * 5249 sqlist_alloc(struct stdata *stp, int kmflag) 5250 { 5251 size_t sqlist_size; 5252 sqlist_t *sqlist; 5253 5254 /* 5255 * Allocate 2 syncql_t's for each pushed module. Note that 5256 * the sqlist_t structure already has 4 syncql_t's built in: 5257 * 2 for the stream head, and 2 for the driver/other stream head. 5258 */ 5259 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt + 5260 sizeof (sqlist_t); 5261 if (STRMATED(stp)) 5262 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt; 5263 sqlist = kmem_alloc(sqlist_size, kmflag); 5264 5265 sqlist->sqlist_head = NULL; 5266 sqlist->sqlist_size = sqlist_size; 5267 sqlist->sqlist_index = 0; 5268 5269 return (sqlist); 5270 } 5271 5272 /* 5273 * Free the list created by sqlist_alloc() 5274 */ 5275 static void 5276 sqlist_free(sqlist_t *sqlist) 5277 { 5278 kmem_free(sqlist, sqlist->sqlist_size); 5279 } 5280 5281 /* 5282 * Prevent any new entries into any syncq in this stream. 5283 * Used by freezestr. 5284 */ 5285 void 5286 strblock(queue_t *q) 5287 { 5288 struct stdata *stp; 5289 syncql_t *sql; 5290 sqlist_t *sqlist; 5291 5292 q = _RD(q); 5293 5294 stp = STREAM(q); 5295 ASSERT(stp != NULL); 5296 5297 /* 5298 * Get a sorted list with all the duplicates removed containing 5299 * all the syncqs referenced by this stream. 5300 */ 5301 sqlist = sqlist_build(q, stp, B_FALSE); 5302 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5303 blocksq(sql->sql_sq, SQ_FROZEN, -1); 5304 sqlist_free(sqlist); 5305 } 5306 5307 /* 5308 * Release the block on new entries into this stream 5309 */ 5310 void 5311 strunblock(queue_t *q) 5312 { 5313 struct stdata *stp; 5314 syncql_t *sql; 5315 sqlist_t *sqlist; 5316 int drain_needed; 5317 5318 q = _RD(q); 5319 5320 /* 5321 * Get a sorted list with all the duplicates removed containing 5322 * all the syncqs referenced by this stream. 5323 * Have to drop the SQ_FROZEN flag on all the syncqs before 5324 * starting to drain them; otherwise the draining might 5325 * cause a freezestr in some module on the stream (which 5326 * would deadlock). 5327 */ 5328 stp = STREAM(q); 5329 ASSERT(stp != NULL); 5330 sqlist = sqlist_build(q, stp, B_FALSE); 5331 drain_needed = 0; 5332 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5333 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN); 5334 if (drain_needed) { 5335 for (sql = sqlist->sqlist_head; sql != NULL; 5336 sql = sql->sql_next) 5337 emptysq(sql->sql_sq); 5338 } 5339 sqlist_free(sqlist); 5340 } 5341 5342 #ifdef DEBUG 5343 static int 5344 qprocsareon(queue_t *rq) 5345 { 5346 if (rq->q_next == NULL) 5347 return (0); 5348 return (_WR(rq->q_next)->q_next == _WR(rq)); 5349 } 5350 5351 int 5352 qclaimed(queue_t *q) 5353 { 5354 uint_t count; 5355 5356 count = q->q_syncq->sq_count; 5357 SUM_SQ_PUTCOUNTS(q->q_syncq, count); 5358 return (count != 0); 5359 } 5360 5361 /* 5362 * Check if anyone has frozen this stream with freezestr 5363 */ 5364 int 5365 frozenstr(queue_t *q) 5366 { 5367 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0); 5368 } 5369 #endif /* DEBUG */ 5370 5371 /* 5372 * Enter a queue. 5373 * Obsoleted interface. Should not be used. 5374 */ 5375 void 5376 enterq(queue_t *q) 5377 { 5378 entersq(q->q_syncq, SQ_CALLBACK); 5379 } 5380 5381 void 5382 leaveq(queue_t *q) 5383 { 5384 leavesq(q->q_syncq, SQ_CALLBACK); 5385 } 5386 5387 /* 5388 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits 5389 * to check. 5390 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter 5391 * calls and the running of open, close and service procedures. 5392 * 5393 * If c_inner bit is set no need to grab sq_putlocks since we don't care 5394 * if other threads have entered or are entering put entry point. 5395 * 5396 * If c_inner bit is set it might have been possible to use 5397 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize 5398 * open/close path for IP) but since the count may need to be decremented in 5399 * qwait() we wouldn't know which counter to decrement. Currently counter is 5400 * selected by current cpu_seqid and current CPU can change at any moment. XXX 5401 * in the future we might use curthread id bits to select the counter and this 5402 * would stay constant across routine calls. 5403 */ 5404 void 5405 entersq(syncq_t *sq, int entrypoint) 5406 { 5407 uint16_t count = 0; 5408 uint16_t flags; 5409 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 5410 uint16_t type; 5411 uint_t c_inner = entrypoint & SQ_CI; 5412 uint_t c_outer = entrypoint & SQ_CO; 5413 5414 /* 5415 * Increment ref count to keep closes out of this queue. 5416 */ 5417 ASSERT(sq); 5418 ASSERT(c_inner && c_outer); 5419 mutex_enter(SQLOCK(sq)); 5420 flags = sq->sq_flags; 5421 type = sq->sq_type; 5422 if (!(type & c_inner)) { 5423 /* Make sure all putcounts now use slowlock. */ 5424 count = sq->sq_count; 5425 SQ_PUTLOCKS_ENTER(sq); 5426 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5427 SUM_SQ_PUTCOUNTS(sq, count); 5428 sq->sq_needexcl++; 5429 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5430 waitflags |= SQ_MESSAGES; 5431 } 5432 /* 5433 * Wait until we can enter the inner perimeter. 5434 * If we want exclusive access we wait until sq_count is 0. 5435 * We have to do this before entering the outer perimeter in order 5436 * to preserve put/close message ordering. 5437 */ 5438 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) { 5439 sq->sq_flags = flags | SQ_WANTWAKEUP; 5440 if (!(type & c_inner)) { 5441 SQ_PUTLOCKS_EXIT(sq); 5442 } 5443 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5444 if (!(type & c_inner)) { 5445 count = sq->sq_count; 5446 SQ_PUTLOCKS_ENTER(sq); 5447 SUM_SQ_PUTCOUNTS(sq, count); 5448 } 5449 flags = sq->sq_flags; 5450 } 5451 5452 if (!(type & c_inner)) { 5453 ASSERT(sq->sq_needexcl > 0); 5454 sq->sq_needexcl--; 5455 if (sq->sq_needexcl == 0) { 5456 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5457 } 5458 } 5459 5460 /* Check if we need to enter the outer perimeter */ 5461 if (!(type & c_outer)) { 5462 /* 5463 * We have to enter the outer perimeter exclusively before 5464 * we can increment sq_count to avoid deadlock. This implies 5465 * that we have to re-check sq_flags and sq_count. 5466 * 5467 * is it possible to have c_inner set when c_outer is not set? 5468 */ 5469 if (!(type & c_inner)) { 5470 SQ_PUTLOCKS_EXIT(sq); 5471 } 5472 mutex_exit(SQLOCK(sq)); 5473 outer_enter(sq->sq_outer, SQ_GOAWAY); 5474 mutex_enter(SQLOCK(sq)); 5475 flags = sq->sq_flags; 5476 /* 5477 * there should be no need to recheck sq_putcounts 5478 * because outer_enter() has already waited for them to clear 5479 * after setting SQ_WRITER. 5480 */ 5481 count = sq->sq_count; 5482 #ifdef DEBUG 5483 /* 5484 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead 5485 * of doing an ASSERT internally. Others should do 5486 * something like 5487 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0); 5488 * without the need to #ifdef DEBUG it. 5489 */ 5490 SUMCHECK_SQ_PUTCOUNTS(sq, 0); 5491 #endif 5492 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) || 5493 (!(type & c_inner) && count != 0)) { 5494 sq->sq_flags = flags | SQ_WANTWAKEUP; 5495 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5496 count = sq->sq_count; 5497 flags = sq->sq_flags; 5498 } 5499 } 5500 5501 sq->sq_count++; 5502 ASSERT(sq->sq_count != 0); /* Wraparound */ 5503 if (!(type & c_inner)) { 5504 /* Exclusive entry */ 5505 ASSERT(sq->sq_count == 1); 5506 sq->sq_flags |= SQ_EXCL; 5507 if (type & c_outer) { 5508 SQ_PUTLOCKS_EXIT(sq); 5509 } 5510 } 5511 mutex_exit(SQLOCK(sq)); 5512 } 5513 5514 /* 5515 * Leave a syncq. Announce to framework that closes may proceed. 5516 * c_inner and c_outer specify which concurrency bits to check. 5517 * 5518 * Must never be called from driver or module put entry point. 5519 * 5520 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5521 * sq_putlocks are used. 5522 */ 5523 void 5524 leavesq(syncq_t *sq, int entrypoint) 5525 { 5526 uint16_t flags; 5527 uint16_t type; 5528 uint_t c_outer = entrypoint & SQ_CO; 5529 #ifdef DEBUG 5530 uint_t c_inner = entrypoint & SQ_CI; 5531 #endif 5532 5533 /* 5534 * Decrement ref count, drain the syncq if possible, and wake up 5535 * any waiting close. 5536 */ 5537 ASSERT(sq); 5538 ASSERT(c_inner && c_outer); 5539 mutex_enter(SQLOCK(sq)); 5540 flags = sq->sq_flags; 5541 type = sq->sq_type; 5542 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) { 5543 5544 if (flags & SQ_WANTWAKEUP) { 5545 flags &= ~SQ_WANTWAKEUP; 5546 cv_broadcast(&sq->sq_wait); 5547 } 5548 if (flags & SQ_WANTEXWAKEUP) { 5549 flags &= ~SQ_WANTEXWAKEUP; 5550 cv_broadcast(&sq->sq_exitwait); 5551 } 5552 5553 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) { 5554 /* 5555 * The syncq needs to be drained. "Exit" the syncq 5556 * before calling drain_syncq. 5557 */ 5558 ASSERT(sq->sq_count != 0); 5559 sq->sq_count--; 5560 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5561 sq->sq_flags = flags & ~SQ_EXCL; 5562 drain_syncq(sq); 5563 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 5564 /* Check if we need to exit the outer perimeter */ 5565 /* XXX will this ever be true? */ 5566 if (!(type & c_outer)) 5567 outer_exit(sq->sq_outer); 5568 return; 5569 } 5570 } 5571 ASSERT(sq->sq_count != 0); 5572 sq->sq_count--; 5573 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5574 sq->sq_flags = flags & ~SQ_EXCL; 5575 mutex_exit(SQLOCK(sq)); 5576 5577 /* Check if we need to exit the outer perimeter */ 5578 if (!(sq->sq_type & c_outer)) 5579 outer_exit(sq->sq_outer); 5580 } 5581 5582 /* 5583 * Prevent q_next from changing in this stream by incrementing sq_count. 5584 * 5585 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5586 * sq_putlocks are used. 5587 */ 5588 void 5589 claimq(queue_t *qp) 5590 { 5591 syncq_t *sq = qp->q_syncq; 5592 5593 mutex_enter(SQLOCK(sq)); 5594 sq->sq_count++; 5595 ASSERT(sq->sq_count != 0); /* Wraparound */ 5596 mutex_exit(SQLOCK(sq)); 5597 } 5598 5599 /* 5600 * Undo claimq. 5601 * 5602 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5603 * sq_putlocks are used. 5604 */ 5605 void 5606 releaseq(queue_t *qp) 5607 { 5608 syncq_t *sq = qp->q_syncq; 5609 uint16_t flags; 5610 5611 mutex_enter(SQLOCK(sq)); 5612 ASSERT(sq->sq_count > 0); 5613 sq->sq_count--; 5614 5615 flags = sq->sq_flags; 5616 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) { 5617 if (flags & SQ_WANTWAKEUP) { 5618 flags &= ~SQ_WANTWAKEUP; 5619 cv_broadcast(&sq->sq_wait); 5620 } 5621 sq->sq_flags = flags; 5622 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5623 /* 5624 * To prevent potential recursive invocation of 5625 * drain_syncq we do not call drain_syncq if count is 5626 * non-zero. 5627 */ 5628 if (sq->sq_count == 0) { 5629 drain_syncq(sq); 5630 return; 5631 } else 5632 sqenable(sq); 5633 } 5634 } 5635 mutex_exit(SQLOCK(sq)); 5636 } 5637 5638 /* 5639 * Prevent q_next from changing in this stream by incrementing sd_refcnt. 5640 */ 5641 void 5642 claimstr(queue_t *qp) 5643 { 5644 struct stdata *stp = STREAM(qp); 5645 5646 mutex_enter(&stp->sd_reflock); 5647 stp->sd_refcnt++; 5648 ASSERT(stp->sd_refcnt != 0); /* Wraparound */ 5649 mutex_exit(&stp->sd_reflock); 5650 } 5651 5652 /* 5653 * Undo claimstr. 5654 */ 5655 void 5656 releasestr(queue_t *qp) 5657 { 5658 struct stdata *stp = STREAM(qp); 5659 5660 mutex_enter(&stp->sd_reflock); 5661 ASSERT(stp->sd_refcnt != 0); 5662 if (--stp->sd_refcnt == 0) 5663 cv_broadcast(&stp->sd_refmonitor); 5664 mutex_exit(&stp->sd_reflock); 5665 } 5666 5667 static syncq_t * 5668 new_syncq(void) 5669 { 5670 return (kmem_cache_alloc(syncq_cache, KM_SLEEP)); 5671 } 5672 5673 static void 5674 free_syncq(syncq_t *sq) 5675 { 5676 ASSERT(sq->sq_head == NULL); 5677 ASSERT(sq->sq_outer == NULL); 5678 ASSERT(sq->sq_callbpend == NULL); 5679 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || 5680 (sq->sq_onext == sq && sq->sq_oprev == sq)); 5681 5682 if (sq->sq_ciputctrl != NULL) { 5683 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 5684 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 5685 sq->sq_nciputctrl, 0); 5686 ASSERT(ciputctrl_cache != NULL); 5687 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 5688 } 5689 5690 sq->sq_tail = NULL; 5691 sq->sq_evhead = NULL; 5692 sq->sq_evtail = NULL; 5693 sq->sq_ciputctrl = NULL; 5694 sq->sq_nciputctrl = 0; 5695 sq->sq_count = 0; 5696 sq->sq_rmqcount = 0; 5697 sq->sq_callbflags = 0; 5698 sq->sq_cancelid = 0; 5699 sq->sq_next = NULL; 5700 sq->sq_needexcl = 0; 5701 sq->sq_svcflags = 0; 5702 sq->sq_nqueues = 0; 5703 sq->sq_pri = 0; 5704 sq->sq_onext = NULL; 5705 sq->sq_oprev = NULL; 5706 sq->sq_flags = 0; 5707 sq->sq_type = 0; 5708 sq->sq_servcount = 0; 5709 5710 kmem_cache_free(syncq_cache, sq); 5711 } 5712 5713 /* Outer perimeter code */ 5714 5715 /* 5716 * The outer syncq uses the fields and flags in the syncq slightly 5717 * differently from the inner syncqs. 5718 * sq_count Incremented when there are pending or running 5719 * writers at the outer perimeter to prevent the set of 5720 * inner syncqs that belong to the outer perimeter from 5721 * changing. 5722 * sq_head/tail List of deferred qwriter(OUTER) operations. 5723 * 5724 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while 5725 * inner syncqs are added to or removed from the 5726 * outer perimeter. 5727 * SQ_QUEUED sq_head/tail has messages or events queued. 5728 * 5729 * SQ_WRITER A thread is currently traversing all the inner syncqs 5730 * setting the SQ_WRITER flag. 5731 */ 5732 5733 /* 5734 * Get write access at the outer perimeter. 5735 * Note that read access is done by entersq, putnext, and put by simply 5736 * incrementing sq_count in the inner syncq. 5737 * 5738 * Waits until "flags" is no longer set in the outer to prevent multiple 5739 * threads from having write access at the same time. SQ_WRITER has to be part 5740 * of "flags". 5741 * 5742 * Increases sq_count on the outer syncq to keep away outer_insert/remove 5743 * until the outer_exit is finished. 5744 * 5745 * outer_enter is vulnerable to starvation since it does not prevent new 5746 * threads from entering the inner syncqs while it is waiting for sq_count to 5747 * go to zero. 5748 */ 5749 void 5750 outer_enter(syncq_t *outer, uint16_t flags) 5751 { 5752 syncq_t *sq; 5753 int wait_needed; 5754 uint16_t count; 5755 5756 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5757 outer->sq_oprev != NULL); 5758 ASSERT(flags & SQ_WRITER); 5759 5760 retry: 5761 mutex_enter(SQLOCK(outer)); 5762 while (outer->sq_flags & flags) { 5763 outer->sq_flags |= SQ_WANTWAKEUP; 5764 cv_wait(&outer->sq_wait, SQLOCK(outer)); 5765 } 5766 5767 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5768 outer->sq_flags |= SQ_WRITER; 5769 outer->sq_count++; 5770 ASSERT(outer->sq_count != 0); /* wraparound */ 5771 wait_needed = 0; 5772 /* 5773 * Set SQ_WRITER on all the inner syncqs while holding 5774 * the SQLOCK on the outer syncq. This ensures that the changing 5775 * of SQ_WRITER is atomic under the outer SQLOCK. 5776 */ 5777 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5778 mutex_enter(SQLOCK(sq)); 5779 count = sq->sq_count; 5780 SQ_PUTLOCKS_ENTER(sq); 5781 sq->sq_flags |= SQ_WRITER; 5782 SUM_SQ_PUTCOUNTS(sq, count); 5783 if (count != 0) 5784 wait_needed = 1; 5785 SQ_PUTLOCKS_EXIT(sq); 5786 mutex_exit(SQLOCK(sq)); 5787 } 5788 mutex_exit(SQLOCK(outer)); 5789 5790 /* 5791 * Get everybody out of the syncqs sequentially. 5792 * Note that we don't actually need to acquire the PUTLOCKS, since 5793 * we have already cleared the fastbit, and set QWRITER. By 5794 * definition, the count can not increase since putnext will 5795 * take the slowlock path (and the purpose of acquiring the 5796 * putlocks was to make sure it didn't increase while we were 5797 * waiting). 5798 * 5799 * Note that we still acquire the PUTLOCKS to be safe. 5800 */ 5801 if (wait_needed) { 5802 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5803 mutex_enter(SQLOCK(sq)); 5804 count = sq->sq_count; 5805 SQ_PUTLOCKS_ENTER(sq); 5806 SUM_SQ_PUTCOUNTS(sq, count); 5807 while (count != 0) { 5808 sq->sq_flags |= SQ_WANTWAKEUP; 5809 SQ_PUTLOCKS_EXIT(sq); 5810 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5811 count = sq->sq_count; 5812 SQ_PUTLOCKS_ENTER(sq); 5813 SUM_SQ_PUTCOUNTS(sq, count); 5814 } 5815 SQ_PUTLOCKS_EXIT(sq); 5816 mutex_exit(SQLOCK(sq)); 5817 } 5818 /* 5819 * Verify that none of the flags got set while we 5820 * were waiting for the sq_counts to drop. 5821 * If this happens we exit and retry entering the 5822 * outer perimeter. 5823 */ 5824 mutex_enter(SQLOCK(outer)); 5825 if (outer->sq_flags & (flags & ~SQ_WRITER)) { 5826 mutex_exit(SQLOCK(outer)); 5827 outer_exit(outer); 5828 goto retry; 5829 } 5830 mutex_exit(SQLOCK(outer)); 5831 } 5832 } 5833 5834 /* 5835 * Drop the write access at the outer perimeter. 5836 * Read access is dropped implicitly (by putnext, put, and leavesq) by 5837 * decrementing sq_count. 5838 */ 5839 void 5840 outer_exit(syncq_t *outer) 5841 { 5842 syncq_t *sq; 5843 int drain_needed; 5844 uint16_t flags; 5845 5846 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5847 outer->sq_oprev != NULL); 5848 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer))); 5849 5850 /* 5851 * Atomically (from the perspective of threads calling become_writer) 5852 * drop the write access at the outer perimeter by holding 5853 * SQLOCK(outer) across all the dropsq calls and the resetting of 5854 * SQ_WRITER. 5855 * This defines a locking order between the outer perimeter 5856 * SQLOCK and the inner perimeter SQLOCKs. 5857 */ 5858 mutex_enter(SQLOCK(outer)); 5859 flags = outer->sq_flags; 5860 ASSERT(outer->sq_flags & SQ_WRITER); 5861 if (flags & SQ_QUEUED) { 5862 write_now(outer); 5863 flags = outer->sq_flags; 5864 } 5865 5866 /* 5867 * sq_onext is stable since sq_count has not yet been decreased. 5868 * Reset the SQ_WRITER flags in all syncqs. 5869 * After dropping SQ_WRITER on the outer syncq we empty all the 5870 * inner syncqs. 5871 */ 5872 drain_needed = 0; 5873 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5874 drain_needed += dropsq(sq, SQ_WRITER); 5875 ASSERT(!(outer->sq_flags & SQ_QUEUED)); 5876 flags &= ~SQ_WRITER; 5877 if (drain_needed) { 5878 outer->sq_flags = flags; 5879 mutex_exit(SQLOCK(outer)); 5880 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5881 emptysq(sq); 5882 mutex_enter(SQLOCK(outer)); 5883 flags = outer->sq_flags; 5884 } 5885 if (flags & SQ_WANTWAKEUP) { 5886 flags &= ~SQ_WANTWAKEUP; 5887 cv_broadcast(&outer->sq_wait); 5888 } 5889 outer->sq_flags = flags; 5890 ASSERT(outer->sq_count > 0); 5891 outer->sq_count--; 5892 mutex_exit(SQLOCK(outer)); 5893 } 5894 5895 /* 5896 * Add another syncq to an outer perimeter. 5897 * Block out all other access to the outer perimeter while it is being 5898 * changed using blocksq. 5899 * Assumes that the caller has *not* done an outer_enter. 5900 * 5901 * Vulnerable to starvation in blocksq. 5902 */ 5903 static void 5904 outer_insert(syncq_t *outer, syncq_t *sq) 5905 { 5906 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5907 outer->sq_oprev != NULL); 5908 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 5909 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ 5910 5911 /* Get exclusive access to the outer perimeter list */ 5912 blocksq(outer, SQ_BLOCKED, 0); 5913 ASSERT(outer->sq_flags & SQ_BLOCKED); 5914 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5915 5916 mutex_enter(SQLOCK(sq)); 5917 sq->sq_outer = outer; 5918 outer->sq_onext->sq_oprev = sq; 5919 sq->sq_onext = outer->sq_onext; 5920 outer->sq_onext = sq; 5921 sq->sq_oprev = outer; 5922 mutex_exit(SQLOCK(sq)); 5923 unblocksq(outer, SQ_BLOCKED, 1); 5924 } 5925 5926 /* 5927 * Remove a syncq from an outer perimeter. 5928 * Block out all other access to the outer perimeter while it is being 5929 * changed using blocksq. 5930 * Assumes that the caller has *not* done an outer_enter. 5931 * 5932 * Vulnerable to starvation in blocksq. 5933 */ 5934 static void 5935 outer_remove(syncq_t *outer, syncq_t *sq) 5936 { 5937 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5938 outer->sq_oprev != NULL); 5939 ASSERT(sq->sq_outer == outer); 5940 5941 /* Get exclusive access to the outer perimeter list */ 5942 blocksq(outer, SQ_BLOCKED, 0); 5943 ASSERT(outer->sq_flags & SQ_BLOCKED); 5944 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5945 5946 mutex_enter(SQLOCK(sq)); 5947 sq->sq_outer = NULL; 5948 sq->sq_onext->sq_oprev = sq->sq_oprev; 5949 sq->sq_oprev->sq_onext = sq->sq_onext; 5950 sq->sq_oprev = sq->sq_onext = NULL; 5951 mutex_exit(SQLOCK(sq)); 5952 unblocksq(outer, SQ_BLOCKED, 1); 5953 } 5954 5955 /* 5956 * Queue a deferred qwriter(OUTER) callback for this outer perimeter. 5957 * If this is the first callback for this outer perimeter then add 5958 * this outer perimeter to the list of outer perimeters that 5959 * the qwriter_outer_thread will process. 5960 * 5961 * Increments sq_count in the outer syncq to prevent the membership 5962 * of the outer perimeter (in terms of inner syncqs) to change while 5963 * the callback is pending. 5964 */ 5965 static void 5966 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp) 5967 { 5968 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5969 5970 mp->b_prev = (mblk_t *)func; 5971 mp->b_queue = q; 5972 mp->b_next = NULL; 5973 outer->sq_count++; /* Decremented when dequeued */ 5974 ASSERT(outer->sq_count != 0); /* Wraparound */ 5975 if (outer->sq_evhead == NULL) { 5976 /* First message. */ 5977 outer->sq_evhead = outer->sq_evtail = mp; 5978 outer->sq_flags |= SQ_EVENTS; 5979 mutex_exit(SQLOCK(outer)); 5980 STRSTAT(qwr_outer); 5981 (void) taskq_dispatch(streams_taskq, 5982 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP); 5983 } else { 5984 ASSERT(outer->sq_flags & SQ_EVENTS); 5985 outer->sq_evtail->b_next = mp; 5986 outer->sq_evtail = mp; 5987 mutex_exit(SQLOCK(outer)); 5988 } 5989 } 5990 5991 /* 5992 * Try and upgrade to write access at the outer perimeter. If this can 5993 * not be done without blocking then queue the callback to be done 5994 * by the qwriter_outer_thread. 5995 * 5996 * This routine can only be called from put or service procedures plus 5997 * asynchronous callback routines that have properly entered the queue (with 5998 * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq 5999 * associated with q. 6000 */ 6001 void 6002 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)()) 6003 { 6004 syncq_t *osq, *sq, *outer; 6005 int failed; 6006 uint16_t flags; 6007 6008 osq = q->q_syncq; 6009 outer = osq->sq_outer; 6010 if (outer == NULL) 6011 panic("qwriter(PERIM_OUTER): no outer perimeter"); 6012 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 6013 outer->sq_oprev != NULL); 6014 6015 mutex_enter(SQLOCK(outer)); 6016 flags = outer->sq_flags; 6017 /* 6018 * If some thread is traversing sq_next, or if we are blocked by 6019 * outer_insert or outer_remove, or if the we already have queued 6020 * callbacks, then queue this callback for later processing. 6021 * 6022 * Also queue the qwriter for an interrupt thread in order 6023 * to reduce the time spent running at high IPL. 6024 * to identify there are events. 6025 */ 6026 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) { 6027 /* 6028 * Queue the become_writer request. 6029 * The queueing is atomic under SQLOCK(outer) in order 6030 * to synchronize with outer_exit. 6031 * queue_writer will drop the outer SQLOCK 6032 */ 6033 if (flags & SQ_BLOCKED) { 6034 /* Must set SQ_WRITER on inner perimeter */ 6035 mutex_enter(SQLOCK(osq)); 6036 osq->sq_flags |= SQ_WRITER; 6037 mutex_exit(SQLOCK(osq)); 6038 } else { 6039 if (!(flags & SQ_WRITER)) { 6040 /* 6041 * The outer could have been SQ_BLOCKED thus 6042 * SQ_WRITER might not be set on the inner. 6043 */ 6044 mutex_enter(SQLOCK(osq)); 6045 osq->sq_flags |= SQ_WRITER; 6046 mutex_exit(SQLOCK(osq)); 6047 } 6048 ASSERT(osq->sq_flags & SQ_WRITER); 6049 } 6050 queue_writer(outer, func, q, mp); 6051 return; 6052 } 6053 /* 6054 * We are half-way to exclusive access to the outer perimeter. 6055 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove 6056 * while the inner syncqs are traversed. 6057 */ 6058 outer->sq_count++; 6059 ASSERT(outer->sq_count != 0); /* wraparound */ 6060 flags |= SQ_WRITER; 6061 /* 6062 * Check if we can run the function immediately. Mark all 6063 * syncqs with the writer flag to prevent new entries into 6064 * put and service procedures. 6065 * 6066 * Set SQ_WRITER on all the inner syncqs while holding 6067 * the SQLOCK on the outer syncq. This ensures that the changing 6068 * of SQ_WRITER is atomic under the outer SQLOCK. 6069 */ 6070 failed = 0; 6071 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 6072 uint16_t count; 6073 uint_t maxcnt = (sq == osq) ? 1 : 0; 6074 6075 mutex_enter(SQLOCK(sq)); 6076 count = sq->sq_count; 6077 SQ_PUTLOCKS_ENTER(sq); 6078 SUM_SQ_PUTCOUNTS(sq, count); 6079 if (sq->sq_count > maxcnt) 6080 failed = 1; 6081 sq->sq_flags |= SQ_WRITER; 6082 SQ_PUTLOCKS_EXIT(sq); 6083 mutex_exit(SQLOCK(sq)); 6084 } 6085 if (failed) { 6086 /* 6087 * Some other thread has a read claim on the outer perimeter. 6088 * Queue the callback for deferred processing. 6089 * 6090 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER 6091 * so that other qwriter(OUTER) calls will queue their 6092 * callbacks as well. queue_writer increments sq_count so we 6093 * decrement to compensate for the our increment. 6094 * 6095 * Dropping SQ_WRITER enables the writer thread to work 6096 * on this outer perimeter. 6097 */ 6098 outer->sq_flags = flags; 6099 queue_writer(outer, func, q, mp); 6100 /* queue_writer dropper the lock */ 6101 mutex_enter(SQLOCK(outer)); 6102 ASSERT(outer->sq_count > 0); 6103 outer->sq_count--; 6104 ASSERT(outer->sq_flags & SQ_WRITER); 6105 flags = outer->sq_flags; 6106 flags &= ~SQ_WRITER; 6107 if (flags & SQ_WANTWAKEUP) { 6108 flags &= ~SQ_WANTWAKEUP; 6109 cv_broadcast(&outer->sq_wait); 6110 } 6111 outer->sq_flags = flags; 6112 mutex_exit(SQLOCK(outer)); 6113 return; 6114 } else { 6115 outer->sq_flags = flags; 6116 mutex_exit(SQLOCK(outer)); 6117 } 6118 6119 /* Can run it immediately */ 6120 (*func)(q, mp); 6121 6122 outer_exit(outer); 6123 } 6124 6125 /* 6126 * Dequeue all writer callbacks from the outer perimeter and run them. 6127 */ 6128 static void 6129 write_now(syncq_t *outer) 6130 { 6131 mblk_t *mp; 6132 queue_t *q; 6133 void (*func)(); 6134 6135 ASSERT(MUTEX_HELD(SQLOCK(outer))); 6136 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 6137 outer->sq_oprev != NULL); 6138 while ((mp = outer->sq_evhead) != NULL) { 6139 /* 6140 * queues cannot be placed on the queuelist on the outer 6141 * perimeter. 6142 */ 6143 ASSERT(!(outer->sq_flags & SQ_MESSAGES)); 6144 ASSERT((outer->sq_flags & SQ_EVENTS)); 6145 6146 outer->sq_evhead = mp->b_next; 6147 if (outer->sq_evhead == NULL) { 6148 outer->sq_evtail = NULL; 6149 outer->sq_flags &= ~SQ_EVENTS; 6150 } 6151 ASSERT(outer->sq_count != 0); 6152 outer->sq_count--; /* Incremented when enqueued. */ 6153 mutex_exit(SQLOCK(outer)); 6154 /* 6155 * Drop the message if the queue is closing. 6156 * Make sure that the queue is "claimed" when the callback 6157 * is run in order to satisfy various ASSERTs. 6158 */ 6159 q = mp->b_queue; 6160 func = (void (*)())mp->b_prev; 6161 ASSERT(func != NULL); 6162 mp->b_next = mp->b_prev = NULL; 6163 if (q->q_flag & QWCLOSE) { 6164 freemsg(mp); 6165 } else { 6166 claimq(q); 6167 (*func)(q, mp); 6168 releaseq(q); 6169 } 6170 mutex_enter(SQLOCK(outer)); 6171 } 6172 ASSERT(MUTEX_HELD(SQLOCK(outer))); 6173 } 6174 6175 /* 6176 * The list of messages on the inner syncq is effectively hashed 6177 * by destination queue. These destination queues are doubly 6178 * linked lists (hopefully) in priority order. Messages are then 6179 * put on the queue referenced by the q_sqhead/q_sqtail elements. 6180 * Additional messages are linked together by the b_next/b_prev 6181 * elements in the mblk, with (similar to putq()) the first message 6182 * having a NULL b_prev and the last message having a NULL b_next. 6183 * 6184 * Events, such as qwriter callbacks, are put onto a list in FIFO 6185 * order referenced by sq_evhead, and sq_evtail. This is a singly 6186 * linked list, and messages here MUST be processed in the order queued. 6187 */ 6188 6189 /* 6190 * Run the events on the syncq event list (sq_evhead). 6191 * Assumes there is only one claim on the syncq, it is 6192 * already exclusive (SQ_EXCL set), and the SQLOCK held. 6193 * Messages here are processed in order, with the SQ_EXCL bit 6194 * held all the way through till the last message is processed. 6195 */ 6196 void 6197 sq_run_events(syncq_t *sq) 6198 { 6199 mblk_t *bp; 6200 queue_t *qp; 6201 uint16_t flags = sq->sq_flags; 6202 void (*func)(); 6203 6204 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6205 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6206 sq->sq_oprev == NULL) || 6207 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6208 sq->sq_oprev != NULL)); 6209 6210 ASSERT(flags & SQ_EXCL); 6211 ASSERT(sq->sq_count == 1); 6212 6213 /* 6214 * We need to process all of the events on this list. It 6215 * is possible that new events will be added while we are 6216 * away processing a callback, so on every loop, we start 6217 * back at the beginning of the list. 6218 */ 6219 /* 6220 * We have to reaccess sq_evhead since there is a 6221 * possibility of a new entry while we were running 6222 * the callback. 6223 */ 6224 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { 6225 ASSERT(bp->b_queue->q_syncq == sq); 6226 ASSERT(sq->sq_flags & SQ_EVENTS); 6227 6228 qp = bp->b_queue; 6229 func = (void (*)())bp->b_prev; 6230 ASSERT(func != NULL); 6231 6232 /* 6233 * Messages from the event queue must be taken off in 6234 * FIFO order. 6235 */ 6236 ASSERT(sq->sq_evhead == bp); 6237 sq->sq_evhead = bp->b_next; 6238 6239 if (bp->b_next == NULL) { 6240 /* Deleting last */ 6241 ASSERT(sq->sq_evtail == bp); 6242 sq->sq_evtail = NULL; 6243 sq->sq_flags &= ~SQ_EVENTS; 6244 } 6245 bp->b_prev = bp->b_next = NULL; 6246 ASSERT(bp->b_datap->db_ref != 0); 6247 6248 mutex_exit(SQLOCK(sq)); 6249 6250 (*func)(qp, bp); 6251 6252 mutex_enter(SQLOCK(sq)); 6253 /* 6254 * re-read the flags, since they could have changed. 6255 */ 6256 flags = sq->sq_flags; 6257 ASSERT(flags & SQ_EXCL); 6258 } 6259 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); 6260 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6261 6262 if (flags & SQ_WANTWAKEUP) { 6263 flags &= ~SQ_WANTWAKEUP; 6264 cv_broadcast(&sq->sq_wait); 6265 } 6266 if (flags & SQ_WANTEXWAKEUP) { 6267 flags &= ~SQ_WANTEXWAKEUP; 6268 cv_broadcast(&sq->sq_exitwait); 6269 } 6270 sq->sq_flags = flags; 6271 } 6272 6273 /* 6274 * Put messages on the event list. 6275 * If we can go exclusive now, do so and process the event list, otherwise 6276 * let the last claim service this list (or wake the sqthread). 6277 * This procedure assumes SQLOCK is held. To run the event list, it 6278 * must be called with no claims. 6279 */ 6280 static void 6281 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) 6282 { 6283 uint16_t count; 6284 6285 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6286 ASSERT(func != NULL); 6287 6288 /* 6289 * This is a callback. Add it to the list of callbacks 6290 * and see about upgrading. 6291 */ 6292 mp->b_prev = (mblk_t *)func; 6293 mp->b_queue = q; 6294 mp->b_next = NULL; 6295 if (sq->sq_evhead == NULL) { 6296 sq->sq_evhead = sq->sq_evtail = mp; 6297 sq->sq_flags |= SQ_EVENTS; 6298 } else { 6299 ASSERT(sq->sq_evtail != NULL); 6300 ASSERT(sq->sq_evtail->b_next == NULL); 6301 ASSERT(sq->sq_flags & SQ_EVENTS); 6302 sq->sq_evtail->b_next = mp; 6303 sq->sq_evtail = mp; 6304 } 6305 /* 6306 * We have set SQ_EVENTS, so threads will have to 6307 * unwind out of the perimeter, and new entries will 6308 * not grab a putlock. But we still need to know 6309 * how many threads have already made a claim to the 6310 * syncq, so grab the putlocks, and sum the counts. 6311 * If there are no claims on the syncq, we can upgrade 6312 * to exclusive, and run the event list. 6313 * NOTE: We hold the SQLOCK, so we can just grab the 6314 * putlocks. 6315 */ 6316 count = sq->sq_count; 6317 SQ_PUTLOCKS_ENTER(sq); 6318 SUM_SQ_PUTCOUNTS(sq, count); 6319 /* 6320 * We have no claim, so we need to check if there 6321 * are no others, then we can upgrade. 6322 */ 6323 /* 6324 * There are currently no claims on 6325 * the syncq by this thread (at least on this entry). The thread who has 6326 * the claim should drain syncq. 6327 */ 6328 if (count > 0) { 6329 /* 6330 * Can't upgrade - other threads inside. 6331 */ 6332 SQ_PUTLOCKS_EXIT(sq); 6333 mutex_exit(SQLOCK(sq)); 6334 return; 6335 } 6336 /* 6337 * Need to set SQ_EXCL and make a claim on the syncq. 6338 */ 6339 ASSERT((sq->sq_flags & SQ_EXCL) == 0); 6340 sq->sq_flags |= SQ_EXCL; 6341 ASSERT(sq->sq_count == 0); 6342 sq->sq_count++; 6343 SQ_PUTLOCKS_EXIT(sq); 6344 6345 /* Process the events list */ 6346 sq_run_events(sq); 6347 6348 /* 6349 * Release our claim... 6350 */ 6351 sq->sq_count--; 6352 6353 /* 6354 * And release SQ_EXCL. 6355 * We don't need to acquire the putlocks to release 6356 * SQ_EXCL, since we are exclusive, and hold the SQLOCK. 6357 */ 6358 sq->sq_flags &= ~SQ_EXCL; 6359 6360 /* 6361 * sq_run_events should have released SQ_EXCL 6362 */ 6363 ASSERT(!(sq->sq_flags & SQ_EXCL)); 6364 6365 /* 6366 * If anything happened while we were running the 6367 * events (or was there before), we need to process 6368 * them now. We shouldn't be exclusive sine we 6369 * released the perimeter above (plus, we asserted 6370 * for it). 6371 */ 6372 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) 6373 drain_syncq(sq); 6374 else 6375 mutex_exit(SQLOCK(sq)); 6376 } 6377 6378 /* 6379 * Perform delayed processing. The caller has to make sure that it is safe 6380 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are 6381 * set). 6382 * 6383 * Assume that the caller has NO claims on the syncq. However, a claim 6384 * on the syncq does not indicate that a thread is draining the syncq. 6385 * There may be more claims on the syncq than there are threads draining 6386 * (i.e. #_threads_draining <= sq_count) 6387 * 6388 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set 6389 * in order to preserve qwriter(OUTER) ordering constraints. 6390 * 6391 * sq_putcount only needs to be checked when dispatching the queued 6392 * writer call for CIPUT sync queue, but this is handled in sq_run_events. 6393 */ 6394 void 6395 drain_syncq(syncq_t *sq) 6396 { 6397 queue_t *qp; 6398 uint16_t count; 6399 uint16_t type = sq->sq_type; 6400 uint16_t flags = sq->sq_flags; 6401 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; 6402 6403 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6404 "drain_syncq start:%p", sq); 6405 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6406 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6407 sq->sq_oprev == NULL) || 6408 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6409 sq->sq_oprev != NULL)); 6410 6411 /* 6412 * Drop SQ_SERVICE flag. 6413 */ 6414 if (bg_service) 6415 sq->sq_svcflags &= ~SQ_SERVICE; 6416 6417 /* 6418 * If SQ_EXCL is set, someone else is processing this syncq - let them 6419 * finish the job. 6420 */ 6421 if (flags & SQ_EXCL) { 6422 if (bg_service) { 6423 ASSERT(sq->sq_servcount != 0); 6424 sq->sq_servcount--; 6425 } 6426 mutex_exit(SQLOCK(sq)); 6427 return; 6428 } 6429 6430 /* 6431 * This routine can be called by a background thread if 6432 * it was scheduled by a hi-priority thread. SO, if there are 6433 * NOT messages queued, return (remember, we have the SQLOCK, 6434 * and it cannot change until we release it). Wakeup any waiters also. 6435 */ 6436 if (!(flags & SQ_QUEUED)) { 6437 if (flags & SQ_WANTWAKEUP) { 6438 flags &= ~SQ_WANTWAKEUP; 6439 cv_broadcast(&sq->sq_wait); 6440 } 6441 if (flags & SQ_WANTEXWAKEUP) { 6442 flags &= ~SQ_WANTEXWAKEUP; 6443 cv_broadcast(&sq->sq_exitwait); 6444 } 6445 sq->sq_flags = flags; 6446 if (bg_service) { 6447 ASSERT(sq->sq_servcount != 0); 6448 sq->sq_servcount--; 6449 } 6450 mutex_exit(SQLOCK(sq)); 6451 return; 6452 } 6453 6454 /* 6455 * If this is not a concurrent put perimeter, we need to 6456 * become exclusive to drain. Also, if not CIPUT, we would 6457 * not have acquired a putlock, so we don't need to check 6458 * the putcounts. If not entering with a claim, we test 6459 * for sq_count == 0. 6460 */ 6461 type = sq->sq_type; 6462 if (!(type & SQ_CIPUT)) { 6463 if (sq->sq_count > 1) { 6464 if (bg_service) { 6465 ASSERT(sq->sq_servcount != 0); 6466 sq->sq_servcount--; 6467 } 6468 mutex_exit(SQLOCK(sq)); 6469 return; 6470 } 6471 sq->sq_flags |= SQ_EXCL; 6472 } 6473 6474 /* 6475 * This is where we make a claim to the syncq. 6476 * This can either be done by incrementing a putlock, or 6477 * the sq_count. But since we already have the SQLOCK 6478 * here, we just bump the sq_count. 6479 * 6480 * Note that after we make a claim, we need to let the code 6481 * fall through to the end of this routine to clean itself 6482 * up. A return in the while loop will put the syncq in a 6483 * very bad state. 6484 */ 6485 sq->sq_count++; 6486 ASSERT(sq->sq_count != 0); /* wraparound */ 6487 6488 while ((flags = sq->sq_flags) & SQ_QUEUED) { 6489 /* 6490 * If we are told to stayaway or went exclusive, 6491 * we are done. 6492 */ 6493 if (flags & (SQ_STAYAWAY)) { 6494 break; 6495 } 6496 6497 /* 6498 * If there are events to run, do so. 6499 * We have one claim to the syncq, so if there are 6500 * more than one, other threads are running. 6501 */ 6502 if (sq->sq_evhead != NULL) { 6503 ASSERT(sq->sq_flags & SQ_EVENTS); 6504 6505 count = sq->sq_count; 6506 SQ_PUTLOCKS_ENTER(sq); 6507 SUM_SQ_PUTCOUNTS(sq, count); 6508 if (count > 1) { 6509 SQ_PUTLOCKS_EXIT(sq); 6510 /* Can't upgrade - other threads inside */ 6511 break; 6512 } 6513 ASSERT((flags & SQ_EXCL) == 0); 6514 sq->sq_flags = flags | SQ_EXCL; 6515 SQ_PUTLOCKS_EXIT(sq); 6516 /* 6517 * we have the only claim, run the events, 6518 * sq_run_events will clear the SQ_EXCL flag. 6519 */ 6520 sq_run_events(sq); 6521 6522 /* 6523 * If this is a CIPUT perimeter, we need 6524 * to drop the SQ_EXCL flag so we can properly 6525 * continue draining the syncq. 6526 */ 6527 if (type & SQ_CIPUT) { 6528 ASSERT(sq->sq_flags & SQ_EXCL); 6529 sq->sq_flags &= ~SQ_EXCL; 6530 } 6531 6532 /* 6533 * And go back to the beginning just in case 6534 * anything changed while we were away. 6535 */ 6536 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); 6537 continue; 6538 } 6539 6540 ASSERT(sq->sq_evhead == NULL); 6541 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6542 6543 /* 6544 * Find the queue that is not draining. 6545 * 6546 * q_draining is protected by QLOCK which we do not hold. 6547 * But if it was set, then a thread was draining, and if it gets 6548 * cleared, then it was because the thread has successfully 6549 * drained the syncq, or a GOAWAY state occurred. For the GOAWAY 6550 * state to happen, a thread needs the SQLOCK which we hold, and 6551 * if there was such a flag, we would have already seen it. 6552 */ 6553 6554 for (qp = sq->sq_head; 6555 qp != NULL && (qp->q_draining || 6556 (qp->q_sqflags & Q_SQDRAINING)); 6557 qp = qp->q_sqnext) 6558 ; 6559 6560 if (qp == NULL) 6561 break; 6562 6563 /* 6564 * We have a queue to work on, and we hold the 6565 * SQLOCK and one claim, call qdrain_syncq. 6566 * This means we need to release the SQLOCK and 6567 * acquire the QLOCK (OK since we have a claim). 6568 * Note that qdrain_syncq will actually dequeue 6569 * this queue from the sq_head list when it is 6570 * convinced all the work is done and release 6571 * the QLOCK before returning. 6572 */ 6573 qp->q_sqflags |= Q_SQDRAINING; 6574 mutex_exit(SQLOCK(sq)); 6575 mutex_enter(QLOCK(qp)); 6576 qdrain_syncq(sq, qp); 6577 mutex_enter(SQLOCK(sq)); 6578 6579 /* The queue is drained */ 6580 ASSERT(qp->q_sqflags & Q_SQDRAINING); 6581 qp->q_sqflags &= ~Q_SQDRAINING; 6582 /* 6583 * NOTE: After this point qp should not be used since it may be 6584 * closed. 6585 */ 6586 } 6587 6588 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6589 flags = sq->sq_flags; 6590 6591 /* 6592 * sq->sq_head cannot change because we hold the 6593 * sqlock. However, a thread CAN decide that it is no longer 6594 * going to drain that queue. However, this should be due to 6595 * a GOAWAY state, and we should see that here. 6596 * 6597 * This loop is not very efficient. One solution may be adding a second 6598 * pointer to the "draining" queue, but it is difficult to do when 6599 * queues are inserted in the middle due to priority ordering. Another 6600 * possibility is to yank the queue out of the sq list and put it onto 6601 * the "draining list" and then put it back if it can't be drained. 6602 */ 6603 6604 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || 6605 (type & SQ_CI) || sq->sq_head->q_draining); 6606 6607 /* Drop SQ_EXCL for non-CIPUT perimeters */ 6608 if (!(type & SQ_CIPUT)) 6609 flags &= ~SQ_EXCL; 6610 ASSERT((flags & SQ_EXCL) == 0); 6611 6612 /* Wake up any waiters. */ 6613 if (flags & SQ_WANTWAKEUP) { 6614 flags &= ~SQ_WANTWAKEUP; 6615 cv_broadcast(&sq->sq_wait); 6616 } 6617 if (flags & SQ_WANTEXWAKEUP) { 6618 flags &= ~SQ_WANTEXWAKEUP; 6619 cv_broadcast(&sq->sq_exitwait); 6620 } 6621 sq->sq_flags = flags; 6622 6623 ASSERT(sq->sq_count != 0); 6624 /* Release our claim. */ 6625 sq->sq_count--; 6626 6627 if (bg_service) { 6628 ASSERT(sq->sq_servcount != 0); 6629 sq->sq_servcount--; 6630 } 6631 6632 mutex_exit(SQLOCK(sq)); 6633 6634 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6635 "drain_syncq end:%p", sq); 6636 } 6637 6638 6639 /* 6640 * 6641 * qdrain_syncq can be called (currently) from only one of two places: 6642 * drain_syncq 6643 * putnext (or some variation of it). 6644 * and eventually 6645 * qwait(_sig) 6646 * 6647 * If called from drain_syncq, we found it in the list of queues needing 6648 * service, so there is work to be done (or it wouldn't be in the list). 6649 * 6650 * If called from some putnext variation, it was because the 6651 * perimeter is open, but messages are blocking a putnext and 6652 * there is not a thread working on it. Now a thread could start 6653 * working on it while we are getting ready to do so ourself, but 6654 * the thread would set the q_draining flag, and we can spin out. 6655 * 6656 * As for qwait(_sig), I think I shall let it continue to call 6657 * drain_syncq directly (after all, it will get here eventually). 6658 * 6659 * qdrain_syncq has to terminate when: 6660 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering 6661 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering 6662 * 6663 * ASSUMES: 6664 * One claim 6665 * QLOCK held 6666 * SQLOCK not held 6667 * Will release QLOCK before returning 6668 */ 6669 void 6670 qdrain_syncq(syncq_t *sq, queue_t *q) 6671 { 6672 mblk_t *bp; 6673 #ifdef DEBUG 6674 uint16_t count; 6675 #endif 6676 6677 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6678 "drain_syncq start:%p", sq); 6679 ASSERT(q->q_syncq == sq); 6680 ASSERT(MUTEX_HELD(QLOCK(q))); 6681 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6682 /* 6683 * For non-CIPUT perimeters, we should be called with the exclusive bit 6684 * set already. For CIPUT perimeters, we will be doing a concurrent 6685 * drain, so it better not be set. 6686 */ 6687 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); 6688 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); 6689 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); 6690 /* 6691 * All outer pointers are set, or none of them are 6692 */ 6693 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6694 sq->sq_oprev == NULL) || 6695 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6696 sq->sq_oprev != NULL)); 6697 #ifdef DEBUG 6698 count = sq->sq_count; 6699 /* 6700 * This is OK without the putlocks, because we have one 6701 * claim either from the sq_count, or a putcount. We could 6702 * get an erroneous value from other counts, but ours won't 6703 * change, so one way or another, we will have at least a 6704 * value of one. 6705 */ 6706 SUM_SQ_PUTCOUNTS(sq, count); 6707 ASSERT(count >= 1); 6708 #endif /* DEBUG */ 6709 6710 /* 6711 * The first thing to do is find out if a thread is already draining 6712 * this queue. If so, we are done, just return. 6713 */ 6714 if (q->q_draining) { 6715 mutex_exit(QLOCK(q)); 6716 return; 6717 } 6718 6719 /* 6720 * If the perimeter is exclusive, there is nothing we can do right now, 6721 * go away. Note that there is nothing to prevent this case from 6722 * changing right after this check, but the spin-out will catch it. 6723 */ 6724 6725 /* Tell other threads that we are draining this queue */ 6726 q->q_draining = 1; /* Protected by QLOCK */ 6727 6728 /* 6729 * If there is nothing to do, clear QFULL as necessary. This caters for 6730 * the case where an empty queue was enqueued onto the syncq. 6731 */ 6732 if (q->q_sqhead == NULL) { 6733 ASSERT(q->q_syncqmsgs == 0); 6734 mutex_exit(QLOCK(q)); 6735 clr_qfull(q); 6736 mutex_enter(QLOCK(q)); 6737 } 6738 6739 /* 6740 * Note that q_sqhead must be re-checked here in case another message 6741 * was enqueued whilst QLOCK was dropped during the call to clr_qfull. 6742 */ 6743 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) { 6744 /* 6745 * Because we can enter this routine just because a putnext is 6746 * blocked, we need to spin out if the perimeter wants to go 6747 * exclusive as well as just blocked. We need to spin out also 6748 * if events are queued on the syncq. 6749 * Don't check for SQ_EXCL, because non-CIPUT perimeters would 6750 * set it, and it can't become exclusive while we hold a claim. 6751 */ 6752 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { 6753 break; 6754 } 6755 6756 #ifdef DEBUG 6757 /* 6758 * Since we are in qdrain_syncq, we already know the queue, 6759 * but for sanity, we want to check this against the qp that 6760 * was passed in by bp->b_queue. 6761 */ 6762 6763 ASSERT(bp->b_queue == q); 6764 ASSERT(bp->b_queue->q_syncq == sq); 6765 bp->b_queue = NULL; 6766 6767 /* 6768 * We would have the following check in the DEBUG code: 6769 * 6770 * if (bp->b_prev != NULL) { 6771 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp); 6772 * } 6773 * 6774 * This can't be done, however, since IP modifies qinfo 6775 * structure at run-time (switching between IPv4 qinfo and IPv6 6776 * qinfo), invalidating the check. 6777 * So the assignment to func is left here, but the ASSERT itself 6778 * is removed until the whole issue is resolved. 6779 */ 6780 #endif 6781 ASSERT(q->q_sqhead == bp); 6782 q->q_sqhead = bp->b_next; 6783 bp->b_prev = bp->b_next = NULL; 6784 ASSERT(q->q_syncqmsgs > 0); 6785 mutex_exit(QLOCK(q)); 6786 6787 ASSERT(bp->b_datap->db_ref != 0); 6788 6789 (void) (*q->q_qinfo->qi_putp)(q, bp); 6790 6791 mutex_enter(QLOCK(q)); 6792 6793 /* 6794 * q_syncqmsgs should only be decremented after executing the 6795 * put procedure to avoid message re-ordering. This is due to an 6796 * optimisation in putnext() which can call the put procedure 6797 * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED 6798 * being set). 6799 * 6800 * We also need to clear QFULL in the next service procedure 6801 * queue if this is the last message destined for that queue. 6802 * 6803 * It would make better sense to have some sort of tunable for 6804 * the low water mark, but these semantics are not yet defined. 6805 * So, alas, we use a constant. 6806 */ 6807 if (--q->q_syncqmsgs == 0) { 6808 mutex_exit(QLOCK(q)); 6809 clr_qfull(q); 6810 mutex_enter(QLOCK(q)); 6811 } 6812 6813 /* 6814 * Always clear SQ_EXCL when CIPUT in order to handle 6815 * qwriter(INNER). The putp() can call qwriter and get exclusive 6816 * access IFF this is the only claim. So, we need to test for 6817 * this possibility, acquire the mutex and clear the bit. 6818 */ 6819 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { 6820 mutex_enter(SQLOCK(sq)); 6821 sq->sq_flags &= ~SQ_EXCL; 6822 mutex_exit(SQLOCK(sq)); 6823 } 6824 } 6825 6826 /* 6827 * We should either have no messages on this queue, or we were told to 6828 * goaway by a waiter (which we will wake up at the end of this 6829 * function). 6830 */ 6831 ASSERT((q->q_sqhead == NULL) || 6832 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); 6833 6834 ASSERT(MUTEX_HELD(QLOCK(q))); 6835 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6836 6837 /* Remove the q from the syncq list if all the messages are drained. */ 6838 if (q->q_sqhead == NULL) { 6839 ASSERT(q->q_syncqmsgs == 0); 6840 mutex_enter(SQLOCK(sq)); 6841 if (q->q_sqflags & Q_SQQUEUED) 6842 SQRM_Q(sq, q); 6843 mutex_exit(SQLOCK(sq)); 6844 /* 6845 * Since the queue is removed from the list, reset its priority. 6846 */ 6847 q->q_spri = 0; 6848 } 6849 6850 /* 6851 * Remember, the q_draining flag is used to let another thread know 6852 * that there is a thread currently draining the messages for a queue. 6853 * Since we are now done with this queue (even if there may be messages 6854 * still there), we need to clear this flag so some thread will work on 6855 * it if needed. 6856 */ 6857 ASSERT(q->q_draining); 6858 q->q_draining = 0; 6859 6860 /* Called with a claim, so OK to drop all locks. */ 6861 mutex_exit(QLOCK(q)); 6862 6863 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6864 "drain_syncq end:%p", sq); 6865 } 6866 /* END OF QDRAIN_SYNCQ */ 6867 6868 6869 /* 6870 * This is the mate to qdrain_syncq, except that it is putting the message onto 6871 * the queue instead of draining. Since the message is destined for the queue 6872 * that is selected, there is no need to identify the function because the 6873 * message is intended for the put routine for the queue. For debug kernels, 6874 * this routine will do it anyway just in case. 6875 * 6876 * After the message is enqueued on the syncq, it calls putnext_tail() 6877 * which will schedule a background thread to actually process the message. 6878 * 6879 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and 6880 * SQLOCK(sq) and QLOCK(q) are not held. 6881 */ 6882 void 6883 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) 6884 { 6885 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6886 ASSERT(MUTEX_NOT_HELD(QLOCK(q))); 6887 ASSERT(sq->sq_count > 0); 6888 ASSERT(q->q_syncq == sq); 6889 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6890 sq->sq_oprev == NULL) || 6891 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6892 sq->sq_oprev != NULL)); 6893 6894 mutex_enter(QLOCK(q)); 6895 6896 #ifdef DEBUG 6897 /* 6898 * This is used for debug in the qfill_syncq/qdrain_syncq case 6899 * to trace the queue that the message is intended for. Note 6900 * that the original use was to identify the queue and function 6901 * to call on the drain. In the new syncq, we have the context 6902 * of the queue that we are draining, so call it's putproc and 6903 * don't rely on the saved values. But for debug this is still 6904 * useful information. 6905 */ 6906 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp; 6907 mp->b_queue = q; 6908 mp->b_next = NULL; 6909 #endif 6910 ASSERT(q->q_syncq == sq); 6911 /* 6912 * Enqueue the message on the list. 6913 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to 6914 * protect it. So it's ok to acquire SQLOCK after SQPUT_MP(). 6915 */ 6916 SQPUT_MP(q, mp); 6917 mutex_enter(SQLOCK(sq)); 6918 6919 /* 6920 * And queue on syncq for scheduling, if not already queued. 6921 * Note that we need the SQLOCK for this, and for testing flags 6922 * at the end to see if we will drain. So grab it now, and 6923 * release it before we call qdrain_syncq or return. 6924 */ 6925 if (!(q->q_sqflags & Q_SQQUEUED)) { 6926 q->q_spri = curthread->t_pri; 6927 SQPUT_Q(sq, q); 6928 } 6929 #ifdef DEBUG 6930 else { 6931 /* 6932 * All of these conditions MUST be true! 6933 */ 6934 ASSERT(sq->sq_tail != NULL); 6935 if (sq->sq_tail == sq->sq_head) { 6936 ASSERT((q->q_sqprev == NULL) && 6937 (q->q_sqnext == NULL)); 6938 } else { 6939 ASSERT((q->q_sqprev != NULL) || 6940 (q->q_sqnext != NULL)); 6941 } 6942 ASSERT(sq->sq_flags & SQ_QUEUED); 6943 ASSERT(q->q_syncqmsgs != 0); 6944 ASSERT(q->q_sqflags & Q_SQQUEUED); 6945 } 6946 #endif 6947 mutex_exit(QLOCK(q)); 6948 /* 6949 * SQLOCK is still held, so sq_count can be safely decremented. 6950 */ 6951 sq->sq_count--; 6952 6953 putnext_tail(sq, q, 0); 6954 /* Should not reference sq or q after this point. */ 6955 } 6956 6957 /* End of qfill_syncq */ 6958 6959 /* 6960 * Remove all messages from a syncq (if qp is NULL) or remove all messages 6961 * that would be put into qp by drain_syncq. 6962 * Used when deleting the syncq (qp == NULL) or when detaching 6963 * a queue (qp != NULL). 6964 * Return non-zero if one or more messages were freed. 6965 * 6966 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 6967 * sq_putlocks are used. 6968 * 6969 * NOTE: This function assumes that it is called from the close() context and 6970 * that all the queues in the syncq are going away. For this reason it doesn't 6971 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is 6972 * currently valid, but it is useful to rethink this function to behave properly 6973 * in other cases. 6974 */ 6975 int 6976 flush_syncq(syncq_t *sq, queue_t *qp) 6977 { 6978 mblk_t *bp, *mp_head, *mp_next, *mp_prev; 6979 queue_t *q; 6980 int ret = 0; 6981 6982 mutex_enter(SQLOCK(sq)); 6983 6984 /* 6985 * Before we leave, we need to make sure there are no 6986 * events listed for this queue. All events for this queue 6987 * will just be freed. 6988 */ 6989 if (qp != NULL && sq->sq_evhead != NULL) { 6990 ASSERT(sq->sq_flags & SQ_EVENTS); 6991 6992 mp_prev = NULL; 6993 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { 6994 mp_next = bp->b_next; 6995 if (bp->b_queue == qp) { 6996 /* Delete this message */ 6997 if (mp_prev != NULL) { 6998 mp_prev->b_next = mp_next; 6999 /* 7000 * Update sq_evtail if the last element 7001 * is removed. 7002 */ 7003 if (bp == sq->sq_evtail) { 7004 ASSERT(mp_next == NULL); 7005 sq->sq_evtail = mp_prev; 7006 } 7007 } else 7008 sq->sq_evhead = mp_next; 7009 if (sq->sq_evhead == NULL) 7010 sq->sq_flags &= ~SQ_EVENTS; 7011 bp->b_prev = bp->b_next = NULL; 7012 freemsg(bp); 7013 ret++; 7014 } else { 7015 mp_prev = bp; 7016 } 7017 } 7018 } 7019 7020 /* 7021 * Walk sq_head and: 7022 * - match qp if qp is set, remove it's messages 7023 * - all if qp is not set 7024 */ 7025 q = sq->sq_head; 7026 while (q != NULL) { 7027 ASSERT(q->q_syncq == sq); 7028 if ((qp == NULL) || (qp == q)) { 7029 /* 7030 * Yank the messages as a list off the queue 7031 */ 7032 mp_head = q->q_sqhead; 7033 /* 7034 * We do not have QLOCK(q) here (which is safe due to 7035 * assumptions mentioned above). To obtain the lock we 7036 * need to release SQLOCK which may allow lots of things 7037 * to change upon us. This place requires more analysis. 7038 */ 7039 q->q_sqhead = q->q_sqtail = NULL; 7040 ASSERT(mp_head->b_queue && 7041 mp_head->b_queue->q_syncq == sq); 7042 7043 /* 7044 * Free each of the messages. 7045 */ 7046 for (bp = mp_head; bp != NULL; bp = mp_next) { 7047 mp_next = bp->b_next; 7048 bp->b_prev = bp->b_next = NULL; 7049 freemsg(bp); 7050 ret++; 7051 } 7052 /* 7053 * Now remove the queue from the syncq. 7054 */ 7055 ASSERT(q->q_sqflags & Q_SQQUEUED); 7056 SQRM_Q(sq, q); 7057 q->q_spri = 0; 7058 q->q_syncqmsgs = 0; 7059 7060 /* 7061 * If qp was specified, we are done with it and are 7062 * going to drop SQLOCK(sq) and return. We wakeup syncq 7063 * waiters while we still have the SQLOCK. 7064 */ 7065 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { 7066 sq->sq_flags &= ~SQ_WANTWAKEUP; 7067 cv_broadcast(&sq->sq_wait); 7068 } 7069 /* Drop SQLOCK across clr_qfull */ 7070 mutex_exit(SQLOCK(sq)); 7071 7072 /* 7073 * We avoid doing the test that drain_syncq does and 7074 * unconditionally clear qfull for every flushed 7075 * message. Since flush_syncq is only called during 7076 * close this should not be a problem. 7077 */ 7078 clr_qfull(q); 7079 if (qp != NULL) { 7080 return (ret); 7081 } else { 7082 mutex_enter(SQLOCK(sq)); 7083 /* 7084 * The head was removed by SQRM_Q above. 7085 * reread the new head and flush it. 7086 */ 7087 q = sq->sq_head; 7088 } 7089 } else { 7090 q = q->q_sqnext; 7091 } 7092 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7093 } 7094 7095 if (sq->sq_flags & SQ_WANTWAKEUP) { 7096 sq->sq_flags &= ~SQ_WANTWAKEUP; 7097 cv_broadcast(&sq->sq_wait); 7098 } 7099 7100 mutex_exit(SQLOCK(sq)); 7101 return (ret); 7102 } 7103 7104 /* 7105 * Propagate all messages from a syncq to the next syncq that are associated 7106 * with the specified queue. If the queue is attached to a driver or if the 7107 * messages have been added due to a qwriter(PERIM_INNER), free the messages. 7108 * 7109 * Assumes that the stream is strlock()'ed. We don't come here if there 7110 * are no messages to propagate. 7111 * 7112 * NOTE : If the queue is attached to a driver, all the messages are freed 7113 * as there is no point in propagating the messages from the driver syncq 7114 * to the closing stream head which will in turn get freed later. 7115 */ 7116 static int 7117 propagate_syncq(queue_t *qp) 7118 { 7119 mblk_t *bp, *head, *tail, *prev, *next; 7120 syncq_t *sq; 7121 queue_t *nqp; 7122 syncq_t *nsq; 7123 boolean_t isdriver; 7124 int moved = 0; 7125 uint16_t flags; 7126 pri_t priority = curthread->t_pri; 7127 #ifdef DEBUG 7128 void (*func)(); 7129 #endif 7130 7131 sq = qp->q_syncq; 7132 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7133 /* debug macro */ 7134 SQ_PUTLOCKS_HELD(sq); 7135 /* 7136 * As entersq() does not increment the sq_count for 7137 * the write side, check sq_count for non-QPERQ 7138 * perimeters alone. 7139 */ 7140 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); 7141 7142 /* 7143 * propagate_syncq() can be called because of either messages on the 7144 * queue syncq or because on events on the queue syncq. Do actual 7145 * message propagations if there are any messages. 7146 */ 7147 if (qp->q_syncqmsgs) { 7148 isdriver = (qp->q_flag & QISDRV); 7149 7150 if (!isdriver) { 7151 nqp = qp->q_next; 7152 nsq = nqp->q_syncq; 7153 ASSERT(MUTEX_HELD(SQLOCK(nsq))); 7154 /* debug macro */ 7155 SQ_PUTLOCKS_HELD(nsq); 7156 #ifdef DEBUG 7157 func = (void (*)())(uintptr_t)nqp->q_qinfo->qi_putp; 7158 #endif 7159 } 7160 7161 SQRM_Q(sq, qp); 7162 priority = MAX(qp->q_spri, priority); 7163 qp->q_spri = 0; 7164 head = qp->q_sqhead; 7165 tail = qp->q_sqtail; 7166 qp->q_sqhead = qp->q_sqtail = NULL; 7167 qp->q_syncqmsgs = 0; 7168 7169 /* 7170 * Walk the list of messages, and free them if this is a driver, 7171 * otherwise reset the b_prev and b_queue value to the new putp. 7172 * Afterward, we will just add the head to the end of the next 7173 * syncq, and point the tail to the end of this one. 7174 */ 7175 7176 for (bp = head; bp != NULL; bp = next) { 7177 next = bp->b_next; 7178 if (isdriver) { 7179 bp->b_prev = bp->b_next = NULL; 7180 freemsg(bp); 7181 continue; 7182 } 7183 /* Change the q values for this message */ 7184 bp->b_queue = nqp; 7185 #ifdef DEBUG 7186 bp->b_prev = (mblk_t *)func; 7187 #endif 7188 moved++; 7189 } 7190 /* 7191 * Attach list of messages to the end of the new queue (if there 7192 * is a list of messages). 7193 */ 7194 7195 if (!isdriver && head != NULL) { 7196 ASSERT(tail != NULL); 7197 if (nqp->q_sqhead == NULL) { 7198 nqp->q_sqhead = head; 7199 } else { 7200 ASSERT(nqp->q_sqtail != NULL); 7201 nqp->q_sqtail->b_next = head; 7202 } 7203 nqp->q_sqtail = tail; 7204 /* 7205 * When messages are moved from high priority queue to 7206 * another queue, the destination queue priority is 7207 * upgraded. 7208 */ 7209 7210 if (priority > nqp->q_spri) 7211 nqp->q_spri = priority; 7212 7213 SQPUT_Q(nsq, nqp); 7214 7215 nqp->q_syncqmsgs += moved; 7216 ASSERT(nqp->q_syncqmsgs != 0); 7217 } 7218 } 7219 7220 /* 7221 * Before we leave, we need to make sure there are no 7222 * events listed for this queue. All events for this queue 7223 * will just be freed. 7224 */ 7225 if (sq->sq_evhead != NULL) { 7226 ASSERT(sq->sq_flags & SQ_EVENTS); 7227 prev = NULL; 7228 for (bp = sq->sq_evhead; bp != NULL; bp = next) { 7229 next = bp->b_next; 7230 if (bp->b_queue == qp) { 7231 /* Delete this message */ 7232 if (prev != NULL) { 7233 prev->b_next = next; 7234 /* 7235 * Update sq_evtail if the last element 7236 * is removed. 7237 */ 7238 if (bp == sq->sq_evtail) { 7239 ASSERT(next == NULL); 7240 sq->sq_evtail = prev; 7241 } 7242 } else 7243 sq->sq_evhead = next; 7244 if (sq->sq_evhead == NULL) 7245 sq->sq_flags &= ~SQ_EVENTS; 7246 bp->b_prev = bp->b_next = NULL; 7247 freemsg(bp); 7248 } else { 7249 prev = bp; 7250 } 7251 } 7252 } 7253 7254 flags = sq->sq_flags; 7255 7256 /* Wake up any waiter before leaving. */ 7257 if (flags & SQ_WANTWAKEUP) { 7258 flags &= ~SQ_WANTWAKEUP; 7259 cv_broadcast(&sq->sq_wait); 7260 } 7261 sq->sq_flags = flags; 7262 7263 return (moved); 7264 } 7265 7266 /* 7267 * Try and upgrade to exclusive access at the inner perimeter. If this can 7268 * not be done without blocking then request will be queued on the syncq 7269 * and drain_syncq will run it later. 7270 * 7271 * This routine can only be called from put or service procedures plus 7272 * asynchronous callback routines that have properly entered the queue (with 7273 * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq 7274 * associated with q. 7275 */ 7276 void 7277 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)()) 7278 { 7279 syncq_t *sq = q->q_syncq; 7280 uint16_t count; 7281 7282 mutex_enter(SQLOCK(sq)); 7283 count = sq->sq_count; 7284 SQ_PUTLOCKS_ENTER(sq); 7285 SUM_SQ_PUTCOUNTS(sq, count); 7286 ASSERT(count >= 1); 7287 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); 7288 7289 if (count == 1) { 7290 /* 7291 * Can upgrade. This case also handles nested qwriter calls 7292 * (when the qwriter callback function calls qwriter). In that 7293 * case SQ_EXCL is already set. 7294 */ 7295 sq->sq_flags |= SQ_EXCL; 7296 SQ_PUTLOCKS_EXIT(sq); 7297 mutex_exit(SQLOCK(sq)); 7298 (*func)(q, mp); 7299 /* 7300 * Assumes that leavesq, putnext, and drain_syncq will reset 7301 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on 7302 * until putnext, leavesq, or drain_syncq drops it. 7303 * That way we handle nested qwriter(INNER) without dropping 7304 * SQ_EXCL until the outermost qwriter callback routine is 7305 * done. 7306 */ 7307 return; 7308 } 7309 SQ_PUTLOCKS_EXIT(sq); 7310 sqfill_events(sq, q, mp, func); 7311 } 7312 7313 /* 7314 * Synchronous callback support functions 7315 */ 7316 7317 /* 7318 * Allocate a callback parameter structure. 7319 * Assumes that caller initializes the flags and the id. 7320 * Acquires SQLOCK(sq) if non-NULL is returned. 7321 */ 7322 callbparams_t * 7323 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) 7324 { 7325 callbparams_t *cbp; 7326 size_t size = sizeof (callbparams_t); 7327 7328 cbp = kmem_alloc(size, kmflags & ~KM_PANIC); 7329 7330 /* 7331 * Only try tryhard allocation if the caller is ready to panic. 7332 * Otherwise just fail. 7333 */ 7334 if (cbp == NULL) { 7335 if (kmflags & KM_PANIC) 7336 cbp = kmem_alloc_tryhard(sizeof (callbparams_t), 7337 &size, kmflags); 7338 else 7339 return (NULL); 7340 } 7341 7342 ASSERT(size >= sizeof (callbparams_t)); 7343 cbp->cbp_size = size; 7344 cbp->cbp_sq = sq; 7345 cbp->cbp_func = func; 7346 cbp->cbp_arg = arg; 7347 mutex_enter(SQLOCK(sq)); 7348 cbp->cbp_next = sq->sq_callbpend; 7349 sq->sq_callbpend = cbp; 7350 return (cbp); 7351 } 7352 7353 void 7354 callbparams_free(syncq_t *sq, callbparams_t *cbp) 7355 { 7356 callbparams_t **pp, *p; 7357 7358 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7359 7360 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7361 if (p == cbp) { 7362 *pp = p->cbp_next; 7363 kmem_free(p, p->cbp_size); 7364 return; 7365 } 7366 } 7367 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7368 "callbparams_free: not found\n")); 7369 } 7370 7371 void 7372 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) 7373 { 7374 callbparams_t **pp, *p; 7375 7376 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7377 7378 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7379 if (p->cbp_id == id && p->cbp_flags == flag) { 7380 *pp = p->cbp_next; 7381 kmem_free(p, p->cbp_size); 7382 return; 7383 } 7384 } 7385 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7386 "callbparams_free_id: not found\n")); 7387 } 7388 7389 /* 7390 * Callback wrapper function used by once-only callbacks that can be 7391 * cancelled (qtimeout and qbufcall) 7392 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be 7393 * cancelled by the qun* functions. 7394 */ 7395 void 7396 qcallbwrapper(void *arg) 7397 { 7398 callbparams_t *cbp = arg; 7399 syncq_t *sq; 7400 uint16_t count = 0; 7401 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 7402 uint16_t type; 7403 7404 sq = cbp->cbp_sq; 7405 mutex_enter(SQLOCK(sq)); 7406 type = sq->sq_type; 7407 if (!(type & SQ_CICB)) { 7408 count = sq->sq_count; 7409 SQ_PUTLOCKS_ENTER(sq); 7410 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 7411 SUM_SQ_PUTCOUNTS(sq, count); 7412 sq->sq_needexcl++; 7413 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 7414 waitflags |= SQ_MESSAGES; 7415 } 7416 /* Can not handle exclusive entry at outer perimeter */ 7417 ASSERT(type & SQ_COCB); 7418 7419 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { 7420 if ((sq->sq_callbflags & cbp->cbp_flags) && 7421 (sq->sq_cancelid == cbp->cbp_id)) { 7422 /* timeout has been cancelled */ 7423 sq->sq_callbflags |= SQ_CALLB_BYPASSED; 7424 callbparams_free(sq, cbp); 7425 if (!(type & SQ_CICB)) { 7426 ASSERT(sq->sq_needexcl > 0); 7427 sq->sq_needexcl--; 7428 if (sq->sq_needexcl == 0) { 7429 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7430 } 7431 SQ_PUTLOCKS_EXIT(sq); 7432 } 7433 mutex_exit(SQLOCK(sq)); 7434 return; 7435 } 7436 sq->sq_flags |= SQ_WANTWAKEUP; 7437 if (!(type & SQ_CICB)) { 7438 SQ_PUTLOCKS_EXIT(sq); 7439 } 7440 cv_wait(&sq->sq_wait, SQLOCK(sq)); 7441 if (!(type & SQ_CICB)) { 7442 count = sq->sq_count; 7443 SQ_PUTLOCKS_ENTER(sq); 7444 SUM_SQ_PUTCOUNTS(sq, count); 7445 } 7446 } 7447 7448 sq->sq_count++; 7449 ASSERT(sq->sq_count != 0); /* Wraparound */ 7450 if (!(type & SQ_CICB)) { 7451 ASSERT(count == 0); 7452 sq->sq_flags |= SQ_EXCL; 7453 ASSERT(sq->sq_needexcl > 0); 7454 sq->sq_needexcl--; 7455 if (sq->sq_needexcl == 0) { 7456 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7457 } 7458 SQ_PUTLOCKS_EXIT(sq); 7459 } 7460 7461 mutex_exit(SQLOCK(sq)); 7462 7463 cbp->cbp_func(cbp->cbp_arg); 7464 7465 /* 7466 * We drop the lock only for leavesq to re-acquire it. 7467 * Possible optimization is inline of leavesq. 7468 */ 7469 mutex_enter(SQLOCK(sq)); 7470 callbparams_free(sq, cbp); 7471 mutex_exit(SQLOCK(sq)); 7472 leavesq(sq, SQ_CALLBACK); 7473 } 7474 7475 /* 7476 * No need to grab sq_putlocks here. See comment in strsubr.h that 7477 * explains when sq_putlocks are used. 7478 * 7479 * sq_count (or one of the sq_putcounts) has already been 7480 * decremented by the caller, and if SQ_QUEUED, we need to call 7481 * drain_syncq (the global syncq drain). 7482 * If putnext_tail is called with the SQ_EXCL bit set, we are in 7483 * one of two states, non-CIPUT perimeter, and we need to clear 7484 * it, or we went exclusive in the put procedure. In any case, 7485 * we want to clear the bit now, and it is probably easier to do 7486 * this at the beginning of this function (remember, we hold 7487 * the SQLOCK). Lastly, if there are other messages queued 7488 * on the syncq (and not for our destination), enable the syncq 7489 * for background work. 7490 */ 7491 7492 /* ARGSUSED */ 7493 void 7494 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) 7495 { 7496 uint16_t flags = sq->sq_flags; 7497 7498 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7499 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 7500 7501 /* Clear SQ_EXCL if set in passflags */ 7502 if (passflags & SQ_EXCL) { 7503 flags &= ~SQ_EXCL; 7504 } 7505 if (flags & SQ_WANTWAKEUP) { 7506 flags &= ~SQ_WANTWAKEUP; 7507 cv_broadcast(&sq->sq_wait); 7508 } 7509 if (flags & SQ_WANTEXWAKEUP) { 7510 flags &= ~SQ_WANTEXWAKEUP; 7511 cv_broadcast(&sq->sq_exitwait); 7512 } 7513 sq->sq_flags = flags; 7514 7515 /* 7516 * We have cleared SQ_EXCL if we were asked to, and started 7517 * the wakeup process for waiters. If there are no writers 7518 * then we need to drain the syncq if we were told to, or 7519 * enable the background thread to do it. 7520 */ 7521 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) { 7522 if ((passflags & SQ_QUEUED) || 7523 (sq->sq_svcflags & SQ_DISABLED)) { 7524 /* drain_syncq will take care of events in the list */ 7525 drain_syncq(sq); 7526 return; 7527 } else if (flags & SQ_QUEUED) { 7528 sqenable(sq); 7529 } 7530 } 7531 /* Drop the SQLOCK on exit */ 7532 mutex_exit(SQLOCK(sq)); 7533 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END, 7534 "putnext_end:(%p, %p, %p) done", NULL, qp, sq); 7535 } 7536 7537 void 7538 set_qend(queue_t *q) 7539 { 7540 mutex_enter(QLOCK(q)); 7541 if (!O_SAMESTR(q)) 7542 q->q_flag |= QEND; 7543 else 7544 q->q_flag &= ~QEND; 7545 mutex_exit(QLOCK(q)); 7546 q = _OTHERQ(q); 7547 mutex_enter(QLOCK(q)); 7548 if (!O_SAMESTR(q)) 7549 q->q_flag |= QEND; 7550 else 7551 q->q_flag &= ~QEND; 7552 mutex_exit(QLOCK(q)); 7553 } 7554 7555 /* 7556 * Set QFULL in next service procedure queue (that cares) if not already 7557 * set and if there are already more messages on the syncq than 7558 * sq_max_size. If sq_max_size is 0, no flow control will be asserted on 7559 * any syncq. 7560 * 7561 * The fq here is the next queue with a service procedure. This is where 7562 * we would fail canputnext, so this is where we need to set QFULL. 7563 * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag. 7564 * 7565 * We already have QLOCK at this point. To avoid cross-locks with 7566 * freezestr() which grabs all QLOCKs and with strlock() which grabs both 7567 * SQLOCK and sd_reflock, we need to drop respective locks first. 7568 */ 7569 void 7570 set_qfull(queue_t *q) 7571 { 7572 queue_t *fq = NULL; 7573 7574 ASSERT(MUTEX_HELD(QLOCK(q))); 7575 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) && 7576 (q->q_syncqmsgs > sq_max_size)) { 7577 if ((fq = q->q_nfsrv) == q) { 7578 fq->q_flag |= QFULL; 7579 } else { 7580 mutex_exit(QLOCK(q)); 7581 mutex_enter(QLOCK(fq)); 7582 fq->q_flag |= QFULL; 7583 mutex_exit(QLOCK(fq)); 7584 mutex_enter(QLOCK(q)); 7585 } 7586 } 7587 } 7588 7589 void 7590 clr_qfull(queue_t *q) 7591 { 7592 queue_t *oq = q; 7593 7594 q = q->q_nfsrv; 7595 /* Fast check if there is any work to do before getting the lock. */ 7596 if ((q->q_flag & (QFULL|QWANTW)) == 0) { 7597 return; 7598 } 7599 7600 /* 7601 * Do not reset QFULL (and backenable) if the q_count is the reason 7602 * for QFULL being set. 7603 */ 7604 mutex_enter(QLOCK(q)); 7605 /* 7606 * If queue is empty i.e q_mblkcnt is zero, queue can not be full. 7607 * Hence clear the QFULL. 7608 * If both q_count and q_mblkcnt are less than the hiwat mark, 7609 * clear the QFULL. 7610 */ 7611 if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) && 7612 (q->q_mblkcnt < q->q_hiwat))) { 7613 q->q_flag &= ~QFULL; 7614 /* 7615 * A little more confusing, how about this way: 7616 * if someone wants to write, 7617 * AND 7618 * both counts are less than the lowat mark 7619 * OR 7620 * the lowat mark is zero 7621 * THEN 7622 * backenable 7623 */ 7624 if ((q->q_flag & QWANTW) && 7625 (((q->q_count < q->q_lowat) && 7626 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) { 7627 q->q_flag &= ~QWANTW; 7628 mutex_exit(QLOCK(q)); 7629 backenable(oq, 0); 7630 } else 7631 mutex_exit(QLOCK(q)); 7632 } else 7633 mutex_exit(QLOCK(q)); 7634 } 7635 7636 /* 7637 * Set the forward service procedure pointer. 7638 * 7639 * Called at insert-time to cache a queue's next forward service procedure in 7640 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted 7641 * has a service procedure then q_nfsrv points to itself. If the queue to be 7642 * inserted does not have a service procedure, then q_nfsrv points to the next 7643 * queue forward that has a service procedure. If the queue is at the logical 7644 * end of the stream (driver for write side, stream head for the read side) 7645 * and does not have a service procedure, then q_nfsrv also points to itself. 7646 */ 7647 void 7648 set_nfsrv_ptr( 7649 queue_t *rnew, /* read queue pointer to new module */ 7650 queue_t *wnew, /* write queue pointer to new module */ 7651 queue_t *prev_rq, /* read queue pointer to the module above */ 7652 queue_t *prev_wq) /* write queue pointer to the module above */ 7653 { 7654 queue_t *qp; 7655 7656 if (prev_wq->q_next == NULL) { 7657 /* 7658 * Insert the driver, initialize the driver and stream head. 7659 * In this case, prev_rq/prev_wq should be the stream head. 7660 * _I_INSERT does not allow inserting a driver. Make sure 7661 * that it is not an insertion. 7662 */ 7663 ASSERT(!(rnew->q_flag & _QINSERTING)); 7664 wnew->q_nfsrv = wnew; 7665 if (rnew->q_qinfo->qi_srvp) 7666 rnew->q_nfsrv = rnew; 7667 else 7668 rnew->q_nfsrv = prev_rq; 7669 prev_rq->q_nfsrv = prev_rq; 7670 prev_wq->q_nfsrv = prev_wq; 7671 } else { 7672 /* 7673 * set up read side q_nfsrv pointer. This MUST be done 7674 * before setting the write side, because the setting of 7675 * the write side for a fifo may depend on it. 7676 * 7677 * Suppose we have a fifo that only has pipemod pushed. 7678 * pipemod has no read or write service procedures, so 7679 * nfsrv for both pipemod queues points to prev_rq (the 7680 * stream read head). Now push bufmod (which has only a 7681 * read service procedure). Doing the write side first, 7682 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which 7683 * is WRONG; the next queue forward from wnew with a 7684 * service procedure will be rnew, not the stream read head. 7685 * Since the downstream queue (which in the case of a fifo 7686 * is the read queue rnew) can affect upstream queues, it 7687 * needs to be done first. Setting up the read side first 7688 * sets nfsrv for both pipemod queues to rnew and then 7689 * when the write side is set up, wnew-q_nfsrv will also 7690 * point to rnew. 7691 */ 7692 if (rnew->q_qinfo->qi_srvp) { 7693 /* 7694 * use _OTHERQ() because, if this is a pipe, next 7695 * module may have been pushed from other end and 7696 * q_next could be a read queue. 7697 */ 7698 qp = _OTHERQ(prev_wq->q_next); 7699 while (qp && qp->q_nfsrv != qp) { 7700 qp->q_nfsrv = rnew; 7701 qp = backq(qp); 7702 } 7703 rnew->q_nfsrv = rnew; 7704 } else 7705 rnew->q_nfsrv = prev_rq->q_nfsrv; 7706 7707 /* set up write side q_nfsrv pointer */ 7708 if (wnew->q_qinfo->qi_srvp) { 7709 wnew->q_nfsrv = wnew; 7710 7711 /* 7712 * For insertion, need to update nfsrv of the modules 7713 * above which do not have a service routine. 7714 */ 7715 if (rnew->q_flag & _QINSERTING) { 7716 for (qp = prev_wq; 7717 qp != NULL && qp->q_nfsrv != qp; 7718 qp = backq(qp)) { 7719 qp->q_nfsrv = wnew->q_nfsrv; 7720 } 7721 } 7722 } else { 7723 if (prev_wq->q_next == prev_rq) 7724 /* 7725 * Since prev_wq/prev_rq are the middle of a 7726 * fifo, wnew/rnew will also be the middle of 7727 * a fifo and wnew's nfsrv is same as rnew's. 7728 */ 7729 wnew->q_nfsrv = rnew->q_nfsrv; 7730 else 7731 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv; 7732 } 7733 } 7734 } 7735 7736 /* 7737 * Reset the forward service procedure pointer; called at remove-time. 7738 */ 7739 void 7740 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp) 7741 { 7742 queue_t *tmp_qp; 7743 7744 /* Reset the write side q_nfsrv pointer for _I_REMOVE */ 7745 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) { 7746 for (tmp_qp = backq(wqp); 7747 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp; 7748 tmp_qp = backq(tmp_qp)) { 7749 tmp_qp->q_nfsrv = wqp->q_nfsrv; 7750 } 7751 } 7752 7753 /* reset the read side q_nfsrv pointer */ 7754 if (rqp->q_qinfo->qi_srvp) { 7755 if (wqp->q_next) { /* non-driver case */ 7756 tmp_qp = _OTHERQ(wqp->q_next); 7757 while (tmp_qp && tmp_qp->q_nfsrv == rqp) { 7758 /* Note that rqp->q_next cannot be NULL */ 7759 ASSERT(rqp->q_next != NULL); 7760 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv; 7761 tmp_qp = backq(tmp_qp); 7762 } 7763 } 7764 } 7765 } 7766 7767 /* 7768 * This routine should be called after all stream geometry changes to update 7769 * the stream head cached struio() rd/wr queue pointers. Note must be called 7770 * with the streamlock()ed. 7771 * 7772 * Note: only enables Synchronous STREAMS for a side of a Stream which has 7773 * an explicit synchronous barrier module queue. That is, a queue that 7774 * has specified a struio() type. 7775 */ 7776 static void 7777 strsetuio(stdata_t *stp) 7778 { 7779 queue_t *wrq; 7780 7781 if (stp->sd_flag & STPLEX) { 7782 /* 7783 * Not streamhead, but a mux, so no Synchronous STREAMS. 7784 */ 7785 stp->sd_struiowrq = NULL; 7786 stp->sd_struiordq = NULL; 7787 return; 7788 } 7789 /* 7790 * Scan the write queue(s) while synchronous 7791 * until we find a qinfo uio type specified. 7792 */ 7793 wrq = stp->sd_wrq->q_next; 7794 while (wrq) { 7795 if (wrq->q_struiot == STRUIOT_NONE) { 7796 wrq = 0; 7797 break; 7798 } 7799 if (wrq->q_struiot != STRUIOT_DONTCARE) 7800 break; 7801 if (! _SAMESTR(wrq)) { 7802 wrq = 0; 7803 break; 7804 } 7805 wrq = wrq->q_next; 7806 } 7807 stp->sd_struiowrq = wrq; 7808 /* 7809 * Scan the read queue(s) while synchronous 7810 * until we find a qinfo uio type specified. 7811 */ 7812 wrq = stp->sd_wrq->q_next; 7813 while (wrq) { 7814 if (_RD(wrq)->q_struiot == STRUIOT_NONE) { 7815 wrq = 0; 7816 break; 7817 } 7818 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE) 7819 break; 7820 if (! _SAMESTR(wrq)) { 7821 wrq = 0; 7822 break; 7823 } 7824 wrq = wrq->q_next; 7825 } 7826 stp->sd_struiordq = wrq ? _RD(wrq) : 0; 7827 } 7828 7829 static int 7830 pass_rput(queue_t *q, mblk_t *mp) 7831 { 7832 putnext(q, mp); 7833 return (0); 7834 } 7835 7836 /* 7837 * pass_wput, unblocks the passthru queues, so that 7838 * messages can arrive at muxs lower read queue, before 7839 * I_LINK/I_UNLINK is acked/nacked. 7840 */ 7841 static int 7842 pass_wput(queue_t *q, mblk_t *mp) 7843 { 7844 syncq_t *sq; 7845 7846 sq = _RD(q)->q_syncq; 7847 if (sq->sq_flags & SQ_BLOCKED) 7848 unblocksq(sq, SQ_BLOCKED, 0); 7849 putnext(q, mp); 7850 return (0); 7851 } 7852 7853 /* 7854 * Set up queues for the link/unlink. 7855 * Create a new queue and block it and then insert it 7856 * below the stream head on the lower stream. 7857 * This prevents any messages from arriving during the setq 7858 * as well as while the mux is processing the LINK/I_UNLINK. 7859 * The blocked passq is unblocked once the LINK/I_UNLINK has 7860 * been acked or nacked or if a message is generated and sent 7861 * down muxs write put procedure. 7862 * See pass_wput(). 7863 * 7864 * After the new queue is inserted, all messages coming from below are 7865 * blocked. The call to strlock will ensure that all activity in the stream head 7866 * read queue syncq is stopped (sq_count drops to zero). 7867 */ 7868 static queue_t * 7869 link_addpassthru(stdata_t *stpdown) 7870 { 7871 queue_t *passq; 7872 sqlist_t sqlist; 7873 7874 passq = allocq(); 7875 STREAM(passq) = STREAM(_WR(passq)) = stpdown; 7876 /* setq might sleep in allocator - avoid holding locks. */ 7877 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ, 7878 SQ_CI|SQ_CO, B_FALSE); 7879 claimq(passq); 7880 blocksq(passq->q_syncq, SQ_BLOCKED, 1); 7881 insertq(STREAM(passq), passq); 7882 7883 /* 7884 * Use strlock() to wait for the stream head sq_count to drop to zero 7885 * since we are going to change q_ptr in the stream head. Note that 7886 * insertq() doesn't wait for any syncq counts to drop to zero. 7887 */ 7888 sqlist.sqlist_head = NULL; 7889 sqlist.sqlist_index = 0; 7890 sqlist.sqlist_size = sizeof (sqlist_t); 7891 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq); 7892 strlock(stpdown, &sqlist); 7893 strunlock(stpdown, &sqlist); 7894 7895 releaseq(passq); 7896 return (passq); 7897 } 7898 7899 /* 7900 * Let messages flow up into the mux by removing 7901 * the passq. 7902 */ 7903 static void 7904 link_rempassthru(queue_t *passq) 7905 { 7906 claimq(passq); 7907 removeq(passq); 7908 releaseq(passq); 7909 freeq(passq); 7910 } 7911 7912 /* 7913 * Wait for the condition variable pointed to by `cvp' to be signaled, 7914 * or for `tim' milliseconds to elapse, whichever comes first. If `tim' 7915 * is negative, then there is no time limit. If `nosigs' is non-zero, 7916 * then the wait will be non-interruptible. 7917 * 7918 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout. 7919 */ 7920 clock_t 7921 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs) 7922 { 7923 clock_t ret; 7924 7925 if (tim < 0) { 7926 if (nosigs) { 7927 cv_wait(cvp, mp); 7928 ret = 1; 7929 } else { 7930 ret = cv_wait_sig(cvp, mp); 7931 } 7932 } else if (tim > 0) { 7933 /* 7934 * convert milliseconds to clock ticks 7935 */ 7936 if (nosigs) { 7937 ret = cv_reltimedwait(cvp, mp, 7938 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK); 7939 } else { 7940 ret = cv_reltimedwait_sig(cvp, mp, 7941 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK); 7942 } 7943 } else { 7944 ret = -1; 7945 } 7946 return (ret); 7947 } 7948 7949 /* 7950 * Wait until the stream head can determine if it is at the mark but 7951 * don't wait forever to prevent a race condition between the "mark" state 7952 * in the stream head and any mark state in the caller/user of this routine. 7953 * 7954 * This is used by sockets and for a socket it would be incorrect 7955 * to return a failure for SIOCATMARK when there is no data in the receive 7956 * queue and the marked urgent data is traveling up the stream. 7957 * 7958 * This routine waits until the mark is known by waiting for one of these 7959 * three events: 7960 * The stream head read queue becoming non-empty (including an EOF). 7961 * The STRATMARK flag being set (due to a MSGMARKNEXT message). 7962 * The STRNOTATMARK flag being set (which indicates that the transport 7963 * has sent a MSGNOTMARKNEXT message to indicate that it is not at 7964 * the mark). 7965 * 7966 * The routine returns 1 if the stream is at the mark; 0 if it can 7967 * be determined that the stream is not at the mark. 7968 * If the wait times out and it can't determine 7969 * whether or not the stream might be at the mark the routine will return -1. 7970 * 7971 * Note: This routine should only be used when a mark is pending i.e., 7972 * in the socket case the SIGURG has been posted. 7973 * Note2: This can not wakeup just because synchronous streams indicate 7974 * that data is available since it is not possible to use the synchronous 7975 * streams interfaces to determine the b_flag value for the data queued below 7976 * the stream head. 7977 */ 7978 int 7979 strwaitmark(vnode_t *vp) 7980 { 7981 struct stdata *stp = vp->v_stream; 7982 queue_t *rq = _RD(stp->sd_wrq); 7983 int mark; 7984 7985 mutex_enter(&stp->sd_lock); 7986 while (rq->q_first == NULL && 7987 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) { 7988 stp->sd_flag |= RSLEEP; 7989 7990 /* Wait for 100 milliseconds for any state change. */ 7991 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) { 7992 mutex_exit(&stp->sd_lock); 7993 return (-1); 7994 } 7995 } 7996 if (stp->sd_flag & STRATMARK) 7997 mark = 1; 7998 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK)) 7999 mark = 1; 8000 else 8001 mark = 0; 8002 8003 mutex_exit(&stp->sd_lock); 8004 return (mark); 8005 } 8006 8007 /* 8008 * Set a read side error. If persist is set change the socket error 8009 * to persistent. If errfunc is set install the function as the exported 8010 * error handler. 8011 */ 8012 void 8013 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 8014 { 8015 struct stdata *stp = vp->v_stream; 8016 8017 mutex_enter(&stp->sd_lock); 8018 stp->sd_rerror = error; 8019 if (error == 0 && errfunc == NULL) 8020 stp->sd_flag &= ~STRDERR; 8021 else 8022 stp->sd_flag |= STRDERR; 8023 if (persist) { 8024 stp->sd_flag &= ~STRDERRNONPERSIST; 8025 } else { 8026 stp->sd_flag |= STRDERRNONPERSIST; 8027 } 8028 stp->sd_rderrfunc = errfunc; 8029 if (error != 0 || errfunc != NULL) { 8030 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 8031 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 8032 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 8033 8034 mutex_exit(&stp->sd_lock); 8035 pollwakeup(&stp->sd_pollist, POLLERR); 8036 mutex_enter(&stp->sd_lock); 8037 8038 if (stp->sd_sigflags & S_ERROR) 8039 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 8040 } 8041 mutex_exit(&stp->sd_lock); 8042 } 8043 8044 /* 8045 * Set a write side error. If persist is set change the socket error 8046 * to persistent. 8047 */ 8048 void 8049 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 8050 { 8051 struct stdata *stp = vp->v_stream; 8052 8053 mutex_enter(&stp->sd_lock); 8054 stp->sd_werror = error; 8055 if (error == 0 && errfunc == NULL) 8056 stp->sd_flag &= ~STWRERR; 8057 else 8058 stp->sd_flag |= STWRERR; 8059 if (persist) { 8060 stp->sd_flag &= ~STWRERRNONPERSIST; 8061 } else { 8062 stp->sd_flag |= STWRERRNONPERSIST; 8063 } 8064 stp->sd_wrerrfunc = errfunc; 8065 if (error != 0 || errfunc != NULL) { 8066 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 8067 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 8068 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 8069 8070 mutex_exit(&stp->sd_lock); 8071 pollwakeup(&stp->sd_pollist, POLLERR); 8072 mutex_enter(&stp->sd_lock); 8073 8074 if (stp->sd_sigflags & S_ERROR) 8075 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 8076 } 8077 mutex_exit(&stp->sd_lock); 8078 } 8079 8080 /* 8081 * Make the stream return 0 (EOF) when all data has been read. 8082 * No effect on write side. 8083 */ 8084 void 8085 strseteof(vnode_t *vp, int eof) 8086 { 8087 struct stdata *stp = vp->v_stream; 8088 8089 mutex_enter(&stp->sd_lock); 8090 if (!eof) { 8091 stp->sd_flag &= ~STREOF; 8092 mutex_exit(&stp->sd_lock); 8093 return; 8094 } 8095 stp->sd_flag |= STREOF; 8096 if (stp->sd_flag & RSLEEP) { 8097 stp->sd_flag &= ~RSLEEP; 8098 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); 8099 } 8100 8101 mutex_exit(&stp->sd_lock); 8102 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM); 8103 mutex_enter(&stp->sd_lock); 8104 8105 if (stp->sd_sigflags & (S_INPUT|S_RDNORM)) 8106 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0); 8107 mutex_exit(&stp->sd_lock); 8108 } 8109 8110 void 8111 strflushrq(vnode_t *vp, int flag) 8112 { 8113 struct stdata *stp = vp->v_stream; 8114 8115 mutex_enter(&stp->sd_lock); 8116 flushq(_RD(stp->sd_wrq), flag); 8117 mutex_exit(&stp->sd_lock); 8118 } 8119 8120 void 8121 strsetrputhooks(vnode_t *vp, uint_t flags, 8122 msgfunc_t protofunc, msgfunc_t miscfunc) 8123 { 8124 struct stdata *stp = vp->v_stream; 8125 8126 mutex_enter(&stp->sd_lock); 8127 8128 if (protofunc == NULL) 8129 stp->sd_rprotofunc = strrput_proto; 8130 else 8131 stp->sd_rprotofunc = protofunc; 8132 8133 if (miscfunc == NULL) 8134 stp->sd_rmiscfunc = strrput_misc; 8135 else 8136 stp->sd_rmiscfunc = miscfunc; 8137 8138 if (flags & SH_CONSOL_DATA) 8139 stp->sd_rput_opt |= SR_CONSOL_DATA; 8140 else 8141 stp->sd_rput_opt &= ~SR_CONSOL_DATA; 8142 8143 if (flags & SH_SIGALLDATA) 8144 stp->sd_rput_opt |= SR_SIGALLDATA; 8145 else 8146 stp->sd_rput_opt &= ~SR_SIGALLDATA; 8147 8148 if (flags & SH_IGN_ZEROLEN) 8149 stp->sd_rput_opt |= SR_IGN_ZEROLEN; 8150 else 8151 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN; 8152 8153 mutex_exit(&stp->sd_lock); 8154 } 8155 8156 void 8157 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime) 8158 { 8159 struct stdata *stp = vp->v_stream; 8160 8161 mutex_enter(&stp->sd_lock); 8162 stp->sd_closetime = closetime; 8163 8164 if (flags & SH_SIGPIPE) 8165 stp->sd_wput_opt |= SW_SIGPIPE; 8166 else 8167 stp->sd_wput_opt &= ~SW_SIGPIPE; 8168 if (flags & SH_RECHECK_ERR) 8169 stp->sd_wput_opt |= SW_RECHECK_ERR; 8170 else 8171 stp->sd_wput_opt &= ~SW_RECHECK_ERR; 8172 8173 mutex_exit(&stp->sd_lock); 8174 } 8175 8176 void 8177 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc) 8178 { 8179 struct stdata *stp = vp->v_stream; 8180 8181 mutex_enter(&stp->sd_lock); 8182 8183 stp->sd_rputdatafunc = rdatafunc; 8184 stp->sd_wputdatafunc = wdatafunc; 8185 8186 mutex_exit(&stp->sd_lock); 8187 } 8188 8189 /* Used within framework when the queue is already locked */ 8190 void 8191 qenable_locked(queue_t *q) 8192 { 8193 stdata_t *stp = STREAM(q); 8194 8195 ASSERT(MUTEX_HELD(QLOCK(q))); 8196 8197 if (!q->q_qinfo->qi_srvp) 8198 return; 8199 8200 /* 8201 * Do not place on run queue if already enabled or closing. 8202 */ 8203 if (q->q_flag & (QWCLOSE|QENAB)) 8204 return; 8205 8206 /* 8207 * mark queue enabled and place on run list if it is not already being 8208 * serviced. If it is serviced, the runservice() function will detect 8209 * that QENAB is set and call service procedure before clearing 8210 * QINSERVICE flag. 8211 */ 8212 q->q_flag |= QENAB; 8213 if (q->q_flag & QINSERVICE) 8214 return; 8215 8216 /* Record the time of qenable */ 8217 q->q_qtstamp = ddi_get_lbolt(); 8218 8219 /* 8220 * Put the queue in the stp list and schedule it for background 8221 * processing if it is not already scheduled or if stream head does not 8222 * intent to process it in the foreground later by setting 8223 * STRS_WILLSERVICE flag. 8224 */ 8225 mutex_enter(&stp->sd_qlock); 8226 /* 8227 * If there are already something on the list, stp flags should show 8228 * intention to drain it. 8229 */ 8230 IMPLY(STREAM_NEEDSERVICE(stp), 8231 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))); 8232 8233 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link); 8234 stp->sd_nqueues++; 8235 8236 /* 8237 * If no one will drain this stream we are the first producer and 8238 * need to schedule it for background thread. 8239 */ 8240 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) { 8241 /* 8242 * No one will service this stream later, so we have to 8243 * schedule it now. 8244 */ 8245 STRSTAT(stenables); 8246 stp->sd_svcflags |= STRS_SCHEDULED; 8247 stp->sd_servid = (void *)taskq_dispatch(streams_taskq, 8248 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE); 8249 8250 if (stp->sd_servid == NULL) { 8251 /* 8252 * Task queue failed so fail over to the backup 8253 * servicing thread. 8254 */ 8255 STRSTAT(taskqfails); 8256 /* 8257 * It is safe to clear STRS_SCHEDULED flag because it 8258 * was set by this thread above. 8259 */ 8260 stp->sd_svcflags &= ~STRS_SCHEDULED; 8261 8262 /* 8263 * Failover scheduling is protected by service_queue 8264 * lock. 8265 */ 8266 mutex_enter(&service_queue); 8267 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q)); 8268 ASSERT(q->q_link == NULL); 8269 /* 8270 * Append the queue to qhead/qtail list. 8271 */ 8272 if (qhead == NULL) 8273 qhead = q; 8274 else 8275 qtail->q_link = q; 8276 qtail = q; 8277 /* 8278 * Clear stp queue list. 8279 */ 8280 stp->sd_qhead = stp->sd_qtail = NULL; 8281 stp->sd_nqueues = 0; 8282 /* 8283 * Wakeup background queue processing thread. 8284 */ 8285 cv_signal(&services_to_run); 8286 mutex_exit(&service_queue); 8287 } 8288 } 8289 mutex_exit(&stp->sd_qlock); 8290 } 8291 8292 static void 8293 queue_service(queue_t *q) 8294 { 8295 /* 8296 * The queue in the list should have 8297 * QENAB flag set and should not have 8298 * QINSERVICE flag set. QINSERVICE is 8299 * set when the queue is dequeued and 8300 * qenable_locked doesn't enqueue a 8301 * queue with QINSERVICE set. 8302 */ 8303 8304 ASSERT(!(q->q_flag & QINSERVICE)); 8305 ASSERT((q->q_flag & QENAB)); 8306 mutex_enter(QLOCK(q)); 8307 q->q_flag &= ~QENAB; 8308 q->q_flag |= QINSERVICE; 8309 mutex_exit(QLOCK(q)); 8310 runservice(q); 8311 } 8312 8313 static void 8314 syncq_service(syncq_t *sq) 8315 { 8316 STRSTAT(syncqservice); 8317 mutex_enter(SQLOCK(sq)); 8318 ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); 8319 ASSERT(sq->sq_servcount != 0); 8320 ASSERT(sq->sq_next == NULL); 8321 8322 /* if we came here from the background thread, clear the flag */ 8323 if (sq->sq_svcflags & SQ_BGTHREAD) 8324 sq->sq_svcflags &= ~SQ_BGTHREAD; 8325 8326 /* let drain_syncq know that it's being called in the background */ 8327 sq->sq_svcflags |= SQ_SERVICE; 8328 drain_syncq(sq); 8329 } 8330 8331 static void 8332 qwriter_outer_service(syncq_t *outer) 8333 { 8334 /* 8335 * Note that SQ_WRITER is used on the outer perimeter 8336 * to signal that a qwriter(OUTER) is either investigating 8337 * running or that it is actually running a function. 8338 */ 8339 outer_enter(outer, SQ_BLOCKED|SQ_WRITER); 8340 8341 /* 8342 * All inner syncq are empty and have SQ_WRITER set 8343 * to block entering the outer perimeter. 8344 * 8345 * We do not need to explicitly call write_now since 8346 * outer_exit does it for us. 8347 */ 8348 outer_exit(outer); 8349 } 8350 8351 static void 8352 mblk_free(mblk_t *mp) 8353 { 8354 dblk_t *dbp = mp->b_datap; 8355 frtn_t *frp = dbp->db_frtnp; 8356 8357 mp->b_next = NULL; 8358 if (dbp->db_fthdr != NULL) 8359 str_ftfree(dbp); 8360 8361 ASSERT(dbp->db_fthdr == NULL); 8362 frp->free_func(frp->free_arg); 8363 ASSERT(dbp->db_mblk == mp); 8364 8365 if (dbp->db_credp != NULL) { 8366 crfree(dbp->db_credp); 8367 dbp->db_credp = NULL; 8368 } 8369 dbp->db_cpid = -1; 8370 dbp->db_struioflag = 0; 8371 dbp->db_struioun.cksum.flags = 0; 8372 8373 kmem_cache_free(dbp->db_cache, dbp); 8374 } 8375 8376 /* 8377 * Background processing of the stream queue list. 8378 */ 8379 static void 8380 stream_service(stdata_t *stp) 8381 { 8382 queue_t *q; 8383 8384 mutex_enter(&stp->sd_qlock); 8385 8386 STR_SERVICE(stp, q); 8387 8388 stp->sd_svcflags &= ~STRS_SCHEDULED; 8389 stp->sd_servid = NULL; 8390 cv_signal(&stp->sd_qcv); 8391 mutex_exit(&stp->sd_qlock); 8392 } 8393 8394 /* 8395 * Foreground processing of the stream queue list. 8396 */ 8397 void 8398 stream_runservice(stdata_t *stp) 8399 { 8400 queue_t *q; 8401 8402 mutex_enter(&stp->sd_qlock); 8403 STRSTAT(rservice); 8404 /* 8405 * We are going to drain this stream queue list, so qenable_locked will 8406 * not schedule it until we finish. 8407 */ 8408 stp->sd_svcflags |= STRS_WILLSERVICE; 8409 8410 STR_SERVICE(stp, q); 8411 8412 stp->sd_svcflags &= ~STRS_WILLSERVICE; 8413 mutex_exit(&stp->sd_qlock); 8414 /* 8415 * Help backup background thread to drain the qhead/qtail list. 8416 */ 8417 while (qhead != NULL) { 8418 STRSTAT(qhelps); 8419 mutex_enter(&service_queue); 8420 DQ(q, qhead, qtail, q_link); 8421 mutex_exit(&service_queue); 8422 if (q != NULL) 8423 queue_service(q); 8424 } 8425 } 8426 8427 void 8428 stream_willservice(stdata_t *stp) 8429 { 8430 mutex_enter(&stp->sd_qlock); 8431 stp->sd_svcflags |= STRS_WILLSERVICE; 8432 mutex_exit(&stp->sd_qlock); 8433 } 8434 8435 /* 8436 * Replace the cred currently in the mblk with a different one. 8437 * Also update db_cpid. 8438 */ 8439 void 8440 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid) 8441 { 8442 dblk_t *dbp = mp->b_datap; 8443 cred_t *ocr = dbp->db_credp; 8444 8445 ASSERT(cr != NULL); 8446 8447 if (cr != ocr) { 8448 crhold(dbp->db_credp = cr); 8449 if (ocr != NULL) 8450 crfree(ocr); 8451 } 8452 /* Don't overwrite with NOPID */ 8453 if (cpid != NOPID) 8454 dbp->db_cpid = cpid; 8455 } 8456 8457 /* 8458 * If the src message has a cred, then replace the cred currently in the mblk 8459 * with it. 8460 * Also update db_cpid. 8461 */ 8462 void 8463 mblk_copycred(mblk_t *mp, const mblk_t *src) 8464 { 8465 dblk_t *dbp = mp->b_datap; 8466 cred_t *cr, *ocr; 8467 pid_t cpid; 8468 8469 cr = msg_getcred(src, &cpid); 8470 if (cr == NULL) 8471 return; 8472 8473 ocr = dbp->db_credp; 8474 if (cr != ocr) { 8475 crhold(dbp->db_credp = cr); 8476 if (ocr != NULL) 8477 crfree(ocr); 8478 } 8479 /* Don't overwrite with NOPID */ 8480 if (cpid != NOPID) 8481 dbp->db_cpid = cpid; 8482 } 8483 8484 8485 /* 8486 * Now that NIC drivers are expected to deal only with M_DATA mblks, the 8487 * hcksum_assoc and hcksum_retrieve functions are deprecated in favor of their 8488 * respective mac_hcksum_set and mac_hcksum_get counterparts. 8489 */ 8490 int 8491 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8492 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value, 8493 uint32_t flags, int km_flags) 8494 { 8495 int rc = 0; 8496 8497 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8498 if (mp->b_datap->db_type == M_DATA) { 8499 /* Associate values for M_DATA type */ 8500 DB_CKSUMSTART(mp) = (intptr_t)start; 8501 DB_CKSUMSTUFF(mp) = (intptr_t)stuff; 8502 DB_CKSUMEND(mp) = (intptr_t)end; 8503 DB_CKSUMFLAGS(mp) = flags; 8504 DB_CKSUM16(mp) = (uint16_t)value; 8505 8506 } else { 8507 pattrinfo_t pa_info; 8508 8509 ASSERT(mmd != NULL); 8510 8511 pa_info.type = PATTR_HCKSUM; 8512 pa_info.len = sizeof (pattr_hcksum_t); 8513 8514 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) { 8515 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf; 8516 8517 hck->hcksum_start_offset = start; 8518 hck->hcksum_stuff_offset = stuff; 8519 hck->hcksum_end_offset = end; 8520 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value; 8521 hck->hcksum_flags = flags; 8522 } else { 8523 rc = -1; 8524 } 8525 } 8526 return (rc); 8527 } 8528 8529 void 8530 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8531 uint32_t *start, uint32_t *stuff, uint32_t *end, 8532 uint32_t *value, uint32_t *flags) 8533 { 8534 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8535 if (mp->b_datap->db_type == M_DATA) { 8536 if (flags != NULL) { 8537 *flags = DB_CKSUMFLAGS(mp) & HCK_FLAGS; 8538 if ((*flags & (HCK_PARTIALCKSUM | 8539 HCK_FULLCKSUM)) != 0) { 8540 if (value != NULL) 8541 *value = (uint32_t)DB_CKSUM16(mp); 8542 if ((*flags & HCK_PARTIALCKSUM) != 0) { 8543 if (start != NULL) 8544 *start = 8545 (uint32_t)DB_CKSUMSTART(mp); 8546 if (stuff != NULL) 8547 *stuff = 8548 (uint32_t)DB_CKSUMSTUFF(mp); 8549 if (end != NULL) 8550 *end = 8551 (uint32_t)DB_CKSUMEND(mp); 8552 } 8553 } 8554 } 8555 } else { 8556 pattrinfo_t hck_attr = {PATTR_HCKSUM}; 8557 8558 ASSERT(mmd != NULL); 8559 8560 /* get hardware checksum attribute */ 8561 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) { 8562 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf; 8563 8564 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t)); 8565 if (flags != NULL) 8566 *flags = hck->hcksum_flags; 8567 if (start != NULL) 8568 *start = hck->hcksum_start_offset; 8569 if (stuff != NULL) 8570 *stuff = hck->hcksum_stuff_offset; 8571 if (end != NULL) 8572 *end = hck->hcksum_end_offset; 8573 if (value != NULL) 8574 *value = (uint32_t) 8575 hck->hcksum_cksum_val.inet_cksum; 8576 } 8577 } 8578 } 8579 8580 void 8581 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags) 8582 { 8583 ASSERT(DB_TYPE(mp) == M_DATA); 8584 ASSERT((flags & ~HW_LSO_FLAGS) == 0); 8585 8586 /* Set the flags */ 8587 DB_LSOFLAGS(mp) |= flags; 8588 DB_LSOMSS(mp) = mss; 8589 } 8590 8591 void 8592 lso_info_cleanup(mblk_t *mp) 8593 { 8594 ASSERT(DB_TYPE(mp) == M_DATA); 8595 8596 /* Clear the flags */ 8597 DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS; 8598 DB_LSOMSS(mp) = 0; 8599 } 8600 8601 /* 8602 * Checksum buffer *bp for len bytes with psum partial checksum, 8603 * or 0 if none, and return the 16 bit partial checksum. 8604 */ 8605 unsigned 8606 bcksum(uchar_t *bp, int len, unsigned int psum) 8607 { 8608 int odd = len & 1; 8609 extern unsigned int ip_ocsum(); 8610 8611 if (((intptr_t)bp & 1) == 0 && !odd) { 8612 /* 8613 * Bp is 16 bit aligned and len is multiple of 16 bit word. 8614 */ 8615 return (ip_ocsum((ushort_t *)bp, len >> 1, psum)); 8616 } 8617 if (((intptr_t)bp & 1) != 0) { 8618 /* 8619 * Bp isn't 16 bit aligned. 8620 */ 8621 unsigned int tsum; 8622 8623 #ifdef _LITTLE_ENDIAN 8624 psum += *bp; 8625 #else 8626 psum += *bp << 8; 8627 #endif 8628 len--; 8629 bp++; 8630 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0); 8631 psum += (tsum << 8) & 0xffff | (tsum >> 8); 8632 if (len & 1) { 8633 bp += len - 1; 8634 #ifdef _LITTLE_ENDIAN 8635 psum += *bp << 8; 8636 #else 8637 psum += *bp; 8638 #endif 8639 } 8640 } else { 8641 /* 8642 * Bp is 16 bit aligned. 8643 */ 8644 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum); 8645 if (odd) { 8646 bp += len - 1; 8647 #ifdef _LITTLE_ENDIAN 8648 psum += *bp; 8649 #else 8650 psum += *bp << 8; 8651 #endif 8652 } 8653 } 8654 /* 8655 * Normalize psum to 16 bits before returning the new partial 8656 * checksum. The max psum value before normalization is 0x3FDFE. 8657 */ 8658 return ((psum >> 16) + (psum & 0xFFFF)); 8659 } 8660 8661 boolean_t 8662 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd) 8663 { 8664 boolean_t rc; 8665 8666 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8667 if (DB_TYPE(mp) == M_DATA) { 8668 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0); 8669 } else { 8670 pattrinfo_t zcopy_attr = {PATTR_ZCOPY}; 8671 8672 ASSERT(mmd != NULL); 8673 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL); 8674 } 8675 return (rc); 8676 } 8677 8678 void 8679 freemsgchain(mblk_t *mp) 8680 { 8681 mblk_t *next; 8682 8683 while (mp != NULL) { 8684 next = mp->b_next; 8685 mp->b_next = NULL; 8686 8687 freemsg(mp); 8688 mp = next; 8689 } 8690 } 8691 8692 mblk_t * 8693 copymsgchain(mblk_t *mp) 8694 { 8695 mblk_t *nmp = NULL; 8696 mblk_t **nmpp = &nmp; 8697 8698 for (; mp != NULL; mp = mp->b_next) { 8699 if ((*nmpp = copymsg(mp)) == NULL) { 8700 freemsgchain(nmp); 8701 return (NULL); 8702 } 8703 8704 nmpp = &((*nmpp)->b_next); 8705 } 8706 8707 return (nmp); 8708 } 8709 8710 /* NOTE: Do not add code after this point. */ 8711 #undef QLOCK 8712 8713 /* 8714 * Replacement for QLOCK macro for those that can't use it. 8715 */ 8716 kmutex_t * 8717 QLOCK(queue_t *q) 8718 { 8719 return (&(q)->q_lock); 8720 } 8721 8722 /* 8723 * Dummy runqueues/queuerun functions functions for backwards compatibility. 8724 */ 8725 #undef runqueues 8726 void 8727 runqueues(void) 8728 { 8729 } 8730 8731 #undef queuerun 8732 void 8733 queuerun(void) 8734 { 8735 } 8736 8737 /* 8738 * Initialize the STR stack instance, which tracks autopush and persistent 8739 * links. 8740 */ 8741 /* ARGSUSED */ 8742 static void * 8743 str_stack_init(netstackid_t stackid, netstack_t *ns) 8744 { 8745 str_stack_t *ss; 8746 int i; 8747 8748 ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP); 8749 ss->ss_netstack = ns; 8750 8751 /* 8752 * set up autopush 8753 */ 8754 sad_initspace(ss); 8755 8756 /* 8757 * set up mux_node structures. 8758 */ 8759 ss->ss_devcnt = devcnt; /* In case it should change before free */ 8760 ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) * 8761 ss->ss_devcnt), KM_SLEEP); 8762 for (i = 0; i < ss->ss_devcnt; i++) 8763 ss->ss_mux_nodes[i].mn_imaj = i; 8764 return (ss); 8765 } 8766 8767 /* 8768 * Note: run at zone shutdown and not destroy so that the PLINKs are 8769 * gone by the time other cleanup happens from the destroy callbacks. 8770 */ 8771 static void 8772 str_stack_shutdown(netstackid_t stackid, void *arg) 8773 { 8774 str_stack_t *ss = (str_stack_t *)arg; 8775 int i; 8776 cred_t *cr; 8777 8778 cr = zone_get_kcred(netstackid_to_zoneid(stackid)); 8779 ASSERT(cr != NULL); 8780 8781 /* Undo all the I_PLINKs for this zone */ 8782 for (i = 0; i < ss->ss_devcnt; i++) { 8783 struct mux_edge *ep; 8784 ldi_handle_t lh; 8785 ldi_ident_t li; 8786 int ret; 8787 int rval; 8788 dev_t rdev; 8789 8790 ep = ss->ss_mux_nodes[i].mn_outp; 8791 if (ep == NULL) 8792 continue; 8793 ret = ldi_ident_from_major((major_t)i, &li); 8794 if (ret != 0) { 8795 continue; 8796 } 8797 rdev = ep->me_dev; 8798 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE, 8799 cr, &lh, li); 8800 if (ret != 0) { 8801 ldi_ident_release(li); 8802 continue; 8803 } 8804 8805 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL, 8806 cr, &rval); 8807 if (ret) { 8808 (void) ldi_close(lh, FREAD|FWRITE, cr); 8809 ldi_ident_release(li); 8810 continue; 8811 } 8812 (void) ldi_close(lh, FREAD|FWRITE, cr); 8813 8814 /* Close layered handles */ 8815 ldi_ident_release(li); 8816 } 8817 crfree(cr); 8818 8819 sad_freespace(ss); 8820 8821 kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt); 8822 ss->ss_mux_nodes = NULL; 8823 } 8824 8825 /* 8826 * Free the structure; str_stack_shutdown did the other cleanup work. 8827 */ 8828 /* ARGSUSED */ 8829 static void 8830 str_stack_fini(netstackid_t stackid, void *arg) 8831 { 8832 str_stack_t *ss = (str_stack_t *)arg; 8833 8834 kmem_free(ss, sizeof (*ss)); 8835 } 8836