1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 22 /* All Rights Reserved */ 23 24 25 /* 26 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 27 * Use is subject to license terms. 28 * Copyright (c) 2016 by Delphix. All rights reserved. 29 * Copyright 2018 OmniOS Community Edition (OmniOSce) Association. 30 */ 31 32 #include <sys/types.h> 33 #include <sys/sysmacros.h> 34 #include <sys/param.h> 35 #include <sys/errno.h> 36 #include <sys/signal.h> 37 #include <sys/proc.h> 38 #include <sys/conf.h> 39 #include <sys/cred.h> 40 #include <sys/user.h> 41 #include <sys/vnode.h> 42 #include <sys/file.h> 43 #include <sys/session.h> 44 #include <sys/stream.h> 45 #include <sys/strsubr.h> 46 #include <sys/stropts.h> 47 #include <sys/poll.h> 48 #include <sys/systm.h> 49 #include <sys/cpuvar.h> 50 #include <sys/uio.h> 51 #include <sys/cmn_err.h> 52 #include <sys/priocntl.h> 53 #include <sys/procset.h> 54 #include <sys/vmem.h> 55 #include <sys/bitmap.h> 56 #include <sys/kmem.h> 57 #include <sys/siginfo.h> 58 #include <sys/vtrace.h> 59 #include <sys/callb.h> 60 #include <sys/debug.h> 61 #include <sys/modctl.h> 62 #include <sys/vmsystm.h> 63 #include <vm/page.h> 64 #include <sys/atomic.h> 65 #include <sys/suntpi.h> 66 #include <sys/strlog.h> 67 #include <sys/promif.h> 68 #include <sys/project.h> 69 #include <sys/vm.h> 70 #include <sys/taskq.h> 71 #include <sys/sunddi.h> 72 #include <sys/sunldi_impl.h> 73 #include <sys/strsun.h> 74 #include <sys/isa_defs.h> 75 #include <sys/multidata.h> 76 #include <sys/pattr.h> 77 #include <sys/strft.h> 78 #include <sys/fs/snode.h> 79 #include <sys/zone.h> 80 #include <sys/open.h> 81 #include <sys/sunldi.h> 82 #include <sys/sad.h> 83 #include <sys/netstack.h> 84 85 #define O_SAMESTR(q) (((q)->q_next) && \ 86 (((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR))) 87 88 /* 89 * WARNING: 90 * The variables and routines in this file are private, belonging 91 * to the STREAMS subsystem. These should not be used by modules 92 * or drivers. Compatibility will not be guaranteed. 93 */ 94 95 /* 96 * Id value used to distinguish between different multiplexor links. 97 */ 98 static int32_t lnk_id = 0; 99 100 #define STREAMS_LOPRI MINCLSYSPRI 101 static pri_t streams_lopri = STREAMS_LOPRI; 102 103 #define STRSTAT(x) (str_statistics.x.value.ui64++) 104 typedef struct str_stat { 105 kstat_named_t sqenables; 106 kstat_named_t stenables; 107 kstat_named_t syncqservice; 108 kstat_named_t freebs; 109 kstat_named_t qwr_outer; 110 kstat_named_t rservice; 111 kstat_named_t strwaits; 112 kstat_named_t taskqfails; 113 kstat_named_t bufcalls; 114 kstat_named_t qhelps; 115 kstat_named_t qremoved; 116 kstat_named_t sqremoved; 117 kstat_named_t bcwaits; 118 kstat_named_t sqtoomany; 119 } str_stat_t; 120 121 static str_stat_t str_statistics = { 122 { "sqenables", KSTAT_DATA_UINT64 }, 123 { "stenables", KSTAT_DATA_UINT64 }, 124 { "syncqservice", KSTAT_DATA_UINT64 }, 125 { "freebs", KSTAT_DATA_UINT64 }, 126 { "qwr_outer", KSTAT_DATA_UINT64 }, 127 { "rservice", KSTAT_DATA_UINT64 }, 128 { "strwaits", KSTAT_DATA_UINT64 }, 129 { "taskqfails", KSTAT_DATA_UINT64 }, 130 { "bufcalls", KSTAT_DATA_UINT64 }, 131 { "qhelps", KSTAT_DATA_UINT64 }, 132 { "qremoved", KSTAT_DATA_UINT64 }, 133 { "sqremoved", KSTAT_DATA_UINT64 }, 134 { "bcwaits", KSTAT_DATA_UINT64 }, 135 { "sqtoomany", KSTAT_DATA_UINT64 }, 136 }; 137 138 static kstat_t *str_kstat; 139 140 /* 141 * qrunflag was used previously to control background scheduling of queues. It 142 * is not used anymore, but kept here in case some module still wants to access 143 * it via qready() and setqsched macros. 144 */ 145 char qrunflag; /* Unused */ 146 147 /* 148 * Most of the streams scheduling is done via task queues. Task queues may fail 149 * for non-sleep dispatches, so there are two backup threads servicing failed 150 * requests for queues and syncqs. Both of these threads also service failed 151 * dispatches freebs requests. Queues are put in the list specified by `qhead' 152 * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs 153 * requests are put into `freebs_list' which has no tail pointer. All three 154 * lists are protected by a single `service_queue' lock and use 155 * `services_to_run' condition variable for signaling background threads. Use of 156 * a single lock should not be a problem because it is only used under heavy 157 * loads when task queues start to fail and at that time it may be a good idea 158 * to throttle scheduling requests. 159 * 160 * NOTE: queues and syncqs should be scheduled by two separate threads because 161 * queue servicing may be blocked waiting for a syncq which may be also 162 * scheduled for background execution. This may create a deadlock when only one 163 * thread is used for both. 164 */ 165 166 static taskq_t *streams_taskq; /* Used for most STREAMS scheduling */ 167 168 static kmutex_t service_queue; /* protects all of servicing vars */ 169 static kcondvar_t services_to_run; /* wake up background service thread */ 170 static kcondvar_t syncqs_to_run; /* wake up background service thread */ 171 172 /* 173 * List of queues scheduled for background processing due to lack of resources 174 * in the task queues. Protected by service_queue lock; 175 */ 176 static struct queue *qhead; 177 static struct queue *qtail; 178 179 /* 180 * Same list for syncqs 181 */ 182 static syncq_t *sqhead; 183 static syncq_t *sqtail; 184 185 static mblk_t *freebs_list; /* list of buffers to free */ 186 187 /* 188 * Backup threads for servicing queues and syncqs 189 */ 190 kthread_t *streams_qbkgrnd_thread; 191 kthread_t *streams_sqbkgrnd_thread; 192 193 /* 194 * Bufcalls related variables. 195 */ 196 struct bclist strbcalls; /* list of waiting bufcalls */ 197 kmutex_t strbcall_lock; /* protects bufcall list (strbcalls) */ 198 kcondvar_t strbcall_cv; /* Signaling when a bufcall is added */ 199 kmutex_t bcall_monitor; /* sleep/wakeup style monitor */ 200 kcondvar_t bcall_cv; /* wait 'till executing bufcall completes */ 201 kthread_t *bc_bkgrnd_thread; /* Thread to service bufcall requests */ 202 203 kmutex_t strresources; /* protects global resources */ 204 kmutex_t muxifier; /* single-threads multiplexor creation */ 205 206 static void *str_stack_init(netstackid_t stackid, netstack_t *ns); 207 static void str_stack_shutdown(netstackid_t stackid, void *arg); 208 static void str_stack_fini(netstackid_t stackid, void *arg); 209 210 /* 211 * run_queues is no longer used, but is kept in case some 3rd party 212 * module/driver decides to use it. 213 */ 214 int run_queues = 0; 215 216 /* 217 * sq_max_size is the depth of the syncq (in number of messages) before 218 * qfill_syncq() starts QFULL'ing destination queues. As its primary 219 * consumer - IP is no longer D_MTPERMOD, but there may be other 220 * modules/drivers depend on this syncq flow control, we prefer to 221 * choose a large number as the default value. For potential 222 * performance gain, this value is tunable in /etc/system. 223 */ 224 int sq_max_size = 10000; 225 226 /* 227 * The number of ciputctrl structures per syncq and stream we create when 228 * needed. 229 */ 230 int n_ciputctrl; 231 int max_n_ciputctrl = 16; 232 /* 233 * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache. 234 */ 235 int min_n_ciputctrl = 2; 236 237 /* 238 * Per-driver/module syncqs 239 * ======================== 240 * 241 * For drivers/modules that use PERMOD or outer syncqs we keep a list of 242 * perdm structures, new entries being added (and new syncqs allocated) when 243 * setq() encounters a module/driver with a streamtab that it hasn't seen 244 * before. 245 * The reason for this mechanism is that some modules and drivers share a 246 * common streamtab and it is necessary for those modules and drivers to also 247 * share a common PERMOD syncq. 248 * 249 * perdm_list --> dm_str == streamtab_1 250 * dm_sq == syncq_1 251 * dm_ref 252 * dm_next --> dm_str == streamtab_2 253 * dm_sq == syncq_2 254 * dm_ref 255 * dm_next --> ... NULL 256 * 257 * The dm_ref field is incremented for each new driver/module that takes 258 * a reference to the perdm structure and hence shares the syncq. 259 * References are held in the fmodsw_impl_t structure for each STREAMS module 260 * or the dev_impl array (indexed by device major number) for each driver. 261 * 262 * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL 263 * ^ ^ ^ ^ 264 * | ______________/ | | 265 * | / | | 266 * dev_impl: ...|x|y|... module A module B 267 * 268 * When a module/driver is unloaded the reference count is decremented and, 269 * when it falls to zero, the perdm structure is removed from the list and 270 * the syncq is freed (see rele_dm()). 271 */ 272 perdm_t *perdm_list = NULL; 273 static krwlock_t perdm_rwlock; 274 cdevsw_impl_t *devimpl; 275 276 extern struct qinit strdata; 277 extern struct qinit stwdata; 278 279 static void runservice(queue_t *); 280 static void streams_bufcall_service(void); 281 static void streams_qbkgrnd_service(void); 282 static void streams_sqbkgrnd_service(void); 283 static syncq_t *new_syncq(void); 284 static void free_syncq(syncq_t *); 285 static void outer_insert(syncq_t *, syncq_t *); 286 static void outer_remove(syncq_t *, syncq_t *); 287 static void write_now(syncq_t *); 288 static void clr_qfull(queue_t *); 289 static void runbufcalls(void); 290 static void sqenable(syncq_t *); 291 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)()); 292 static void wait_q_syncq(queue_t *); 293 static void backenable_insertedq(queue_t *); 294 295 static void queue_service(queue_t *); 296 static void stream_service(stdata_t *); 297 static void syncq_service(syncq_t *); 298 static void qwriter_outer_service(syncq_t *); 299 static void mblk_free(mblk_t *); 300 #ifdef DEBUG 301 static int qprocsareon(queue_t *); 302 #endif 303 304 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *); 305 static void reset_nfsrv_ptr(queue_t *, queue_t *); 306 void set_qfull(queue_t *); 307 308 static void sq_run_events(syncq_t *); 309 static int propagate_syncq(queue_t *); 310 311 static void blocksq(syncq_t *, ushort_t, int); 312 static void unblocksq(syncq_t *, ushort_t, int); 313 static int dropsq(syncq_t *, uint16_t); 314 static void emptysq(syncq_t *); 315 static sqlist_t *sqlist_alloc(struct stdata *, int); 316 static void sqlist_free(sqlist_t *); 317 static sqlist_t *sqlist_build(queue_t *, struct stdata *, boolean_t); 318 static void sqlist_insert(sqlist_t *, syncq_t *); 319 static void sqlist_insertall(sqlist_t *, queue_t *); 320 321 static void strsetuio(stdata_t *); 322 323 struct kmem_cache *stream_head_cache; 324 struct kmem_cache *queue_cache; 325 struct kmem_cache *syncq_cache; 326 struct kmem_cache *qband_cache; 327 struct kmem_cache *linkinfo_cache; 328 struct kmem_cache *ciputctrl_cache = NULL; 329 330 static linkinfo_t *linkinfo_list; 331 332 /* Global esballoc throttling queue */ 333 static esb_queue_t system_esbq; 334 335 /* Array of esballoc throttling queues, of length esbq_nelem */ 336 static esb_queue_t *volatile system_esbq_array; 337 static int esbq_nelem; 338 static kmutex_t esbq_lock; 339 static int esbq_log2_cpus_per_q = 0; 340 341 /* Scale the system_esbq length by setting number of CPUs per queue. */ 342 uint_t esbq_cpus_per_q = 1; 343 344 /* 345 * esballoc tunable parameters. 346 */ 347 int esbq_max_qlen = 0x16; /* throttled queue length */ 348 clock_t esbq_timeout = 0x8; /* timeout to process esb queue */ 349 350 /* 351 * Routines to handle esballoc queueing. 352 */ 353 static void esballoc_process_queue(esb_queue_t *); 354 static void esballoc_enqueue_mblk(mblk_t *); 355 static void esballoc_timer(void *); 356 static void esballoc_set_timer(esb_queue_t *, clock_t); 357 static void esballoc_mblk_free(mblk_t *); 358 359 /* 360 * Qinit structure and Module_info structures 361 * for passthru read and write queues 362 */ 363 364 static int pass_rput(queue_t *, mblk_t *); 365 static int pass_wput(queue_t *, mblk_t *); 366 static queue_t *link_addpassthru(stdata_t *); 367 static void link_rempassthru(queue_t *); 368 369 struct module_info passthru_info = { 370 0, 371 "passthru", 372 0, 373 INFPSZ, 374 STRHIGH, 375 STRLOW 376 }; 377 378 struct qinit passthru_rinit = { 379 pass_rput, 380 NULL, 381 NULL, 382 NULL, 383 NULL, 384 &passthru_info, 385 NULL 386 }; 387 388 struct qinit passthru_winit = { 389 pass_wput, 390 NULL, 391 NULL, 392 NULL, 393 NULL, 394 &passthru_info, 395 NULL 396 }; 397 398 /* 399 * Verify correctness of list head/tail pointers. 400 */ 401 #define LISTCHECK(head, tail, link) { \ 402 EQUIV(head, tail); \ 403 IMPLY(tail != NULL, tail->link == NULL); \ 404 } 405 406 /* 407 * Enqueue a list element `el' in the end of a list denoted by `head' and `tail' 408 * using a `link' field. 409 */ 410 #define ENQUEUE(el, head, tail, link) { \ 411 ASSERT(el->link == NULL); \ 412 LISTCHECK(head, tail, link); \ 413 if (head == NULL) \ 414 head = el; \ 415 else \ 416 tail->link = el; \ 417 tail = el; \ 418 } 419 420 /* 421 * Dequeue the first element of the list denoted by `head' and `tail' pointers 422 * using a `link' field and put result into `el'. 423 */ 424 #define DQ(el, head, tail, link) { \ 425 LISTCHECK(head, tail, link); \ 426 el = head; \ 427 if (head != NULL) { \ 428 head = head->link; \ 429 if (head == NULL) \ 430 tail = NULL; \ 431 el->link = NULL; \ 432 } \ 433 } 434 435 /* 436 * Remove `el' from the list using `chase' and `curr' pointers and return result 437 * in `succeed'. 438 */ 439 #define RMQ(el, head, tail, link, chase, curr, succeed) { \ 440 LISTCHECK(head, tail, link); \ 441 chase = NULL; \ 442 succeed = 0; \ 443 for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \ 444 chase = curr; \ 445 if (curr != NULL) { \ 446 succeed = 1; \ 447 ASSERT(curr == el); \ 448 if (chase != NULL) \ 449 chase->link = curr->link; \ 450 else \ 451 head = curr->link; \ 452 curr->link = NULL; \ 453 if (curr == tail) \ 454 tail = chase; \ 455 } \ 456 LISTCHECK(head, tail, link); \ 457 } 458 459 /* Handling of delayed messages on the inner syncq. */ 460 461 /* 462 * DEBUG versions should use function versions (to simplify tracing) and 463 * non-DEBUG kernels should use macro versions. 464 */ 465 466 /* 467 * Put a queue on the syncq list of queues. 468 * Assumes SQLOCK held. 469 */ 470 #define SQPUT_Q(sq, qp) \ 471 { \ 472 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 473 if (!(qp->q_sqflags & Q_SQQUEUED)) { \ 474 /* The queue should not be linked anywhere */ \ 475 ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \ 476 /* Head and tail may only be NULL simultaneously */ \ 477 EQUIV(sq->sq_head, sq->sq_tail); \ 478 /* Queue may be only enqueued on its syncq */ \ 479 ASSERT(sq == qp->q_syncq); \ 480 /* Check the correctness of SQ_MESSAGES flag */ \ 481 EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES)); \ 482 /* Sanity check first/last elements of the list */ \ 483 IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\ 484 IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\ 485 /* \ 486 * Sanity check of priority field: empty queue should \ 487 * have zero priority \ 488 * and nqueues equal to zero. \ 489 */ \ 490 IMPLY(sq->sq_head == NULL, sq->sq_pri == 0); \ 491 /* Sanity check of sq_nqueues field */ \ 492 EQUIV(sq->sq_head, sq->sq_nqueues); \ 493 if (sq->sq_head == NULL) { \ 494 sq->sq_head = sq->sq_tail = qp; \ 495 sq->sq_flags |= SQ_MESSAGES; \ 496 } else if (qp->q_spri == 0) { \ 497 qp->q_sqprev = sq->sq_tail; \ 498 sq->sq_tail->q_sqnext = qp; \ 499 sq->sq_tail = qp; \ 500 } else { \ 501 /* \ 502 * Put this queue in priority order: higher \ 503 * priority gets closer to the head. \ 504 */ \ 505 queue_t **qpp = &sq->sq_tail; \ 506 queue_t *qnext = NULL; \ 507 \ 508 while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \ 509 qnext = *qpp; \ 510 qpp = &(*qpp)->q_sqprev; \ 511 } \ 512 qp->q_sqnext = qnext; \ 513 qp->q_sqprev = *qpp; \ 514 if (*qpp != NULL) { \ 515 (*qpp)->q_sqnext = qp; \ 516 } else { \ 517 sq->sq_head = qp; \ 518 sq->sq_pri = sq->sq_head->q_spri; \ 519 } \ 520 *qpp = qp; \ 521 } \ 522 qp->q_sqflags |= Q_SQQUEUED; \ 523 qp->q_sqtstamp = ddi_get_lbolt(); \ 524 sq->sq_nqueues++; \ 525 } \ 526 } 527 528 /* 529 * Remove a queue from the syncq list 530 * Assumes SQLOCK held. 531 */ 532 #define SQRM_Q(sq, qp) \ 533 { \ 534 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 535 ASSERT(qp->q_sqflags & Q_SQQUEUED); \ 536 ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL); \ 537 ASSERT((sq->sq_flags & SQ_MESSAGES) != 0); \ 538 /* Check that the queue is actually in the list */ \ 539 ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp); \ 540 ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp); \ 541 ASSERT(sq->sq_nqueues != 0); \ 542 if (qp->q_sqprev == NULL) { \ 543 /* First queue on list, make head q_sqnext */ \ 544 sq->sq_head = qp->q_sqnext; \ 545 } else { \ 546 /* Make prev->next == next */ \ 547 qp->q_sqprev->q_sqnext = qp->q_sqnext; \ 548 } \ 549 if (qp->q_sqnext == NULL) { \ 550 /* Last queue on list, make tail sqprev */ \ 551 sq->sq_tail = qp->q_sqprev; \ 552 } else { \ 553 /* Make next->prev == prev */ \ 554 qp->q_sqnext->q_sqprev = qp->q_sqprev; \ 555 } \ 556 /* clear out references on this queue */ \ 557 qp->q_sqprev = qp->q_sqnext = NULL; \ 558 qp->q_sqflags &= ~Q_SQQUEUED; \ 559 /* If there is nothing queued, clear SQ_MESSAGES */ \ 560 if (sq->sq_head != NULL) { \ 561 sq->sq_pri = sq->sq_head->q_spri; \ 562 } else { \ 563 sq->sq_flags &= ~SQ_MESSAGES; \ 564 sq->sq_pri = 0; \ 565 } \ 566 sq->sq_nqueues--; \ 567 ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL || \ 568 (sq->sq_flags & SQ_QUEUED) == 0); \ 569 } 570 571 /* Hide the definition from the header file. */ 572 #ifdef SQPUT_MP 573 #undef SQPUT_MP 574 #endif 575 576 /* 577 * Put a message on the queue syncq. 578 * Assumes QLOCK held. 579 */ 580 #define SQPUT_MP(qp, mp) \ 581 { \ 582 ASSERT(MUTEX_HELD(QLOCK(qp))); \ 583 ASSERT(qp->q_sqhead == NULL || \ 584 (qp->q_sqtail != NULL && \ 585 qp->q_sqtail->b_next == NULL)); \ 586 qp->q_syncqmsgs++; \ 587 ASSERT(qp->q_syncqmsgs != 0); /* Wraparound */ \ 588 if (qp->q_sqhead == NULL) { \ 589 qp->q_sqhead = qp->q_sqtail = mp; \ 590 } else { \ 591 qp->q_sqtail->b_next = mp; \ 592 qp->q_sqtail = mp; \ 593 } \ 594 ASSERT(qp->q_syncqmsgs > 0); \ 595 set_qfull(qp); \ 596 } 597 598 #define SQ_PUTCOUNT_SETFAST_LOCKED(sq) { \ 599 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 600 if ((sq)->sq_ciputctrl != NULL) { \ 601 int i; \ 602 int nlocks = (sq)->sq_nciputctrl; \ 603 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 604 ASSERT((sq)->sq_type & SQ_CIPUT); \ 605 for (i = 0; i <= nlocks; i++) { \ 606 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 607 cip[i].ciputctrl_count |= SQ_FASTPUT; \ 608 } \ 609 } \ 610 } 611 612 613 #define SQ_PUTCOUNT_CLRFAST_LOCKED(sq) { \ 614 ASSERT(MUTEX_HELD(SQLOCK(sq))); \ 615 if ((sq)->sq_ciputctrl != NULL) { \ 616 int i; \ 617 int nlocks = (sq)->sq_nciputctrl; \ 618 ciputctrl_t *cip = (sq)->sq_ciputctrl; \ 619 ASSERT((sq)->sq_type & SQ_CIPUT); \ 620 for (i = 0; i <= nlocks; i++) { \ 621 ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \ 622 cip[i].ciputctrl_count &= ~SQ_FASTPUT; \ 623 } \ 624 } \ 625 } 626 627 /* 628 * Run service procedures for all queues in the stream head. 629 */ 630 #define STR_SERVICE(stp, q) { \ 631 ASSERT(MUTEX_HELD(&stp->sd_qlock)); \ 632 while (stp->sd_qhead != NULL) { \ 633 DQ(q, stp->sd_qhead, stp->sd_qtail, q_link); \ 634 ASSERT(stp->sd_nqueues > 0); \ 635 stp->sd_nqueues--; \ 636 ASSERT(!(q->q_flag & QINSERVICE)); \ 637 mutex_exit(&stp->sd_qlock); \ 638 queue_service(q); \ 639 mutex_enter(&stp->sd_qlock); \ 640 } \ 641 ASSERT(stp->sd_nqueues == 0); \ 642 ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL)); \ 643 } 644 645 /* 646 * Constructor/destructor routines for the stream head cache 647 */ 648 /* ARGSUSED */ 649 static int 650 stream_head_constructor(void *buf, void *cdrarg, int kmflags) 651 { 652 stdata_t *stp = buf; 653 654 mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL); 655 mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL); 656 mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL); 657 cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL); 658 cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL); 659 cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL); 660 cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL); 661 cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL); 662 stp->sd_wrq = NULL; 663 664 return (0); 665 } 666 667 /* ARGSUSED */ 668 static void 669 stream_head_destructor(void *buf, void *cdrarg) 670 { 671 stdata_t *stp = buf; 672 673 mutex_destroy(&stp->sd_lock); 674 mutex_destroy(&stp->sd_reflock); 675 mutex_destroy(&stp->sd_qlock); 676 cv_destroy(&stp->sd_monitor); 677 cv_destroy(&stp->sd_iocmonitor); 678 cv_destroy(&stp->sd_refmonitor); 679 cv_destroy(&stp->sd_qcv); 680 cv_destroy(&stp->sd_zcopy_wait); 681 } 682 683 /* 684 * Constructor/destructor routines for the queue cache 685 */ 686 /* ARGSUSED */ 687 static int 688 queue_constructor(void *buf, void *cdrarg, int kmflags) 689 { 690 queinfo_t *qip = buf; 691 queue_t *qp = &qip->qu_rqueue; 692 queue_t *wqp = &qip->qu_wqueue; 693 syncq_t *sq = &qip->qu_syncq; 694 695 qp->q_first = NULL; 696 qp->q_link = NULL; 697 qp->q_count = 0; 698 qp->q_mblkcnt = 0; 699 qp->q_sqhead = NULL; 700 qp->q_sqtail = NULL; 701 qp->q_sqnext = NULL; 702 qp->q_sqprev = NULL; 703 qp->q_sqflags = 0; 704 qp->q_rwcnt = 0; 705 qp->q_spri = 0; 706 707 mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL); 708 cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL); 709 710 wqp->q_first = NULL; 711 wqp->q_link = NULL; 712 wqp->q_count = 0; 713 wqp->q_mblkcnt = 0; 714 wqp->q_sqhead = NULL; 715 wqp->q_sqtail = NULL; 716 wqp->q_sqnext = NULL; 717 wqp->q_sqprev = NULL; 718 wqp->q_sqflags = 0; 719 wqp->q_rwcnt = 0; 720 wqp->q_spri = 0; 721 722 mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL); 723 cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL); 724 725 sq->sq_head = NULL; 726 sq->sq_tail = NULL; 727 sq->sq_evhead = NULL; 728 sq->sq_evtail = NULL; 729 sq->sq_callbpend = NULL; 730 sq->sq_outer = NULL; 731 sq->sq_onext = NULL; 732 sq->sq_oprev = NULL; 733 sq->sq_next = NULL; 734 sq->sq_svcflags = 0; 735 sq->sq_servcount = 0; 736 sq->sq_needexcl = 0; 737 sq->sq_nqueues = 0; 738 sq->sq_pri = 0; 739 740 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 741 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 742 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 743 744 return (0); 745 } 746 747 /* ARGSUSED */ 748 static void 749 queue_destructor(void *buf, void *cdrarg) 750 { 751 queinfo_t *qip = buf; 752 queue_t *qp = &qip->qu_rqueue; 753 queue_t *wqp = &qip->qu_wqueue; 754 syncq_t *sq = &qip->qu_syncq; 755 756 ASSERT(qp->q_sqhead == NULL); 757 ASSERT(wqp->q_sqhead == NULL); 758 ASSERT(qp->q_sqnext == NULL); 759 ASSERT(wqp->q_sqnext == NULL); 760 ASSERT(qp->q_rwcnt == 0); 761 ASSERT(wqp->q_rwcnt == 0); 762 763 mutex_destroy(&qp->q_lock); 764 cv_destroy(&qp->q_wait); 765 766 mutex_destroy(&wqp->q_lock); 767 cv_destroy(&wqp->q_wait); 768 769 mutex_destroy(&sq->sq_lock); 770 cv_destroy(&sq->sq_wait); 771 cv_destroy(&sq->sq_exitwait); 772 } 773 774 /* 775 * Constructor/destructor routines for the syncq cache 776 */ 777 /* ARGSUSED */ 778 static int 779 syncq_constructor(void *buf, void *cdrarg, int kmflags) 780 { 781 syncq_t *sq = buf; 782 783 bzero(buf, sizeof (syncq_t)); 784 785 mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL); 786 cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL); 787 cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL); 788 789 return (0); 790 } 791 792 /* ARGSUSED */ 793 static void 794 syncq_destructor(void *buf, void *cdrarg) 795 { 796 syncq_t *sq = buf; 797 798 ASSERT(sq->sq_head == NULL); 799 ASSERT(sq->sq_tail == NULL); 800 ASSERT(sq->sq_evhead == NULL); 801 ASSERT(sq->sq_evtail == NULL); 802 ASSERT(sq->sq_callbpend == NULL); 803 ASSERT(sq->sq_callbflags == 0); 804 ASSERT(sq->sq_outer == NULL); 805 ASSERT(sq->sq_onext == NULL); 806 ASSERT(sq->sq_oprev == NULL); 807 ASSERT(sq->sq_next == NULL); 808 ASSERT(sq->sq_needexcl == 0); 809 ASSERT(sq->sq_svcflags == 0); 810 ASSERT(sq->sq_servcount == 0); 811 ASSERT(sq->sq_nqueues == 0); 812 ASSERT(sq->sq_pri == 0); 813 ASSERT(sq->sq_count == 0); 814 ASSERT(sq->sq_rmqcount == 0); 815 ASSERT(sq->sq_cancelid == 0); 816 ASSERT(sq->sq_ciputctrl == NULL); 817 ASSERT(sq->sq_nciputctrl == 0); 818 ASSERT(sq->sq_type == 0); 819 ASSERT(sq->sq_flags == 0); 820 821 mutex_destroy(&sq->sq_lock); 822 cv_destroy(&sq->sq_wait); 823 cv_destroy(&sq->sq_exitwait); 824 } 825 826 /* ARGSUSED */ 827 static int 828 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags) 829 { 830 ciputctrl_t *cip = buf; 831 int i; 832 833 for (i = 0; i < n_ciputctrl; i++) { 834 cip[i].ciputctrl_count = SQ_FASTPUT; 835 mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL); 836 } 837 838 return (0); 839 } 840 841 /* ARGSUSED */ 842 static void 843 ciputctrl_destructor(void *buf, void *cdrarg) 844 { 845 ciputctrl_t *cip = buf; 846 int i; 847 848 for (i = 0; i < n_ciputctrl; i++) { 849 ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT); 850 mutex_destroy(&cip[i].ciputctrl_lock); 851 } 852 } 853 854 /* 855 * Init routine run from main at boot time. 856 */ 857 void 858 strinit(void) 859 { 860 int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); 861 862 stream_head_cache = kmem_cache_create("stream_head_cache", 863 sizeof (stdata_t), 0, 864 stream_head_constructor, stream_head_destructor, NULL, 865 NULL, NULL, 0); 866 867 queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0, 868 queue_constructor, queue_destructor, NULL, NULL, NULL, 0); 869 870 syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0, 871 syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0); 872 873 qband_cache = kmem_cache_create("qband_cache", 874 sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 875 876 linkinfo_cache = kmem_cache_create("linkinfo_cache", 877 sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 878 879 n_ciputctrl = ncpus; 880 n_ciputctrl = 1 << highbit(n_ciputctrl - 1); 881 ASSERT(n_ciputctrl >= 1); 882 n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl); 883 if (n_ciputctrl >= min_n_ciputctrl) { 884 ciputctrl_cache = kmem_cache_create("ciputctrl_cache", 885 sizeof (ciputctrl_t) * n_ciputctrl, 886 sizeof (ciputctrl_t), ciputctrl_constructor, 887 ciputctrl_destructor, NULL, NULL, NULL, 0); 888 } 889 890 streams_taskq = system_taskq; 891 892 if (streams_taskq == NULL) 893 panic("strinit: no memory for streams taskq!"); 894 895 bc_bkgrnd_thread = thread_create(NULL, 0, 896 streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri); 897 898 streams_qbkgrnd_thread = thread_create(NULL, 0, 899 streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 900 901 streams_sqbkgrnd_thread = thread_create(NULL, 0, 902 streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri); 903 904 /* 905 * Create STREAMS kstats. 906 */ 907 str_kstat = kstat_create("streams", 0, "strstat", 908 "net", KSTAT_TYPE_NAMED, 909 sizeof (str_statistics) / sizeof (kstat_named_t), 910 KSTAT_FLAG_VIRTUAL); 911 912 if (str_kstat != NULL) { 913 str_kstat->ks_data = &str_statistics; 914 kstat_install(str_kstat); 915 } 916 917 /* 918 * TPI support routine initialisation. 919 */ 920 tpi_init(); 921 922 /* 923 * Handle to have autopush and persistent link information per 924 * zone. 925 * Note: uses shutdown hook instead of destroy hook so that the 926 * persistent links can be torn down before the destroy hooks 927 * in the TCP/IP stack are called. 928 */ 929 netstack_register(NS_STR, str_stack_init, str_stack_shutdown, 930 str_stack_fini); 931 } 932 933 void 934 str_sendsig(vnode_t *vp, int event, uchar_t band, int error) 935 { 936 struct stdata *stp; 937 938 ASSERT(vp->v_stream); 939 stp = vp->v_stream; 940 /* Have to hold sd_lock to prevent siglist from changing */ 941 mutex_enter(&stp->sd_lock); 942 if (stp->sd_sigflags & event) 943 strsendsig(stp->sd_siglist, event, band, error); 944 mutex_exit(&stp->sd_lock); 945 } 946 947 /* 948 * Send the "sevent" set of signals to a process. 949 * This might send more than one signal if the process is registered 950 * for multiple events. The caller should pass in an sevent that only 951 * includes the events for which the process has registered. 952 */ 953 static void 954 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info, 955 uchar_t band, int error) 956 { 957 ASSERT(MUTEX_HELD(&proc->p_lock)); 958 959 info->si_band = 0; 960 info->si_errno = 0; 961 962 if (sevent & S_ERROR) { 963 sevent &= ~S_ERROR; 964 info->si_code = POLL_ERR; 965 info->si_errno = error; 966 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 967 "strsendsig:proc %p info %p", proc, info); 968 sigaddq(proc, NULL, info, KM_NOSLEEP); 969 info->si_errno = 0; 970 } 971 if (sevent & S_HANGUP) { 972 sevent &= ~S_HANGUP; 973 info->si_code = POLL_HUP; 974 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 975 "strsendsig:proc %p info %p", proc, info); 976 sigaddq(proc, NULL, info, KM_NOSLEEP); 977 } 978 if (sevent & S_HIPRI) { 979 sevent &= ~S_HIPRI; 980 info->si_code = POLL_PRI; 981 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 982 "strsendsig:proc %p info %p", proc, info); 983 sigaddq(proc, NULL, info, KM_NOSLEEP); 984 } 985 if (sevent & S_RDBAND) { 986 sevent &= ~S_RDBAND; 987 if (events & S_BANDURG) 988 sigtoproc(proc, NULL, SIGURG); 989 else 990 sigtoproc(proc, NULL, SIGPOLL); 991 } 992 if (sevent & S_WRBAND) { 993 sevent &= ~S_WRBAND; 994 sigtoproc(proc, NULL, SIGPOLL); 995 } 996 if (sevent & S_INPUT) { 997 sevent &= ~S_INPUT; 998 info->si_code = POLL_IN; 999 info->si_band = band; 1000 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1001 "strsendsig:proc %p info %p", proc, info); 1002 sigaddq(proc, NULL, info, KM_NOSLEEP); 1003 info->si_band = 0; 1004 } 1005 if (sevent & S_OUTPUT) { 1006 sevent &= ~S_OUTPUT; 1007 info->si_code = POLL_OUT; 1008 info->si_band = band; 1009 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1010 "strsendsig:proc %p info %p", proc, info); 1011 sigaddq(proc, NULL, info, KM_NOSLEEP); 1012 info->si_band = 0; 1013 } 1014 if (sevent & S_MSG) { 1015 sevent &= ~S_MSG; 1016 info->si_code = POLL_MSG; 1017 info->si_band = band; 1018 TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG, 1019 "strsendsig:proc %p info %p", proc, info); 1020 sigaddq(proc, NULL, info, KM_NOSLEEP); 1021 info->si_band = 0; 1022 } 1023 if (sevent & S_RDNORM) { 1024 sevent &= ~S_RDNORM; 1025 sigtoproc(proc, NULL, SIGPOLL); 1026 } 1027 if (sevent != 0) { 1028 panic("strsendsig: unknown event(s) %x", sevent); 1029 } 1030 } 1031 1032 /* 1033 * Send SIGPOLL/SIGURG signal to all processes and process groups 1034 * registered on the given signal list that want a signal for at 1035 * least one of the specified events. 1036 * 1037 * Must be called with exclusive access to siglist (caller holding sd_lock). 1038 * 1039 * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding 1040 * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure 1041 * while it is in the siglist. 1042 * 1043 * For performance reasons (MP scalability) the code drops pidlock 1044 * when sending signals to a single process. 1045 * When sending to a process group the code holds 1046 * pidlock to prevent the membership in the process group from changing 1047 * while walking the p_pglink list. 1048 */ 1049 void 1050 strsendsig(strsig_t *siglist, int event, uchar_t band, int error) 1051 { 1052 strsig_t *ssp; 1053 k_siginfo_t info; 1054 struct pid *pidp; 1055 proc_t *proc; 1056 1057 info.si_signo = SIGPOLL; 1058 info.si_errno = 0; 1059 for (ssp = siglist; ssp; ssp = ssp->ss_next) { 1060 int sevent; 1061 1062 sevent = ssp->ss_events & event; 1063 if (sevent == 0) 1064 continue; 1065 1066 if ((pidp = ssp->ss_pidp) == NULL) { 1067 /* pid was released but still on event list */ 1068 continue; 1069 } 1070 1071 1072 if (ssp->ss_pid > 0) { 1073 /* 1074 * XXX This unfortunately still generates 1075 * a signal when a fd is closed but 1076 * the proc is active. 1077 */ 1078 ASSERT(ssp->ss_pid == pidp->pid_id); 1079 1080 mutex_enter(&pidlock); 1081 proc = prfind_zone(pidp->pid_id, ALL_ZONES); 1082 if (proc == NULL) { 1083 mutex_exit(&pidlock); 1084 continue; 1085 } 1086 mutex_enter(&proc->p_lock); 1087 mutex_exit(&pidlock); 1088 dosendsig(proc, ssp->ss_events, sevent, &info, 1089 band, error); 1090 mutex_exit(&proc->p_lock); 1091 } else { 1092 /* 1093 * Send to process group. Hold pidlock across 1094 * calls to dosendsig(). 1095 */ 1096 pid_t pgrp = -ssp->ss_pid; 1097 1098 mutex_enter(&pidlock); 1099 proc = pgfind_zone(pgrp, ALL_ZONES); 1100 while (proc != NULL) { 1101 mutex_enter(&proc->p_lock); 1102 dosendsig(proc, ssp->ss_events, sevent, 1103 &info, band, error); 1104 mutex_exit(&proc->p_lock); 1105 proc = proc->p_pglink; 1106 } 1107 mutex_exit(&pidlock); 1108 } 1109 } 1110 } 1111 1112 /* 1113 * Attach a stream device or module. 1114 * qp is a read queue; the new queue goes in so its next 1115 * read ptr is the argument, and the write queue corresponding 1116 * to the argument points to this queue. Return 0 on success, 1117 * or a non-zero errno on failure. 1118 */ 1119 int 1120 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp, 1121 boolean_t is_insert) 1122 { 1123 major_t major; 1124 cdevsw_impl_t *dp; 1125 struct streamtab *str; 1126 queue_t *rq; 1127 queue_t *wrq; 1128 uint32_t qflag; 1129 uint32_t sqtype; 1130 perdm_t *dmp; 1131 int error; 1132 int sflag; 1133 1134 rq = allocq(); 1135 wrq = _WR(rq); 1136 STREAM(rq) = STREAM(wrq) = STREAM(qp); 1137 1138 if (fp != NULL) { 1139 str = fp->f_str; 1140 qflag = fp->f_qflag; 1141 sqtype = fp->f_sqtype; 1142 dmp = fp->f_dmp; 1143 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 1144 sflag = MODOPEN; 1145 1146 /* 1147 * stash away a pointer to the module structure so we can 1148 * unref it in qdetach. 1149 */ 1150 rq->q_fp = fp; 1151 } else { 1152 ASSERT(!is_insert); 1153 1154 major = getmajor(*devp); 1155 dp = &devimpl[major]; 1156 1157 str = dp->d_str; 1158 ASSERT(str == STREAMSTAB(major)); 1159 1160 qflag = dp->d_qflag; 1161 ASSERT(qflag & QISDRV); 1162 sqtype = dp->d_sqtype; 1163 1164 /* create perdm_t if needed */ 1165 if (NEED_DM(dp->d_dmp, qflag)) 1166 dp->d_dmp = hold_dm(str, qflag, sqtype); 1167 1168 dmp = dp->d_dmp; 1169 sflag = 0; 1170 } 1171 1172 TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS, 1173 "qattach:qflag == %X(%X)", qflag, *devp); 1174 1175 /* setq might sleep in allocator - avoid holding locks. */ 1176 setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE); 1177 1178 /* 1179 * Before calling the module's open routine, set up the q_next 1180 * pointer for inserting a module in the middle of a stream. 1181 * 1182 * Note that we can always set _QINSERTING and set up q_next 1183 * pointer for both inserting and pushing a module. Then there 1184 * is no need for the is_insert parameter. In insertq(), called 1185 * by qprocson(), assume that q_next of the new module always points 1186 * to the correct queue and use it for insertion. Everything should 1187 * work out fine. But in the first release of _I_INSERT, we 1188 * distinguish between inserting and pushing to make sure that 1189 * pushing a module follows the same code path as before. 1190 */ 1191 if (is_insert) { 1192 rq->q_flag |= _QINSERTING; 1193 rq->q_next = qp; 1194 } 1195 1196 /* 1197 * If there is an outer perimeter get exclusive access during 1198 * the open procedure. Bump up the reference count on the queue. 1199 */ 1200 entersq(rq->q_syncq, SQ_OPENCLOSE); 1201 error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp); 1202 if (error != 0) 1203 goto failed; 1204 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1205 ASSERT(qprocsareon(rq)); 1206 return (0); 1207 1208 failed: 1209 rq->q_flag &= ~_QINSERTING; 1210 if (backq(wrq) != NULL && backq(wrq)->q_next == wrq) 1211 qprocsoff(rq); 1212 leavesq(rq->q_syncq, SQ_OPENCLOSE); 1213 rq->q_next = wrq->q_next = NULL; 1214 qdetach(rq, 0, 0, crp, B_FALSE); 1215 return (error); 1216 } 1217 1218 /* 1219 * Handle second open of stream. For modules, set the 1220 * last argument to MODOPEN and do not pass any open flags. 1221 * Ignore dummydev since this is not the first open. 1222 */ 1223 int 1224 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp) 1225 { 1226 int error; 1227 dev_t dummydev; 1228 queue_t *wqp = _WR(qp); 1229 1230 ASSERT(qp->q_flag & QREADR); 1231 entersq(qp->q_syncq, SQ_OPENCLOSE); 1232 1233 dummydev = *devp; 1234 if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev, 1235 (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) { 1236 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1237 mutex_enter(&STREAM(qp)->sd_lock); 1238 qp->q_stream->sd_flag |= STREOPENFAIL; 1239 mutex_exit(&STREAM(qp)->sd_lock); 1240 return (error); 1241 } 1242 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1243 1244 /* 1245 * successful open should have done qprocson() 1246 */ 1247 ASSERT(qprocsareon(_RD(qp))); 1248 return (0); 1249 } 1250 1251 /* 1252 * Detach a stream module or device. 1253 * If clmode == 1 then the module or driver was opened and its 1254 * close routine must be called. If clmode == 0, the module 1255 * or driver was never opened or the open failed, and so its close 1256 * should not be called. 1257 */ 1258 void 1259 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove) 1260 { 1261 queue_t *wqp = _WR(qp); 1262 ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB)); 1263 1264 if (STREAM_NEEDSERVICE(STREAM(qp))) 1265 stream_runservice(STREAM(qp)); 1266 1267 if (clmode) { 1268 /* 1269 * Make sure that all the messages on the write side syncq are 1270 * processed and nothing is left. Since we are closing, no new 1271 * messages may appear there. 1272 */ 1273 wait_q_syncq(wqp); 1274 1275 entersq(qp->q_syncq, SQ_OPENCLOSE); 1276 if (is_remove) { 1277 mutex_enter(QLOCK(qp)); 1278 qp->q_flag |= _QREMOVING; 1279 mutex_exit(QLOCK(qp)); 1280 } 1281 (*qp->q_qinfo->qi_qclose)(qp, flag, crp); 1282 /* 1283 * Check that qprocsoff() was actually called. 1284 */ 1285 ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE)); 1286 1287 leavesq(qp->q_syncq, SQ_OPENCLOSE); 1288 } else { 1289 disable_svc(qp); 1290 } 1291 1292 /* 1293 * Allow any threads blocked in entersq to proceed and discover 1294 * the QWCLOSE is set. 1295 * Note: This assumes that all users of entersq check QWCLOSE. 1296 * Currently runservice is the only entersq that can happen 1297 * after removeq has finished. 1298 * Removeq will have discarded all messages destined to the closing 1299 * pair of queues from the syncq. 1300 * NOTE: Calling a function inside an assert is unconventional. 1301 * However, it does not cause any problem since flush_syncq() does 1302 * not change any state except when it returns non-zero i.e. 1303 * when the assert will trigger. 1304 */ 1305 ASSERT(flush_syncq(qp->q_syncq, qp) == 0); 1306 ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0); 1307 ASSERT((qp->q_flag & QPERMOD) || 1308 ((qp->q_syncq->sq_head == NULL) && 1309 (wqp->q_syncq->sq_head == NULL))); 1310 1311 /* release any fmodsw_impl_t structure held on behalf of the queue */ 1312 ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV); 1313 if (qp->q_fp != NULL) 1314 fmodsw_rele(qp->q_fp); 1315 1316 /* freeq removes us from the outer perimeter if any */ 1317 freeq(qp); 1318 } 1319 1320 /* Prevent service procedures from being called */ 1321 void 1322 disable_svc(queue_t *qp) 1323 { 1324 queue_t *wqp = _WR(qp); 1325 1326 ASSERT(qp->q_flag & QREADR); 1327 mutex_enter(QLOCK(qp)); 1328 qp->q_flag |= QWCLOSE; 1329 mutex_exit(QLOCK(qp)); 1330 mutex_enter(QLOCK(wqp)); 1331 wqp->q_flag |= QWCLOSE; 1332 mutex_exit(QLOCK(wqp)); 1333 } 1334 1335 /* Allow service procedures to be called again */ 1336 void 1337 enable_svc(queue_t *qp) 1338 { 1339 queue_t *wqp = _WR(qp); 1340 1341 ASSERT(qp->q_flag & QREADR); 1342 mutex_enter(QLOCK(qp)); 1343 qp->q_flag &= ~QWCLOSE; 1344 mutex_exit(QLOCK(qp)); 1345 mutex_enter(QLOCK(wqp)); 1346 wqp->q_flag &= ~QWCLOSE; 1347 mutex_exit(QLOCK(wqp)); 1348 } 1349 1350 /* 1351 * Remove queue from qhead/qtail if it is enabled. 1352 * Only reset QENAB if the queue was removed from the runlist. 1353 * A queue goes through 3 stages: 1354 * It is on the service list and QENAB is set. 1355 * It is removed from the service list but QENAB is still set. 1356 * QENAB gets changed to QINSERVICE. 1357 * QINSERVICE is reset (when the service procedure is done) 1358 * Thus we can not reset QENAB unless we actually removed it from the service 1359 * queue. 1360 */ 1361 void 1362 remove_runlist(queue_t *qp) 1363 { 1364 if (qp->q_flag & QENAB && qhead != NULL) { 1365 queue_t *q_chase; 1366 queue_t *q_curr; 1367 int removed; 1368 1369 mutex_enter(&service_queue); 1370 RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed); 1371 mutex_exit(&service_queue); 1372 if (removed) { 1373 STRSTAT(qremoved); 1374 qp->q_flag &= ~QENAB; 1375 } 1376 } 1377 } 1378 1379 1380 /* 1381 * Wait for any pending service processing to complete. 1382 * The removal of queues from the runlist is not atomic with the 1383 * clearing of the QENABLED flag and setting the INSERVICE flag. 1384 * consequently it is possible for remove_runlist in strclose 1385 * to not find the queue on the runlist but for it to be QENABLED 1386 * and not yet INSERVICE -> hence wait_svc needs to check QENABLED 1387 * as well as INSERVICE. 1388 */ 1389 void 1390 wait_svc(queue_t *qp) 1391 { 1392 queue_t *wqp = _WR(qp); 1393 1394 ASSERT(qp->q_flag & QREADR); 1395 1396 /* 1397 * Try to remove queues from qhead/qtail list. 1398 */ 1399 if (qhead != NULL) { 1400 remove_runlist(qp); 1401 remove_runlist(wqp); 1402 } 1403 /* 1404 * Wait till the syncqs associated with the queue disappear from the 1405 * background processing list. 1406 * This only needs to be done for non-PERMOD perimeters since 1407 * for PERMOD perimeters the syncq may be shared and will only be freed 1408 * when the last module/driver is unloaded. 1409 * If for PERMOD perimeters queue was on the syncq list, removeq() 1410 * should call propagate_syncq() or drain_syncq() for it. Both of these 1411 * functions remove the queue from its syncq list, so sqthread will not 1412 * try to access the queue. 1413 */ 1414 if (!(qp->q_flag & QPERMOD)) { 1415 syncq_t *rsq = qp->q_syncq; 1416 syncq_t *wsq = wqp->q_syncq; 1417 1418 /* 1419 * Disable rsq and wsq and wait for any background processing of 1420 * syncq to complete. 1421 */ 1422 wait_sq_svc(rsq); 1423 if (wsq != rsq) 1424 wait_sq_svc(wsq); 1425 } 1426 1427 mutex_enter(QLOCK(qp)); 1428 while (qp->q_flag & (QINSERVICE|QENAB)) 1429 cv_wait(&qp->q_wait, QLOCK(qp)); 1430 mutex_exit(QLOCK(qp)); 1431 mutex_enter(QLOCK(wqp)); 1432 while (wqp->q_flag & (QINSERVICE|QENAB)) 1433 cv_wait(&wqp->q_wait, QLOCK(wqp)); 1434 mutex_exit(QLOCK(wqp)); 1435 } 1436 1437 /* 1438 * Put ioctl data from userland buffer `arg' into the mblk chain `bp'. 1439 * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may 1440 * also be set, and is passed through to allocb_cred_wait(). 1441 * 1442 * Returns errno on failure, zero on success. 1443 */ 1444 int 1445 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr) 1446 { 1447 mblk_t *tmp; 1448 ssize_t count; 1449 int error = 0; 1450 1451 ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K || 1452 (flag & (U_TO_K | K_TO_K)) == K_TO_K); 1453 1454 if (bp->b_datap->db_type == M_IOCTL) { 1455 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1456 } else { 1457 ASSERT(bp->b_datap->db_type == M_COPYIN); 1458 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1459 } 1460 /* 1461 * strdoioctl validates ioc_count, so if this assert fails it 1462 * cannot be due to user error. 1463 */ 1464 ASSERT(count >= 0); 1465 1466 if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr, 1467 curproc->p_pid)) == NULL) { 1468 return (error); 1469 } 1470 error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K)); 1471 if (error != 0) { 1472 freeb(tmp); 1473 return (error); 1474 } 1475 DB_CPID(tmp) = curproc->p_pid; 1476 tmp->b_wptr += count; 1477 bp->b_cont = tmp; 1478 1479 return (0); 1480 } 1481 1482 /* 1483 * Copy ioctl data to user-land. Return non-zero errno on failure, 1484 * 0 for success. 1485 */ 1486 int 1487 getiocd(mblk_t *bp, char *arg, int copymode) 1488 { 1489 ssize_t count; 1490 size_t n; 1491 int error; 1492 1493 if (bp->b_datap->db_type == M_IOCACK) 1494 count = ((struct iocblk *)bp->b_rptr)->ioc_count; 1495 else { 1496 ASSERT(bp->b_datap->db_type == M_COPYOUT); 1497 count = ((struct copyreq *)bp->b_rptr)->cq_size; 1498 } 1499 ASSERT(count >= 0); 1500 1501 for (bp = bp->b_cont; bp && count; 1502 count -= n, bp = bp->b_cont, arg += n) { 1503 n = MIN(count, bp->b_wptr - bp->b_rptr); 1504 error = strcopyout(bp->b_rptr, arg, n, copymode); 1505 if (error) 1506 return (error); 1507 } 1508 ASSERT(count == 0); 1509 return (0); 1510 } 1511 1512 /* 1513 * Allocate a linkinfo entry given the write queue of the 1514 * bottom module of the top stream and the write queue of the 1515 * stream head of the bottom stream. 1516 */ 1517 linkinfo_t * 1518 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown) 1519 { 1520 linkinfo_t *linkp; 1521 1522 linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP); 1523 1524 linkp->li_lblk.l_qtop = qup; 1525 linkp->li_lblk.l_qbot = qdown; 1526 linkp->li_fpdown = fpdown; 1527 1528 mutex_enter(&strresources); 1529 linkp->li_next = linkinfo_list; 1530 linkp->li_prev = NULL; 1531 if (linkp->li_next) 1532 linkp->li_next->li_prev = linkp; 1533 linkinfo_list = linkp; 1534 linkp->li_lblk.l_index = ++lnk_id; 1535 ASSERT(lnk_id != 0); /* this should never wrap in practice */ 1536 mutex_exit(&strresources); 1537 1538 return (linkp); 1539 } 1540 1541 /* 1542 * Free a linkinfo entry. 1543 */ 1544 void 1545 lbfree(linkinfo_t *linkp) 1546 { 1547 mutex_enter(&strresources); 1548 if (linkp->li_next) 1549 linkp->li_next->li_prev = linkp->li_prev; 1550 if (linkp->li_prev) 1551 linkp->li_prev->li_next = linkp->li_next; 1552 else 1553 linkinfo_list = linkp->li_next; 1554 mutex_exit(&strresources); 1555 1556 kmem_cache_free(linkinfo_cache, linkp); 1557 } 1558 1559 /* 1560 * Check for a potential linking cycle. 1561 * Return 1 if a link will result in a cycle, 1562 * and 0 otherwise. 1563 */ 1564 int 1565 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss) 1566 { 1567 struct mux_node *np; 1568 struct mux_edge *ep; 1569 int i; 1570 major_t lomaj; 1571 major_t upmaj; 1572 /* 1573 * if the lower stream is a pipe/FIFO, return, since link 1574 * cycles can not happen on pipes/FIFOs 1575 */ 1576 if (lostp->sd_vnode->v_type == VFIFO) 1577 return (0); 1578 1579 for (i = 0; i < ss->ss_devcnt; i++) { 1580 np = &ss->ss_mux_nodes[i]; 1581 MUX_CLEAR(np); 1582 } 1583 lomaj = getmajor(lostp->sd_vnode->v_rdev); 1584 upmaj = getmajor(upstp->sd_vnode->v_rdev); 1585 np = &ss->ss_mux_nodes[lomaj]; 1586 for (;;) { 1587 if (!MUX_DIDVISIT(np)) { 1588 if (np->mn_imaj == upmaj) 1589 return (1); 1590 if (np->mn_outp == NULL) { 1591 MUX_VISIT(np); 1592 if (np->mn_originp == NULL) 1593 return (0); 1594 np = np->mn_originp; 1595 continue; 1596 } 1597 MUX_VISIT(np); 1598 np->mn_startp = np->mn_outp; 1599 } else { 1600 if (np->mn_startp == NULL) { 1601 if (np->mn_originp == NULL) 1602 return (0); 1603 else { 1604 np = np->mn_originp; 1605 continue; 1606 } 1607 } 1608 /* 1609 * If ep->me_nodep is a FIFO (me_nodep == NULL), 1610 * ignore the edge and move on. ep->me_nodep gets 1611 * set to NULL in mux_addedge() if it is a FIFO. 1612 * 1613 */ 1614 ep = np->mn_startp; 1615 np->mn_startp = ep->me_nextp; 1616 if (ep->me_nodep == NULL) 1617 continue; 1618 ep->me_nodep->mn_originp = np; 1619 np = ep->me_nodep; 1620 } 1621 } 1622 } 1623 1624 /* 1625 * Find linkinfo entry corresponding to the parameters. 1626 */ 1627 linkinfo_t * 1628 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss) 1629 { 1630 linkinfo_t *linkp; 1631 struct mux_edge *mep; 1632 struct mux_node *mnp; 1633 queue_t *qup; 1634 1635 mutex_enter(&strresources); 1636 if ((type & LINKTYPEMASK) == LINKNORMAL) { 1637 qup = getendq(stp->sd_wrq); 1638 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1639 if ((qup == linkp->li_lblk.l_qtop) && 1640 (!index || (index == linkp->li_lblk.l_index))) { 1641 mutex_exit(&strresources); 1642 return (linkp); 1643 } 1644 } 1645 } else { 1646 ASSERT((type & LINKTYPEMASK) == LINKPERSIST); 1647 mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)]; 1648 mep = mnp->mn_outp; 1649 while (mep) { 1650 if ((index == 0) || (index == mep->me_muxid)) 1651 break; 1652 mep = mep->me_nextp; 1653 } 1654 if (!mep) { 1655 mutex_exit(&strresources); 1656 return (NULL); 1657 } 1658 for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) { 1659 if ((!linkp->li_lblk.l_qtop) && 1660 (mep->me_muxid == linkp->li_lblk.l_index)) { 1661 mutex_exit(&strresources); 1662 return (linkp); 1663 } 1664 } 1665 } 1666 mutex_exit(&strresources); 1667 return (NULL); 1668 } 1669 1670 /* 1671 * Given a queue ptr, follow the chain of q_next pointers until you reach the 1672 * last queue on the chain and return it. 1673 */ 1674 queue_t * 1675 getendq(queue_t *q) 1676 { 1677 ASSERT(q != NULL); 1678 while (_SAMESTR(q)) 1679 q = q->q_next; 1680 return (q); 1681 } 1682 1683 /* 1684 * Wait for the syncq count to drop to zero. 1685 * sq could be either outer or inner. 1686 */ 1687 1688 static void 1689 wait_syncq(syncq_t *sq) 1690 { 1691 uint16_t count; 1692 1693 mutex_enter(SQLOCK(sq)); 1694 count = sq->sq_count; 1695 SQ_PUTLOCKS_ENTER(sq); 1696 SUM_SQ_PUTCOUNTS(sq, count); 1697 while (count != 0) { 1698 sq->sq_flags |= SQ_WANTWAKEUP; 1699 SQ_PUTLOCKS_EXIT(sq); 1700 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1701 count = sq->sq_count; 1702 SQ_PUTLOCKS_ENTER(sq); 1703 SUM_SQ_PUTCOUNTS(sq, count); 1704 } 1705 SQ_PUTLOCKS_EXIT(sq); 1706 mutex_exit(SQLOCK(sq)); 1707 } 1708 1709 /* 1710 * Wait while there are any messages for the queue in its syncq. 1711 */ 1712 static void 1713 wait_q_syncq(queue_t *q) 1714 { 1715 if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1716 syncq_t *sq = q->q_syncq; 1717 1718 mutex_enter(SQLOCK(sq)); 1719 while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) { 1720 sq->sq_flags |= SQ_WANTWAKEUP; 1721 cv_wait(&sq->sq_wait, SQLOCK(sq)); 1722 } 1723 mutex_exit(SQLOCK(sq)); 1724 } 1725 } 1726 1727 1728 int 1729 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp, 1730 int lhlink) 1731 { 1732 struct stdata *stp; 1733 struct strioctl strioc; 1734 struct linkinfo *linkp; 1735 struct stdata *stpdown; 1736 struct streamtab *str; 1737 queue_t *passq; 1738 syncq_t *passyncq; 1739 queue_t *rq; 1740 cdevsw_impl_t *dp; 1741 uint32_t qflag; 1742 uint32_t sqtype; 1743 perdm_t *dmp; 1744 int error = 0; 1745 netstack_t *ns; 1746 str_stack_t *ss; 1747 1748 stp = vp->v_stream; 1749 TRACE_1(TR_FAC_STREAMS_FR, 1750 TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp); 1751 /* 1752 * Test for invalid upper stream 1753 */ 1754 if (stp->sd_flag & STRHUP) { 1755 return (ENXIO); 1756 } 1757 if (vp->v_type == VFIFO) { 1758 return (EINVAL); 1759 } 1760 if (stp->sd_strtab == NULL) { 1761 return (EINVAL); 1762 } 1763 if (!stp->sd_strtab->st_muxwinit) { 1764 return (EINVAL); 1765 } 1766 if (fpdown == NULL) { 1767 return (EBADF); 1768 } 1769 ns = netstack_find_by_cred(crp); 1770 ASSERT(ns != NULL); 1771 ss = ns->netstack_str; 1772 ASSERT(ss != NULL); 1773 1774 if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) { 1775 netstack_rele(ss->ss_netstack); 1776 return (EINVAL); 1777 } 1778 mutex_enter(&muxifier); 1779 if (stp->sd_flag & STPLEX) { 1780 mutex_exit(&muxifier); 1781 netstack_rele(ss->ss_netstack); 1782 return (ENXIO); 1783 } 1784 1785 /* 1786 * Test for invalid lower stream. 1787 * The check for the v_type != VFIFO and having a major 1788 * number not >= devcnt is done to avoid problems with 1789 * adding mux_node entry past the end of mux_nodes[]. 1790 * For FIFO's we don't add an entry so this isn't a 1791 * problem. 1792 */ 1793 if (((stpdown = fpdown->f_vnode->v_stream) == NULL) || 1794 (stpdown == stp) || (stpdown->sd_flag & 1795 (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) || 1796 ((stpdown->sd_vnode->v_type != VFIFO) && 1797 (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) || 1798 linkcycle(stp, stpdown, ss)) { 1799 mutex_exit(&muxifier); 1800 netstack_rele(ss->ss_netstack); 1801 return (EINVAL); 1802 } 1803 TRACE_1(TR_FAC_STREAMS_FR, 1804 TR_STPDOWN, "stpdown:%p", stpdown); 1805 rq = getendq(stp->sd_wrq); 1806 if (cmd == I_PLINK) 1807 rq = NULL; 1808 1809 linkp = alloclink(rq, stpdown->sd_wrq, fpdown); 1810 1811 strioc.ic_cmd = cmd; 1812 strioc.ic_timout = INFTIM; 1813 strioc.ic_len = sizeof (struct linkblk); 1814 strioc.ic_dp = (char *)&linkp->li_lblk; 1815 1816 /* 1817 * STRPLUMB protects plumbing changes and should be set before 1818 * link_addpassthru()/link_rempassthru() are called, so it is set here 1819 * and cleared in the end of mlink when passthru queue is removed. 1820 * Setting of STRPLUMB prevents reopens of the stream while passthru 1821 * queue is in-place (it is not a proper module and doesn't have open 1822 * entry point). 1823 * 1824 * STPLEX prevents any threads from entering the stream from above. It 1825 * can't be set before the call to link_addpassthru() because putnext 1826 * from below may cause stream head I/O routines to be called and these 1827 * routines assert that STPLEX is not set. After link_addpassthru() 1828 * nothing may come from below since the pass queue syncq is blocked. 1829 * Note also that STPLEX should be cleared before the call to 1830 * link_rempassthru() since when messages start flowing to the stream 1831 * head (e.g. because of message propagation from the pass queue) stream 1832 * head I/O routines may be called with STPLEX flag set. 1833 * 1834 * When STPLEX is set, nothing may come into the stream from above and 1835 * it is safe to do a setq which will change stream head. So, the 1836 * correct sequence of actions is: 1837 * 1838 * 1) Set STRPLUMB 1839 * 2) Call link_addpassthru() 1840 * 3) Set STPLEX 1841 * 4) Call setq and update the stream state 1842 * 5) Clear STPLEX 1843 * 6) Call link_rempassthru() 1844 * 7) Clear STRPLUMB 1845 * 1846 * The same sequence applies to munlink() code. 1847 */ 1848 mutex_enter(&stpdown->sd_lock); 1849 stpdown->sd_flag |= STRPLUMB; 1850 mutex_exit(&stpdown->sd_lock); 1851 /* 1852 * Add passthru queue below lower mux. This will block 1853 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 1854 */ 1855 passq = link_addpassthru(stpdown); 1856 1857 mutex_enter(&stpdown->sd_lock); 1858 stpdown->sd_flag |= STPLEX; 1859 mutex_exit(&stpdown->sd_lock); 1860 1861 rq = _RD(stpdown->sd_wrq); 1862 /* 1863 * There may be messages in the streamhead's syncq due to messages 1864 * that arrived before link_addpassthru() was done. To avoid 1865 * background processing of the syncq happening simultaneous with 1866 * setq processing, we disable the streamhead syncq and wait until 1867 * existing background thread finishes working on it. 1868 */ 1869 wait_sq_svc(rq->q_syncq); 1870 passyncq = passq->q_syncq; 1871 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1872 blocksq(passyncq, SQ_BLOCKED, 0); 1873 1874 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 1875 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 1876 rq->q_ptr = _WR(rq)->q_ptr = NULL; 1877 1878 /* setq might sleep in allocator - avoid holding locks. */ 1879 /* Note: we are holding muxifier here. */ 1880 1881 str = stp->sd_strtab; 1882 dp = &devimpl[getmajor(vp->v_rdev)]; 1883 ASSERT(dp->d_str == str); 1884 1885 qflag = dp->d_qflag; 1886 sqtype = dp->d_sqtype; 1887 1888 /* create perdm_t if needed */ 1889 if (NEED_DM(dp->d_dmp, qflag)) 1890 dp->d_dmp = hold_dm(str, qflag, sqtype); 1891 1892 dmp = dp->d_dmp; 1893 1894 setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype, 1895 B_TRUE); 1896 1897 /* 1898 * XXX Remove any "odd" messages from the queue. 1899 * Keep only M_DATA, M_PROTO, M_PCPROTO. 1900 */ 1901 error = strdoioctl(stp, &strioc, FNATIVE, 1902 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 1903 if (error != 0) { 1904 lbfree(linkp); 1905 1906 if (!(passyncq->sq_flags & SQ_BLOCKED)) 1907 blocksq(passyncq, SQ_BLOCKED, 0); 1908 /* 1909 * Restore the stream head queue and then remove 1910 * the passq. Turn off STPLEX before we turn on 1911 * the stream by removing the passq. 1912 */ 1913 rq->q_ptr = _WR(rq)->q_ptr = stpdown; 1914 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, 1915 B_TRUE); 1916 1917 mutex_enter(&stpdown->sd_lock); 1918 stpdown->sd_flag &= ~STPLEX; 1919 mutex_exit(&stpdown->sd_lock); 1920 1921 link_rempassthru(passq); 1922 1923 mutex_enter(&stpdown->sd_lock); 1924 stpdown->sd_flag &= ~STRPLUMB; 1925 /* Wakeup anyone waiting for STRPLUMB to clear. */ 1926 cv_broadcast(&stpdown->sd_monitor); 1927 mutex_exit(&stpdown->sd_lock); 1928 1929 mutex_exit(&muxifier); 1930 netstack_rele(ss->ss_netstack); 1931 return (error); 1932 } 1933 mutex_enter(&fpdown->f_tlock); 1934 fpdown->f_count++; 1935 mutex_exit(&fpdown->f_tlock); 1936 1937 /* 1938 * if we've made it here the linkage is all set up so we should also 1939 * set up the layered driver linkages 1940 */ 1941 1942 ASSERT((cmd == I_LINK) || (cmd == I_PLINK)); 1943 if (cmd == I_LINK) { 1944 ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL); 1945 } else { 1946 ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST); 1947 } 1948 1949 link_rempassthru(passq); 1950 1951 mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss); 1952 1953 /* 1954 * Mark the upper stream as having dependent links 1955 * so that strclose can clean it up. 1956 */ 1957 if (cmd == I_LINK) { 1958 mutex_enter(&stp->sd_lock); 1959 stp->sd_flag |= STRHASLINKS; 1960 mutex_exit(&stp->sd_lock); 1961 } 1962 /* 1963 * Wake up any other processes that may have been 1964 * waiting on the lower stream. These will all 1965 * error out. 1966 */ 1967 mutex_enter(&stpdown->sd_lock); 1968 /* The passthru module is removed so we may release STRPLUMB */ 1969 stpdown->sd_flag &= ~STRPLUMB; 1970 cv_broadcast(&rq->q_wait); 1971 cv_broadcast(&_WR(rq)->q_wait); 1972 cv_broadcast(&stpdown->sd_monitor); 1973 mutex_exit(&stpdown->sd_lock); 1974 mutex_exit(&muxifier); 1975 *rvalp = linkp->li_lblk.l_index; 1976 netstack_rele(ss->ss_netstack); 1977 return (0); 1978 } 1979 1980 int 1981 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink) 1982 { 1983 int ret; 1984 struct file *fpdown; 1985 1986 fpdown = getf(arg); 1987 ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink); 1988 if (fpdown != NULL) 1989 releasef(arg); 1990 return (ret); 1991 } 1992 1993 /* 1994 * Unlink a multiplexor link. Stp is the controlling stream for the 1995 * link, and linkp points to the link's entry in the linkinfo list. 1996 * The muxifier lock must be held on entry and is dropped on exit. 1997 * 1998 * NOTE : Currently it is assumed that mux would process all the messages 1999 * sitting on it's queue before ACKing the UNLINK. It is the responsibility 2000 * of the mux to handle all the messages that arrive before UNLINK. 2001 * If the mux has to send down messages on its lower stream before 2002 * ACKing I_UNLINK, then it *should* know to handle messages even 2003 * after the UNLINK is acked (actually it should be able to handle till we 2004 * re-block the read side of the pass queue here). If the mux does not 2005 * open up the lower stream, any messages that arrive during UNLINK 2006 * will be put in the stream head. In the case of lower stream opening 2007 * up, some messages might land in the stream head depending on when 2008 * the message arrived and when the read side of the pass queue was 2009 * re-blocked. 2010 */ 2011 int 2012 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp, 2013 str_stack_t *ss) 2014 { 2015 struct strioctl strioc; 2016 struct stdata *stpdown; 2017 queue_t *rq, *wrq; 2018 queue_t *passq; 2019 syncq_t *passyncq; 2020 int error = 0; 2021 file_t *fpdown; 2022 2023 ASSERT(MUTEX_HELD(&muxifier)); 2024 2025 stpdown = linkp->li_fpdown->f_vnode->v_stream; 2026 2027 /* 2028 * See the comment in mlink() concerning STRPLUMB/STPLEX flags. 2029 */ 2030 mutex_enter(&stpdown->sd_lock); 2031 stpdown->sd_flag |= STRPLUMB; 2032 mutex_exit(&stpdown->sd_lock); 2033 2034 /* 2035 * Add passthru queue below lower mux. This will block 2036 * syncqs of lower muxs read queue during I_LINK/I_UNLINK. 2037 */ 2038 passq = link_addpassthru(stpdown); 2039 2040 if ((flag & LINKTYPEMASK) == LINKNORMAL) 2041 strioc.ic_cmd = I_UNLINK; 2042 else 2043 strioc.ic_cmd = I_PUNLINK; 2044 strioc.ic_timout = INFTIM; 2045 strioc.ic_len = sizeof (struct linkblk); 2046 strioc.ic_dp = (char *)&linkp->li_lblk; 2047 2048 error = strdoioctl(stp, &strioc, FNATIVE, 2049 K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp); 2050 2051 /* 2052 * If there was an error and this is not called via strclose, 2053 * return to the user. Otherwise, pretend there was no error 2054 * and close the link. 2055 */ 2056 if (error) { 2057 if (flag & LINKCLOSE) { 2058 cmn_err(CE_WARN, "KERNEL: munlink: could not perform " 2059 "unlink ioctl, closing anyway (%d)\n", error); 2060 } else { 2061 link_rempassthru(passq); 2062 mutex_enter(&stpdown->sd_lock); 2063 stpdown->sd_flag &= ~STRPLUMB; 2064 cv_broadcast(&stpdown->sd_monitor); 2065 mutex_exit(&stpdown->sd_lock); 2066 mutex_exit(&muxifier); 2067 return (error); 2068 } 2069 } 2070 2071 mux_rmvedge(stp, linkp->li_lblk.l_index, ss); 2072 fpdown = linkp->li_fpdown; 2073 lbfree(linkp); 2074 2075 /* 2076 * We go ahead and drop muxifier here--it's a nasty global lock that 2077 * can slow others down. It's okay to since attempts to mlink() this 2078 * stream will be stopped because STPLEX is still set in the stdata 2079 * structure, and munlink() is stopped because mux_rmvedge() and 2080 * lbfree() have removed it from mux_nodes[] and linkinfo_list, 2081 * respectively. Note that we defer the closef() of fpdown until 2082 * after we drop muxifier since strclose() can call munlinkall(). 2083 */ 2084 mutex_exit(&muxifier); 2085 2086 wrq = stpdown->sd_wrq; 2087 rq = _RD(wrq); 2088 2089 /* 2090 * Get rid of outstanding service procedure runs, before we make 2091 * it a stream head, since a stream head doesn't have any service 2092 * procedure. 2093 */ 2094 disable_svc(rq); 2095 wait_svc(rq); 2096 2097 /* 2098 * Since we don't disable the syncq for QPERMOD, we wait for whatever 2099 * is queued up to be finished. mux should take care that nothing is 2100 * send down to this queue. We should do it now as we're going to block 2101 * passyncq if it was unblocked. 2102 */ 2103 if (wrq->q_flag & QPERMOD) { 2104 syncq_t *sq = wrq->q_syncq; 2105 2106 mutex_enter(SQLOCK(sq)); 2107 while (wrq->q_sqflags & Q_SQQUEUED) { 2108 sq->sq_flags |= SQ_WANTWAKEUP; 2109 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2110 } 2111 mutex_exit(SQLOCK(sq)); 2112 } 2113 passyncq = passq->q_syncq; 2114 if (!(passyncq->sq_flags & SQ_BLOCKED)) { 2115 2116 syncq_t *sq, *outer; 2117 2118 /* 2119 * Messages could be flowing from underneath. We will 2120 * block the read side of the passq. This would be 2121 * sufficient for QPAIR and QPERQ muxes to ensure 2122 * that no data is flowing up into this queue 2123 * and hence no thread active in this instance of 2124 * lower mux. But for QPERMOD and QMTOUTPERIM there 2125 * could be messages on the inner and outer/inner 2126 * syncqs respectively. We will wait for them to drain. 2127 * Because passq is blocked messages end up in the syncq 2128 * And qfill_syncq could possibly end up setting QFULL 2129 * which will access the rq->q_flag. Hence, we have to 2130 * acquire the QLOCK in setq. 2131 * 2132 * XXX Messages can also flow from top into this 2133 * queue though the unlink is over (Ex. some instance 2134 * in putnext() called from top that has still not 2135 * accessed this queue. And also putq(lowerq) ?). 2136 * Solution : How about blocking the l_qtop queue ? 2137 * Do we really care about such pure D_MP muxes ? 2138 */ 2139 2140 blocksq(passyncq, SQ_BLOCKED, 0); 2141 2142 sq = rq->q_syncq; 2143 if ((outer = sq->sq_outer) != NULL) { 2144 2145 /* 2146 * We have to just wait for the outer sq_count 2147 * drop to zero. As this does not prevent new 2148 * messages to enter the outer perimeter, this 2149 * is subject to starvation. 2150 * 2151 * NOTE :Because of blocksq above, messages could 2152 * be in the inner syncq only because of some 2153 * thread holding the outer perimeter exclusively. 2154 * Hence it would be sufficient to wait for the 2155 * exclusive holder of the outer perimeter to drain 2156 * the inner and outer syncqs. But we will not depend 2157 * on this feature and hence check the inner syncqs 2158 * separately. 2159 */ 2160 wait_syncq(outer); 2161 } 2162 2163 2164 /* 2165 * There could be messages destined for 2166 * this queue. Let the exclusive holder 2167 * drain it. 2168 */ 2169 2170 wait_syncq(sq); 2171 ASSERT((rq->q_flag & QPERMOD) || 2172 ((rq->q_syncq->sq_head == NULL) && 2173 (_WR(rq)->q_syncq->sq_head == NULL))); 2174 } 2175 2176 /* 2177 * We haven't taken care of QPERMOD case yet. QPERMOD is a special 2178 * case as we don't disable its syncq or remove it off the syncq 2179 * service list. 2180 */ 2181 if (rq->q_flag & QPERMOD) { 2182 syncq_t *sq = rq->q_syncq; 2183 2184 mutex_enter(SQLOCK(sq)); 2185 while (rq->q_sqflags & Q_SQQUEUED) { 2186 sq->sq_flags |= SQ_WANTWAKEUP; 2187 cv_wait(&sq->sq_wait, SQLOCK(sq)); 2188 } 2189 mutex_exit(SQLOCK(sq)); 2190 } 2191 2192 /* 2193 * flush_syncq changes states only when there are some messages to 2194 * free, i.e. when it returns non-zero value to return. 2195 */ 2196 ASSERT(flush_syncq(rq->q_syncq, rq) == 0); 2197 ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0); 2198 2199 /* 2200 * Nobody else should know about this queue now. 2201 * If the mux did not process the messages before 2202 * acking the I_UNLINK, free them now. 2203 */ 2204 2205 flushq(rq, FLUSHALL); 2206 flushq(_WR(rq), FLUSHALL); 2207 2208 /* 2209 * Convert the mux lower queue into a stream head queue. 2210 * Turn off STPLEX before we turn on the stream by removing the passq. 2211 */ 2212 rq->q_ptr = wrq->q_ptr = stpdown; 2213 setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE); 2214 2215 ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE); 2216 ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq)); 2217 2218 enable_svc(rq); 2219 2220 /* 2221 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still 2222 * needs to be set to prevent reopen() of the stream - such reopen may 2223 * try to call non-existent pass queue open routine and panic. 2224 */ 2225 mutex_enter(&stpdown->sd_lock); 2226 stpdown->sd_flag &= ~STPLEX; 2227 mutex_exit(&stpdown->sd_lock); 2228 2229 ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) || 2230 ((flag & LINKTYPEMASK) == LINKPERSIST)); 2231 2232 /* clean up the layered driver linkages */ 2233 if ((flag & LINKTYPEMASK) == LINKNORMAL) { 2234 ldi_munlink_fp(stp, fpdown, LINKNORMAL); 2235 } else { 2236 ldi_munlink_fp(stp, fpdown, LINKPERSIST); 2237 } 2238 2239 link_rempassthru(passq); 2240 2241 /* 2242 * Now all plumbing changes are finished and STRPLUMB is no 2243 * longer needed. 2244 */ 2245 mutex_enter(&stpdown->sd_lock); 2246 stpdown->sd_flag &= ~STRPLUMB; 2247 cv_broadcast(&stpdown->sd_monitor); 2248 mutex_exit(&stpdown->sd_lock); 2249 2250 (void) closef(fpdown); 2251 return (0); 2252 } 2253 2254 /* 2255 * Unlink all multiplexor links for which stp is the controlling stream. 2256 * Return 0, or a non-zero errno on failure. 2257 */ 2258 int 2259 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss) 2260 { 2261 linkinfo_t *linkp; 2262 int error = 0; 2263 2264 mutex_enter(&muxifier); 2265 while (linkp = findlinks(stp, 0, flag, ss)) { 2266 /* 2267 * munlink() releases the muxifier lock. 2268 */ 2269 if (error = munlink(stp, linkp, flag, crp, rvalp, ss)) 2270 return (error); 2271 mutex_enter(&muxifier); 2272 } 2273 mutex_exit(&muxifier); 2274 return (0); 2275 } 2276 2277 /* 2278 * A multiplexor link has been made. Add an 2279 * edge to the directed graph. 2280 */ 2281 void 2282 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss) 2283 { 2284 struct mux_node *np; 2285 struct mux_edge *ep; 2286 major_t upmaj; 2287 major_t lomaj; 2288 2289 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2290 lomaj = getmajor(lostp->sd_vnode->v_rdev); 2291 np = &ss->ss_mux_nodes[upmaj]; 2292 if (np->mn_outp) { 2293 ep = np->mn_outp; 2294 while (ep->me_nextp) 2295 ep = ep->me_nextp; 2296 ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2297 ep = ep->me_nextp; 2298 } else { 2299 np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP); 2300 ep = np->mn_outp; 2301 } 2302 ep->me_nextp = NULL; 2303 ep->me_muxid = muxid; 2304 /* 2305 * Save the dev_t for the purposes of str_stack_shutdown. 2306 * str_stack_shutdown assumes that the device allows reopen, since 2307 * this dev_t is the one after any cloning by xx_open(). 2308 * Would prefer finding the dev_t from before any cloning, 2309 * but specfs doesn't retain that. 2310 */ 2311 ep->me_dev = upstp->sd_vnode->v_rdev; 2312 if (lostp->sd_vnode->v_type == VFIFO) 2313 ep->me_nodep = NULL; 2314 else 2315 ep->me_nodep = &ss->ss_mux_nodes[lomaj]; 2316 } 2317 2318 /* 2319 * A multiplexor link has been removed. Remove the 2320 * edge in the directed graph. 2321 */ 2322 void 2323 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss) 2324 { 2325 struct mux_node *np; 2326 struct mux_edge *ep; 2327 struct mux_edge *pep = NULL; 2328 major_t upmaj; 2329 2330 upmaj = getmajor(upstp->sd_vnode->v_rdev); 2331 np = &ss->ss_mux_nodes[upmaj]; 2332 ASSERT(np->mn_outp != NULL); 2333 ep = np->mn_outp; 2334 while (ep) { 2335 if (ep->me_muxid == muxid) { 2336 if (pep) 2337 pep->me_nextp = ep->me_nextp; 2338 else 2339 np->mn_outp = ep->me_nextp; 2340 kmem_free(ep, sizeof (struct mux_edge)); 2341 return; 2342 } 2343 pep = ep; 2344 ep = ep->me_nextp; 2345 } 2346 ASSERT(0); /* should not reach here */ 2347 } 2348 2349 /* 2350 * Translate the device flags (from conf.h) to the corresponding 2351 * qflag and sq_flag (type) values. 2352 */ 2353 int 2354 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp, 2355 uint32_t *sqtypep) 2356 { 2357 uint32_t qflag = 0; 2358 uint32_t sqtype = 0; 2359 2360 if (devflag & _D_OLD) 2361 goto bad; 2362 2363 /* Inner perimeter presence and scope */ 2364 switch (devflag & D_MTINNER_MASK) { 2365 case D_MP: 2366 qflag |= QMTSAFE; 2367 sqtype |= SQ_CI; 2368 break; 2369 case D_MTPERQ|D_MP: 2370 qflag |= QPERQ; 2371 break; 2372 case D_MTQPAIR|D_MP: 2373 qflag |= QPAIR; 2374 break; 2375 case D_MTPERMOD|D_MP: 2376 qflag |= QPERMOD; 2377 break; 2378 default: 2379 goto bad; 2380 } 2381 2382 /* Outer perimeter */ 2383 if (devflag & D_MTOUTPERIM) { 2384 switch (devflag & D_MTINNER_MASK) { 2385 case D_MP: 2386 case D_MTPERQ|D_MP: 2387 case D_MTQPAIR|D_MP: 2388 break; 2389 default: 2390 goto bad; 2391 } 2392 qflag |= QMTOUTPERIM; 2393 } 2394 2395 /* Inner perimeter modifiers */ 2396 if (devflag & D_MTINNER_MOD) { 2397 switch (devflag & D_MTINNER_MASK) { 2398 case D_MP: 2399 goto bad; 2400 default: 2401 break; 2402 } 2403 if (devflag & D_MTPUTSHARED) 2404 sqtype |= SQ_CIPUT; 2405 if (devflag & _D_MTOCSHARED) { 2406 /* 2407 * The code in putnext assumes that it has the 2408 * highest concurrency by not checking sq_count. 2409 * Thus _D_MTOCSHARED can only be supported when 2410 * D_MTPUTSHARED is set. 2411 */ 2412 if (!(devflag & D_MTPUTSHARED)) 2413 goto bad; 2414 sqtype |= SQ_CIOC; 2415 } 2416 if (devflag & _D_MTCBSHARED) { 2417 /* 2418 * The code in putnext assumes that it has the 2419 * highest concurrency by not checking sq_count. 2420 * Thus _D_MTCBSHARED can only be supported when 2421 * D_MTPUTSHARED is set. 2422 */ 2423 if (!(devflag & D_MTPUTSHARED)) 2424 goto bad; 2425 sqtype |= SQ_CICB; 2426 } 2427 if (devflag & _D_MTSVCSHARED) { 2428 /* 2429 * The code in putnext assumes that it has the 2430 * highest concurrency by not checking sq_count. 2431 * Thus _D_MTSVCSHARED can only be supported when 2432 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is 2433 * supported only for QPERMOD. 2434 */ 2435 if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD)) 2436 goto bad; 2437 sqtype |= SQ_CISVC; 2438 } 2439 } 2440 2441 /* Default outer perimeter concurrency */ 2442 sqtype |= SQ_CO; 2443 2444 /* Outer perimeter modifiers */ 2445 if (devflag & D_MTOCEXCL) { 2446 if (!(devflag & D_MTOUTPERIM)) { 2447 /* No outer perimeter */ 2448 goto bad; 2449 } 2450 sqtype &= ~SQ_COOC; 2451 } 2452 2453 /* Synchronous Streams extended qinit structure */ 2454 if (devflag & D_SYNCSTR) 2455 qflag |= QSYNCSTR; 2456 2457 /* 2458 * Private flag used by a transport module to indicate 2459 * to sockfs that it supports direct-access mode without 2460 * having to go through STREAMS. 2461 */ 2462 if (devflag & _D_DIRECT) { 2463 /* Reject unless the module is fully-MT (no perimeter) */ 2464 if ((qflag & QMT_TYPEMASK) != QMTSAFE) 2465 goto bad; 2466 qflag |= _QDIRECT; 2467 } 2468 2469 /* 2470 * Private flag used to indicate that a streams module should only 2471 * be pushed once. The TTY streams modules have this flag since if 2472 * libc believes itself to be an xpg4 process then it will 2473 * automatically and unconditionally push them when a PTS device is 2474 * opened. If an application is not aware of this then without this 2475 * flag we would end up with duplicate modules. 2476 */ 2477 if (devflag & _D_SINGLE_INSTANCE) 2478 qflag |= _QSINGLE_INSTANCE; 2479 2480 *qflagp = qflag; 2481 *sqtypep = sqtype; 2482 return (0); 2483 2484 bad: 2485 cmn_err(CE_WARN, 2486 "stropen: bad MT flags (0x%x) in driver '%s'", 2487 (int)(qflag & D_MTSAFETY_MASK), 2488 stp->st_rdinit->qi_minfo->mi_idname); 2489 2490 return (EINVAL); 2491 } 2492 2493 /* 2494 * Set the interface values for a pair of queues (qinit structure, 2495 * packet sizes, water marks). 2496 * setq assumes that the caller does not have a claim (entersq or claimq) 2497 * on the queue. 2498 */ 2499 void 2500 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit, 2501 perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed) 2502 { 2503 queue_t *wq; 2504 syncq_t *sq, *outer; 2505 2506 ASSERT(rq->q_flag & QREADR); 2507 ASSERT((qflag & QMT_TYPEMASK) != 0); 2508 IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL); 2509 2510 wq = _WR(rq); 2511 rq->q_qinfo = rinit; 2512 rq->q_hiwat = rinit->qi_minfo->mi_hiwat; 2513 rq->q_lowat = rinit->qi_minfo->mi_lowat; 2514 rq->q_minpsz = rinit->qi_minfo->mi_minpsz; 2515 rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz; 2516 wq->q_qinfo = winit; 2517 wq->q_hiwat = winit->qi_minfo->mi_hiwat; 2518 wq->q_lowat = winit->qi_minfo->mi_lowat; 2519 wq->q_minpsz = winit->qi_minfo->mi_minpsz; 2520 wq->q_maxpsz = winit->qi_minfo->mi_maxpsz; 2521 2522 /* Remove old syncqs */ 2523 sq = rq->q_syncq; 2524 outer = sq->sq_outer; 2525 if (outer != NULL) { 2526 ASSERT(wq->q_syncq->sq_outer == outer); 2527 outer_remove(outer, rq->q_syncq); 2528 if (wq->q_syncq != rq->q_syncq) 2529 outer_remove(outer, wq->q_syncq); 2530 } 2531 ASSERT(sq->sq_outer == NULL); 2532 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2533 2534 if (sq != SQ(rq)) { 2535 if (!(rq->q_flag & QPERMOD)) 2536 free_syncq(sq); 2537 if (wq->q_syncq == rq->q_syncq) 2538 wq->q_syncq = NULL; 2539 rq->q_syncq = NULL; 2540 } 2541 if (wq->q_syncq != NULL && wq->q_syncq != sq && 2542 wq->q_syncq != SQ(rq)) { 2543 free_syncq(wq->q_syncq); 2544 wq->q_syncq = NULL; 2545 } 2546 ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL && 2547 rq->q_syncq->sq_tail == NULL)); 2548 ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL && 2549 wq->q_syncq->sq_tail == NULL)); 2550 2551 if (!(rq->q_flag & QPERMOD) && 2552 rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) { 2553 ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2554 SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl, 2555 rq->q_syncq->sq_nciputctrl, 0); 2556 ASSERT(ciputctrl_cache != NULL); 2557 kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl); 2558 rq->q_syncq->sq_ciputctrl = NULL; 2559 rq->q_syncq->sq_nciputctrl = 0; 2560 } 2561 2562 if (!(wq->q_flag & QPERMOD) && 2563 wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) { 2564 ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1); 2565 SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl, 2566 wq->q_syncq->sq_nciputctrl, 0); 2567 ASSERT(ciputctrl_cache != NULL); 2568 kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl); 2569 wq->q_syncq->sq_ciputctrl = NULL; 2570 wq->q_syncq->sq_nciputctrl = 0; 2571 } 2572 2573 sq = SQ(rq); 2574 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 2575 ASSERT(sq->sq_outer == NULL); 2576 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 2577 2578 /* 2579 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS 2580 * bits in sq_flag based on the sqtype. 2581 */ 2582 ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0); 2583 2584 rq->q_syncq = wq->q_syncq = sq; 2585 sq->sq_type = sqtype; 2586 sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS); 2587 2588 /* 2589 * We are making sq_svcflags zero, 2590 * resetting SQ_DISABLED in case it was set by 2591 * wait_svc() in the munlink path. 2592 * 2593 */ 2594 ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0); 2595 sq->sq_svcflags = 0; 2596 2597 /* 2598 * We need to acquire the lock here for the mlink and munlink case, 2599 * where canputnext, backenable, etc can access the q_flag. 2600 */ 2601 if (lock_needed) { 2602 mutex_enter(QLOCK(rq)); 2603 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2604 mutex_exit(QLOCK(rq)); 2605 mutex_enter(QLOCK(wq)); 2606 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2607 mutex_exit(QLOCK(wq)); 2608 } else { 2609 rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2610 wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag; 2611 } 2612 2613 if (qflag & QPERQ) { 2614 /* Allocate a separate syncq for the write side */ 2615 sq = new_syncq(); 2616 sq->sq_type = rq->q_syncq->sq_type; 2617 sq->sq_flags = rq->q_syncq->sq_flags; 2618 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2619 sq->sq_oprev == NULL); 2620 wq->q_syncq = sq; 2621 } 2622 if (qflag & QPERMOD) { 2623 sq = dmp->dm_sq; 2624 2625 /* 2626 * Assert that we do have an inner perimeter syncq and that it 2627 * does not have an outer perimeter associated with it. 2628 */ 2629 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 2630 sq->sq_oprev == NULL); 2631 rq->q_syncq = wq->q_syncq = sq; 2632 } 2633 if (qflag & QMTOUTPERIM) { 2634 outer = dmp->dm_sq; 2635 2636 ASSERT(outer->sq_outer == NULL); 2637 outer_insert(outer, rq->q_syncq); 2638 if (wq->q_syncq != rq->q_syncq) 2639 outer_insert(outer, wq->q_syncq); 2640 } 2641 ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2642 (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2643 ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) == 2644 (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS)); 2645 ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK)); 2646 2647 /* 2648 * Initialize struio() types. 2649 */ 2650 rq->q_struiot = 2651 (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE; 2652 wq->q_struiot = 2653 (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE; 2654 } 2655 2656 perdm_t * 2657 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype) 2658 { 2659 syncq_t *sq; 2660 perdm_t **pp; 2661 perdm_t *p; 2662 perdm_t *dmp; 2663 2664 ASSERT(str != NULL); 2665 ASSERT(qflag & (QPERMOD | QMTOUTPERIM)); 2666 2667 rw_enter(&perdm_rwlock, RW_READER); 2668 for (p = perdm_list; p != NULL; p = p->dm_next) { 2669 if (p->dm_str == str) { /* found one */ 2670 atomic_inc_32(&(p->dm_ref)); 2671 rw_exit(&perdm_rwlock); 2672 return (p); 2673 } 2674 } 2675 rw_exit(&perdm_rwlock); 2676 2677 sq = new_syncq(); 2678 if (qflag & QPERMOD) { 2679 sq->sq_type = sqtype | SQ_PERMOD; 2680 sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS; 2681 } else { 2682 ASSERT(qflag & QMTOUTPERIM); 2683 sq->sq_onext = sq->sq_oprev = sq; 2684 } 2685 2686 dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP); 2687 dmp->dm_sq = sq; 2688 dmp->dm_str = str; 2689 dmp->dm_ref = 1; 2690 dmp->dm_next = NULL; 2691 2692 rw_enter(&perdm_rwlock, RW_WRITER); 2693 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) { 2694 if (p->dm_str == str) { /* already present */ 2695 p->dm_ref++; 2696 rw_exit(&perdm_rwlock); 2697 free_syncq(sq); 2698 kmem_free(dmp, sizeof (perdm_t)); 2699 return (p); 2700 } 2701 } 2702 2703 *pp = dmp; 2704 rw_exit(&perdm_rwlock); 2705 return (dmp); 2706 } 2707 2708 void 2709 rele_dm(perdm_t *dmp) 2710 { 2711 perdm_t **pp; 2712 perdm_t *p; 2713 2714 rw_enter(&perdm_rwlock, RW_WRITER); 2715 ASSERT(dmp->dm_ref > 0); 2716 2717 if (--dmp->dm_ref > 0) { 2718 rw_exit(&perdm_rwlock); 2719 return; 2720 } 2721 2722 for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) 2723 if (p == dmp) 2724 break; 2725 ASSERT(p == dmp); 2726 *pp = p->dm_next; 2727 rw_exit(&perdm_rwlock); 2728 2729 /* 2730 * Wait for any background processing that relies on the 2731 * syncq to complete before it is freed. 2732 */ 2733 wait_sq_svc(p->dm_sq); 2734 free_syncq(p->dm_sq); 2735 kmem_free(p, sizeof (perdm_t)); 2736 } 2737 2738 /* 2739 * Make a protocol message given control and data buffers. 2740 * n.b., this can block; be careful of what locks you hold when calling it. 2741 * 2742 * If sd_maxblk is less than *iosize this routine can fail part way through 2743 * (due to an allocation failure). In this case on return *iosize will contain 2744 * the amount that was consumed. Otherwise *iosize will not be modified 2745 * i.e. it will contain the amount that was consumed. 2746 */ 2747 int 2748 strmakemsg( 2749 struct strbuf *mctl, 2750 ssize_t *iosize, 2751 struct uio *uiop, 2752 stdata_t *stp, 2753 int32_t flag, 2754 mblk_t **mpp) 2755 { 2756 mblk_t *mpctl = NULL; 2757 mblk_t *mpdata = NULL; 2758 int error; 2759 2760 ASSERT(uiop != NULL); 2761 2762 *mpp = NULL; 2763 /* Create control part, if any */ 2764 if ((mctl != NULL) && (mctl->len >= 0)) { 2765 error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl); 2766 if (error) 2767 return (error); 2768 } 2769 /* Create data part, if any */ 2770 if (*iosize >= 0) { 2771 error = strmakedata(iosize, uiop, stp, flag, &mpdata); 2772 if (error) { 2773 freemsg(mpctl); 2774 return (error); 2775 } 2776 } 2777 if (mpctl != NULL) { 2778 if (mpdata != NULL) 2779 linkb(mpctl, mpdata); 2780 *mpp = mpctl; 2781 } else { 2782 *mpp = mpdata; 2783 } 2784 return (0); 2785 } 2786 2787 /* 2788 * Make the control part of a protocol message given a control buffer. 2789 * n.b., this can block; be careful of what locks you hold when calling it. 2790 */ 2791 int 2792 strmakectl( 2793 struct strbuf *mctl, 2794 int32_t flag, 2795 int32_t fflag, 2796 mblk_t **mpp) 2797 { 2798 mblk_t *bp = NULL; 2799 unsigned char msgtype; 2800 int error = 0; 2801 cred_t *cr = CRED(); 2802 2803 /* We do not support interrupt threads using the stream head to send */ 2804 ASSERT(cr != NULL); 2805 2806 *mpp = NULL; 2807 /* 2808 * Create control part of message, if any. 2809 */ 2810 if ((mctl != NULL) && (mctl->len >= 0)) { 2811 caddr_t base; 2812 int ctlcount; 2813 int allocsz; 2814 2815 if (flag & RS_HIPRI) 2816 msgtype = M_PCPROTO; 2817 else 2818 msgtype = M_PROTO; 2819 2820 ctlcount = mctl->len; 2821 base = mctl->buf; 2822 2823 /* 2824 * Give modules a better chance to reuse M_PROTO/M_PCPROTO 2825 * blocks by increasing the size to something more usable. 2826 */ 2827 allocsz = MAX(ctlcount, 64); 2828 2829 /* 2830 * Range checking has already been done; simply try 2831 * to allocate a message block for the ctl part. 2832 */ 2833 while ((bp = allocb_cred(allocsz, cr, 2834 curproc->p_pid)) == NULL) { 2835 if (fflag & (FNDELAY|FNONBLOCK)) 2836 return (EAGAIN); 2837 if (error = strwaitbuf(allocsz, BPRI_MED)) 2838 return (error); 2839 } 2840 2841 bp->b_datap->db_type = msgtype; 2842 if (copyin(base, bp->b_wptr, ctlcount)) { 2843 freeb(bp); 2844 return (EFAULT); 2845 } 2846 bp->b_wptr += ctlcount; 2847 } 2848 *mpp = bp; 2849 return (0); 2850 } 2851 2852 /* 2853 * Make a protocol message given data buffers. 2854 * n.b., this can block; be careful of what locks you hold when calling it. 2855 * 2856 * If sd_maxblk is less than *iosize this routine can fail part way through 2857 * (due to an allocation failure). In this case on return *iosize will contain 2858 * the amount that was consumed. Otherwise *iosize will not be modified 2859 * i.e. it will contain the amount that was consumed. 2860 */ 2861 int 2862 strmakedata( 2863 ssize_t *iosize, 2864 struct uio *uiop, 2865 stdata_t *stp, 2866 int32_t flag, 2867 mblk_t **mpp) 2868 { 2869 mblk_t *mp = NULL; 2870 mblk_t *bp; 2871 int wroff = (int)stp->sd_wroff; 2872 int tail_len = (int)stp->sd_tail; 2873 int extra = wroff + tail_len; 2874 int error = 0; 2875 ssize_t maxblk; 2876 ssize_t count = *iosize; 2877 cred_t *cr; 2878 2879 *mpp = NULL; 2880 if (count < 0) 2881 return (0); 2882 2883 /* We do not support interrupt threads using the stream head to send */ 2884 cr = CRED(); 2885 ASSERT(cr != NULL); 2886 2887 maxblk = stp->sd_maxblk; 2888 if (maxblk == INFPSZ) 2889 maxblk = count; 2890 2891 /* 2892 * Create data part of message, if any. 2893 */ 2894 do { 2895 ssize_t size; 2896 dblk_t *dp; 2897 2898 ASSERT(uiop); 2899 2900 size = MIN(count, maxblk); 2901 2902 while ((bp = allocb_cred(size + extra, cr, 2903 curproc->p_pid)) == NULL) { 2904 error = EAGAIN; 2905 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 2906 (error = strwaitbuf(size + extra, BPRI_MED)) != 0) { 2907 if (count == *iosize) { 2908 freemsg(mp); 2909 return (error); 2910 } else { 2911 *iosize -= count; 2912 *mpp = mp; 2913 return (0); 2914 } 2915 } 2916 } 2917 dp = bp->b_datap; 2918 dp->db_cpid = curproc->p_pid; 2919 ASSERT(wroff <= dp->db_lim - bp->b_wptr); 2920 bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff; 2921 2922 if (flag & STRUIO_POSTPONE) { 2923 /* 2924 * Setup the stream uio portion of the 2925 * dblk for subsequent use by struioget(). 2926 */ 2927 dp->db_struioflag = STRUIO_SPEC; 2928 dp->db_cksumstart = 0; 2929 dp->db_cksumstuff = 0; 2930 dp->db_cksumend = size; 2931 *(long long *)dp->db_struioun.data = 0ll; 2932 bp->b_wptr += size; 2933 } else { 2934 if (stp->sd_copyflag & STRCOPYCACHED) 2935 uiop->uio_extflg |= UIO_COPY_CACHED; 2936 2937 if (size != 0) { 2938 error = uiomove(bp->b_wptr, size, UIO_WRITE, 2939 uiop); 2940 if (error != 0) { 2941 freeb(bp); 2942 freemsg(mp); 2943 return (error); 2944 } 2945 } 2946 bp->b_wptr += size; 2947 2948 if (stp->sd_wputdatafunc != NULL) { 2949 mblk_t *newbp; 2950 2951 newbp = (stp->sd_wputdatafunc)(stp->sd_vnode, 2952 bp, NULL, NULL, NULL, NULL); 2953 if (newbp == NULL) { 2954 freeb(bp); 2955 freemsg(mp); 2956 return (ECOMM); 2957 } 2958 bp = newbp; 2959 } 2960 } 2961 2962 count -= size; 2963 2964 if (mp == NULL) 2965 mp = bp; 2966 else 2967 linkb(mp, bp); 2968 } while (count > 0); 2969 2970 *mpp = mp; 2971 return (0); 2972 } 2973 2974 /* 2975 * Wait for a buffer to become available. Return non-zero errno 2976 * if not able to wait, 0 if buffer is probably there. 2977 */ 2978 int 2979 strwaitbuf(size_t size, int pri) 2980 { 2981 bufcall_id_t id; 2982 2983 mutex_enter(&bcall_monitor); 2984 if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast, 2985 &ttoproc(curthread)->p_flag_cv)) == 0) { 2986 mutex_exit(&bcall_monitor); 2987 return (ENOSR); 2988 } 2989 if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) { 2990 unbufcall(id); 2991 mutex_exit(&bcall_monitor); 2992 return (EINTR); 2993 } 2994 unbufcall(id); 2995 mutex_exit(&bcall_monitor); 2996 return (0); 2997 } 2998 2999 /* 3000 * This function waits for a read or write event to happen on a stream. 3001 * fmode can specify FNDELAY and/or FNONBLOCK. 3002 * The timeout is in ms with -1 meaning infinite. 3003 * The flag values work as follows: 3004 * READWAIT Check for read side errors, send M_READ 3005 * GETWAIT Check for read side errors, no M_READ 3006 * WRITEWAIT Check for write side errors. 3007 * NOINTR Do not return error if nonblocking or timeout. 3008 * STR_NOERROR Ignore all errors except STPLEX. 3009 * STR_NOSIG Ignore/hold signals during the duration of the call. 3010 * STR_PEEK Pass through the strgeterr(). 3011 */ 3012 int 3013 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout, 3014 int *done) 3015 { 3016 int slpflg, errs; 3017 int error; 3018 kcondvar_t *sleepon; 3019 mblk_t *mp; 3020 ssize_t *rd_count; 3021 clock_t rval; 3022 3023 ASSERT(MUTEX_HELD(&stp->sd_lock)); 3024 if ((flag & READWAIT) || (flag & GETWAIT)) { 3025 slpflg = RSLEEP; 3026 sleepon = &_RD(stp->sd_wrq)->q_wait; 3027 errs = STRDERR|STPLEX; 3028 } else { 3029 slpflg = WSLEEP; 3030 sleepon = &stp->sd_wrq->q_wait; 3031 errs = STWRERR|STRHUP|STPLEX; 3032 } 3033 if (flag & STR_NOERROR) 3034 errs = STPLEX; 3035 3036 if (stp->sd_wakeq & slpflg) { 3037 /* 3038 * A strwakeq() is pending, no need to sleep. 3039 */ 3040 stp->sd_wakeq &= ~slpflg; 3041 *done = 0; 3042 return (0); 3043 } 3044 3045 if (stp->sd_flag & errs) { 3046 /* 3047 * Check for errors before going to sleep since the 3048 * caller might not have checked this while holding 3049 * sd_lock. 3050 */ 3051 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3052 if (error != 0) { 3053 *done = 1; 3054 return (error); 3055 } 3056 } 3057 3058 /* 3059 * If any module downstream has requested read notification 3060 * by setting SNDMREAD flag using M_SETOPTS, send a message 3061 * down stream. 3062 */ 3063 if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) { 3064 mutex_exit(&stp->sd_lock); 3065 if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED, 3066 (flag & STR_NOSIG), &error))) { 3067 mutex_enter(&stp->sd_lock); 3068 *done = 1; 3069 return (error); 3070 } 3071 mp->b_datap->db_type = M_READ; 3072 rd_count = (ssize_t *)mp->b_wptr; 3073 *rd_count = count; 3074 mp->b_wptr += sizeof (ssize_t); 3075 /* 3076 * Send the number of bytes requested by the 3077 * read as the argument to M_READ. 3078 */ 3079 stream_willservice(stp); 3080 putnext(stp->sd_wrq, mp); 3081 stream_runservice(stp); 3082 mutex_enter(&stp->sd_lock); 3083 3084 /* 3085 * If any data arrived due to inline processing 3086 * of putnext(), don't sleep. 3087 */ 3088 if (_RD(stp->sd_wrq)->q_first != NULL) { 3089 *done = 0; 3090 return (0); 3091 } 3092 } 3093 3094 if (fmode & (FNDELAY|FNONBLOCK)) { 3095 if (!(flag & NOINTR)) 3096 error = EAGAIN; 3097 else 3098 error = 0; 3099 *done = 1; 3100 return (error); 3101 } 3102 3103 stp->sd_flag |= slpflg; 3104 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2, 3105 "strwaitq sleeps (2):%p, %X, %lX, %X, %p", 3106 stp, flag, count, fmode, done); 3107 3108 rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG); 3109 if (rval > 0) { 3110 /* EMPTY */ 3111 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2, 3112 "strwaitq awakes(2):%X, %X, %X, %X, %X", 3113 stp, flag, count, fmode, done); 3114 } else if (rval == 0) { 3115 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2, 3116 "strwaitq interrupt #2:%p, %X, %lX, %X, %p", 3117 stp, flag, count, fmode, done); 3118 stp->sd_flag &= ~slpflg; 3119 cv_broadcast(sleepon); 3120 if (!(flag & NOINTR)) 3121 error = EINTR; 3122 else 3123 error = 0; 3124 *done = 1; 3125 return (error); 3126 } else { 3127 /* timeout */ 3128 TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME, 3129 "strwaitq timeout:%p, %X, %lX, %X, %p", 3130 stp, flag, count, fmode, done); 3131 *done = 1; 3132 if (!(flag & NOINTR)) 3133 return (ETIME); 3134 else 3135 return (0); 3136 } 3137 /* 3138 * If the caller implements delayed errors (i.e. queued after data) 3139 * we can not check for errors here since data as well as an 3140 * error might have arrived at the stream head. We return to 3141 * have the caller check the read queue before checking for errors. 3142 */ 3143 if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) { 3144 error = strgeterr(stp, errs, (flag & STR_PEEK)); 3145 if (error != 0) { 3146 *done = 1; 3147 return (error); 3148 } 3149 } 3150 *done = 0; 3151 return (0); 3152 } 3153 3154 /* 3155 * Perform job control discipline access checks. 3156 * Return 0 for success and the errno for failure. 3157 */ 3158 3159 #define cantsend(p, t, sig) \ 3160 (sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig)) 3161 3162 int 3163 straccess(struct stdata *stp, enum jcaccess mode) 3164 { 3165 extern kcondvar_t lbolt_cv; /* XXX: should be in a header file */ 3166 kthread_t *t = curthread; 3167 proc_t *p = ttoproc(t); 3168 sess_t *sp; 3169 3170 ASSERT(mutex_owned(&stp->sd_lock)); 3171 3172 if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO) 3173 return (0); 3174 3175 mutex_enter(&p->p_lock); /* protects p_pgidp */ 3176 3177 for (;;) { 3178 mutex_enter(&p->p_splock); /* protects p->p_sessp */ 3179 sp = p->p_sessp; 3180 mutex_enter(&sp->s_lock); /* protects sp->* */ 3181 3182 /* 3183 * If this is not the calling process's controlling terminal 3184 * or if the calling process is already in the foreground 3185 * then allow access. 3186 */ 3187 if (sp->s_dev != stp->sd_vnode->v_rdev || 3188 p->p_pgidp == stp->sd_pgidp) { 3189 mutex_exit(&sp->s_lock); 3190 mutex_exit(&p->p_splock); 3191 mutex_exit(&p->p_lock); 3192 return (0); 3193 } 3194 3195 /* 3196 * Check to see if controlling terminal has been deallocated. 3197 */ 3198 if (sp->s_vp == NULL) { 3199 if (!cantsend(p, t, SIGHUP)) 3200 sigtoproc(p, t, SIGHUP); 3201 mutex_exit(&sp->s_lock); 3202 mutex_exit(&p->p_splock); 3203 mutex_exit(&p->p_lock); 3204 return (EIO); 3205 } 3206 3207 mutex_exit(&sp->s_lock); 3208 mutex_exit(&p->p_splock); 3209 3210 if (mode == JCGETP) { 3211 mutex_exit(&p->p_lock); 3212 return (0); 3213 } 3214 3215 if (mode == JCREAD) { 3216 if (p->p_detached || cantsend(p, t, SIGTTIN)) { 3217 mutex_exit(&p->p_lock); 3218 return (EIO); 3219 } 3220 mutex_exit(&p->p_lock); 3221 mutex_exit(&stp->sd_lock); 3222 pgsignal(p->p_pgidp, SIGTTIN); 3223 mutex_enter(&stp->sd_lock); 3224 mutex_enter(&p->p_lock); 3225 } else { /* mode == JCWRITE or JCSETP */ 3226 if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) || 3227 cantsend(p, t, SIGTTOU)) { 3228 mutex_exit(&p->p_lock); 3229 return (0); 3230 } 3231 if (p->p_detached) { 3232 mutex_exit(&p->p_lock); 3233 return (EIO); 3234 } 3235 mutex_exit(&p->p_lock); 3236 mutex_exit(&stp->sd_lock); 3237 pgsignal(p->p_pgidp, SIGTTOU); 3238 mutex_enter(&stp->sd_lock); 3239 mutex_enter(&p->p_lock); 3240 } 3241 3242 /* 3243 * We call cv_wait_sig_swap() to cause the appropriate 3244 * action for the jobcontrol signal to take place. 3245 * If the signal is being caught, we will take the 3246 * EINTR error return. Otherwise, the default action 3247 * of causing the process to stop will take place. 3248 * In this case, we rely on the periodic cv_broadcast() on 3249 * &lbolt_cv to wake us up to loop around and test again. 3250 * We can't get here if the signal is ignored or 3251 * if the current thread is blocking the signal. 3252 */ 3253 mutex_exit(&stp->sd_lock); 3254 if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) { 3255 mutex_exit(&p->p_lock); 3256 mutex_enter(&stp->sd_lock); 3257 return (EINTR); 3258 } 3259 mutex_exit(&p->p_lock); 3260 mutex_enter(&stp->sd_lock); 3261 mutex_enter(&p->p_lock); 3262 } 3263 } 3264 3265 /* 3266 * Return size of message of block type (bp->b_datap->db_type) 3267 */ 3268 size_t 3269 xmsgsize(mblk_t *bp) 3270 { 3271 unsigned char type; 3272 size_t count = 0; 3273 3274 type = bp->b_datap->db_type; 3275 3276 for (; bp; bp = bp->b_cont) { 3277 if (type != bp->b_datap->db_type) 3278 break; 3279 ASSERT(bp->b_wptr >= bp->b_rptr); 3280 count += bp->b_wptr - bp->b_rptr; 3281 } 3282 return (count); 3283 } 3284 3285 /* 3286 * Allocate a stream head. 3287 */ 3288 struct stdata * 3289 shalloc(queue_t *qp) 3290 { 3291 stdata_t *stp; 3292 3293 stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP); 3294 3295 stp->sd_wrq = _WR(qp); 3296 stp->sd_strtab = NULL; 3297 stp->sd_iocid = 0; 3298 stp->sd_mate = NULL; 3299 stp->sd_freezer = NULL; 3300 stp->sd_refcnt = 0; 3301 stp->sd_wakeq = 0; 3302 stp->sd_anchor = 0; 3303 stp->sd_struiowrq = NULL; 3304 stp->sd_struiordq = NULL; 3305 stp->sd_struiodnak = 0; 3306 stp->sd_struionak = NULL; 3307 stp->sd_t_audit_data = NULL; 3308 stp->sd_rput_opt = 0; 3309 stp->sd_wput_opt = 0; 3310 stp->sd_read_opt = 0; 3311 stp->sd_rprotofunc = strrput_proto; 3312 stp->sd_rmiscfunc = strrput_misc; 3313 stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL; 3314 stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL; 3315 stp->sd_ciputctrl = NULL; 3316 stp->sd_nciputctrl = 0; 3317 stp->sd_qhead = NULL; 3318 stp->sd_qtail = NULL; 3319 stp->sd_servid = NULL; 3320 stp->sd_nqueues = 0; 3321 stp->sd_svcflags = 0; 3322 stp->sd_copyflag = 0; 3323 3324 return (stp); 3325 } 3326 3327 /* 3328 * Free a stream head. 3329 */ 3330 void 3331 shfree(stdata_t *stp) 3332 { 3333 ASSERT(MUTEX_NOT_HELD(&stp->sd_lock)); 3334 3335 stp->sd_wrq = NULL; 3336 3337 mutex_enter(&stp->sd_qlock); 3338 while (stp->sd_svcflags & STRS_SCHEDULED) { 3339 STRSTAT(strwaits); 3340 cv_wait(&stp->sd_qcv, &stp->sd_qlock); 3341 } 3342 mutex_exit(&stp->sd_qlock); 3343 3344 if (stp->sd_ciputctrl != NULL) { 3345 ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1); 3346 SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl, 3347 stp->sd_nciputctrl, 0); 3348 ASSERT(ciputctrl_cache != NULL); 3349 kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl); 3350 stp->sd_ciputctrl = NULL; 3351 stp->sd_nciputctrl = 0; 3352 } 3353 ASSERT(stp->sd_qhead == NULL); 3354 ASSERT(stp->sd_qtail == NULL); 3355 ASSERT(stp->sd_nqueues == 0); 3356 kmem_cache_free(stream_head_cache, stp); 3357 } 3358 3359 /* 3360 * Allocate a pair of queues and a syncq for the pair 3361 */ 3362 queue_t * 3363 allocq(void) 3364 { 3365 queinfo_t *qip; 3366 queue_t *qp, *wqp; 3367 syncq_t *sq; 3368 3369 qip = kmem_cache_alloc(queue_cache, KM_SLEEP); 3370 3371 qp = &qip->qu_rqueue; 3372 wqp = &qip->qu_wqueue; 3373 sq = &qip->qu_syncq; 3374 3375 qp->q_last = NULL; 3376 qp->q_next = NULL; 3377 qp->q_ptr = NULL; 3378 qp->q_flag = QUSE | QREADR; 3379 qp->q_bandp = NULL; 3380 qp->q_stream = NULL; 3381 qp->q_syncq = sq; 3382 qp->q_nband = 0; 3383 qp->q_nfsrv = NULL; 3384 qp->q_draining = 0; 3385 qp->q_syncqmsgs = 0; 3386 qp->q_spri = 0; 3387 qp->q_qtstamp = 0; 3388 qp->q_sqtstamp = 0; 3389 qp->q_fp = NULL; 3390 3391 wqp->q_last = NULL; 3392 wqp->q_next = NULL; 3393 wqp->q_ptr = NULL; 3394 wqp->q_flag = QUSE; 3395 wqp->q_bandp = NULL; 3396 wqp->q_stream = NULL; 3397 wqp->q_syncq = sq; 3398 wqp->q_nband = 0; 3399 wqp->q_nfsrv = NULL; 3400 wqp->q_draining = 0; 3401 wqp->q_syncqmsgs = 0; 3402 wqp->q_qtstamp = 0; 3403 wqp->q_sqtstamp = 0; 3404 wqp->q_spri = 0; 3405 3406 sq->sq_count = 0; 3407 sq->sq_rmqcount = 0; 3408 sq->sq_flags = 0; 3409 sq->sq_type = 0; 3410 sq->sq_callbflags = 0; 3411 sq->sq_cancelid = 0; 3412 sq->sq_ciputctrl = NULL; 3413 sq->sq_nciputctrl = 0; 3414 sq->sq_needexcl = 0; 3415 sq->sq_svcflags = 0; 3416 3417 return (qp); 3418 } 3419 3420 /* 3421 * Free a pair of queues and the "attached" syncq. 3422 * Discard any messages left on the syncq(s), remove the syncq(s) from the 3423 * outer perimeter, and free the syncq(s) if they are not the "attached" syncq. 3424 */ 3425 void 3426 freeq(queue_t *qp) 3427 { 3428 qband_t *qbp, *nqbp; 3429 syncq_t *sq, *outer; 3430 queue_t *wqp = _WR(qp); 3431 3432 ASSERT(qp->q_flag & QREADR); 3433 3434 /* 3435 * If a previously dispatched taskq job is scheduled to run 3436 * sync_service() or a service routine is scheduled for the 3437 * queues about to be freed, wait here until all service is 3438 * done on the queue and all associated queues and syncqs. 3439 */ 3440 wait_svc(qp); 3441 3442 (void) flush_syncq(qp->q_syncq, qp); 3443 (void) flush_syncq(wqp->q_syncq, wqp); 3444 ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0); 3445 3446 /* 3447 * Flush the queues before q_next is set to NULL This is needed 3448 * in order to backenable any downstream queue before we go away. 3449 * Note: we are already removed from the stream so that the 3450 * backenabling will not cause any messages to be delivered to our 3451 * put procedures. 3452 */ 3453 flushq(qp, FLUSHALL); 3454 flushq(wqp, FLUSHALL); 3455 3456 /* Tidy up - removeq only does a half-remove from stream */ 3457 qp->q_next = wqp->q_next = NULL; 3458 ASSERT(!(qp->q_flag & QENAB)); 3459 ASSERT(!(wqp->q_flag & QENAB)); 3460 3461 outer = qp->q_syncq->sq_outer; 3462 if (outer != NULL) { 3463 outer_remove(outer, qp->q_syncq); 3464 if (wqp->q_syncq != qp->q_syncq) 3465 outer_remove(outer, wqp->q_syncq); 3466 } 3467 /* 3468 * Free any syncqs that are outside what allocq returned. 3469 */ 3470 if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD)) 3471 free_syncq(qp->q_syncq); 3472 if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp)) 3473 free_syncq(wqp->q_syncq); 3474 3475 ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3476 ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0); 3477 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 3478 ASSERT(MUTEX_NOT_HELD(QLOCK(wqp))); 3479 sq = SQ(qp); 3480 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 3481 ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL); 3482 ASSERT(sq->sq_outer == NULL); 3483 ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL); 3484 ASSERT(sq->sq_callbpend == NULL); 3485 ASSERT(sq->sq_needexcl == 0); 3486 3487 if (sq->sq_ciputctrl != NULL) { 3488 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 3489 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 3490 sq->sq_nciputctrl, 0); 3491 ASSERT(ciputctrl_cache != NULL); 3492 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 3493 sq->sq_ciputctrl = NULL; 3494 sq->sq_nciputctrl = 0; 3495 } 3496 3497 ASSERT(qp->q_first == NULL && wqp->q_first == NULL); 3498 ASSERT(qp->q_count == 0 && wqp->q_count == 0); 3499 ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0); 3500 3501 qp->q_flag &= ~QUSE; 3502 wqp->q_flag &= ~QUSE; 3503 3504 /* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */ 3505 /* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */ 3506 3507 qbp = qp->q_bandp; 3508 while (qbp) { 3509 nqbp = qbp->qb_next; 3510 freeband(qbp); 3511 qbp = nqbp; 3512 } 3513 qbp = wqp->q_bandp; 3514 while (qbp) { 3515 nqbp = qbp->qb_next; 3516 freeband(qbp); 3517 qbp = nqbp; 3518 } 3519 kmem_cache_free(queue_cache, qp); 3520 } 3521 3522 /* 3523 * Allocate a qband structure. 3524 */ 3525 qband_t * 3526 allocband(void) 3527 { 3528 qband_t *qbp; 3529 3530 qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP); 3531 if (qbp == NULL) 3532 return (NULL); 3533 3534 qbp->qb_next = NULL; 3535 qbp->qb_count = 0; 3536 qbp->qb_mblkcnt = 0; 3537 qbp->qb_first = NULL; 3538 qbp->qb_last = NULL; 3539 qbp->qb_flag = 0; 3540 3541 return (qbp); 3542 } 3543 3544 /* 3545 * Free a qband structure. 3546 */ 3547 void 3548 freeband(qband_t *qbp) 3549 { 3550 kmem_cache_free(qband_cache, qbp); 3551 } 3552 3553 /* 3554 * Just like putnextctl(9F), except that allocb_wait() is used. 3555 * 3556 * Consolidation Private, and of course only callable from the stream head or 3557 * routines that may block. 3558 */ 3559 int 3560 putnextctl_wait(queue_t *q, int type) 3561 { 3562 mblk_t *bp; 3563 int error; 3564 3565 if ((datamsg(type) && (type != M_DELAY)) || 3566 (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL) 3567 return (0); 3568 3569 bp->b_datap->db_type = (unsigned char)type; 3570 putnext(q, bp); 3571 return (1); 3572 } 3573 3574 /* 3575 * Run any possible bufcalls. 3576 */ 3577 void 3578 runbufcalls(void) 3579 { 3580 strbufcall_t *bcp; 3581 3582 mutex_enter(&bcall_monitor); 3583 mutex_enter(&strbcall_lock); 3584 3585 if (strbcalls.bc_head) { 3586 size_t count; 3587 int nevent; 3588 3589 /* 3590 * count how many events are on the list 3591 * now so we can check to avoid looping 3592 * in low memory situations 3593 */ 3594 nevent = 0; 3595 for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next) 3596 nevent++; 3597 3598 /* 3599 * get estimate of available memory from kmem_avail(). 3600 * awake all bufcall functions waiting for 3601 * memory whose request could be satisfied 3602 * by 'count' memory and let 'em fight for it. 3603 */ 3604 count = kmem_avail(); 3605 while ((bcp = strbcalls.bc_head) != NULL && nevent) { 3606 STRSTAT(bufcalls); 3607 --nevent; 3608 if (bcp->bc_size <= count) { 3609 bcp->bc_executor = curthread; 3610 mutex_exit(&strbcall_lock); 3611 (*bcp->bc_func)(bcp->bc_arg); 3612 mutex_enter(&strbcall_lock); 3613 bcp->bc_executor = NULL; 3614 cv_broadcast(&bcall_cv); 3615 strbcalls.bc_head = bcp->bc_next; 3616 kmem_free(bcp, sizeof (strbufcall_t)); 3617 } else { 3618 /* 3619 * too big, try again later - note 3620 * that nevent was decremented above 3621 * so we won't retry this one on this 3622 * iteration of the loop 3623 */ 3624 if (bcp->bc_next != NULL) { 3625 strbcalls.bc_head = bcp->bc_next; 3626 bcp->bc_next = NULL; 3627 strbcalls.bc_tail->bc_next = bcp; 3628 strbcalls.bc_tail = bcp; 3629 } 3630 } 3631 } 3632 if (strbcalls.bc_head == NULL) 3633 strbcalls.bc_tail = NULL; 3634 } 3635 3636 mutex_exit(&strbcall_lock); 3637 mutex_exit(&bcall_monitor); 3638 } 3639 3640 3641 /* 3642 * Actually run queue's service routine. 3643 */ 3644 static void 3645 runservice(queue_t *q) 3646 { 3647 qband_t *qbp; 3648 3649 ASSERT(q->q_qinfo->qi_srvp); 3650 again: 3651 entersq(q->q_syncq, SQ_SVC); 3652 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START, 3653 "runservice starts:%p", q); 3654 3655 if (!(q->q_flag & QWCLOSE)) 3656 (*q->q_qinfo->qi_srvp)(q); 3657 3658 TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END, 3659 "runservice ends:(%p)", q); 3660 3661 leavesq(q->q_syncq, SQ_SVC); 3662 3663 mutex_enter(QLOCK(q)); 3664 if (q->q_flag & QENAB) { 3665 q->q_flag &= ~QENAB; 3666 mutex_exit(QLOCK(q)); 3667 goto again; 3668 } 3669 q->q_flag &= ~QINSERVICE; 3670 q->q_flag &= ~QBACK; 3671 for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next) 3672 qbp->qb_flag &= ~QB_BACK; 3673 /* 3674 * Wakeup thread waiting for the service procedure 3675 * to be run (strclose and qdetach). 3676 */ 3677 cv_broadcast(&q->q_wait); 3678 3679 mutex_exit(QLOCK(q)); 3680 } 3681 3682 /* 3683 * Background processing of bufcalls. 3684 */ 3685 void 3686 streams_bufcall_service(void) 3687 { 3688 callb_cpr_t cprinfo; 3689 3690 CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr, 3691 "streams_bufcall_service"); 3692 3693 mutex_enter(&strbcall_lock); 3694 3695 for (;;) { 3696 if (strbcalls.bc_head != NULL && kmem_avail() > 0) { 3697 mutex_exit(&strbcall_lock); 3698 runbufcalls(); 3699 mutex_enter(&strbcall_lock); 3700 } 3701 if (strbcalls.bc_head != NULL) { 3702 STRSTAT(bcwaits); 3703 /* Wait for memory to become available */ 3704 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3705 (void) cv_reltimedwait(&memavail_cv, &strbcall_lock, 3706 SEC_TO_TICK(60), TR_CLOCK_TICK); 3707 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3708 } 3709 3710 /* Wait for new work to arrive */ 3711 if (strbcalls.bc_head == NULL) { 3712 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3713 cv_wait(&strbcall_cv, &strbcall_lock); 3714 CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock); 3715 } 3716 } 3717 } 3718 3719 /* 3720 * Background processing of streams background tasks which failed 3721 * taskq_dispatch. 3722 */ 3723 static void 3724 streams_qbkgrnd_service(void) 3725 { 3726 callb_cpr_t cprinfo; 3727 queue_t *q; 3728 3729 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3730 "streams_bkgrnd_service"); 3731 3732 mutex_enter(&service_queue); 3733 3734 for (;;) { 3735 /* 3736 * Wait for work to arrive. 3737 */ 3738 while ((freebs_list == NULL) && (qhead == NULL)) { 3739 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3740 cv_wait(&services_to_run, &service_queue); 3741 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3742 } 3743 /* 3744 * Handle all pending freebs requests to free memory. 3745 */ 3746 while (freebs_list != NULL) { 3747 mblk_t *mp = freebs_list; 3748 freebs_list = mp->b_next; 3749 mutex_exit(&service_queue); 3750 mblk_free(mp); 3751 mutex_enter(&service_queue); 3752 } 3753 /* 3754 * Run pending queues. 3755 */ 3756 while (qhead != NULL) { 3757 DQ(q, qhead, qtail, q_link); 3758 ASSERT(q != NULL); 3759 mutex_exit(&service_queue); 3760 queue_service(q); 3761 mutex_enter(&service_queue); 3762 } 3763 ASSERT(qhead == NULL && qtail == NULL); 3764 } 3765 } 3766 3767 /* 3768 * Background processing of streams background tasks which failed 3769 * taskq_dispatch. 3770 */ 3771 static void 3772 streams_sqbkgrnd_service(void) 3773 { 3774 callb_cpr_t cprinfo; 3775 syncq_t *sq; 3776 3777 CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr, 3778 "streams_sqbkgrnd_service"); 3779 3780 mutex_enter(&service_queue); 3781 3782 for (;;) { 3783 /* 3784 * Wait for work to arrive. 3785 */ 3786 while (sqhead == NULL) { 3787 CALLB_CPR_SAFE_BEGIN(&cprinfo); 3788 cv_wait(&syncqs_to_run, &service_queue); 3789 CALLB_CPR_SAFE_END(&cprinfo, &service_queue); 3790 } 3791 3792 /* 3793 * Run pending syncqs. 3794 */ 3795 while (sqhead != NULL) { 3796 DQ(sq, sqhead, sqtail, sq_next); 3797 ASSERT(sq != NULL); 3798 ASSERT(sq->sq_svcflags & SQ_BGTHREAD); 3799 mutex_exit(&service_queue); 3800 syncq_service(sq); 3801 mutex_enter(&service_queue); 3802 } 3803 } 3804 } 3805 3806 /* 3807 * Disable the syncq and wait for background syncq processing to complete. 3808 * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the 3809 * list. 3810 */ 3811 void 3812 wait_sq_svc(syncq_t *sq) 3813 { 3814 mutex_enter(SQLOCK(sq)); 3815 sq->sq_svcflags |= SQ_DISABLED; 3816 if (sq->sq_svcflags & SQ_BGTHREAD) { 3817 syncq_t *sq_chase; 3818 syncq_t *sq_curr; 3819 int removed; 3820 3821 ASSERT(sq->sq_servcount == 1); 3822 mutex_enter(&service_queue); 3823 RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed); 3824 mutex_exit(&service_queue); 3825 if (removed) { 3826 sq->sq_svcflags &= ~SQ_BGTHREAD; 3827 sq->sq_servcount = 0; 3828 STRSTAT(sqremoved); 3829 goto done; 3830 } 3831 } 3832 while (sq->sq_servcount != 0) { 3833 sq->sq_flags |= SQ_WANTWAKEUP; 3834 cv_wait(&sq->sq_wait, SQLOCK(sq)); 3835 } 3836 done: 3837 mutex_exit(SQLOCK(sq)); 3838 } 3839 3840 /* 3841 * Put a syncq on the list of syncq's to be serviced by the sqthread. 3842 * Add the argument to the end of the sqhead list and set the flag 3843 * indicating this syncq has been enabled. If it has already been 3844 * enabled, don't do anything. 3845 * This routine assumes that SQLOCK is held. 3846 * NOTE that the lock order is to have the SQLOCK first, 3847 * so if the service_syncq lock is held, we need to release it 3848 * before acquiring the SQLOCK (mostly relevant for the background 3849 * thread, and this seems to be common among the STREAMS global locks). 3850 * Note that the sq_svcflags are protected by the SQLOCK. 3851 */ 3852 void 3853 sqenable(syncq_t *sq) 3854 { 3855 /* 3856 * This is probably not important except for where I believe it 3857 * is being called. At that point, it should be held (and it 3858 * is a pain to release it just for this routine, so don't do 3859 * it). 3860 */ 3861 ASSERT(MUTEX_HELD(SQLOCK(sq))); 3862 3863 IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL); 3864 IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD); 3865 3866 /* 3867 * Do not put on list if background thread is scheduled or 3868 * syncq is disabled. 3869 */ 3870 if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD)) 3871 return; 3872 3873 /* 3874 * Check whether we should enable sq at all. 3875 * Non PERMOD syncqs may be drained by at most one thread. 3876 * PERMOD syncqs may be drained by several threads but we limit the 3877 * total amount to the lesser of 3878 * Number of queues on the squeue and 3879 * Number of CPUs. 3880 */ 3881 if (sq->sq_servcount != 0) { 3882 if (((sq->sq_type & SQ_PERMOD) == 0) || 3883 (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) { 3884 STRSTAT(sqtoomany); 3885 return; 3886 } 3887 } 3888 3889 sq->sq_tstamp = ddi_get_lbolt(); 3890 STRSTAT(sqenables); 3891 3892 /* Attempt a taskq dispatch */ 3893 sq->sq_servid = (void *)taskq_dispatch(streams_taskq, 3894 (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE); 3895 if (sq->sq_servid != NULL) { 3896 sq->sq_servcount++; 3897 return; 3898 } 3899 3900 /* 3901 * This taskq dispatch failed, but a previous one may have succeeded. 3902 * Don't try to schedule on the background thread whilst there is 3903 * outstanding taskq processing. 3904 */ 3905 if (sq->sq_servcount != 0) 3906 return; 3907 3908 /* 3909 * System is low on resources and can't perform a non-sleeping 3910 * dispatch. Schedule the syncq for a background thread and mark the 3911 * syncq to avoid any further taskq dispatch attempts. 3912 */ 3913 mutex_enter(&service_queue); 3914 STRSTAT(taskqfails); 3915 ENQUEUE(sq, sqhead, sqtail, sq_next); 3916 sq->sq_svcflags |= SQ_BGTHREAD; 3917 sq->sq_servcount = 1; 3918 cv_signal(&syncqs_to_run); 3919 mutex_exit(&service_queue); 3920 } 3921 3922 /* 3923 * Note: fifo_close() depends on the mblk_t on the queue being freed 3924 * asynchronously. The asynchronous freeing of messages breaks the 3925 * recursive call chain of fifo_close() while there are I_SENDFD type of 3926 * messages referring to other file pointers on the queue. Then when 3927 * closing pipes it can avoid stack overflow in case of daisy-chained 3928 * pipes, and also avoid deadlock in case of fifonode_t pairs (which 3929 * share the same fifolock_t). 3930 * 3931 * No need to kpreempt_disable to access cpu_seqid. If we migrate and 3932 * the esb queue does not match the new CPU, that is OK. 3933 */ 3934 void 3935 freebs_enqueue(mblk_t *mp, dblk_t *dbp) 3936 { 3937 int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q; 3938 esb_queue_t *eqp; 3939 3940 ASSERT(dbp->db_mblk == mp); 3941 ASSERT(qindex < esbq_nelem); 3942 3943 eqp = system_esbq_array; 3944 if (eqp != NULL) { 3945 eqp += qindex; 3946 } else { 3947 mutex_enter(&esbq_lock); 3948 if (kmem_ready && system_esbq_array == NULL) 3949 system_esbq_array = (esb_queue_t *)kmem_zalloc( 3950 esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP); 3951 mutex_exit(&esbq_lock); 3952 eqp = system_esbq_array; 3953 if (eqp != NULL) 3954 eqp += qindex; 3955 else 3956 eqp = &system_esbq; 3957 } 3958 3959 /* 3960 * Check data sanity. The dblock should have non-empty free function. 3961 * It is better to panic here then later when the dblock is freed 3962 * asynchronously when the context is lost. 3963 */ 3964 if (dbp->db_frtnp->free_func == NULL) { 3965 panic("freebs_enqueue: dblock %p has a NULL free callback", 3966 (void *)dbp); 3967 } 3968 3969 mutex_enter(&eqp->eq_lock); 3970 /* queue the new mblk on the esballoc queue */ 3971 if (eqp->eq_head == NULL) { 3972 eqp->eq_head = eqp->eq_tail = mp; 3973 } else { 3974 eqp->eq_tail->b_next = mp; 3975 eqp->eq_tail = mp; 3976 } 3977 eqp->eq_len++; 3978 3979 /* If we're the first thread to reach the threshold, process */ 3980 if (eqp->eq_len >= esbq_max_qlen && 3981 !(eqp->eq_flags & ESBQ_PROCESSING)) 3982 esballoc_process_queue(eqp); 3983 3984 esballoc_set_timer(eqp, esbq_timeout); 3985 mutex_exit(&eqp->eq_lock); 3986 } 3987 3988 static void 3989 esballoc_process_queue(esb_queue_t *eqp) 3990 { 3991 mblk_t *mp; 3992 3993 ASSERT(MUTEX_HELD(&eqp->eq_lock)); 3994 3995 eqp->eq_flags |= ESBQ_PROCESSING; 3996 3997 do { 3998 /* 3999 * Detach the message chain for processing. 4000 */ 4001 mp = eqp->eq_head; 4002 eqp->eq_tail->b_next = NULL; 4003 eqp->eq_head = eqp->eq_tail = NULL; 4004 eqp->eq_len = 0; 4005 mutex_exit(&eqp->eq_lock); 4006 4007 /* 4008 * Process the message chain. 4009 */ 4010 esballoc_enqueue_mblk(mp); 4011 mutex_enter(&eqp->eq_lock); 4012 } while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0)); 4013 4014 eqp->eq_flags &= ~ESBQ_PROCESSING; 4015 } 4016 4017 /* 4018 * taskq callback routine to free esballoced mblk's 4019 */ 4020 static void 4021 esballoc_mblk_free(mblk_t *mp) 4022 { 4023 mblk_t *nextmp; 4024 4025 for (; mp != NULL; mp = nextmp) { 4026 nextmp = mp->b_next; 4027 mp->b_next = NULL; 4028 mblk_free(mp); 4029 } 4030 } 4031 4032 static void 4033 esballoc_enqueue_mblk(mblk_t *mp) 4034 { 4035 4036 if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp, 4037 TQ_NOSLEEP) == TASKQID_INVALID) { 4038 mblk_t *first_mp = mp; 4039 /* 4040 * System is low on resources and can't perform a non-sleeping 4041 * dispatch. Schedule for a background thread. 4042 */ 4043 mutex_enter(&service_queue); 4044 STRSTAT(taskqfails); 4045 4046 while (mp->b_next != NULL) 4047 mp = mp->b_next; 4048 4049 mp->b_next = freebs_list; 4050 freebs_list = first_mp; 4051 cv_signal(&services_to_run); 4052 mutex_exit(&service_queue); 4053 } 4054 } 4055 4056 static void 4057 esballoc_timer(void *arg) 4058 { 4059 esb_queue_t *eqp = arg; 4060 4061 mutex_enter(&eqp->eq_lock); 4062 eqp->eq_flags &= ~ESBQ_TIMER; 4063 4064 if (!(eqp->eq_flags & ESBQ_PROCESSING) && 4065 eqp->eq_len > 0) 4066 esballoc_process_queue(eqp); 4067 4068 esballoc_set_timer(eqp, esbq_timeout); 4069 mutex_exit(&eqp->eq_lock); 4070 } 4071 4072 static void 4073 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout) 4074 { 4075 ASSERT(MUTEX_HELD(&eqp->eq_lock)); 4076 4077 if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) { 4078 (void) timeout(esballoc_timer, eqp, eq_timeout); 4079 eqp->eq_flags |= ESBQ_TIMER; 4080 } 4081 } 4082 4083 /* 4084 * Setup esbq array length based upon NCPU scaled by CPUs per 4085 * queue. Use static system_esbq until kmem_ready and we can 4086 * create an array in freebs_enqueue(). 4087 */ 4088 void 4089 esballoc_queue_init(void) 4090 { 4091 esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1); 4092 esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q; 4093 esbq_nelem = howmany(NCPU, esbq_cpus_per_q); 4094 system_esbq.eq_len = 0; 4095 system_esbq.eq_head = system_esbq.eq_tail = NULL; 4096 system_esbq.eq_flags = 0; 4097 } 4098 4099 /* 4100 * Set the QBACK or QB_BACK flag in the given queue for 4101 * the given priority band. 4102 */ 4103 void 4104 setqback(queue_t *q, unsigned char pri) 4105 { 4106 int i; 4107 qband_t *qbp; 4108 qband_t **qbpp; 4109 4110 ASSERT(MUTEX_HELD(QLOCK(q))); 4111 if (pri != 0) { 4112 if (pri > q->q_nband) { 4113 qbpp = &q->q_bandp; 4114 while (*qbpp) 4115 qbpp = &(*qbpp)->qb_next; 4116 while (pri > q->q_nband) { 4117 if ((*qbpp = allocband()) == NULL) { 4118 cmn_err(CE_WARN, 4119 "setqback: can't allocate qband\n"); 4120 return; 4121 } 4122 (*qbpp)->qb_hiwat = q->q_hiwat; 4123 (*qbpp)->qb_lowat = q->q_lowat; 4124 q->q_nband++; 4125 qbpp = &(*qbpp)->qb_next; 4126 } 4127 } 4128 qbp = q->q_bandp; 4129 i = pri; 4130 while (--i) 4131 qbp = qbp->qb_next; 4132 qbp->qb_flag |= QB_BACK; 4133 } else { 4134 q->q_flag |= QBACK; 4135 } 4136 } 4137 4138 int 4139 strcopyin(void *from, void *to, size_t len, int copyflag) 4140 { 4141 if (copyflag & U_TO_K) { 4142 ASSERT((copyflag & K_TO_K) == 0); 4143 if (copyin(from, to, len)) 4144 return (EFAULT); 4145 } else { 4146 ASSERT(copyflag & K_TO_K); 4147 bcopy(from, to, len); 4148 } 4149 return (0); 4150 } 4151 4152 int 4153 strcopyout(void *from, void *to, size_t len, int copyflag) 4154 { 4155 if (copyflag & U_TO_K) { 4156 if (copyout(from, to, len)) 4157 return (EFAULT); 4158 } else { 4159 ASSERT(copyflag & K_TO_K); 4160 bcopy(from, to, len); 4161 } 4162 return (0); 4163 } 4164 4165 /* 4166 * strsignal_nolock() posts a signal to the process(es) at the stream head. 4167 * It assumes that the stream head lock is already held, whereas strsignal() 4168 * acquires the lock first. This routine was created because a few callers 4169 * release the stream head lock before calling only to re-acquire it after 4170 * it returns. 4171 */ 4172 void 4173 strsignal_nolock(stdata_t *stp, int sig, uchar_t band) 4174 { 4175 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4176 switch (sig) { 4177 case SIGPOLL: 4178 if (stp->sd_sigflags & S_MSG) 4179 strsendsig(stp->sd_siglist, S_MSG, band, 0); 4180 break; 4181 default: 4182 if (stp->sd_pgidp) 4183 pgsignal(stp->sd_pgidp, sig); 4184 break; 4185 } 4186 } 4187 4188 void 4189 strsignal(stdata_t *stp, int sig, int32_t band) 4190 { 4191 TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG, 4192 "strsignal:%p, %X, %X", stp, sig, band); 4193 4194 mutex_enter(&stp->sd_lock); 4195 switch (sig) { 4196 case SIGPOLL: 4197 if (stp->sd_sigflags & S_MSG) 4198 strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0); 4199 break; 4200 4201 default: 4202 if (stp->sd_pgidp) { 4203 pgsignal(stp->sd_pgidp, sig); 4204 } 4205 break; 4206 } 4207 mutex_exit(&stp->sd_lock); 4208 } 4209 4210 void 4211 strhup(stdata_t *stp) 4212 { 4213 ASSERT(mutex_owned(&stp->sd_lock)); 4214 pollwakeup(&stp->sd_pollist, POLLHUP); 4215 if (stp->sd_sigflags & S_HANGUP) 4216 strsendsig(stp->sd_siglist, S_HANGUP, 0, 0); 4217 } 4218 4219 /* 4220 * Backenable the first queue upstream from `q' with a service procedure. 4221 */ 4222 void 4223 backenable(queue_t *q, uchar_t pri) 4224 { 4225 queue_t *nq; 4226 4227 /* 4228 * Our presence might not prevent other modules in our own 4229 * stream from popping/pushing since the caller of getq might not 4230 * have a claim on the queue (some drivers do a getq on somebody 4231 * else's queue - they know that the queue itself is not going away 4232 * but the framework has to guarantee q_next in that stream). 4233 */ 4234 claimstr(q); 4235 4236 /* Find nearest back queue with service proc */ 4237 for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) { 4238 ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq)); 4239 } 4240 4241 if (nq) { 4242 kthread_t *freezer; 4243 /* 4244 * backenable can be called either with no locks held 4245 * or with the stream frozen (the latter occurs when a module 4246 * calls rmvq with the stream frozen). If the stream is frozen 4247 * by the caller the caller will hold all qlocks in the stream. 4248 * Note that a frozen stream doesn't freeze a mated stream, 4249 * so we explicitly check for that. 4250 */ 4251 freezer = STREAM(q)->sd_freezer; 4252 if (freezer != curthread || STREAM(q) != STREAM(nq)) { 4253 mutex_enter(QLOCK(nq)); 4254 } 4255 #ifdef DEBUG 4256 else { 4257 ASSERT(frozenstr(q)); 4258 ASSERT(MUTEX_HELD(QLOCK(q))); 4259 ASSERT(MUTEX_HELD(QLOCK(nq))); 4260 } 4261 #endif 4262 setqback(nq, pri); 4263 qenable_locked(nq); 4264 if (freezer != curthread || STREAM(q) != STREAM(nq)) 4265 mutex_exit(QLOCK(nq)); 4266 } 4267 releasestr(q); 4268 } 4269 4270 /* 4271 * Return the appropriate errno when one of flags_to_check is set 4272 * in sd_flags. Uses the exported error routines if they are set. 4273 * Will return 0 if non error is set (or if the exported error routines 4274 * do not return an error). 4275 * 4276 * If there is both a read and write error to check, we prefer the read error. 4277 * Also, give preference to recorded errno's over the error functions. 4278 * The flags that are handled are: 4279 * STPLEX return EINVAL 4280 * STRDERR return sd_rerror (and clear if STRDERRNONPERSIST) 4281 * STWRERR return sd_werror (and clear if STWRERRNONPERSIST) 4282 * STRHUP return sd_werror 4283 * 4284 * If the caller indicates that the operation is a peek, a nonpersistent error 4285 * is not cleared. 4286 */ 4287 int 4288 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek) 4289 { 4290 int32_t sd_flag = stp->sd_flag & flags_to_check; 4291 int error = 0; 4292 4293 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4294 ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0); 4295 if (sd_flag & STPLEX) 4296 error = EINVAL; 4297 else if (sd_flag & STRDERR) { 4298 error = stp->sd_rerror; 4299 if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) { 4300 /* 4301 * Read errors are non-persistent i.e. discarded once 4302 * returned to a non-peeking caller, 4303 */ 4304 stp->sd_rerror = 0; 4305 stp->sd_flag &= ~STRDERR; 4306 } 4307 if (error == 0 && stp->sd_rderrfunc != NULL) { 4308 int clearerr = 0; 4309 4310 error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek, 4311 &clearerr); 4312 if (clearerr) { 4313 stp->sd_flag &= ~STRDERR; 4314 stp->sd_rderrfunc = NULL; 4315 } 4316 } 4317 } else if (sd_flag & STWRERR) { 4318 error = stp->sd_werror; 4319 if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) { 4320 /* 4321 * Write errors are non-persistent i.e. discarded once 4322 * returned to a non-peeking caller, 4323 */ 4324 stp->sd_werror = 0; 4325 stp->sd_flag &= ~STWRERR; 4326 } 4327 if (error == 0 && stp->sd_wrerrfunc != NULL) { 4328 int clearerr = 0; 4329 4330 error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek, 4331 &clearerr); 4332 if (clearerr) { 4333 stp->sd_flag &= ~STWRERR; 4334 stp->sd_wrerrfunc = NULL; 4335 } 4336 } 4337 } else if (sd_flag & STRHUP) { 4338 /* sd_werror set when STRHUP */ 4339 error = stp->sd_werror; 4340 } 4341 return (error); 4342 } 4343 4344 4345 /* 4346 * Single-thread open/close/push/pop 4347 * for twisted streams also 4348 */ 4349 int 4350 strstartplumb(stdata_t *stp, int flag, int cmd) 4351 { 4352 int waited = 1; 4353 int error = 0; 4354 4355 if (STRMATED(stp)) { 4356 struct stdata *stmatep = stp->sd_mate; 4357 4358 STRLOCKMATES(stp); 4359 while (waited) { 4360 waited = 0; 4361 while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4362 if ((cmd == I_POP) && 4363 (flag & (FNDELAY|FNONBLOCK))) { 4364 STRUNLOCKMATES(stp); 4365 return (EAGAIN); 4366 } 4367 waited = 1; 4368 mutex_exit(&stp->sd_lock); 4369 if (!cv_wait_sig(&stmatep->sd_monitor, 4370 &stmatep->sd_lock)) { 4371 mutex_exit(&stmatep->sd_lock); 4372 return (EINTR); 4373 } 4374 mutex_exit(&stmatep->sd_lock); 4375 STRLOCKMATES(stp); 4376 } 4377 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4378 if ((cmd == I_POP) && 4379 (flag & (FNDELAY|FNONBLOCK))) { 4380 STRUNLOCKMATES(stp); 4381 return (EAGAIN); 4382 } 4383 waited = 1; 4384 mutex_exit(&stmatep->sd_lock); 4385 if (!cv_wait_sig(&stp->sd_monitor, 4386 &stp->sd_lock)) { 4387 mutex_exit(&stp->sd_lock); 4388 return (EINTR); 4389 } 4390 mutex_exit(&stp->sd_lock); 4391 STRLOCKMATES(stp); 4392 } 4393 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4394 error = strgeterr(stp, 4395 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4396 if (error != 0) { 4397 STRUNLOCKMATES(stp); 4398 return (error); 4399 } 4400 } 4401 } 4402 stp->sd_flag |= STRPLUMB; 4403 STRUNLOCKMATES(stp); 4404 } else { 4405 mutex_enter(&stp->sd_lock); 4406 while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) { 4407 if (((cmd == I_POP) || (cmd == _I_REMOVE)) && 4408 (flag & (FNDELAY|FNONBLOCK))) { 4409 mutex_exit(&stp->sd_lock); 4410 return (EAGAIN); 4411 } 4412 if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) { 4413 mutex_exit(&stp->sd_lock); 4414 return (EINTR); 4415 } 4416 if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) { 4417 error = strgeterr(stp, 4418 STRDERR|STWRERR|STRHUP|STPLEX, 0); 4419 if (error != 0) { 4420 mutex_exit(&stp->sd_lock); 4421 return (error); 4422 } 4423 } 4424 } 4425 stp->sd_flag |= STRPLUMB; 4426 mutex_exit(&stp->sd_lock); 4427 } 4428 return (0); 4429 } 4430 4431 /* 4432 * Complete the plumbing operation associated with stream `stp'. 4433 */ 4434 void 4435 strendplumb(stdata_t *stp) 4436 { 4437 ASSERT(MUTEX_HELD(&stp->sd_lock)); 4438 ASSERT(stp->sd_flag & STRPLUMB); 4439 stp->sd_flag &= ~STRPLUMB; 4440 cv_broadcast(&stp->sd_monitor); 4441 } 4442 4443 /* 4444 * This describes how the STREAMS framework handles synchronization 4445 * during open/push and close/pop. 4446 * The key interfaces for open and close are qprocson and qprocsoff, 4447 * respectively. While the close case in general is harder both open 4448 * have close have significant similarities. 4449 * 4450 * During close the STREAMS framework has to both ensure that there 4451 * are no stale references to the queue pair (and syncq) that 4452 * are being closed and also provide the guarantees that are documented 4453 * in qprocsoff(9F). 4454 * If there are stale references to the queue that is closing it can 4455 * result in kernel memory corruption or kernel panics. 4456 * 4457 * Note that is it up to the module/driver to ensure that it itself 4458 * does not have any stale references to the closing queues once its close 4459 * routine returns. This includes: 4460 * - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines 4461 * associated with the queues. For timeout and bufcall callbacks the 4462 * module/driver also has to ensure (or wait for) any callbacks that 4463 * are in progress. 4464 * - If the module/driver is using esballoc it has to ensure that any 4465 * esballoc free functions do not refer to a queue that has closed. 4466 * (Note that in general the close routine can not wait for the esballoc'ed 4467 * messages to be freed since that can cause a deadlock.) 4468 * - Cancelling any interrupts that refer to the closing queues and 4469 * also ensuring that there are no interrupts in progress that will 4470 * refer to the closing queues once the close routine returns. 4471 * - For multiplexors removing any driver global state that refers to 4472 * the closing queue and also ensuring that there are no threads in 4473 * the multiplexor that has picked up a queue pointer but not yet 4474 * finished using it. 4475 * 4476 * In addition, a driver/module can only reference the q_next pointer 4477 * in its open, close, put, or service procedures or in a 4478 * qtimeout/qbufcall callback procedure executing "on" the correct 4479 * stream. Thus it can not reference the q_next pointer in an interrupt 4480 * routine or a timeout, bufcall or esballoc callback routine. Likewise 4481 * it can not reference q_next of a different queue e.g. in a mux that 4482 * passes messages from one queues put/service procedure to another queue. 4483 * In all the cases when the driver/module can not access the q_next 4484 * field it must use the *next* versions e.g. canputnext instead of 4485 * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...). 4486 * 4487 * 4488 * Assuming that the driver/module conforms to the above constraints 4489 * the STREAMS framework has to avoid stale references to q_next for all 4490 * the framework internal cases which include (but are not limited to): 4491 * - Threads in canput/canputnext/backenable and elsewhere that are 4492 * walking q_next. 4493 * - Messages on a syncq that have a reference to the queue through b_queue. 4494 * - Messages on an outer perimeter (syncq) that have a reference to the 4495 * queue through b_queue. 4496 * - Threads that use q_nfsrv (e.g. canput) to find a queue. 4497 * Note that only canput and bcanput use q_nfsrv without any locking. 4498 * 4499 * The STREAMS framework providing the qprocsoff(9F) guarantees means that 4500 * after qprocsoff returns, the framework has to ensure that no threads can 4501 * enter the put or service routines for the closing read or write-side queue. 4502 * In addition to preventing "direct" entry into the put procedures 4503 * the framework also has to prevent messages being drained from 4504 * the syncq or the outer perimeter. 4505 * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only 4506 * mechanism to prevent qwriter(PERIM_OUTER) from running after 4507 * qprocsoff has returned. 4508 * Note that if a module/driver uses put(9F) on one of its own queues 4509 * it is up to the module/driver to ensure that the put() doesn't 4510 * get called when the queue is closing. 4511 * 4512 * 4513 * The framework aspects of the above "contract" is implemented by 4514 * qprocsoff, removeq, and strlock: 4515 * - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from 4516 * entering the service procedures. 4517 * - strlock acquires the sd_lock and sd_reflock to prevent putnext, 4518 * canputnext, backenable etc from dereferencing the q_next that will 4519 * soon change. 4520 * - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext 4521 * or other q_next walker that uses claimstr/releasestr to finish. 4522 * - optionally for every syncq in the stream strlock acquires all the 4523 * sq_lock's and waits for all sq_counts to drop to a value that indicates 4524 * that no thread executes in the put or service procedures and that no 4525 * thread is draining into the module/driver. This ensures that no 4526 * open, close, put, service, or qtimeout/qbufcall callback procedure is 4527 * currently executing hence no such thread can end up with the old stale 4528 * q_next value and no canput/backenable can have the old stale 4529 * q_nfsrv/q_next. 4530 * - qdetach (wait_svc) makes sure that any scheduled or running threads 4531 * have either finished or observed the QWCLOSE flag and gone away. 4532 */ 4533 4534 4535 /* 4536 * Get all the locks necessary to change q_next. 4537 * 4538 * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the 4539 * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that 4540 * the only threads inside the syncq are threads currently calling removeq(). 4541 * Since threads calling removeq() are in the process of removing their queues 4542 * from the stream, we do not need to worry about them accessing a stale q_next 4543 * pointer and thus we do not need to wait for them to exit (in fact, waiting 4544 * for them can cause deadlock). 4545 * 4546 * This routine is subject to starvation since it does not set any flag to 4547 * prevent threads from entering a module in the stream (i.e. sq_count can 4548 * increase on some syncq while it is waiting on some other syncq). 4549 * 4550 * Assumes that only one thread attempts to call strlock for a given 4551 * stream. If this is not the case the two threads would deadlock. 4552 * This assumption is guaranteed since strlock is only called by insertq 4553 * and removeq and streams plumbing changes are single-threaded for 4554 * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags. 4555 * 4556 * For pipes, it is not difficult to atomically designate a pair of streams 4557 * to be mated. Once mated atomically by the framework the twisted pair remain 4558 * configured that way until dismantled atomically by the framework. 4559 * When plumbing takes place on a twisted stream it is necessary to ensure that 4560 * this operation is done exclusively on the twisted stream since two such 4561 * operations, each initiated on different ends of the pipe will deadlock 4562 * waiting for each other to complete. 4563 * 4564 * On entry, no locks should be held. 4565 * The locks acquired and held by strlock depends on a few factors. 4566 * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired 4567 * and held on exit and all sq_count are at an acceptable level. 4568 * - In all cases, sd_lock and sd_reflock are acquired and held on exit with 4569 * sd_refcnt being zero. 4570 */ 4571 4572 static void 4573 strlock(struct stdata *stp, sqlist_t *sqlist) 4574 { 4575 syncql_t *sql, *sql2; 4576 retry: 4577 /* 4578 * Wait for any claimstr to go away. 4579 */ 4580 if (STRMATED(stp)) { 4581 struct stdata *stp1, *stp2; 4582 4583 STRLOCKMATES(stp); 4584 /* 4585 * Note that the selection of locking order is not 4586 * important, just that they are always acquired in 4587 * the same order. To assure this, we choose this 4588 * order based on the value of the pointer, and since 4589 * the pointer will not change for the life of this 4590 * pair, we will always grab the locks in the same 4591 * order (and hence, prevent deadlocks). 4592 */ 4593 if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) { 4594 stp1 = stp; 4595 stp2 = stp->sd_mate; 4596 } else { 4597 stp2 = stp; 4598 stp1 = stp->sd_mate; 4599 } 4600 mutex_enter(&stp1->sd_reflock); 4601 if (stp1->sd_refcnt > 0) { 4602 STRUNLOCKMATES(stp); 4603 cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock); 4604 mutex_exit(&stp1->sd_reflock); 4605 goto retry; 4606 } 4607 mutex_enter(&stp2->sd_reflock); 4608 if (stp2->sd_refcnt > 0) { 4609 STRUNLOCKMATES(stp); 4610 mutex_exit(&stp1->sd_reflock); 4611 cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock); 4612 mutex_exit(&stp2->sd_reflock); 4613 goto retry; 4614 } 4615 STREAM_PUTLOCKS_ENTER(stp1); 4616 STREAM_PUTLOCKS_ENTER(stp2); 4617 } else { 4618 mutex_enter(&stp->sd_lock); 4619 mutex_enter(&stp->sd_reflock); 4620 while (stp->sd_refcnt > 0) { 4621 mutex_exit(&stp->sd_lock); 4622 cv_wait(&stp->sd_refmonitor, &stp->sd_reflock); 4623 if (mutex_tryenter(&stp->sd_lock) == 0) { 4624 mutex_exit(&stp->sd_reflock); 4625 mutex_enter(&stp->sd_lock); 4626 mutex_enter(&stp->sd_reflock); 4627 } 4628 } 4629 STREAM_PUTLOCKS_ENTER(stp); 4630 } 4631 4632 if (sqlist == NULL) 4633 return; 4634 4635 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4636 syncq_t *sq = sql->sql_sq; 4637 uint16_t count; 4638 4639 mutex_enter(SQLOCK(sq)); 4640 count = sq->sq_count; 4641 ASSERT(sq->sq_rmqcount <= count); 4642 SQ_PUTLOCKS_ENTER(sq); 4643 SUM_SQ_PUTCOUNTS(sq, count); 4644 if (count == sq->sq_rmqcount) 4645 continue; 4646 4647 /* Failed - drop all locks that we have acquired so far */ 4648 if (STRMATED(stp)) { 4649 STREAM_PUTLOCKS_EXIT(stp); 4650 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4651 STRUNLOCKMATES(stp); 4652 mutex_exit(&stp->sd_reflock); 4653 mutex_exit(&stp->sd_mate->sd_reflock); 4654 } else { 4655 STREAM_PUTLOCKS_EXIT(stp); 4656 mutex_exit(&stp->sd_lock); 4657 mutex_exit(&stp->sd_reflock); 4658 } 4659 for (sql2 = sqlist->sqlist_head; sql2 != sql; 4660 sql2 = sql2->sql_next) { 4661 SQ_PUTLOCKS_EXIT(sql2->sql_sq); 4662 mutex_exit(SQLOCK(sql2->sql_sq)); 4663 } 4664 4665 /* 4666 * The wait loop below may starve when there are many threads 4667 * claiming the syncq. This is especially a problem with permod 4668 * syncqs (IP). To lessen the impact of the problem we increment 4669 * sq_needexcl and clear fastbits so that putnexts will slow 4670 * down and call sqenable instead of draining right away. 4671 */ 4672 sq->sq_needexcl++; 4673 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 4674 while (count > sq->sq_rmqcount) { 4675 sq->sq_flags |= SQ_WANTWAKEUP; 4676 SQ_PUTLOCKS_EXIT(sq); 4677 cv_wait(&sq->sq_wait, SQLOCK(sq)); 4678 count = sq->sq_count; 4679 SQ_PUTLOCKS_ENTER(sq); 4680 SUM_SQ_PUTCOUNTS(sq, count); 4681 } 4682 sq->sq_needexcl--; 4683 if (sq->sq_needexcl == 0) 4684 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 4685 SQ_PUTLOCKS_EXIT(sq); 4686 ASSERT(count == sq->sq_rmqcount); 4687 mutex_exit(SQLOCK(sq)); 4688 goto retry; 4689 } 4690 } 4691 4692 /* 4693 * Drop all the locks that strlock acquired. 4694 */ 4695 static void 4696 strunlock(struct stdata *stp, sqlist_t *sqlist) 4697 { 4698 syncql_t *sql; 4699 4700 if (STRMATED(stp)) { 4701 STREAM_PUTLOCKS_EXIT(stp); 4702 STREAM_PUTLOCKS_EXIT(stp->sd_mate); 4703 STRUNLOCKMATES(stp); 4704 mutex_exit(&stp->sd_reflock); 4705 mutex_exit(&stp->sd_mate->sd_reflock); 4706 } else { 4707 STREAM_PUTLOCKS_EXIT(stp); 4708 mutex_exit(&stp->sd_lock); 4709 mutex_exit(&stp->sd_reflock); 4710 } 4711 4712 if (sqlist == NULL) 4713 return; 4714 4715 for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) { 4716 SQ_PUTLOCKS_EXIT(sql->sql_sq); 4717 mutex_exit(SQLOCK(sql->sql_sq)); 4718 } 4719 } 4720 4721 /* 4722 * When the module has service procedure, we need check if the next 4723 * module which has service procedure is in flow control to trigger 4724 * the backenable. 4725 */ 4726 static void 4727 backenable_insertedq(queue_t *q) 4728 { 4729 qband_t *qbp; 4730 4731 claimstr(q); 4732 if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) { 4733 if (q->q_next->q_nfsrv->q_flag & QWANTW) 4734 backenable(q, 0); 4735 4736 qbp = q->q_next->q_nfsrv->q_bandp; 4737 for (; qbp != NULL; qbp = qbp->qb_next) 4738 if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL) 4739 backenable(q, qbp->qb_first->b_band); 4740 } 4741 releasestr(q); 4742 } 4743 4744 /* 4745 * Given two read queues, insert a new single one after another. 4746 * 4747 * This routine acquires all the necessary locks in order to change 4748 * q_next and related pointer using strlock(). 4749 * It depends on the stream head ensuring that there are no concurrent 4750 * insertq or removeq on the same stream. The stream head ensures this 4751 * using the flags STWOPEN, STRCLOSE, and STRPLUMB. 4752 * 4753 * Note that no syncq locks are held during the q_next change. This is 4754 * applied to all streams since, unlike removeq, there is no problem of stale 4755 * pointers when adding a module to the stream. Thus drivers/modules that do a 4756 * canput(rq->q_next) would never get a closed/freed queue pointer even if we 4757 * applied this optimization to all streams. 4758 */ 4759 void 4760 insertq(struct stdata *stp, queue_t *new) 4761 { 4762 queue_t *after; 4763 queue_t *wafter; 4764 queue_t *wnew = _WR(new); 4765 boolean_t have_fifo = B_FALSE; 4766 4767 if (new->q_flag & _QINSERTING) { 4768 ASSERT(stp->sd_vnode->v_type != VFIFO); 4769 after = new->q_next; 4770 wafter = _WR(new->q_next); 4771 } else { 4772 after = _RD(stp->sd_wrq); 4773 wafter = stp->sd_wrq; 4774 } 4775 4776 TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ, 4777 "insertq:%p, %p", after, new); 4778 ASSERT(after->q_flag & QREADR); 4779 ASSERT(new->q_flag & QREADR); 4780 4781 strlock(stp, NULL); 4782 4783 /* Do we have a FIFO? */ 4784 if (wafter->q_next == after) { 4785 have_fifo = B_TRUE; 4786 wnew->q_next = new; 4787 } else { 4788 wnew->q_next = wafter->q_next; 4789 } 4790 new->q_next = after; 4791 4792 set_nfsrv_ptr(new, wnew, after, wafter); 4793 /* 4794 * set_nfsrv_ptr() needs to know if this is an insertion or not, 4795 * so only reset this flag after calling it. 4796 */ 4797 new->q_flag &= ~_QINSERTING; 4798 4799 if (have_fifo) { 4800 wafter->q_next = wnew; 4801 } else { 4802 if (wafter->q_next) 4803 _OTHERQ(wafter->q_next)->q_next = new; 4804 wafter->q_next = wnew; 4805 } 4806 4807 set_qend(new); 4808 /* The QEND flag might have to be updated for the upstream guy */ 4809 set_qend(after); 4810 4811 ASSERT(_SAMESTR(new) == O_SAMESTR(new)); 4812 ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew)); 4813 ASSERT(_SAMESTR(after) == O_SAMESTR(after)); 4814 ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter)); 4815 strsetuio(stp); 4816 4817 /* 4818 * If this was a module insertion, bump the push count. 4819 */ 4820 if (!(new->q_flag & QISDRV)) 4821 stp->sd_pushcnt++; 4822 4823 strunlock(stp, NULL); 4824 4825 /* check if the write Q needs backenable */ 4826 backenable_insertedq(wnew); 4827 4828 /* check if the read Q needs backenable */ 4829 backenable_insertedq(new); 4830 } 4831 4832 /* 4833 * Given a read queue, unlink it from any neighbors. 4834 * 4835 * This routine acquires all the necessary locks in order to 4836 * change q_next and related pointers and also guard against 4837 * stale references (e.g. through q_next) to the queue that 4838 * is being removed. It also plays part of the role in ensuring 4839 * that the module's/driver's put procedure doesn't get called 4840 * after qprocsoff returns. 4841 * 4842 * Removeq depends on the stream head ensuring that there are 4843 * no concurrent insertq or removeq on the same stream. The 4844 * stream head ensures this using the flags STWOPEN, STRCLOSE and 4845 * STRPLUMB. 4846 * 4847 * The set of locks needed to remove the queue is different in 4848 * different cases: 4849 * 4850 * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after 4851 * waiting for the syncq reference count to drop to 0 indicating that no 4852 * non-close threads are present anywhere in the stream. This ensures that any 4853 * module/driver can reference q_next in its open, close, put, or service 4854 * procedures. 4855 * 4856 * The sq_rmqcount counter tracks the number of threads inside removeq(). 4857 * strlock() ensures that there is either no threads executing inside perimeter 4858 * or there is only a thread calling qprocsoff(). 4859 * 4860 * strlock() compares the value of sq_count with the number of threads inside 4861 * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup 4862 * any threads waiting in strlock() when the sq_rmqcount increases. 4863 */ 4864 4865 void 4866 removeq(queue_t *qp) 4867 { 4868 queue_t *wqp = _WR(qp); 4869 struct stdata *stp = STREAM(qp); 4870 sqlist_t *sqlist = NULL; 4871 boolean_t isdriver; 4872 int moved; 4873 syncq_t *sq = qp->q_syncq; 4874 syncq_t *wsq = wqp->q_syncq; 4875 4876 ASSERT(stp); 4877 4878 TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ, 4879 "removeq:%p %p", qp, wqp); 4880 ASSERT(qp->q_flag&QREADR); 4881 4882 /* 4883 * For queues using Synchronous streams, we must wait for all threads in 4884 * rwnext() to drain out before proceeding. 4885 */ 4886 if (qp->q_flag & QSYNCSTR) { 4887 /* First, we need wakeup any threads blocked in rwnext() */ 4888 mutex_enter(SQLOCK(sq)); 4889 if (sq->sq_flags & SQ_WANTWAKEUP) { 4890 sq->sq_flags &= ~SQ_WANTWAKEUP; 4891 cv_broadcast(&sq->sq_wait); 4892 } 4893 mutex_exit(SQLOCK(sq)); 4894 4895 if (wsq != sq) { 4896 mutex_enter(SQLOCK(wsq)); 4897 if (wsq->sq_flags & SQ_WANTWAKEUP) { 4898 wsq->sq_flags &= ~SQ_WANTWAKEUP; 4899 cv_broadcast(&wsq->sq_wait); 4900 } 4901 mutex_exit(SQLOCK(wsq)); 4902 } 4903 4904 mutex_enter(QLOCK(qp)); 4905 while (qp->q_rwcnt > 0) { 4906 qp->q_flag |= QWANTRMQSYNC; 4907 cv_wait(&qp->q_wait, QLOCK(qp)); 4908 } 4909 mutex_exit(QLOCK(qp)); 4910 4911 mutex_enter(QLOCK(wqp)); 4912 while (wqp->q_rwcnt > 0) { 4913 wqp->q_flag |= QWANTRMQSYNC; 4914 cv_wait(&wqp->q_wait, QLOCK(wqp)); 4915 } 4916 mutex_exit(QLOCK(wqp)); 4917 } 4918 4919 mutex_enter(SQLOCK(sq)); 4920 sq->sq_rmqcount++; 4921 if (sq->sq_flags & SQ_WANTWAKEUP) { 4922 sq->sq_flags &= ~SQ_WANTWAKEUP; 4923 cv_broadcast(&sq->sq_wait); 4924 } 4925 mutex_exit(SQLOCK(sq)); 4926 4927 isdriver = (qp->q_flag & QISDRV); 4928 4929 sqlist = sqlist_build(qp, stp, STRMATED(stp)); 4930 strlock(stp, sqlist); 4931 4932 reset_nfsrv_ptr(qp, wqp); 4933 4934 ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp); 4935 ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp); 4936 /* Do we have a FIFO? */ 4937 if (wqp->q_next == qp) { 4938 stp->sd_wrq->q_next = _RD(stp->sd_wrq); 4939 } else { 4940 if (wqp->q_next) 4941 backq(qp)->q_next = qp->q_next; 4942 if (qp->q_next) 4943 backq(wqp)->q_next = wqp->q_next; 4944 } 4945 4946 /* The QEND flag might have to be updated for the upstream guy */ 4947 if (qp->q_next) 4948 set_qend(qp->q_next); 4949 4950 ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq)); 4951 ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq))); 4952 4953 /* 4954 * Move any messages destined for the put procedures to the next 4955 * syncq in line. Otherwise free them. 4956 */ 4957 moved = 0; 4958 /* 4959 * Quick check to see whether there are any messages or events. 4960 */ 4961 if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS)) 4962 moved += propagate_syncq(qp); 4963 if (wqp->q_syncqmsgs != 0 || 4964 (wqp->q_syncq->sq_flags & SQ_EVENTS)) 4965 moved += propagate_syncq(wqp); 4966 4967 strsetuio(stp); 4968 4969 /* 4970 * If this was a module removal, decrement the push count. 4971 */ 4972 if (!isdriver) 4973 stp->sd_pushcnt--; 4974 4975 strunlock(stp, sqlist); 4976 sqlist_free(sqlist); 4977 4978 /* 4979 * Make sure any messages that were propagated are drained. 4980 * Also clear any QFULL bit caused by messages that were propagated. 4981 */ 4982 4983 if (qp->q_next != NULL) { 4984 clr_qfull(qp); 4985 /* 4986 * For the driver calling qprocsoff, propagate_syncq 4987 * frees all the messages instead of putting it in 4988 * the stream head 4989 */ 4990 if (!isdriver && (moved > 0)) 4991 emptysq(qp->q_next->q_syncq); 4992 } 4993 if (wqp->q_next != NULL) { 4994 clr_qfull(wqp); 4995 /* 4996 * We come here for any pop of a module except for the 4997 * case of driver being removed. We don't call emptysq 4998 * if we did not move any messages. This will avoid holding 4999 * PERMOD syncq locks in emptysq 5000 */ 5001 if (moved > 0) 5002 emptysq(wqp->q_next->q_syncq); 5003 } 5004 5005 mutex_enter(SQLOCK(sq)); 5006 sq->sq_rmqcount--; 5007 mutex_exit(SQLOCK(sq)); 5008 } 5009 5010 /* 5011 * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or 5012 * SQ_WRITER) on a syncq. 5013 * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the 5014 * sync queue and waits until sq_count reaches maxcnt. 5015 * 5016 * If maxcnt is -1 there's no need to grab sq_putlocks since the caller 5017 * does not care about putnext threads that are in the middle of calling put 5018 * entry points. 5019 * 5020 * This routine is used for both inner and outer syncqs. 5021 */ 5022 static void 5023 blocksq(syncq_t *sq, ushort_t flag, int maxcnt) 5024 { 5025 uint16_t count = 0; 5026 5027 mutex_enter(SQLOCK(sq)); 5028 /* 5029 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset. 5030 * SQ_FROZEN will be set if there is a frozen stream that has a 5031 * queue which also refers to this "shared" syncq. 5032 * SQ_BLOCKED will be set if there is "off" queue which also 5033 * refers to this "shared" syncq. 5034 */ 5035 if (maxcnt != -1) { 5036 count = sq->sq_count; 5037 SQ_PUTLOCKS_ENTER(sq); 5038 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5039 SUM_SQ_PUTCOUNTS(sq, count); 5040 } 5041 sq->sq_needexcl++; 5042 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5043 5044 while ((sq->sq_flags & flag) || 5045 (maxcnt != -1 && count > (unsigned)maxcnt)) { 5046 sq->sq_flags |= SQ_WANTWAKEUP; 5047 if (maxcnt != -1) { 5048 SQ_PUTLOCKS_EXIT(sq); 5049 } 5050 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5051 if (maxcnt != -1) { 5052 count = sq->sq_count; 5053 SQ_PUTLOCKS_ENTER(sq); 5054 SUM_SQ_PUTCOUNTS(sq, count); 5055 } 5056 } 5057 sq->sq_needexcl--; 5058 sq->sq_flags |= flag; 5059 ASSERT(maxcnt == -1 || count == maxcnt); 5060 if (maxcnt != -1) { 5061 if (sq->sq_needexcl == 0) { 5062 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5063 } 5064 SQ_PUTLOCKS_EXIT(sq); 5065 } else if (sq->sq_needexcl == 0) { 5066 SQ_PUTCOUNT_SETFAST(sq); 5067 } 5068 5069 mutex_exit(SQLOCK(sq)); 5070 } 5071 5072 /* 5073 * Reset a flag that was set with blocksq. 5074 * 5075 * Can not use this routine to reset SQ_WRITER. 5076 * 5077 * If "isouter" is set then the syncq is assumed to be an outer perimeter 5078 * and drain_syncq is not called. Instead we rely on the qwriter_outer thread 5079 * to handle the queued qwriter operations. 5080 * 5081 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5082 * sq_putlocks are used. 5083 */ 5084 static void 5085 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter) 5086 { 5087 uint16_t flags; 5088 5089 mutex_enter(SQLOCK(sq)); 5090 ASSERT(resetflag != SQ_WRITER); 5091 ASSERT(sq->sq_flags & resetflag); 5092 flags = sq->sq_flags & ~resetflag; 5093 sq->sq_flags = flags; 5094 if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) { 5095 if (flags & SQ_WANTWAKEUP) { 5096 flags &= ~SQ_WANTWAKEUP; 5097 cv_broadcast(&sq->sq_wait); 5098 } 5099 sq->sq_flags = flags; 5100 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5101 if (!isouter) { 5102 /* drain_syncq drops SQLOCK */ 5103 drain_syncq(sq); 5104 return; 5105 } 5106 } 5107 } 5108 mutex_exit(SQLOCK(sq)); 5109 } 5110 5111 /* 5112 * Reset a flag that was set with blocksq. 5113 * Does not drain the syncq. Use emptysq() for that. 5114 * Returns 1 if SQ_QUEUED is set. Otherwise 0. 5115 * 5116 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5117 * sq_putlocks are used. 5118 */ 5119 static int 5120 dropsq(syncq_t *sq, uint16_t resetflag) 5121 { 5122 uint16_t flags; 5123 5124 mutex_enter(SQLOCK(sq)); 5125 ASSERT(sq->sq_flags & resetflag); 5126 flags = sq->sq_flags & ~resetflag; 5127 if (flags & SQ_WANTWAKEUP) { 5128 flags &= ~SQ_WANTWAKEUP; 5129 cv_broadcast(&sq->sq_wait); 5130 } 5131 sq->sq_flags = flags; 5132 mutex_exit(SQLOCK(sq)); 5133 if (flags & SQ_QUEUED) 5134 return (1); 5135 return (0); 5136 } 5137 5138 /* 5139 * Empty all the messages on a syncq. 5140 * 5141 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5142 * sq_putlocks are used. 5143 */ 5144 static void 5145 emptysq(syncq_t *sq) 5146 { 5147 uint16_t flags; 5148 5149 mutex_enter(SQLOCK(sq)); 5150 flags = sq->sq_flags; 5151 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5152 /* 5153 * To prevent potential recursive invocation of drain_syncq we 5154 * do not call drain_syncq if count is non-zero. 5155 */ 5156 if (sq->sq_count == 0) { 5157 /* drain_syncq() drops SQLOCK */ 5158 drain_syncq(sq); 5159 return; 5160 } else 5161 sqenable(sq); 5162 } 5163 mutex_exit(SQLOCK(sq)); 5164 } 5165 5166 /* 5167 * Ordered insert while removing duplicates. 5168 */ 5169 static void 5170 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp) 5171 { 5172 syncql_t *sqlp, **prev_sqlpp, *new_sqlp; 5173 5174 prev_sqlpp = &sqlist->sqlist_head; 5175 while ((sqlp = *prev_sqlpp) != NULL) { 5176 if (sqlp->sql_sq >= sqp) { 5177 if (sqlp->sql_sq == sqp) /* duplicate */ 5178 return; 5179 break; 5180 } 5181 prev_sqlpp = &sqlp->sql_next; 5182 } 5183 new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++]; 5184 ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size); 5185 new_sqlp->sql_next = sqlp; 5186 new_sqlp->sql_sq = sqp; 5187 *prev_sqlpp = new_sqlp; 5188 } 5189 5190 /* 5191 * Walk the write side queues until we hit either the driver 5192 * or a twist in the stream (_SAMESTR will return false in both 5193 * these cases) then turn around and walk the read side queues 5194 * back up to the stream head. 5195 */ 5196 static void 5197 sqlist_insertall(sqlist_t *sqlist, queue_t *q) 5198 { 5199 while (q != NULL) { 5200 sqlist_insert(sqlist, q->q_syncq); 5201 5202 if (_SAMESTR(q)) 5203 q = q->q_next; 5204 else if (!(q->q_flag & QREADR)) 5205 q = _RD(q); 5206 else 5207 q = NULL; 5208 } 5209 } 5210 5211 /* 5212 * Allocate and build a list of all syncqs in a stream and the syncq(s) 5213 * associated with the "q" parameter. The resulting list is sorted in a 5214 * canonical order and is free of duplicates. 5215 * Assumes the passed queue is a _RD(q). 5216 */ 5217 static sqlist_t * 5218 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist) 5219 { 5220 sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP); 5221 5222 /* 5223 * start with the current queue/qpair 5224 */ 5225 ASSERT(q->q_flag & QREADR); 5226 5227 sqlist_insert(sqlist, q->q_syncq); 5228 sqlist_insert(sqlist, _WR(q)->q_syncq); 5229 5230 sqlist_insertall(sqlist, stp->sd_wrq); 5231 if (do_twist) 5232 sqlist_insertall(sqlist, stp->sd_mate->sd_wrq); 5233 5234 return (sqlist); 5235 } 5236 5237 static sqlist_t * 5238 sqlist_alloc(struct stdata *stp, int kmflag) 5239 { 5240 size_t sqlist_size; 5241 sqlist_t *sqlist; 5242 5243 /* 5244 * Allocate 2 syncql_t's for each pushed module. Note that 5245 * the sqlist_t structure already has 4 syncql_t's built in: 5246 * 2 for the stream head, and 2 for the driver/other stream head. 5247 */ 5248 sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt + 5249 sizeof (sqlist_t); 5250 if (STRMATED(stp)) 5251 sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt; 5252 sqlist = kmem_alloc(sqlist_size, kmflag); 5253 5254 sqlist->sqlist_head = NULL; 5255 sqlist->sqlist_size = sqlist_size; 5256 sqlist->sqlist_index = 0; 5257 5258 return (sqlist); 5259 } 5260 5261 /* 5262 * Free the list created by sqlist_alloc() 5263 */ 5264 static void 5265 sqlist_free(sqlist_t *sqlist) 5266 { 5267 kmem_free(sqlist, sqlist->sqlist_size); 5268 } 5269 5270 /* 5271 * Prevent any new entries into any syncq in this stream. 5272 * Used by freezestr. 5273 */ 5274 void 5275 strblock(queue_t *q) 5276 { 5277 struct stdata *stp; 5278 syncql_t *sql; 5279 sqlist_t *sqlist; 5280 5281 q = _RD(q); 5282 5283 stp = STREAM(q); 5284 ASSERT(stp != NULL); 5285 5286 /* 5287 * Get a sorted list with all the duplicates removed containing 5288 * all the syncqs referenced by this stream. 5289 */ 5290 sqlist = sqlist_build(q, stp, B_FALSE); 5291 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5292 blocksq(sql->sql_sq, SQ_FROZEN, -1); 5293 sqlist_free(sqlist); 5294 } 5295 5296 /* 5297 * Release the block on new entries into this stream 5298 */ 5299 void 5300 strunblock(queue_t *q) 5301 { 5302 struct stdata *stp; 5303 syncql_t *sql; 5304 sqlist_t *sqlist; 5305 int drain_needed; 5306 5307 q = _RD(q); 5308 5309 /* 5310 * Get a sorted list with all the duplicates removed containing 5311 * all the syncqs referenced by this stream. 5312 * Have to drop the SQ_FROZEN flag on all the syncqs before 5313 * starting to drain them; otherwise the draining might 5314 * cause a freezestr in some module on the stream (which 5315 * would deadlock). 5316 */ 5317 stp = STREAM(q); 5318 ASSERT(stp != NULL); 5319 sqlist = sqlist_build(q, stp, B_FALSE); 5320 drain_needed = 0; 5321 for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next) 5322 drain_needed += dropsq(sql->sql_sq, SQ_FROZEN); 5323 if (drain_needed) { 5324 for (sql = sqlist->sqlist_head; sql != NULL; 5325 sql = sql->sql_next) 5326 emptysq(sql->sql_sq); 5327 } 5328 sqlist_free(sqlist); 5329 } 5330 5331 #ifdef DEBUG 5332 static int 5333 qprocsareon(queue_t *rq) 5334 { 5335 if (rq->q_next == NULL) 5336 return (0); 5337 return (_WR(rq->q_next)->q_next == _WR(rq)); 5338 } 5339 5340 int 5341 qclaimed(queue_t *q) 5342 { 5343 uint_t count; 5344 5345 count = q->q_syncq->sq_count; 5346 SUM_SQ_PUTCOUNTS(q->q_syncq, count); 5347 return (count != 0); 5348 } 5349 5350 /* 5351 * Check if anyone has frozen this stream with freezestr 5352 */ 5353 int 5354 frozenstr(queue_t *q) 5355 { 5356 return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0); 5357 } 5358 #endif /* DEBUG */ 5359 5360 /* 5361 * Enter a queue. 5362 * Obsoleted interface. Should not be used. 5363 */ 5364 void 5365 enterq(queue_t *q) 5366 { 5367 entersq(q->q_syncq, SQ_CALLBACK); 5368 } 5369 5370 void 5371 leaveq(queue_t *q) 5372 { 5373 leavesq(q->q_syncq, SQ_CALLBACK); 5374 } 5375 5376 /* 5377 * Enter a perimeter. c_inner and c_outer specifies which concurrency bits 5378 * to check. 5379 * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter 5380 * calls and the running of open, close and service procedures. 5381 * 5382 * If c_inner bit is set no need to grab sq_putlocks since we don't care 5383 * if other threads have entered or are entering put entry point. 5384 * 5385 * If c_inner bit is set it might have been possible to use 5386 * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize 5387 * open/close path for IP) but since the count may need to be decremented in 5388 * qwait() we wouldn't know which counter to decrement. Currently counter is 5389 * selected by current cpu_seqid and current CPU can change at any moment. XXX 5390 * in the future we might use curthread id bits to select the counter and this 5391 * would stay constant across routine calls. 5392 */ 5393 void 5394 entersq(syncq_t *sq, int entrypoint) 5395 { 5396 uint16_t count = 0; 5397 uint16_t flags; 5398 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 5399 uint16_t type; 5400 uint_t c_inner = entrypoint & SQ_CI; 5401 uint_t c_outer = entrypoint & SQ_CO; 5402 5403 /* 5404 * Increment ref count to keep closes out of this queue. 5405 */ 5406 ASSERT(sq); 5407 ASSERT(c_inner && c_outer); 5408 mutex_enter(SQLOCK(sq)); 5409 flags = sq->sq_flags; 5410 type = sq->sq_type; 5411 if (!(type & c_inner)) { 5412 /* Make sure all putcounts now use slowlock. */ 5413 count = sq->sq_count; 5414 SQ_PUTLOCKS_ENTER(sq); 5415 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 5416 SUM_SQ_PUTCOUNTS(sq, count); 5417 sq->sq_needexcl++; 5418 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 5419 waitflags |= SQ_MESSAGES; 5420 } 5421 /* 5422 * Wait until we can enter the inner perimeter. 5423 * If we want exclusive access we wait until sq_count is 0. 5424 * We have to do this before entering the outer perimeter in order 5425 * to preserve put/close message ordering. 5426 */ 5427 while ((flags & waitflags) || (!(type & c_inner) && count != 0)) { 5428 sq->sq_flags = flags | SQ_WANTWAKEUP; 5429 if (!(type & c_inner)) { 5430 SQ_PUTLOCKS_EXIT(sq); 5431 } 5432 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5433 if (!(type & c_inner)) { 5434 count = sq->sq_count; 5435 SQ_PUTLOCKS_ENTER(sq); 5436 SUM_SQ_PUTCOUNTS(sq, count); 5437 } 5438 flags = sq->sq_flags; 5439 } 5440 5441 if (!(type & c_inner)) { 5442 ASSERT(sq->sq_needexcl > 0); 5443 sq->sq_needexcl--; 5444 if (sq->sq_needexcl == 0) { 5445 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 5446 } 5447 } 5448 5449 /* Check if we need to enter the outer perimeter */ 5450 if (!(type & c_outer)) { 5451 /* 5452 * We have to enter the outer perimeter exclusively before 5453 * we can increment sq_count to avoid deadlock. This implies 5454 * that we have to re-check sq_flags and sq_count. 5455 * 5456 * is it possible to have c_inner set when c_outer is not set? 5457 */ 5458 if (!(type & c_inner)) { 5459 SQ_PUTLOCKS_EXIT(sq); 5460 } 5461 mutex_exit(SQLOCK(sq)); 5462 outer_enter(sq->sq_outer, SQ_GOAWAY); 5463 mutex_enter(SQLOCK(sq)); 5464 flags = sq->sq_flags; 5465 /* 5466 * there should be no need to recheck sq_putcounts 5467 * because outer_enter() has already waited for them to clear 5468 * after setting SQ_WRITER. 5469 */ 5470 count = sq->sq_count; 5471 #ifdef DEBUG 5472 /* 5473 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead 5474 * of doing an ASSERT internally. Others should do 5475 * something like 5476 * ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0); 5477 * without the need to #ifdef DEBUG it. 5478 */ 5479 SUMCHECK_SQ_PUTCOUNTS(sq, 0); 5480 #endif 5481 while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) || 5482 (!(type & c_inner) && count != 0)) { 5483 sq->sq_flags = flags | SQ_WANTWAKEUP; 5484 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5485 count = sq->sq_count; 5486 flags = sq->sq_flags; 5487 } 5488 } 5489 5490 sq->sq_count++; 5491 ASSERT(sq->sq_count != 0); /* Wraparound */ 5492 if (!(type & c_inner)) { 5493 /* Exclusive entry */ 5494 ASSERT(sq->sq_count == 1); 5495 sq->sq_flags |= SQ_EXCL; 5496 if (type & c_outer) { 5497 SQ_PUTLOCKS_EXIT(sq); 5498 } 5499 } 5500 mutex_exit(SQLOCK(sq)); 5501 } 5502 5503 /* 5504 * Leave a syncq. Announce to framework that closes may proceed. 5505 * c_inner and c_outer specify which concurrency bits to check. 5506 * 5507 * Must never be called from driver or module put entry point. 5508 * 5509 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5510 * sq_putlocks are used. 5511 */ 5512 void 5513 leavesq(syncq_t *sq, int entrypoint) 5514 { 5515 uint16_t flags; 5516 uint16_t type; 5517 uint_t c_outer = entrypoint & SQ_CO; 5518 #ifdef DEBUG 5519 uint_t c_inner = entrypoint & SQ_CI; 5520 #endif 5521 5522 /* 5523 * Decrement ref count, drain the syncq if possible, and wake up 5524 * any waiting close. 5525 */ 5526 ASSERT(sq); 5527 ASSERT(c_inner && c_outer); 5528 mutex_enter(SQLOCK(sq)); 5529 flags = sq->sq_flags; 5530 type = sq->sq_type; 5531 if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) { 5532 5533 if (flags & SQ_WANTWAKEUP) { 5534 flags &= ~SQ_WANTWAKEUP; 5535 cv_broadcast(&sq->sq_wait); 5536 } 5537 if (flags & SQ_WANTEXWAKEUP) { 5538 flags &= ~SQ_WANTEXWAKEUP; 5539 cv_broadcast(&sq->sq_exitwait); 5540 } 5541 5542 if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) { 5543 /* 5544 * The syncq needs to be drained. "Exit" the syncq 5545 * before calling drain_syncq. 5546 */ 5547 ASSERT(sq->sq_count != 0); 5548 sq->sq_count--; 5549 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5550 sq->sq_flags = flags & ~SQ_EXCL; 5551 drain_syncq(sq); 5552 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 5553 /* Check if we need to exit the outer perimeter */ 5554 /* XXX will this ever be true? */ 5555 if (!(type & c_outer)) 5556 outer_exit(sq->sq_outer); 5557 return; 5558 } 5559 } 5560 ASSERT(sq->sq_count != 0); 5561 sq->sq_count--; 5562 ASSERT((flags & SQ_EXCL) || (type & c_inner)); 5563 sq->sq_flags = flags & ~SQ_EXCL; 5564 mutex_exit(SQLOCK(sq)); 5565 5566 /* Check if we need to exit the outer perimeter */ 5567 if (!(sq->sq_type & c_outer)) 5568 outer_exit(sq->sq_outer); 5569 } 5570 5571 /* 5572 * Prevent q_next from changing in this stream by incrementing sq_count. 5573 * 5574 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5575 * sq_putlocks are used. 5576 */ 5577 void 5578 claimq(queue_t *qp) 5579 { 5580 syncq_t *sq = qp->q_syncq; 5581 5582 mutex_enter(SQLOCK(sq)); 5583 sq->sq_count++; 5584 ASSERT(sq->sq_count != 0); /* Wraparound */ 5585 mutex_exit(SQLOCK(sq)); 5586 } 5587 5588 /* 5589 * Undo claimq. 5590 * 5591 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 5592 * sq_putlocks are used. 5593 */ 5594 void 5595 releaseq(queue_t *qp) 5596 { 5597 syncq_t *sq = qp->q_syncq; 5598 uint16_t flags; 5599 5600 mutex_enter(SQLOCK(sq)); 5601 ASSERT(sq->sq_count > 0); 5602 sq->sq_count--; 5603 5604 flags = sq->sq_flags; 5605 if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) { 5606 if (flags & SQ_WANTWAKEUP) { 5607 flags &= ~SQ_WANTWAKEUP; 5608 cv_broadcast(&sq->sq_wait); 5609 } 5610 sq->sq_flags = flags; 5611 if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) { 5612 /* 5613 * To prevent potential recursive invocation of 5614 * drain_syncq we do not call drain_syncq if count is 5615 * non-zero. 5616 */ 5617 if (sq->sq_count == 0) { 5618 drain_syncq(sq); 5619 return; 5620 } else 5621 sqenable(sq); 5622 } 5623 } 5624 mutex_exit(SQLOCK(sq)); 5625 } 5626 5627 /* 5628 * Prevent q_next from changing in this stream by incrementing sd_refcnt. 5629 */ 5630 void 5631 claimstr(queue_t *qp) 5632 { 5633 struct stdata *stp = STREAM(qp); 5634 5635 mutex_enter(&stp->sd_reflock); 5636 stp->sd_refcnt++; 5637 ASSERT(stp->sd_refcnt != 0); /* Wraparound */ 5638 mutex_exit(&stp->sd_reflock); 5639 } 5640 5641 /* 5642 * Undo claimstr. 5643 */ 5644 void 5645 releasestr(queue_t *qp) 5646 { 5647 struct stdata *stp = STREAM(qp); 5648 5649 mutex_enter(&stp->sd_reflock); 5650 ASSERT(stp->sd_refcnt != 0); 5651 if (--stp->sd_refcnt == 0) 5652 cv_broadcast(&stp->sd_refmonitor); 5653 mutex_exit(&stp->sd_reflock); 5654 } 5655 5656 static syncq_t * 5657 new_syncq(void) 5658 { 5659 return (kmem_cache_alloc(syncq_cache, KM_SLEEP)); 5660 } 5661 5662 static void 5663 free_syncq(syncq_t *sq) 5664 { 5665 ASSERT(sq->sq_head == NULL); 5666 ASSERT(sq->sq_outer == NULL); 5667 ASSERT(sq->sq_callbpend == NULL); 5668 ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) || 5669 (sq->sq_onext == sq && sq->sq_oprev == sq)); 5670 5671 if (sq->sq_ciputctrl != NULL) { 5672 ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1); 5673 SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl, 5674 sq->sq_nciputctrl, 0); 5675 ASSERT(ciputctrl_cache != NULL); 5676 kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl); 5677 } 5678 5679 sq->sq_tail = NULL; 5680 sq->sq_evhead = NULL; 5681 sq->sq_evtail = NULL; 5682 sq->sq_ciputctrl = NULL; 5683 sq->sq_nciputctrl = 0; 5684 sq->sq_count = 0; 5685 sq->sq_rmqcount = 0; 5686 sq->sq_callbflags = 0; 5687 sq->sq_cancelid = 0; 5688 sq->sq_next = NULL; 5689 sq->sq_needexcl = 0; 5690 sq->sq_svcflags = 0; 5691 sq->sq_nqueues = 0; 5692 sq->sq_pri = 0; 5693 sq->sq_onext = NULL; 5694 sq->sq_oprev = NULL; 5695 sq->sq_flags = 0; 5696 sq->sq_type = 0; 5697 sq->sq_servcount = 0; 5698 5699 kmem_cache_free(syncq_cache, sq); 5700 } 5701 5702 /* Outer perimeter code */ 5703 5704 /* 5705 * The outer syncq uses the fields and flags in the syncq slightly 5706 * differently from the inner syncqs. 5707 * sq_count Incremented when there are pending or running 5708 * writers at the outer perimeter to prevent the set of 5709 * inner syncqs that belong to the outer perimeter from 5710 * changing. 5711 * sq_head/tail List of deferred qwriter(OUTER) operations. 5712 * 5713 * SQ_BLOCKED Set to prevent traversing of sq_next,sq_prev while 5714 * inner syncqs are added to or removed from the 5715 * outer perimeter. 5716 * SQ_QUEUED sq_head/tail has messages or events queued. 5717 * 5718 * SQ_WRITER A thread is currently traversing all the inner syncqs 5719 * setting the SQ_WRITER flag. 5720 */ 5721 5722 /* 5723 * Get write access at the outer perimeter. 5724 * Note that read access is done by entersq, putnext, and put by simply 5725 * incrementing sq_count in the inner syncq. 5726 * 5727 * Waits until "flags" is no longer set in the outer to prevent multiple 5728 * threads from having write access at the same time. SQ_WRITER has to be part 5729 * of "flags". 5730 * 5731 * Increases sq_count on the outer syncq to keep away outer_insert/remove 5732 * until the outer_exit is finished. 5733 * 5734 * outer_enter is vulnerable to starvation since it does not prevent new 5735 * threads from entering the inner syncqs while it is waiting for sq_count to 5736 * go to zero. 5737 */ 5738 void 5739 outer_enter(syncq_t *outer, uint16_t flags) 5740 { 5741 syncq_t *sq; 5742 int wait_needed; 5743 uint16_t count; 5744 5745 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5746 outer->sq_oprev != NULL); 5747 ASSERT(flags & SQ_WRITER); 5748 5749 retry: 5750 mutex_enter(SQLOCK(outer)); 5751 while (outer->sq_flags & flags) { 5752 outer->sq_flags |= SQ_WANTWAKEUP; 5753 cv_wait(&outer->sq_wait, SQLOCK(outer)); 5754 } 5755 5756 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5757 outer->sq_flags |= SQ_WRITER; 5758 outer->sq_count++; 5759 ASSERT(outer->sq_count != 0); /* wraparound */ 5760 wait_needed = 0; 5761 /* 5762 * Set SQ_WRITER on all the inner syncqs while holding 5763 * the SQLOCK on the outer syncq. This ensures that the changing 5764 * of SQ_WRITER is atomic under the outer SQLOCK. 5765 */ 5766 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5767 mutex_enter(SQLOCK(sq)); 5768 count = sq->sq_count; 5769 SQ_PUTLOCKS_ENTER(sq); 5770 sq->sq_flags |= SQ_WRITER; 5771 SUM_SQ_PUTCOUNTS(sq, count); 5772 if (count != 0) 5773 wait_needed = 1; 5774 SQ_PUTLOCKS_EXIT(sq); 5775 mutex_exit(SQLOCK(sq)); 5776 } 5777 mutex_exit(SQLOCK(outer)); 5778 5779 /* 5780 * Get everybody out of the syncqs sequentially. 5781 * Note that we don't actually need to acquire the PUTLOCKS, since 5782 * we have already cleared the fastbit, and set QWRITER. By 5783 * definition, the count can not increase since putnext will 5784 * take the slowlock path (and the purpose of acquiring the 5785 * putlocks was to make sure it didn't increase while we were 5786 * waiting). 5787 * 5788 * Note that we still acquire the PUTLOCKS to be safe. 5789 */ 5790 if (wait_needed) { 5791 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 5792 mutex_enter(SQLOCK(sq)); 5793 count = sq->sq_count; 5794 SQ_PUTLOCKS_ENTER(sq); 5795 SUM_SQ_PUTCOUNTS(sq, count); 5796 while (count != 0) { 5797 sq->sq_flags |= SQ_WANTWAKEUP; 5798 SQ_PUTLOCKS_EXIT(sq); 5799 cv_wait(&sq->sq_wait, SQLOCK(sq)); 5800 count = sq->sq_count; 5801 SQ_PUTLOCKS_ENTER(sq); 5802 SUM_SQ_PUTCOUNTS(sq, count); 5803 } 5804 SQ_PUTLOCKS_EXIT(sq); 5805 mutex_exit(SQLOCK(sq)); 5806 } 5807 /* 5808 * Verify that none of the flags got set while we 5809 * were waiting for the sq_counts to drop. 5810 * If this happens we exit and retry entering the 5811 * outer perimeter. 5812 */ 5813 mutex_enter(SQLOCK(outer)); 5814 if (outer->sq_flags & (flags & ~SQ_WRITER)) { 5815 mutex_exit(SQLOCK(outer)); 5816 outer_exit(outer); 5817 goto retry; 5818 } 5819 mutex_exit(SQLOCK(outer)); 5820 } 5821 } 5822 5823 /* 5824 * Drop the write access at the outer perimeter. 5825 * Read access is dropped implicitly (by putnext, put, and leavesq) by 5826 * decrementing sq_count. 5827 */ 5828 void 5829 outer_exit(syncq_t *outer) 5830 { 5831 syncq_t *sq; 5832 int drain_needed; 5833 uint16_t flags; 5834 5835 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5836 outer->sq_oprev != NULL); 5837 ASSERT(MUTEX_NOT_HELD(SQLOCK(outer))); 5838 5839 /* 5840 * Atomically (from the perspective of threads calling become_writer) 5841 * drop the write access at the outer perimeter by holding 5842 * SQLOCK(outer) across all the dropsq calls and the resetting of 5843 * SQ_WRITER. 5844 * This defines a locking order between the outer perimeter 5845 * SQLOCK and the inner perimeter SQLOCKs. 5846 */ 5847 mutex_enter(SQLOCK(outer)); 5848 flags = outer->sq_flags; 5849 ASSERT(outer->sq_flags & SQ_WRITER); 5850 if (flags & SQ_QUEUED) { 5851 write_now(outer); 5852 flags = outer->sq_flags; 5853 } 5854 5855 /* 5856 * sq_onext is stable since sq_count has not yet been decreased. 5857 * Reset the SQ_WRITER flags in all syncqs. 5858 * After dropping SQ_WRITER on the outer syncq we empty all the 5859 * inner syncqs. 5860 */ 5861 drain_needed = 0; 5862 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5863 drain_needed += dropsq(sq, SQ_WRITER); 5864 ASSERT(!(outer->sq_flags & SQ_QUEUED)); 5865 flags &= ~SQ_WRITER; 5866 if (drain_needed) { 5867 outer->sq_flags = flags; 5868 mutex_exit(SQLOCK(outer)); 5869 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) 5870 emptysq(sq); 5871 mutex_enter(SQLOCK(outer)); 5872 flags = outer->sq_flags; 5873 } 5874 if (flags & SQ_WANTWAKEUP) { 5875 flags &= ~SQ_WANTWAKEUP; 5876 cv_broadcast(&outer->sq_wait); 5877 } 5878 outer->sq_flags = flags; 5879 ASSERT(outer->sq_count > 0); 5880 outer->sq_count--; 5881 mutex_exit(SQLOCK(outer)); 5882 } 5883 5884 /* 5885 * Add another syncq to an outer perimeter. 5886 * Block out all other access to the outer perimeter while it is being 5887 * changed using blocksq. 5888 * Assumes that the caller has *not* done an outer_enter. 5889 * 5890 * Vulnerable to starvation in blocksq. 5891 */ 5892 static void 5893 outer_insert(syncq_t *outer, syncq_t *sq) 5894 { 5895 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5896 outer->sq_oprev != NULL); 5897 ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL && 5898 sq->sq_oprev == NULL); /* Can't be in an outer perimeter */ 5899 5900 /* Get exclusive access to the outer perimeter list */ 5901 blocksq(outer, SQ_BLOCKED, 0); 5902 ASSERT(outer->sq_flags & SQ_BLOCKED); 5903 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5904 5905 mutex_enter(SQLOCK(sq)); 5906 sq->sq_outer = outer; 5907 outer->sq_onext->sq_oprev = sq; 5908 sq->sq_onext = outer->sq_onext; 5909 outer->sq_onext = sq; 5910 sq->sq_oprev = outer; 5911 mutex_exit(SQLOCK(sq)); 5912 unblocksq(outer, SQ_BLOCKED, 1); 5913 } 5914 5915 /* 5916 * Remove a syncq from an outer perimeter. 5917 * Block out all other access to the outer perimeter while it is being 5918 * changed using blocksq. 5919 * Assumes that the caller has *not* done an outer_enter. 5920 * 5921 * Vulnerable to starvation in blocksq. 5922 */ 5923 static void 5924 outer_remove(syncq_t *outer, syncq_t *sq) 5925 { 5926 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 5927 outer->sq_oprev != NULL); 5928 ASSERT(sq->sq_outer == outer); 5929 5930 /* Get exclusive access to the outer perimeter list */ 5931 blocksq(outer, SQ_BLOCKED, 0); 5932 ASSERT(outer->sq_flags & SQ_BLOCKED); 5933 ASSERT(!(outer->sq_flags & SQ_WRITER)); 5934 5935 mutex_enter(SQLOCK(sq)); 5936 sq->sq_outer = NULL; 5937 sq->sq_onext->sq_oprev = sq->sq_oprev; 5938 sq->sq_oprev->sq_onext = sq->sq_onext; 5939 sq->sq_oprev = sq->sq_onext = NULL; 5940 mutex_exit(SQLOCK(sq)); 5941 unblocksq(outer, SQ_BLOCKED, 1); 5942 } 5943 5944 /* 5945 * Queue a deferred qwriter(OUTER) callback for this outer perimeter. 5946 * If this is the first callback for this outer perimeter then add 5947 * this outer perimeter to the list of outer perimeters that 5948 * the qwriter_outer_thread will process. 5949 * 5950 * Increments sq_count in the outer syncq to prevent the membership 5951 * of the outer perimeter (in terms of inner syncqs) to change while 5952 * the callback is pending. 5953 */ 5954 static void 5955 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp) 5956 { 5957 ASSERT(MUTEX_HELD(SQLOCK(outer))); 5958 5959 mp->b_prev = (mblk_t *)func; 5960 mp->b_queue = q; 5961 mp->b_next = NULL; 5962 outer->sq_count++; /* Decremented when dequeued */ 5963 ASSERT(outer->sq_count != 0); /* Wraparound */ 5964 if (outer->sq_evhead == NULL) { 5965 /* First message. */ 5966 outer->sq_evhead = outer->sq_evtail = mp; 5967 outer->sq_flags |= SQ_EVENTS; 5968 mutex_exit(SQLOCK(outer)); 5969 STRSTAT(qwr_outer); 5970 (void) taskq_dispatch(streams_taskq, 5971 (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP); 5972 } else { 5973 ASSERT(outer->sq_flags & SQ_EVENTS); 5974 outer->sq_evtail->b_next = mp; 5975 outer->sq_evtail = mp; 5976 mutex_exit(SQLOCK(outer)); 5977 } 5978 } 5979 5980 /* 5981 * Try and upgrade to write access at the outer perimeter. If this can 5982 * not be done without blocking then queue the callback to be done 5983 * by the qwriter_outer_thread. 5984 * 5985 * This routine can only be called from put or service procedures plus 5986 * asynchronous callback routines that have properly entered the queue (with 5987 * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq 5988 * associated with q. 5989 */ 5990 void 5991 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)()) 5992 { 5993 syncq_t *osq, *sq, *outer; 5994 int failed; 5995 uint16_t flags; 5996 5997 osq = q->q_syncq; 5998 outer = osq->sq_outer; 5999 if (outer == NULL) 6000 panic("qwriter(PERIM_OUTER): no outer perimeter"); 6001 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 6002 outer->sq_oprev != NULL); 6003 6004 mutex_enter(SQLOCK(outer)); 6005 flags = outer->sq_flags; 6006 /* 6007 * If some thread is traversing sq_next, or if we are blocked by 6008 * outer_insert or outer_remove, or if the we already have queued 6009 * callbacks, then queue this callback for later processing. 6010 * 6011 * Also queue the qwriter for an interrupt thread in order 6012 * to reduce the time spent running at high IPL. 6013 * to identify there are events. 6014 */ 6015 if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) { 6016 /* 6017 * Queue the become_writer request. 6018 * The queueing is atomic under SQLOCK(outer) in order 6019 * to synchronize with outer_exit. 6020 * queue_writer will drop the outer SQLOCK 6021 */ 6022 if (flags & SQ_BLOCKED) { 6023 /* Must set SQ_WRITER on inner perimeter */ 6024 mutex_enter(SQLOCK(osq)); 6025 osq->sq_flags |= SQ_WRITER; 6026 mutex_exit(SQLOCK(osq)); 6027 } else { 6028 if (!(flags & SQ_WRITER)) { 6029 /* 6030 * The outer could have been SQ_BLOCKED thus 6031 * SQ_WRITER might not be set on the inner. 6032 */ 6033 mutex_enter(SQLOCK(osq)); 6034 osq->sq_flags |= SQ_WRITER; 6035 mutex_exit(SQLOCK(osq)); 6036 } 6037 ASSERT(osq->sq_flags & SQ_WRITER); 6038 } 6039 queue_writer(outer, func, q, mp); 6040 return; 6041 } 6042 /* 6043 * We are half-way to exclusive access to the outer perimeter. 6044 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove 6045 * while the inner syncqs are traversed. 6046 */ 6047 outer->sq_count++; 6048 ASSERT(outer->sq_count != 0); /* wraparound */ 6049 flags |= SQ_WRITER; 6050 /* 6051 * Check if we can run the function immediately. Mark all 6052 * syncqs with the writer flag to prevent new entries into 6053 * put and service procedures. 6054 * 6055 * Set SQ_WRITER on all the inner syncqs while holding 6056 * the SQLOCK on the outer syncq. This ensures that the changing 6057 * of SQ_WRITER is atomic under the outer SQLOCK. 6058 */ 6059 failed = 0; 6060 for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) { 6061 uint16_t count; 6062 uint_t maxcnt = (sq == osq) ? 1 : 0; 6063 6064 mutex_enter(SQLOCK(sq)); 6065 count = sq->sq_count; 6066 SQ_PUTLOCKS_ENTER(sq); 6067 SUM_SQ_PUTCOUNTS(sq, count); 6068 if (sq->sq_count > maxcnt) 6069 failed = 1; 6070 sq->sq_flags |= SQ_WRITER; 6071 SQ_PUTLOCKS_EXIT(sq); 6072 mutex_exit(SQLOCK(sq)); 6073 } 6074 if (failed) { 6075 /* 6076 * Some other thread has a read claim on the outer perimeter. 6077 * Queue the callback for deferred processing. 6078 * 6079 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER 6080 * so that other qwriter(OUTER) calls will queue their 6081 * callbacks as well. queue_writer increments sq_count so we 6082 * decrement to compensate for the our increment. 6083 * 6084 * Dropping SQ_WRITER enables the writer thread to work 6085 * on this outer perimeter. 6086 */ 6087 outer->sq_flags = flags; 6088 queue_writer(outer, func, q, mp); 6089 /* queue_writer dropper the lock */ 6090 mutex_enter(SQLOCK(outer)); 6091 ASSERT(outer->sq_count > 0); 6092 outer->sq_count--; 6093 ASSERT(outer->sq_flags & SQ_WRITER); 6094 flags = outer->sq_flags; 6095 flags &= ~SQ_WRITER; 6096 if (flags & SQ_WANTWAKEUP) { 6097 flags &= ~SQ_WANTWAKEUP; 6098 cv_broadcast(&outer->sq_wait); 6099 } 6100 outer->sq_flags = flags; 6101 mutex_exit(SQLOCK(outer)); 6102 return; 6103 } else { 6104 outer->sq_flags = flags; 6105 mutex_exit(SQLOCK(outer)); 6106 } 6107 6108 /* Can run it immediately */ 6109 (*func)(q, mp); 6110 6111 outer_exit(outer); 6112 } 6113 6114 /* 6115 * Dequeue all writer callbacks from the outer perimeter and run them. 6116 */ 6117 static void 6118 write_now(syncq_t *outer) 6119 { 6120 mblk_t *mp; 6121 queue_t *q; 6122 void (*func)(); 6123 6124 ASSERT(MUTEX_HELD(SQLOCK(outer))); 6125 ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL && 6126 outer->sq_oprev != NULL); 6127 while ((mp = outer->sq_evhead) != NULL) { 6128 /* 6129 * queues cannot be placed on the queuelist on the outer 6130 * perimeter. 6131 */ 6132 ASSERT(!(outer->sq_flags & SQ_MESSAGES)); 6133 ASSERT((outer->sq_flags & SQ_EVENTS)); 6134 6135 outer->sq_evhead = mp->b_next; 6136 if (outer->sq_evhead == NULL) { 6137 outer->sq_evtail = NULL; 6138 outer->sq_flags &= ~SQ_EVENTS; 6139 } 6140 ASSERT(outer->sq_count != 0); 6141 outer->sq_count--; /* Incremented when enqueued. */ 6142 mutex_exit(SQLOCK(outer)); 6143 /* 6144 * Drop the message if the queue is closing. 6145 * Make sure that the queue is "claimed" when the callback 6146 * is run in order to satisfy various ASSERTs. 6147 */ 6148 q = mp->b_queue; 6149 func = (void (*)())mp->b_prev; 6150 ASSERT(func != NULL); 6151 mp->b_next = mp->b_prev = NULL; 6152 if (q->q_flag & QWCLOSE) { 6153 freemsg(mp); 6154 } else { 6155 claimq(q); 6156 (*func)(q, mp); 6157 releaseq(q); 6158 } 6159 mutex_enter(SQLOCK(outer)); 6160 } 6161 ASSERT(MUTEX_HELD(SQLOCK(outer))); 6162 } 6163 6164 /* 6165 * The list of messages on the inner syncq is effectively hashed 6166 * by destination queue. These destination queues are doubly 6167 * linked lists (hopefully) in priority order. Messages are then 6168 * put on the queue referenced by the q_sqhead/q_sqtail elements. 6169 * Additional messages are linked together by the b_next/b_prev 6170 * elements in the mblk, with (similar to putq()) the first message 6171 * having a NULL b_prev and the last message having a NULL b_next. 6172 * 6173 * Events, such as qwriter callbacks, are put onto a list in FIFO 6174 * order referenced by sq_evhead, and sq_evtail. This is a singly 6175 * linked list, and messages here MUST be processed in the order queued. 6176 */ 6177 6178 /* 6179 * Run the events on the syncq event list (sq_evhead). 6180 * Assumes there is only one claim on the syncq, it is 6181 * already exclusive (SQ_EXCL set), and the SQLOCK held. 6182 * Messages here are processed in order, with the SQ_EXCL bit 6183 * held all the way through till the last message is processed. 6184 */ 6185 void 6186 sq_run_events(syncq_t *sq) 6187 { 6188 mblk_t *bp; 6189 queue_t *qp; 6190 uint16_t flags = sq->sq_flags; 6191 void (*func)(); 6192 6193 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6194 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6195 sq->sq_oprev == NULL) || 6196 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6197 sq->sq_oprev != NULL)); 6198 6199 ASSERT(flags & SQ_EXCL); 6200 ASSERT(sq->sq_count == 1); 6201 6202 /* 6203 * We need to process all of the events on this list. It 6204 * is possible that new events will be added while we are 6205 * away processing a callback, so on every loop, we start 6206 * back at the beginning of the list. 6207 */ 6208 /* 6209 * We have to reaccess sq_evhead since there is a 6210 * possibility of a new entry while we were running 6211 * the callback. 6212 */ 6213 for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) { 6214 ASSERT(bp->b_queue->q_syncq == sq); 6215 ASSERT(sq->sq_flags & SQ_EVENTS); 6216 6217 qp = bp->b_queue; 6218 func = (void (*)())bp->b_prev; 6219 ASSERT(func != NULL); 6220 6221 /* 6222 * Messages from the event queue must be taken off in 6223 * FIFO order. 6224 */ 6225 ASSERT(sq->sq_evhead == bp); 6226 sq->sq_evhead = bp->b_next; 6227 6228 if (bp->b_next == NULL) { 6229 /* Deleting last */ 6230 ASSERT(sq->sq_evtail == bp); 6231 sq->sq_evtail = NULL; 6232 sq->sq_flags &= ~SQ_EVENTS; 6233 } 6234 bp->b_prev = bp->b_next = NULL; 6235 ASSERT(bp->b_datap->db_ref != 0); 6236 6237 mutex_exit(SQLOCK(sq)); 6238 6239 (*func)(qp, bp); 6240 6241 mutex_enter(SQLOCK(sq)); 6242 /* 6243 * re-read the flags, since they could have changed. 6244 */ 6245 flags = sq->sq_flags; 6246 ASSERT(flags & SQ_EXCL); 6247 } 6248 ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL); 6249 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6250 6251 if (flags & SQ_WANTWAKEUP) { 6252 flags &= ~SQ_WANTWAKEUP; 6253 cv_broadcast(&sq->sq_wait); 6254 } 6255 if (flags & SQ_WANTEXWAKEUP) { 6256 flags &= ~SQ_WANTEXWAKEUP; 6257 cv_broadcast(&sq->sq_exitwait); 6258 } 6259 sq->sq_flags = flags; 6260 } 6261 6262 /* 6263 * Put messages on the event list. 6264 * If we can go exclusive now, do so and process the event list, otherwise 6265 * let the last claim service this list (or wake the sqthread). 6266 * This procedure assumes SQLOCK is held. To run the event list, it 6267 * must be called with no claims. 6268 */ 6269 static void 6270 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)()) 6271 { 6272 uint16_t count; 6273 6274 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6275 ASSERT(func != NULL); 6276 6277 /* 6278 * This is a callback. Add it to the list of callbacks 6279 * and see about upgrading. 6280 */ 6281 mp->b_prev = (mblk_t *)func; 6282 mp->b_queue = q; 6283 mp->b_next = NULL; 6284 if (sq->sq_evhead == NULL) { 6285 sq->sq_evhead = sq->sq_evtail = mp; 6286 sq->sq_flags |= SQ_EVENTS; 6287 } else { 6288 ASSERT(sq->sq_evtail != NULL); 6289 ASSERT(sq->sq_evtail->b_next == NULL); 6290 ASSERT(sq->sq_flags & SQ_EVENTS); 6291 sq->sq_evtail->b_next = mp; 6292 sq->sq_evtail = mp; 6293 } 6294 /* 6295 * We have set SQ_EVENTS, so threads will have to 6296 * unwind out of the perimeter, and new entries will 6297 * not grab a putlock. But we still need to know 6298 * how many threads have already made a claim to the 6299 * syncq, so grab the putlocks, and sum the counts. 6300 * If there are no claims on the syncq, we can upgrade 6301 * to exclusive, and run the event list. 6302 * NOTE: We hold the SQLOCK, so we can just grab the 6303 * putlocks. 6304 */ 6305 count = sq->sq_count; 6306 SQ_PUTLOCKS_ENTER(sq); 6307 SUM_SQ_PUTCOUNTS(sq, count); 6308 /* 6309 * We have no claim, so we need to check if there 6310 * are no others, then we can upgrade. 6311 */ 6312 /* 6313 * There are currently no claims on 6314 * the syncq by this thread (at least on this entry). The thread who has 6315 * the claim should drain syncq. 6316 */ 6317 if (count > 0) { 6318 /* 6319 * Can't upgrade - other threads inside. 6320 */ 6321 SQ_PUTLOCKS_EXIT(sq); 6322 mutex_exit(SQLOCK(sq)); 6323 return; 6324 } 6325 /* 6326 * Need to set SQ_EXCL and make a claim on the syncq. 6327 */ 6328 ASSERT((sq->sq_flags & SQ_EXCL) == 0); 6329 sq->sq_flags |= SQ_EXCL; 6330 ASSERT(sq->sq_count == 0); 6331 sq->sq_count++; 6332 SQ_PUTLOCKS_EXIT(sq); 6333 6334 /* Process the events list */ 6335 sq_run_events(sq); 6336 6337 /* 6338 * Release our claim... 6339 */ 6340 sq->sq_count--; 6341 6342 /* 6343 * And release SQ_EXCL. 6344 * We don't need to acquire the putlocks to release 6345 * SQ_EXCL, since we are exclusive, and hold the SQLOCK. 6346 */ 6347 sq->sq_flags &= ~SQ_EXCL; 6348 6349 /* 6350 * sq_run_events should have released SQ_EXCL 6351 */ 6352 ASSERT(!(sq->sq_flags & SQ_EXCL)); 6353 6354 /* 6355 * If anything happened while we were running the 6356 * events (or was there before), we need to process 6357 * them now. We shouldn't be exclusive sine we 6358 * released the perimeter above (plus, we asserted 6359 * for it). 6360 */ 6361 if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED)) 6362 drain_syncq(sq); 6363 else 6364 mutex_exit(SQLOCK(sq)); 6365 } 6366 6367 /* 6368 * Perform delayed processing. The caller has to make sure that it is safe 6369 * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are 6370 * set). 6371 * 6372 * Assume that the caller has NO claims on the syncq. However, a claim 6373 * on the syncq does not indicate that a thread is draining the syncq. 6374 * There may be more claims on the syncq than there are threads draining 6375 * (i.e. #_threads_draining <= sq_count) 6376 * 6377 * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set 6378 * in order to preserve qwriter(OUTER) ordering constraints. 6379 * 6380 * sq_putcount only needs to be checked when dispatching the queued 6381 * writer call for CIPUT sync queue, but this is handled in sq_run_events. 6382 */ 6383 void 6384 drain_syncq(syncq_t *sq) 6385 { 6386 queue_t *qp; 6387 uint16_t count; 6388 uint16_t type = sq->sq_type; 6389 uint16_t flags = sq->sq_flags; 6390 boolean_t bg_service = sq->sq_svcflags & SQ_SERVICE; 6391 6392 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6393 "drain_syncq start:%p", sq); 6394 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6395 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6396 sq->sq_oprev == NULL) || 6397 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6398 sq->sq_oprev != NULL)); 6399 6400 /* 6401 * Drop SQ_SERVICE flag. 6402 */ 6403 if (bg_service) 6404 sq->sq_svcflags &= ~SQ_SERVICE; 6405 6406 /* 6407 * If SQ_EXCL is set, someone else is processing this syncq - let them 6408 * finish the job. 6409 */ 6410 if (flags & SQ_EXCL) { 6411 if (bg_service) { 6412 ASSERT(sq->sq_servcount != 0); 6413 sq->sq_servcount--; 6414 } 6415 mutex_exit(SQLOCK(sq)); 6416 return; 6417 } 6418 6419 /* 6420 * This routine can be called by a background thread if 6421 * it was scheduled by a hi-priority thread. SO, if there are 6422 * NOT messages queued, return (remember, we have the SQLOCK, 6423 * and it cannot change until we release it). Wakeup any waiters also. 6424 */ 6425 if (!(flags & SQ_QUEUED)) { 6426 if (flags & SQ_WANTWAKEUP) { 6427 flags &= ~SQ_WANTWAKEUP; 6428 cv_broadcast(&sq->sq_wait); 6429 } 6430 if (flags & SQ_WANTEXWAKEUP) { 6431 flags &= ~SQ_WANTEXWAKEUP; 6432 cv_broadcast(&sq->sq_exitwait); 6433 } 6434 sq->sq_flags = flags; 6435 if (bg_service) { 6436 ASSERT(sq->sq_servcount != 0); 6437 sq->sq_servcount--; 6438 } 6439 mutex_exit(SQLOCK(sq)); 6440 return; 6441 } 6442 6443 /* 6444 * If this is not a concurrent put perimeter, we need to 6445 * become exclusive to drain. Also, if not CIPUT, we would 6446 * not have acquired a putlock, so we don't need to check 6447 * the putcounts. If not entering with a claim, we test 6448 * for sq_count == 0. 6449 */ 6450 type = sq->sq_type; 6451 if (!(type & SQ_CIPUT)) { 6452 if (sq->sq_count > 1) { 6453 if (bg_service) { 6454 ASSERT(sq->sq_servcount != 0); 6455 sq->sq_servcount--; 6456 } 6457 mutex_exit(SQLOCK(sq)); 6458 return; 6459 } 6460 sq->sq_flags |= SQ_EXCL; 6461 } 6462 6463 /* 6464 * This is where we make a claim to the syncq. 6465 * This can either be done by incrementing a putlock, or 6466 * the sq_count. But since we already have the SQLOCK 6467 * here, we just bump the sq_count. 6468 * 6469 * Note that after we make a claim, we need to let the code 6470 * fall through to the end of this routine to clean itself 6471 * up. A return in the while loop will put the syncq in a 6472 * very bad state. 6473 */ 6474 sq->sq_count++; 6475 ASSERT(sq->sq_count != 0); /* wraparound */ 6476 6477 while ((flags = sq->sq_flags) & SQ_QUEUED) { 6478 /* 6479 * If we are told to stayaway or went exclusive, 6480 * we are done. 6481 */ 6482 if (flags & (SQ_STAYAWAY)) { 6483 break; 6484 } 6485 6486 /* 6487 * If there are events to run, do so. 6488 * We have one claim to the syncq, so if there are 6489 * more than one, other threads are running. 6490 */ 6491 if (sq->sq_evhead != NULL) { 6492 ASSERT(sq->sq_flags & SQ_EVENTS); 6493 6494 count = sq->sq_count; 6495 SQ_PUTLOCKS_ENTER(sq); 6496 SUM_SQ_PUTCOUNTS(sq, count); 6497 if (count > 1) { 6498 SQ_PUTLOCKS_EXIT(sq); 6499 /* Can't upgrade - other threads inside */ 6500 break; 6501 } 6502 ASSERT((flags & SQ_EXCL) == 0); 6503 sq->sq_flags = flags | SQ_EXCL; 6504 SQ_PUTLOCKS_EXIT(sq); 6505 /* 6506 * we have the only claim, run the events, 6507 * sq_run_events will clear the SQ_EXCL flag. 6508 */ 6509 sq_run_events(sq); 6510 6511 /* 6512 * If this is a CIPUT perimeter, we need 6513 * to drop the SQ_EXCL flag so we can properly 6514 * continue draining the syncq. 6515 */ 6516 if (type & SQ_CIPUT) { 6517 ASSERT(sq->sq_flags & SQ_EXCL); 6518 sq->sq_flags &= ~SQ_EXCL; 6519 } 6520 6521 /* 6522 * And go back to the beginning just in case 6523 * anything changed while we were away. 6524 */ 6525 ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT)); 6526 continue; 6527 } 6528 6529 ASSERT(sq->sq_evhead == NULL); 6530 ASSERT(!(sq->sq_flags & SQ_EVENTS)); 6531 6532 /* 6533 * Find the queue that is not draining. 6534 * 6535 * q_draining is protected by QLOCK which we do not hold. 6536 * But if it was set, then a thread was draining, and if it gets 6537 * cleared, then it was because the thread has successfully 6538 * drained the syncq, or a GOAWAY state occurred. For the GOAWAY 6539 * state to happen, a thread needs the SQLOCK which we hold, and 6540 * if there was such a flag, we would have already seen it. 6541 */ 6542 6543 for (qp = sq->sq_head; 6544 qp != NULL && (qp->q_draining || 6545 (qp->q_sqflags & Q_SQDRAINING)); 6546 qp = qp->q_sqnext) 6547 ; 6548 6549 if (qp == NULL) 6550 break; 6551 6552 /* 6553 * We have a queue to work on, and we hold the 6554 * SQLOCK and one claim, call qdrain_syncq. 6555 * This means we need to release the SQLOCK and 6556 * acquire the QLOCK (OK since we have a claim). 6557 * Note that qdrain_syncq will actually dequeue 6558 * this queue from the sq_head list when it is 6559 * convinced all the work is done and release 6560 * the QLOCK before returning. 6561 */ 6562 qp->q_sqflags |= Q_SQDRAINING; 6563 mutex_exit(SQLOCK(sq)); 6564 mutex_enter(QLOCK(qp)); 6565 qdrain_syncq(sq, qp); 6566 mutex_enter(SQLOCK(sq)); 6567 6568 /* The queue is drained */ 6569 ASSERT(qp->q_sqflags & Q_SQDRAINING); 6570 qp->q_sqflags &= ~Q_SQDRAINING; 6571 /* 6572 * NOTE: After this point qp should not be used since it may be 6573 * closed. 6574 */ 6575 } 6576 6577 ASSERT(MUTEX_HELD(SQLOCK(sq))); 6578 flags = sq->sq_flags; 6579 6580 /* 6581 * sq->sq_head cannot change because we hold the 6582 * sqlock. However, a thread CAN decide that it is no longer 6583 * going to drain that queue. However, this should be due to 6584 * a GOAWAY state, and we should see that here. 6585 * 6586 * This loop is not very efficient. One solution may be adding a second 6587 * pointer to the "draining" queue, but it is difficult to do when 6588 * queues are inserted in the middle due to priority ordering. Another 6589 * possibility is to yank the queue out of the sq list and put it onto 6590 * the "draining list" and then put it back if it can't be drained. 6591 */ 6592 6593 ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) || 6594 (type & SQ_CI) || sq->sq_head->q_draining); 6595 6596 /* Drop SQ_EXCL for non-CIPUT perimeters */ 6597 if (!(type & SQ_CIPUT)) 6598 flags &= ~SQ_EXCL; 6599 ASSERT((flags & SQ_EXCL) == 0); 6600 6601 /* Wake up any waiters. */ 6602 if (flags & SQ_WANTWAKEUP) { 6603 flags &= ~SQ_WANTWAKEUP; 6604 cv_broadcast(&sq->sq_wait); 6605 } 6606 if (flags & SQ_WANTEXWAKEUP) { 6607 flags &= ~SQ_WANTEXWAKEUP; 6608 cv_broadcast(&sq->sq_exitwait); 6609 } 6610 sq->sq_flags = flags; 6611 6612 ASSERT(sq->sq_count != 0); 6613 /* Release our claim. */ 6614 sq->sq_count--; 6615 6616 if (bg_service) { 6617 ASSERT(sq->sq_servcount != 0); 6618 sq->sq_servcount--; 6619 } 6620 6621 mutex_exit(SQLOCK(sq)); 6622 6623 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6624 "drain_syncq end:%p", sq); 6625 } 6626 6627 6628 /* 6629 * 6630 * qdrain_syncq can be called (currently) from only one of two places: 6631 * drain_syncq 6632 * putnext (or some variation of it). 6633 * and eventually 6634 * qwait(_sig) 6635 * 6636 * If called from drain_syncq, we found it in the list of queues needing 6637 * service, so there is work to be done (or it wouldn't be in the list). 6638 * 6639 * If called from some putnext variation, it was because the 6640 * perimeter is open, but messages are blocking a putnext and 6641 * there is not a thread working on it. Now a thread could start 6642 * working on it while we are getting ready to do so ourself, but 6643 * the thread would set the q_draining flag, and we can spin out. 6644 * 6645 * As for qwait(_sig), I think I shall let it continue to call 6646 * drain_syncq directly (after all, it will get here eventually). 6647 * 6648 * qdrain_syncq has to terminate when: 6649 * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering 6650 * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering 6651 * 6652 * ASSUMES: 6653 * One claim 6654 * QLOCK held 6655 * SQLOCK not held 6656 * Will release QLOCK before returning 6657 */ 6658 void 6659 qdrain_syncq(syncq_t *sq, queue_t *q) 6660 { 6661 mblk_t *bp; 6662 #ifdef DEBUG 6663 uint16_t count; 6664 #endif 6665 6666 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START, 6667 "drain_syncq start:%p", sq); 6668 ASSERT(q->q_syncq == sq); 6669 ASSERT(MUTEX_HELD(QLOCK(q))); 6670 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6671 /* 6672 * For non-CIPUT perimeters, we should be called with the exclusive bit 6673 * set already. For CIPUT perimeters, we will be doing a concurrent 6674 * drain, so it better not be set. 6675 */ 6676 ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT))); 6677 ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL))); 6678 ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL)); 6679 /* 6680 * All outer pointers are set, or none of them are 6681 */ 6682 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6683 sq->sq_oprev == NULL) || 6684 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6685 sq->sq_oprev != NULL)); 6686 #ifdef DEBUG 6687 count = sq->sq_count; 6688 /* 6689 * This is OK without the putlocks, because we have one 6690 * claim either from the sq_count, or a putcount. We could 6691 * get an erroneous value from other counts, but ours won't 6692 * change, so one way or another, we will have at least a 6693 * value of one. 6694 */ 6695 SUM_SQ_PUTCOUNTS(sq, count); 6696 ASSERT(count >= 1); 6697 #endif /* DEBUG */ 6698 6699 /* 6700 * The first thing to do is find out if a thread is already draining 6701 * this queue. If so, we are done, just return. 6702 */ 6703 if (q->q_draining) { 6704 mutex_exit(QLOCK(q)); 6705 return; 6706 } 6707 6708 /* 6709 * If the perimeter is exclusive, there is nothing we can do right now, 6710 * go away. Note that there is nothing to prevent this case from 6711 * changing right after this check, but the spin-out will catch it. 6712 */ 6713 6714 /* Tell other threads that we are draining this queue */ 6715 q->q_draining = 1; /* Protected by QLOCK */ 6716 6717 /* 6718 * If there is nothing to do, clear QFULL as necessary. This caters for 6719 * the case where an empty queue was enqueued onto the syncq. 6720 */ 6721 if (q->q_sqhead == NULL) { 6722 ASSERT(q->q_syncqmsgs == 0); 6723 mutex_exit(QLOCK(q)); 6724 clr_qfull(q); 6725 mutex_enter(QLOCK(q)); 6726 } 6727 6728 /* 6729 * Note that q_sqhead must be re-checked here in case another message 6730 * was enqueued whilst QLOCK was dropped during the call to clr_qfull. 6731 */ 6732 for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) { 6733 /* 6734 * Because we can enter this routine just because a putnext is 6735 * blocked, we need to spin out if the perimeter wants to go 6736 * exclusive as well as just blocked. We need to spin out also 6737 * if events are queued on the syncq. 6738 * Don't check for SQ_EXCL, because non-CIPUT perimeters would 6739 * set it, and it can't become exclusive while we hold a claim. 6740 */ 6741 if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) { 6742 break; 6743 } 6744 6745 #ifdef DEBUG 6746 /* 6747 * Since we are in qdrain_syncq, we already know the queue, 6748 * but for sanity, we want to check this against the qp that 6749 * was passed in by bp->b_queue. 6750 */ 6751 6752 ASSERT(bp->b_queue == q); 6753 ASSERT(bp->b_queue->q_syncq == sq); 6754 bp->b_queue = NULL; 6755 6756 /* 6757 * We would have the following check in the DEBUG code: 6758 * 6759 * if (bp->b_prev != NULL) { 6760 * ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp); 6761 * } 6762 * 6763 * This can't be done, however, since IP modifies qinfo 6764 * structure at run-time (switching between IPv4 qinfo and IPv6 6765 * qinfo), invalidating the check. 6766 * So the assignment to func is left here, but the ASSERT itself 6767 * is removed until the whole issue is resolved. 6768 */ 6769 #endif 6770 ASSERT(q->q_sqhead == bp); 6771 q->q_sqhead = bp->b_next; 6772 bp->b_prev = bp->b_next = NULL; 6773 ASSERT(q->q_syncqmsgs > 0); 6774 mutex_exit(QLOCK(q)); 6775 6776 ASSERT(bp->b_datap->db_ref != 0); 6777 6778 (void) (*q->q_qinfo->qi_putp)(q, bp); 6779 6780 mutex_enter(QLOCK(q)); 6781 6782 /* 6783 * q_syncqmsgs should only be decremented after executing the 6784 * put procedure to avoid message re-ordering. This is due to an 6785 * optimisation in putnext() which can call the put procedure 6786 * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED 6787 * being set). 6788 * 6789 * We also need to clear QFULL in the next service procedure 6790 * queue if this is the last message destined for that queue. 6791 * 6792 * It would make better sense to have some sort of tunable for 6793 * the low water mark, but these semantics are not yet defined. 6794 * So, alas, we use a constant. 6795 */ 6796 if (--q->q_syncqmsgs == 0) { 6797 mutex_exit(QLOCK(q)); 6798 clr_qfull(q); 6799 mutex_enter(QLOCK(q)); 6800 } 6801 6802 /* 6803 * Always clear SQ_EXCL when CIPUT in order to handle 6804 * qwriter(INNER). The putp() can call qwriter and get exclusive 6805 * access IFF this is the only claim. So, we need to test for 6806 * this possibility, acquire the mutex and clear the bit. 6807 */ 6808 if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) { 6809 mutex_enter(SQLOCK(sq)); 6810 sq->sq_flags &= ~SQ_EXCL; 6811 mutex_exit(SQLOCK(sq)); 6812 } 6813 } 6814 6815 /* 6816 * We should either have no messages on this queue, or we were told to 6817 * goaway by a waiter (which we will wake up at the end of this 6818 * function). 6819 */ 6820 ASSERT((q->q_sqhead == NULL) || 6821 (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS))); 6822 6823 ASSERT(MUTEX_HELD(QLOCK(q))); 6824 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6825 6826 /* Remove the q from the syncq list if all the messages are drained. */ 6827 if (q->q_sqhead == NULL) { 6828 ASSERT(q->q_syncqmsgs == 0); 6829 mutex_enter(SQLOCK(sq)); 6830 if (q->q_sqflags & Q_SQQUEUED) 6831 SQRM_Q(sq, q); 6832 mutex_exit(SQLOCK(sq)); 6833 /* 6834 * Since the queue is removed from the list, reset its priority. 6835 */ 6836 q->q_spri = 0; 6837 } 6838 6839 /* 6840 * Remember, the q_draining flag is used to let another thread know 6841 * that there is a thread currently draining the messages for a queue. 6842 * Since we are now done with this queue (even if there may be messages 6843 * still there), we need to clear this flag so some thread will work on 6844 * it if needed. 6845 */ 6846 ASSERT(q->q_draining); 6847 q->q_draining = 0; 6848 6849 /* Called with a claim, so OK to drop all locks. */ 6850 mutex_exit(QLOCK(q)); 6851 6852 TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END, 6853 "drain_syncq end:%p", sq); 6854 } 6855 /* END OF QDRAIN_SYNCQ */ 6856 6857 6858 /* 6859 * This is the mate to qdrain_syncq, except that it is putting the message onto 6860 * the queue instead of draining. Since the message is destined for the queue 6861 * that is selected, there is no need to identify the function because the 6862 * message is intended for the put routine for the queue. For debug kernels, 6863 * this routine will do it anyway just in case. 6864 * 6865 * After the message is enqueued on the syncq, it calls putnext_tail() 6866 * which will schedule a background thread to actually process the message. 6867 * 6868 * Assumes that there is a claim on the syncq (sq->sq_count > 0) and 6869 * SQLOCK(sq) and QLOCK(q) are not held. 6870 */ 6871 void 6872 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp) 6873 { 6874 ASSERT(MUTEX_NOT_HELD(SQLOCK(sq))); 6875 ASSERT(MUTEX_NOT_HELD(QLOCK(q))); 6876 ASSERT(sq->sq_count > 0); 6877 ASSERT(q->q_syncq == sq); 6878 ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL && 6879 sq->sq_oprev == NULL) || 6880 (sq->sq_outer != NULL && sq->sq_onext != NULL && 6881 sq->sq_oprev != NULL)); 6882 6883 mutex_enter(QLOCK(q)); 6884 6885 #ifdef DEBUG 6886 /* 6887 * This is used for debug in the qfill_syncq/qdrain_syncq case 6888 * to trace the queue that the message is intended for. Note 6889 * that the original use was to identify the queue and function 6890 * to call on the drain. In the new syncq, we have the context 6891 * of the queue that we are draining, so call it's putproc and 6892 * don't rely on the saved values. But for debug this is still 6893 * useful information. 6894 */ 6895 mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp; 6896 mp->b_queue = q; 6897 mp->b_next = NULL; 6898 #endif 6899 ASSERT(q->q_syncq == sq); 6900 /* 6901 * Enqueue the message on the list. 6902 * SQPUT_MP() accesses q_syncqmsgs. We are already holding QLOCK to 6903 * protect it. So it's ok to acquire SQLOCK after SQPUT_MP(). 6904 */ 6905 SQPUT_MP(q, mp); 6906 mutex_enter(SQLOCK(sq)); 6907 6908 /* 6909 * And queue on syncq for scheduling, if not already queued. 6910 * Note that we need the SQLOCK for this, and for testing flags 6911 * at the end to see if we will drain. So grab it now, and 6912 * release it before we call qdrain_syncq or return. 6913 */ 6914 if (!(q->q_sqflags & Q_SQQUEUED)) { 6915 q->q_spri = curthread->t_pri; 6916 SQPUT_Q(sq, q); 6917 } 6918 #ifdef DEBUG 6919 else { 6920 /* 6921 * All of these conditions MUST be true! 6922 */ 6923 ASSERT(sq->sq_tail != NULL); 6924 if (sq->sq_tail == sq->sq_head) { 6925 ASSERT((q->q_sqprev == NULL) && 6926 (q->q_sqnext == NULL)); 6927 } else { 6928 ASSERT((q->q_sqprev != NULL) || 6929 (q->q_sqnext != NULL)); 6930 } 6931 ASSERT(sq->sq_flags & SQ_QUEUED); 6932 ASSERT(q->q_syncqmsgs != 0); 6933 ASSERT(q->q_sqflags & Q_SQQUEUED); 6934 } 6935 #endif 6936 mutex_exit(QLOCK(q)); 6937 /* 6938 * SQLOCK is still held, so sq_count can be safely decremented. 6939 */ 6940 sq->sq_count--; 6941 6942 putnext_tail(sq, q, 0); 6943 /* Should not reference sq or q after this point. */ 6944 } 6945 6946 /* End of qfill_syncq */ 6947 6948 /* 6949 * Remove all messages from a syncq (if qp is NULL) or remove all messages 6950 * that would be put into qp by drain_syncq. 6951 * Used when deleting the syncq (qp == NULL) or when detaching 6952 * a queue (qp != NULL). 6953 * Return non-zero if one or more messages were freed. 6954 * 6955 * No need to grab sq_putlocks here. See comment in strsubr.h that explains when 6956 * sq_putlocks are used. 6957 * 6958 * NOTE: This function assumes that it is called from the close() context and 6959 * that all the queues in the syncq are going away. For this reason it doesn't 6960 * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is 6961 * currently valid, but it is useful to rethink this function to behave properly 6962 * in other cases. 6963 */ 6964 int 6965 flush_syncq(syncq_t *sq, queue_t *qp) 6966 { 6967 mblk_t *bp, *mp_head, *mp_next, *mp_prev; 6968 queue_t *q; 6969 int ret = 0; 6970 6971 mutex_enter(SQLOCK(sq)); 6972 6973 /* 6974 * Before we leave, we need to make sure there are no 6975 * events listed for this queue. All events for this queue 6976 * will just be freed. 6977 */ 6978 if (qp != NULL && sq->sq_evhead != NULL) { 6979 ASSERT(sq->sq_flags & SQ_EVENTS); 6980 6981 mp_prev = NULL; 6982 for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) { 6983 mp_next = bp->b_next; 6984 if (bp->b_queue == qp) { 6985 /* Delete this message */ 6986 if (mp_prev != NULL) { 6987 mp_prev->b_next = mp_next; 6988 /* 6989 * Update sq_evtail if the last element 6990 * is removed. 6991 */ 6992 if (bp == sq->sq_evtail) { 6993 ASSERT(mp_next == NULL); 6994 sq->sq_evtail = mp_prev; 6995 } 6996 } else 6997 sq->sq_evhead = mp_next; 6998 if (sq->sq_evhead == NULL) 6999 sq->sq_flags &= ~SQ_EVENTS; 7000 bp->b_prev = bp->b_next = NULL; 7001 freemsg(bp); 7002 ret++; 7003 } else { 7004 mp_prev = bp; 7005 } 7006 } 7007 } 7008 7009 /* 7010 * Walk sq_head and: 7011 * - match qp if qp is set, remove it's messages 7012 * - all if qp is not set 7013 */ 7014 q = sq->sq_head; 7015 while (q != NULL) { 7016 ASSERT(q->q_syncq == sq); 7017 if ((qp == NULL) || (qp == q)) { 7018 /* 7019 * Yank the messages as a list off the queue 7020 */ 7021 mp_head = q->q_sqhead; 7022 /* 7023 * We do not have QLOCK(q) here (which is safe due to 7024 * assumptions mentioned above). To obtain the lock we 7025 * need to release SQLOCK which may allow lots of things 7026 * to change upon us. This place requires more analysis. 7027 */ 7028 q->q_sqhead = q->q_sqtail = NULL; 7029 ASSERT(mp_head->b_queue && 7030 mp_head->b_queue->q_syncq == sq); 7031 7032 /* 7033 * Free each of the messages. 7034 */ 7035 for (bp = mp_head; bp != NULL; bp = mp_next) { 7036 mp_next = bp->b_next; 7037 bp->b_prev = bp->b_next = NULL; 7038 freemsg(bp); 7039 ret++; 7040 } 7041 /* 7042 * Now remove the queue from the syncq. 7043 */ 7044 ASSERT(q->q_sqflags & Q_SQQUEUED); 7045 SQRM_Q(sq, q); 7046 q->q_spri = 0; 7047 q->q_syncqmsgs = 0; 7048 7049 /* 7050 * If qp was specified, we are done with it and are 7051 * going to drop SQLOCK(sq) and return. We wakeup syncq 7052 * waiters while we still have the SQLOCK. 7053 */ 7054 if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) { 7055 sq->sq_flags &= ~SQ_WANTWAKEUP; 7056 cv_broadcast(&sq->sq_wait); 7057 } 7058 /* Drop SQLOCK across clr_qfull */ 7059 mutex_exit(SQLOCK(sq)); 7060 7061 /* 7062 * We avoid doing the test that drain_syncq does and 7063 * unconditionally clear qfull for every flushed 7064 * message. Since flush_syncq is only called during 7065 * close this should not be a problem. 7066 */ 7067 clr_qfull(q); 7068 if (qp != NULL) { 7069 return (ret); 7070 } else { 7071 mutex_enter(SQLOCK(sq)); 7072 /* 7073 * The head was removed by SQRM_Q above. 7074 * reread the new head and flush it. 7075 */ 7076 q = sq->sq_head; 7077 } 7078 } else { 7079 q = q->q_sqnext; 7080 } 7081 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7082 } 7083 7084 if (sq->sq_flags & SQ_WANTWAKEUP) { 7085 sq->sq_flags &= ~SQ_WANTWAKEUP; 7086 cv_broadcast(&sq->sq_wait); 7087 } 7088 7089 mutex_exit(SQLOCK(sq)); 7090 return (ret); 7091 } 7092 7093 /* 7094 * Propagate all messages from a syncq to the next syncq that are associated 7095 * with the specified queue. If the queue is attached to a driver or if the 7096 * messages have been added due to a qwriter(PERIM_INNER), free the messages. 7097 * 7098 * Assumes that the stream is strlock()'ed. We don't come here if there 7099 * are no messages to propagate. 7100 * 7101 * NOTE : If the queue is attached to a driver, all the messages are freed 7102 * as there is no point in propagating the messages from the driver syncq 7103 * to the closing stream head which will in turn get freed later. 7104 */ 7105 static int 7106 propagate_syncq(queue_t *qp) 7107 { 7108 mblk_t *bp, *head, *tail, *prev, *next; 7109 syncq_t *sq; 7110 queue_t *nqp; 7111 syncq_t *nsq; 7112 boolean_t isdriver; 7113 int moved = 0; 7114 uint16_t flags; 7115 pri_t priority = curthread->t_pri; 7116 #ifdef DEBUG 7117 void (*func)(); 7118 #endif 7119 7120 sq = qp->q_syncq; 7121 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7122 /* debug macro */ 7123 SQ_PUTLOCKS_HELD(sq); 7124 /* 7125 * As entersq() does not increment the sq_count for 7126 * the write side, check sq_count for non-QPERQ 7127 * perimeters alone. 7128 */ 7129 ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1)); 7130 7131 /* 7132 * propagate_syncq() can be called because of either messages on the 7133 * queue syncq or because on events on the queue syncq. Do actual 7134 * message propagations if there are any messages. 7135 */ 7136 if (qp->q_syncqmsgs) { 7137 isdriver = (qp->q_flag & QISDRV); 7138 7139 if (!isdriver) { 7140 nqp = qp->q_next; 7141 nsq = nqp->q_syncq; 7142 ASSERT(MUTEX_HELD(SQLOCK(nsq))); 7143 /* debug macro */ 7144 SQ_PUTLOCKS_HELD(nsq); 7145 #ifdef DEBUG 7146 func = (void (*)())nqp->q_qinfo->qi_putp; 7147 #endif 7148 } 7149 7150 SQRM_Q(sq, qp); 7151 priority = MAX(qp->q_spri, priority); 7152 qp->q_spri = 0; 7153 head = qp->q_sqhead; 7154 tail = qp->q_sqtail; 7155 qp->q_sqhead = qp->q_sqtail = NULL; 7156 qp->q_syncqmsgs = 0; 7157 7158 /* 7159 * Walk the list of messages, and free them if this is a driver, 7160 * otherwise reset the b_prev and b_queue value to the new putp. 7161 * Afterward, we will just add the head to the end of the next 7162 * syncq, and point the tail to the end of this one. 7163 */ 7164 7165 for (bp = head; bp != NULL; bp = next) { 7166 next = bp->b_next; 7167 if (isdriver) { 7168 bp->b_prev = bp->b_next = NULL; 7169 freemsg(bp); 7170 continue; 7171 } 7172 /* Change the q values for this message */ 7173 bp->b_queue = nqp; 7174 #ifdef DEBUG 7175 bp->b_prev = (mblk_t *)func; 7176 #endif 7177 moved++; 7178 } 7179 /* 7180 * Attach list of messages to the end of the new queue (if there 7181 * is a list of messages). 7182 */ 7183 7184 if (!isdriver && head != NULL) { 7185 ASSERT(tail != NULL); 7186 if (nqp->q_sqhead == NULL) { 7187 nqp->q_sqhead = head; 7188 } else { 7189 ASSERT(nqp->q_sqtail != NULL); 7190 nqp->q_sqtail->b_next = head; 7191 } 7192 nqp->q_sqtail = tail; 7193 /* 7194 * When messages are moved from high priority queue to 7195 * another queue, the destination queue priority is 7196 * upgraded. 7197 */ 7198 7199 if (priority > nqp->q_spri) 7200 nqp->q_spri = priority; 7201 7202 SQPUT_Q(nsq, nqp); 7203 7204 nqp->q_syncqmsgs += moved; 7205 ASSERT(nqp->q_syncqmsgs != 0); 7206 } 7207 } 7208 7209 /* 7210 * Before we leave, we need to make sure there are no 7211 * events listed for this queue. All events for this queue 7212 * will just be freed. 7213 */ 7214 if (sq->sq_evhead != NULL) { 7215 ASSERT(sq->sq_flags & SQ_EVENTS); 7216 prev = NULL; 7217 for (bp = sq->sq_evhead; bp != NULL; bp = next) { 7218 next = bp->b_next; 7219 if (bp->b_queue == qp) { 7220 /* Delete this message */ 7221 if (prev != NULL) { 7222 prev->b_next = next; 7223 /* 7224 * Update sq_evtail if the last element 7225 * is removed. 7226 */ 7227 if (bp == sq->sq_evtail) { 7228 ASSERT(next == NULL); 7229 sq->sq_evtail = prev; 7230 } 7231 } else 7232 sq->sq_evhead = next; 7233 if (sq->sq_evhead == NULL) 7234 sq->sq_flags &= ~SQ_EVENTS; 7235 bp->b_prev = bp->b_next = NULL; 7236 freemsg(bp); 7237 } else { 7238 prev = bp; 7239 } 7240 } 7241 } 7242 7243 flags = sq->sq_flags; 7244 7245 /* Wake up any waiter before leaving. */ 7246 if (flags & SQ_WANTWAKEUP) { 7247 flags &= ~SQ_WANTWAKEUP; 7248 cv_broadcast(&sq->sq_wait); 7249 } 7250 sq->sq_flags = flags; 7251 7252 return (moved); 7253 } 7254 7255 /* 7256 * Try and upgrade to exclusive access at the inner perimeter. If this can 7257 * not be done without blocking then request will be queued on the syncq 7258 * and drain_syncq will run it later. 7259 * 7260 * This routine can only be called from put or service procedures plus 7261 * asynchronous callback routines that have properly entered the queue (with 7262 * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq 7263 * associated with q. 7264 */ 7265 void 7266 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)()) 7267 { 7268 syncq_t *sq = q->q_syncq; 7269 uint16_t count; 7270 7271 mutex_enter(SQLOCK(sq)); 7272 count = sq->sq_count; 7273 SQ_PUTLOCKS_ENTER(sq); 7274 SUM_SQ_PUTCOUNTS(sq, count); 7275 ASSERT(count >= 1); 7276 ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC)); 7277 7278 if (count == 1) { 7279 /* 7280 * Can upgrade. This case also handles nested qwriter calls 7281 * (when the qwriter callback function calls qwriter). In that 7282 * case SQ_EXCL is already set. 7283 */ 7284 sq->sq_flags |= SQ_EXCL; 7285 SQ_PUTLOCKS_EXIT(sq); 7286 mutex_exit(SQLOCK(sq)); 7287 (*func)(q, mp); 7288 /* 7289 * Assumes that leavesq, putnext, and drain_syncq will reset 7290 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on 7291 * until putnext, leavesq, or drain_syncq drops it. 7292 * That way we handle nested qwriter(INNER) without dropping 7293 * SQ_EXCL until the outermost qwriter callback routine is 7294 * done. 7295 */ 7296 return; 7297 } 7298 SQ_PUTLOCKS_EXIT(sq); 7299 sqfill_events(sq, q, mp, func); 7300 } 7301 7302 /* 7303 * Synchronous callback support functions 7304 */ 7305 7306 /* 7307 * Allocate a callback parameter structure. 7308 * Assumes that caller initializes the flags and the id. 7309 * Acquires SQLOCK(sq) if non-NULL is returned. 7310 */ 7311 callbparams_t * 7312 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags) 7313 { 7314 callbparams_t *cbp; 7315 size_t size = sizeof (callbparams_t); 7316 7317 cbp = kmem_alloc(size, kmflags & ~KM_PANIC); 7318 7319 /* 7320 * Only try tryhard allocation if the caller is ready to panic. 7321 * Otherwise just fail. 7322 */ 7323 if (cbp == NULL) { 7324 if (kmflags & KM_PANIC) 7325 cbp = kmem_alloc_tryhard(sizeof (callbparams_t), 7326 &size, kmflags); 7327 else 7328 return (NULL); 7329 } 7330 7331 ASSERT(size >= sizeof (callbparams_t)); 7332 cbp->cbp_size = size; 7333 cbp->cbp_sq = sq; 7334 cbp->cbp_func = func; 7335 cbp->cbp_arg = arg; 7336 mutex_enter(SQLOCK(sq)); 7337 cbp->cbp_next = sq->sq_callbpend; 7338 sq->sq_callbpend = cbp; 7339 return (cbp); 7340 } 7341 7342 void 7343 callbparams_free(syncq_t *sq, callbparams_t *cbp) 7344 { 7345 callbparams_t **pp, *p; 7346 7347 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7348 7349 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7350 if (p == cbp) { 7351 *pp = p->cbp_next; 7352 kmem_free(p, p->cbp_size); 7353 return; 7354 } 7355 } 7356 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7357 "callbparams_free: not found\n")); 7358 } 7359 7360 void 7361 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag) 7362 { 7363 callbparams_t **pp, *p; 7364 7365 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7366 7367 for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) { 7368 if (p->cbp_id == id && p->cbp_flags == flag) { 7369 *pp = p->cbp_next; 7370 kmem_free(p, p->cbp_size); 7371 return; 7372 } 7373 } 7374 (void) (STRLOG(0, 0, 0, SL_CONSOLE, 7375 "callbparams_free_id: not found\n")); 7376 } 7377 7378 /* 7379 * Callback wrapper function used by once-only callbacks that can be 7380 * cancelled (qtimeout and qbufcall) 7381 * Contains inline version of entersq(sq, SQ_CALLBACK) that can be 7382 * cancelled by the qun* functions. 7383 */ 7384 void 7385 qcallbwrapper(void *arg) 7386 { 7387 callbparams_t *cbp = arg; 7388 syncq_t *sq; 7389 uint16_t count = 0; 7390 uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL; 7391 uint16_t type; 7392 7393 sq = cbp->cbp_sq; 7394 mutex_enter(SQLOCK(sq)); 7395 type = sq->sq_type; 7396 if (!(type & SQ_CICB)) { 7397 count = sq->sq_count; 7398 SQ_PUTLOCKS_ENTER(sq); 7399 SQ_PUTCOUNT_CLRFAST_LOCKED(sq); 7400 SUM_SQ_PUTCOUNTS(sq, count); 7401 sq->sq_needexcl++; 7402 ASSERT(sq->sq_needexcl != 0); /* wraparound */ 7403 waitflags |= SQ_MESSAGES; 7404 } 7405 /* Can not handle exclusive entry at outer perimeter */ 7406 ASSERT(type & SQ_COCB); 7407 7408 while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) { 7409 if ((sq->sq_callbflags & cbp->cbp_flags) && 7410 (sq->sq_cancelid == cbp->cbp_id)) { 7411 /* timeout has been cancelled */ 7412 sq->sq_callbflags |= SQ_CALLB_BYPASSED; 7413 callbparams_free(sq, cbp); 7414 if (!(type & SQ_CICB)) { 7415 ASSERT(sq->sq_needexcl > 0); 7416 sq->sq_needexcl--; 7417 if (sq->sq_needexcl == 0) { 7418 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7419 } 7420 SQ_PUTLOCKS_EXIT(sq); 7421 } 7422 mutex_exit(SQLOCK(sq)); 7423 return; 7424 } 7425 sq->sq_flags |= SQ_WANTWAKEUP; 7426 if (!(type & SQ_CICB)) { 7427 SQ_PUTLOCKS_EXIT(sq); 7428 } 7429 cv_wait(&sq->sq_wait, SQLOCK(sq)); 7430 if (!(type & SQ_CICB)) { 7431 count = sq->sq_count; 7432 SQ_PUTLOCKS_ENTER(sq); 7433 SUM_SQ_PUTCOUNTS(sq, count); 7434 } 7435 } 7436 7437 sq->sq_count++; 7438 ASSERT(sq->sq_count != 0); /* Wraparound */ 7439 if (!(type & SQ_CICB)) { 7440 ASSERT(count == 0); 7441 sq->sq_flags |= SQ_EXCL; 7442 ASSERT(sq->sq_needexcl > 0); 7443 sq->sq_needexcl--; 7444 if (sq->sq_needexcl == 0) { 7445 SQ_PUTCOUNT_SETFAST_LOCKED(sq); 7446 } 7447 SQ_PUTLOCKS_EXIT(sq); 7448 } 7449 7450 mutex_exit(SQLOCK(sq)); 7451 7452 cbp->cbp_func(cbp->cbp_arg); 7453 7454 /* 7455 * We drop the lock only for leavesq to re-acquire it. 7456 * Possible optimization is inline of leavesq. 7457 */ 7458 mutex_enter(SQLOCK(sq)); 7459 callbparams_free(sq, cbp); 7460 mutex_exit(SQLOCK(sq)); 7461 leavesq(sq, SQ_CALLBACK); 7462 } 7463 7464 /* 7465 * No need to grab sq_putlocks here. See comment in strsubr.h that 7466 * explains when sq_putlocks are used. 7467 * 7468 * sq_count (or one of the sq_putcounts) has already been 7469 * decremented by the caller, and if SQ_QUEUED, we need to call 7470 * drain_syncq (the global syncq drain). 7471 * If putnext_tail is called with the SQ_EXCL bit set, we are in 7472 * one of two states, non-CIPUT perimeter, and we need to clear 7473 * it, or we went exclusive in the put procedure. In any case, 7474 * we want to clear the bit now, and it is probably easier to do 7475 * this at the beginning of this function (remember, we hold 7476 * the SQLOCK). Lastly, if there are other messages queued 7477 * on the syncq (and not for our destination), enable the syncq 7478 * for background work. 7479 */ 7480 7481 /* ARGSUSED */ 7482 void 7483 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags) 7484 { 7485 uint16_t flags = sq->sq_flags; 7486 7487 ASSERT(MUTEX_HELD(SQLOCK(sq))); 7488 ASSERT(MUTEX_NOT_HELD(QLOCK(qp))); 7489 7490 /* Clear SQ_EXCL if set in passflags */ 7491 if (passflags & SQ_EXCL) { 7492 flags &= ~SQ_EXCL; 7493 } 7494 if (flags & SQ_WANTWAKEUP) { 7495 flags &= ~SQ_WANTWAKEUP; 7496 cv_broadcast(&sq->sq_wait); 7497 } 7498 if (flags & SQ_WANTEXWAKEUP) { 7499 flags &= ~SQ_WANTEXWAKEUP; 7500 cv_broadcast(&sq->sq_exitwait); 7501 } 7502 sq->sq_flags = flags; 7503 7504 /* 7505 * We have cleared SQ_EXCL if we were asked to, and started 7506 * the wakeup process for waiters. If there are no writers 7507 * then we need to drain the syncq if we were told to, or 7508 * enable the background thread to do it. 7509 */ 7510 if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) { 7511 if ((passflags & SQ_QUEUED) || 7512 (sq->sq_svcflags & SQ_DISABLED)) { 7513 /* drain_syncq will take care of events in the list */ 7514 drain_syncq(sq); 7515 return; 7516 } else if (flags & SQ_QUEUED) { 7517 sqenable(sq); 7518 } 7519 } 7520 /* Drop the SQLOCK on exit */ 7521 mutex_exit(SQLOCK(sq)); 7522 TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END, 7523 "putnext_end:(%p, %p, %p) done", NULL, qp, sq); 7524 } 7525 7526 void 7527 set_qend(queue_t *q) 7528 { 7529 mutex_enter(QLOCK(q)); 7530 if (!O_SAMESTR(q)) 7531 q->q_flag |= QEND; 7532 else 7533 q->q_flag &= ~QEND; 7534 mutex_exit(QLOCK(q)); 7535 q = _OTHERQ(q); 7536 mutex_enter(QLOCK(q)); 7537 if (!O_SAMESTR(q)) 7538 q->q_flag |= QEND; 7539 else 7540 q->q_flag &= ~QEND; 7541 mutex_exit(QLOCK(q)); 7542 } 7543 7544 /* 7545 * Set QFULL in next service procedure queue (that cares) if not already 7546 * set and if there are already more messages on the syncq than 7547 * sq_max_size. If sq_max_size is 0, no flow control will be asserted on 7548 * any syncq. 7549 * 7550 * The fq here is the next queue with a service procedure. This is where 7551 * we would fail canputnext, so this is where we need to set QFULL. 7552 * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag. 7553 * 7554 * We already have QLOCK at this point. To avoid cross-locks with 7555 * freezestr() which grabs all QLOCKs and with strlock() which grabs both 7556 * SQLOCK and sd_reflock, we need to drop respective locks first. 7557 */ 7558 void 7559 set_qfull(queue_t *q) 7560 { 7561 queue_t *fq = NULL; 7562 7563 ASSERT(MUTEX_HELD(QLOCK(q))); 7564 if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) && 7565 (q->q_syncqmsgs > sq_max_size)) { 7566 if ((fq = q->q_nfsrv) == q) { 7567 fq->q_flag |= QFULL; 7568 } else { 7569 mutex_exit(QLOCK(q)); 7570 mutex_enter(QLOCK(fq)); 7571 fq->q_flag |= QFULL; 7572 mutex_exit(QLOCK(fq)); 7573 mutex_enter(QLOCK(q)); 7574 } 7575 } 7576 } 7577 7578 void 7579 clr_qfull(queue_t *q) 7580 { 7581 queue_t *oq = q; 7582 7583 q = q->q_nfsrv; 7584 /* Fast check if there is any work to do before getting the lock. */ 7585 if ((q->q_flag & (QFULL|QWANTW)) == 0) { 7586 return; 7587 } 7588 7589 /* 7590 * Do not reset QFULL (and backenable) if the q_count is the reason 7591 * for QFULL being set. 7592 */ 7593 mutex_enter(QLOCK(q)); 7594 /* 7595 * If queue is empty i.e q_mblkcnt is zero, queue can not be full. 7596 * Hence clear the QFULL. 7597 * If both q_count and q_mblkcnt are less than the hiwat mark, 7598 * clear the QFULL. 7599 */ 7600 if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) && 7601 (q->q_mblkcnt < q->q_hiwat))) { 7602 q->q_flag &= ~QFULL; 7603 /* 7604 * A little more confusing, how about this way: 7605 * if someone wants to write, 7606 * AND 7607 * both counts are less than the lowat mark 7608 * OR 7609 * the lowat mark is zero 7610 * THEN 7611 * backenable 7612 */ 7613 if ((q->q_flag & QWANTW) && 7614 (((q->q_count < q->q_lowat) && 7615 (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) { 7616 q->q_flag &= ~QWANTW; 7617 mutex_exit(QLOCK(q)); 7618 backenable(oq, 0); 7619 } else 7620 mutex_exit(QLOCK(q)); 7621 } else 7622 mutex_exit(QLOCK(q)); 7623 } 7624 7625 /* 7626 * Set the forward service procedure pointer. 7627 * 7628 * Called at insert-time to cache a queue's next forward service procedure in 7629 * q_nfsrv; used by canput() and canputnext(). If the queue to be inserted 7630 * has a service procedure then q_nfsrv points to itself. If the queue to be 7631 * inserted does not have a service procedure, then q_nfsrv points to the next 7632 * queue forward that has a service procedure. If the queue is at the logical 7633 * end of the stream (driver for write side, stream head for the read side) 7634 * and does not have a service procedure, then q_nfsrv also points to itself. 7635 */ 7636 void 7637 set_nfsrv_ptr( 7638 queue_t *rnew, /* read queue pointer to new module */ 7639 queue_t *wnew, /* write queue pointer to new module */ 7640 queue_t *prev_rq, /* read queue pointer to the module above */ 7641 queue_t *prev_wq) /* write queue pointer to the module above */ 7642 { 7643 queue_t *qp; 7644 7645 if (prev_wq->q_next == NULL) { 7646 /* 7647 * Insert the driver, initialize the driver and stream head. 7648 * In this case, prev_rq/prev_wq should be the stream head. 7649 * _I_INSERT does not allow inserting a driver. Make sure 7650 * that it is not an insertion. 7651 */ 7652 ASSERT(!(rnew->q_flag & _QINSERTING)); 7653 wnew->q_nfsrv = wnew; 7654 if (rnew->q_qinfo->qi_srvp) 7655 rnew->q_nfsrv = rnew; 7656 else 7657 rnew->q_nfsrv = prev_rq; 7658 prev_rq->q_nfsrv = prev_rq; 7659 prev_wq->q_nfsrv = prev_wq; 7660 } else { 7661 /* 7662 * set up read side q_nfsrv pointer. This MUST be done 7663 * before setting the write side, because the setting of 7664 * the write side for a fifo may depend on it. 7665 * 7666 * Suppose we have a fifo that only has pipemod pushed. 7667 * pipemod has no read or write service procedures, so 7668 * nfsrv for both pipemod queues points to prev_rq (the 7669 * stream read head). Now push bufmod (which has only a 7670 * read service procedure). Doing the write side first, 7671 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which 7672 * is WRONG; the next queue forward from wnew with a 7673 * service procedure will be rnew, not the stream read head. 7674 * Since the downstream queue (which in the case of a fifo 7675 * is the read queue rnew) can affect upstream queues, it 7676 * needs to be done first. Setting up the read side first 7677 * sets nfsrv for both pipemod queues to rnew and then 7678 * when the write side is set up, wnew-q_nfsrv will also 7679 * point to rnew. 7680 */ 7681 if (rnew->q_qinfo->qi_srvp) { 7682 /* 7683 * use _OTHERQ() because, if this is a pipe, next 7684 * module may have been pushed from other end and 7685 * q_next could be a read queue. 7686 */ 7687 qp = _OTHERQ(prev_wq->q_next); 7688 while (qp && qp->q_nfsrv != qp) { 7689 qp->q_nfsrv = rnew; 7690 qp = backq(qp); 7691 } 7692 rnew->q_nfsrv = rnew; 7693 } else 7694 rnew->q_nfsrv = prev_rq->q_nfsrv; 7695 7696 /* set up write side q_nfsrv pointer */ 7697 if (wnew->q_qinfo->qi_srvp) { 7698 wnew->q_nfsrv = wnew; 7699 7700 /* 7701 * For insertion, need to update nfsrv of the modules 7702 * above which do not have a service routine. 7703 */ 7704 if (rnew->q_flag & _QINSERTING) { 7705 for (qp = prev_wq; 7706 qp != NULL && qp->q_nfsrv != qp; 7707 qp = backq(qp)) { 7708 qp->q_nfsrv = wnew->q_nfsrv; 7709 } 7710 } 7711 } else { 7712 if (prev_wq->q_next == prev_rq) 7713 /* 7714 * Since prev_wq/prev_rq are the middle of a 7715 * fifo, wnew/rnew will also be the middle of 7716 * a fifo and wnew's nfsrv is same as rnew's. 7717 */ 7718 wnew->q_nfsrv = rnew->q_nfsrv; 7719 else 7720 wnew->q_nfsrv = prev_wq->q_next->q_nfsrv; 7721 } 7722 } 7723 } 7724 7725 /* 7726 * Reset the forward service procedure pointer; called at remove-time. 7727 */ 7728 void 7729 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp) 7730 { 7731 queue_t *tmp_qp; 7732 7733 /* Reset the write side q_nfsrv pointer for _I_REMOVE */ 7734 if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) { 7735 for (tmp_qp = backq(wqp); 7736 tmp_qp != NULL && tmp_qp->q_nfsrv == wqp; 7737 tmp_qp = backq(tmp_qp)) { 7738 tmp_qp->q_nfsrv = wqp->q_nfsrv; 7739 } 7740 } 7741 7742 /* reset the read side q_nfsrv pointer */ 7743 if (rqp->q_qinfo->qi_srvp) { 7744 if (wqp->q_next) { /* non-driver case */ 7745 tmp_qp = _OTHERQ(wqp->q_next); 7746 while (tmp_qp && tmp_qp->q_nfsrv == rqp) { 7747 /* Note that rqp->q_next cannot be NULL */ 7748 ASSERT(rqp->q_next != NULL); 7749 tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv; 7750 tmp_qp = backq(tmp_qp); 7751 } 7752 } 7753 } 7754 } 7755 7756 /* 7757 * This routine should be called after all stream geometry changes to update 7758 * the stream head cached struio() rd/wr queue pointers. Note must be called 7759 * with the streamlock()ed. 7760 * 7761 * Note: only enables Synchronous STREAMS for a side of a Stream which has 7762 * an explicit synchronous barrier module queue. That is, a queue that 7763 * has specified a struio() type. 7764 */ 7765 static void 7766 strsetuio(stdata_t *stp) 7767 { 7768 queue_t *wrq; 7769 7770 if (stp->sd_flag & STPLEX) { 7771 /* 7772 * Not streamhead, but a mux, so no Synchronous STREAMS. 7773 */ 7774 stp->sd_struiowrq = NULL; 7775 stp->sd_struiordq = NULL; 7776 return; 7777 } 7778 /* 7779 * Scan the write queue(s) while synchronous 7780 * until we find a qinfo uio type specified. 7781 */ 7782 wrq = stp->sd_wrq->q_next; 7783 while (wrq) { 7784 if (wrq->q_struiot == STRUIOT_NONE) { 7785 wrq = 0; 7786 break; 7787 } 7788 if (wrq->q_struiot != STRUIOT_DONTCARE) 7789 break; 7790 if (! _SAMESTR(wrq)) { 7791 wrq = 0; 7792 break; 7793 } 7794 wrq = wrq->q_next; 7795 } 7796 stp->sd_struiowrq = wrq; 7797 /* 7798 * Scan the read queue(s) while synchronous 7799 * until we find a qinfo uio type specified. 7800 */ 7801 wrq = stp->sd_wrq->q_next; 7802 while (wrq) { 7803 if (_RD(wrq)->q_struiot == STRUIOT_NONE) { 7804 wrq = 0; 7805 break; 7806 } 7807 if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE) 7808 break; 7809 if (! _SAMESTR(wrq)) { 7810 wrq = 0; 7811 break; 7812 } 7813 wrq = wrq->q_next; 7814 } 7815 stp->sd_struiordq = wrq ? _RD(wrq) : 0; 7816 } 7817 7818 static int 7819 pass_rput(queue_t *q, mblk_t *mp) 7820 { 7821 putnext(q, mp); 7822 return (0); 7823 } 7824 7825 /* 7826 * pass_wput, unblocks the passthru queues, so that 7827 * messages can arrive at muxs lower read queue, before 7828 * I_LINK/I_UNLINK is acked/nacked. 7829 */ 7830 static int 7831 pass_wput(queue_t *q, mblk_t *mp) 7832 { 7833 syncq_t *sq; 7834 7835 sq = _RD(q)->q_syncq; 7836 if (sq->sq_flags & SQ_BLOCKED) 7837 unblocksq(sq, SQ_BLOCKED, 0); 7838 putnext(q, mp); 7839 return (0); 7840 } 7841 7842 /* 7843 * Set up queues for the link/unlink. 7844 * Create a new queue and block it and then insert it 7845 * below the stream head on the lower stream. 7846 * This prevents any messages from arriving during the setq 7847 * as well as while the mux is processing the LINK/I_UNLINK. 7848 * The blocked passq is unblocked once the LINK/I_UNLINK has 7849 * been acked or nacked or if a message is generated and sent 7850 * down muxs write put procedure. 7851 * See pass_wput(). 7852 * 7853 * After the new queue is inserted, all messages coming from below are 7854 * blocked. The call to strlock will ensure that all activity in the stream head 7855 * read queue syncq is stopped (sq_count drops to zero). 7856 */ 7857 static queue_t * 7858 link_addpassthru(stdata_t *stpdown) 7859 { 7860 queue_t *passq; 7861 sqlist_t sqlist; 7862 7863 passq = allocq(); 7864 STREAM(passq) = STREAM(_WR(passq)) = stpdown; 7865 /* setq might sleep in allocator - avoid holding locks. */ 7866 setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ, 7867 SQ_CI|SQ_CO, B_FALSE); 7868 claimq(passq); 7869 blocksq(passq->q_syncq, SQ_BLOCKED, 1); 7870 insertq(STREAM(passq), passq); 7871 7872 /* 7873 * Use strlock() to wait for the stream head sq_count to drop to zero 7874 * since we are going to change q_ptr in the stream head. Note that 7875 * insertq() doesn't wait for any syncq counts to drop to zero. 7876 */ 7877 sqlist.sqlist_head = NULL; 7878 sqlist.sqlist_index = 0; 7879 sqlist.sqlist_size = sizeof (sqlist_t); 7880 sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq); 7881 strlock(stpdown, &sqlist); 7882 strunlock(stpdown, &sqlist); 7883 7884 releaseq(passq); 7885 return (passq); 7886 } 7887 7888 /* 7889 * Let messages flow up into the mux by removing 7890 * the passq. 7891 */ 7892 static void 7893 link_rempassthru(queue_t *passq) 7894 { 7895 claimq(passq); 7896 removeq(passq); 7897 releaseq(passq); 7898 freeq(passq); 7899 } 7900 7901 /* 7902 * Wait for the condition variable pointed to by `cvp' to be signaled, 7903 * or for `tim' milliseconds to elapse, whichever comes first. If `tim' 7904 * is negative, then there is no time limit. If `nosigs' is non-zero, 7905 * then the wait will be non-interruptible. 7906 * 7907 * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout. 7908 */ 7909 clock_t 7910 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs) 7911 { 7912 clock_t ret; 7913 7914 if (tim < 0) { 7915 if (nosigs) { 7916 cv_wait(cvp, mp); 7917 ret = 1; 7918 } else { 7919 ret = cv_wait_sig(cvp, mp); 7920 } 7921 } else if (tim > 0) { 7922 /* 7923 * convert milliseconds to clock ticks 7924 */ 7925 if (nosigs) { 7926 ret = cv_reltimedwait(cvp, mp, 7927 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK); 7928 } else { 7929 ret = cv_reltimedwait_sig(cvp, mp, 7930 MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK); 7931 } 7932 } else { 7933 ret = -1; 7934 } 7935 return (ret); 7936 } 7937 7938 /* 7939 * Wait until the stream head can determine if it is at the mark but 7940 * don't wait forever to prevent a race condition between the "mark" state 7941 * in the stream head and any mark state in the caller/user of this routine. 7942 * 7943 * This is used by sockets and for a socket it would be incorrect 7944 * to return a failure for SIOCATMARK when there is no data in the receive 7945 * queue and the marked urgent data is traveling up the stream. 7946 * 7947 * This routine waits until the mark is known by waiting for one of these 7948 * three events: 7949 * The stream head read queue becoming non-empty (including an EOF). 7950 * The STRATMARK flag being set (due to a MSGMARKNEXT message). 7951 * The STRNOTATMARK flag being set (which indicates that the transport 7952 * has sent a MSGNOTMARKNEXT message to indicate that it is not at 7953 * the mark). 7954 * 7955 * The routine returns 1 if the stream is at the mark; 0 if it can 7956 * be determined that the stream is not at the mark. 7957 * If the wait times out and it can't determine 7958 * whether or not the stream might be at the mark the routine will return -1. 7959 * 7960 * Note: This routine should only be used when a mark is pending i.e., 7961 * in the socket case the SIGURG has been posted. 7962 * Note2: This can not wakeup just because synchronous streams indicate 7963 * that data is available since it is not possible to use the synchronous 7964 * streams interfaces to determine the b_flag value for the data queued below 7965 * the stream head. 7966 */ 7967 int 7968 strwaitmark(vnode_t *vp) 7969 { 7970 struct stdata *stp = vp->v_stream; 7971 queue_t *rq = _RD(stp->sd_wrq); 7972 int mark; 7973 7974 mutex_enter(&stp->sd_lock); 7975 while (rq->q_first == NULL && 7976 !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) { 7977 stp->sd_flag |= RSLEEP; 7978 7979 /* Wait for 100 milliseconds for any state change. */ 7980 if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) { 7981 mutex_exit(&stp->sd_lock); 7982 return (-1); 7983 } 7984 } 7985 if (stp->sd_flag & STRATMARK) 7986 mark = 1; 7987 else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK)) 7988 mark = 1; 7989 else 7990 mark = 0; 7991 7992 mutex_exit(&stp->sd_lock); 7993 return (mark); 7994 } 7995 7996 /* 7997 * Set a read side error. If persist is set change the socket error 7998 * to persistent. If errfunc is set install the function as the exported 7999 * error handler. 8000 */ 8001 void 8002 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 8003 { 8004 struct stdata *stp = vp->v_stream; 8005 8006 mutex_enter(&stp->sd_lock); 8007 stp->sd_rerror = error; 8008 if (error == 0 && errfunc == NULL) 8009 stp->sd_flag &= ~STRDERR; 8010 else 8011 stp->sd_flag |= STRDERR; 8012 if (persist) { 8013 stp->sd_flag &= ~STRDERRNONPERSIST; 8014 } else { 8015 stp->sd_flag |= STRDERRNONPERSIST; 8016 } 8017 stp->sd_rderrfunc = errfunc; 8018 if (error != 0 || errfunc != NULL) { 8019 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 8020 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 8021 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 8022 8023 mutex_exit(&stp->sd_lock); 8024 pollwakeup(&stp->sd_pollist, POLLERR); 8025 mutex_enter(&stp->sd_lock); 8026 8027 if (stp->sd_sigflags & S_ERROR) 8028 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 8029 } 8030 mutex_exit(&stp->sd_lock); 8031 } 8032 8033 /* 8034 * Set a write side error. If persist is set change the socket error 8035 * to persistent. 8036 */ 8037 void 8038 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc) 8039 { 8040 struct stdata *stp = vp->v_stream; 8041 8042 mutex_enter(&stp->sd_lock); 8043 stp->sd_werror = error; 8044 if (error == 0 && errfunc == NULL) 8045 stp->sd_flag &= ~STWRERR; 8046 else 8047 stp->sd_flag |= STWRERR; 8048 if (persist) { 8049 stp->sd_flag &= ~STWRERRNONPERSIST; 8050 } else { 8051 stp->sd_flag |= STWRERRNONPERSIST; 8052 } 8053 stp->sd_wrerrfunc = errfunc; 8054 if (error != 0 || errfunc != NULL) { 8055 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); /* readers */ 8056 cv_broadcast(&stp->sd_wrq->q_wait); /* writers */ 8057 cv_broadcast(&stp->sd_monitor); /* ioctllers */ 8058 8059 mutex_exit(&stp->sd_lock); 8060 pollwakeup(&stp->sd_pollist, POLLERR); 8061 mutex_enter(&stp->sd_lock); 8062 8063 if (stp->sd_sigflags & S_ERROR) 8064 strsendsig(stp->sd_siglist, S_ERROR, 0, error); 8065 } 8066 mutex_exit(&stp->sd_lock); 8067 } 8068 8069 /* 8070 * Make the stream return 0 (EOF) when all data has been read. 8071 * No effect on write side. 8072 */ 8073 void 8074 strseteof(vnode_t *vp, int eof) 8075 { 8076 struct stdata *stp = vp->v_stream; 8077 8078 mutex_enter(&stp->sd_lock); 8079 if (!eof) { 8080 stp->sd_flag &= ~STREOF; 8081 mutex_exit(&stp->sd_lock); 8082 return; 8083 } 8084 stp->sd_flag |= STREOF; 8085 if (stp->sd_flag & RSLEEP) { 8086 stp->sd_flag &= ~RSLEEP; 8087 cv_broadcast(&_RD(stp->sd_wrq)->q_wait); 8088 } 8089 8090 mutex_exit(&stp->sd_lock); 8091 pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM); 8092 mutex_enter(&stp->sd_lock); 8093 8094 if (stp->sd_sigflags & (S_INPUT|S_RDNORM)) 8095 strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0); 8096 mutex_exit(&stp->sd_lock); 8097 } 8098 8099 void 8100 strflushrq(vnode_t *vp, int flag) 8101 { 8102 struct stdata *stp = vp->v_stream; 8103 8104 mutex_enter(&stp->sd_lock); 8105 flushq(_RD(stp->sd_wrq), flag); 8106 mutex_exit(&stp->sd_lock); 8107 } 8108 8109 void 8110 strsetrputhooks(vnode_t *vp, uint_t flags, 8111 msgfunc_t protofunc, msgfunc_t miscfunc) 8112 { 8113 struct stdata *stp = vp->v_stream; 8114 8115 mutex_enter(&stp->sd_lock); 8116 8117 if (protofunc == NULL) 8118 stp->sd_rprotofunc = strrput_proto; 8119 else 8120 stp->sd_rprotofunc = protofunc; 8121 8122 if (miscfunc == NULL) 8123 stp->sd_rmiscfunc = strrput_misc; 8124 else 8125 stp->sd_rmiscfunc = miscfunc; 8126 8127 if (flags & SH_CONSOL_DATA) 8128 stp->sd_rput_opt |= SR_CONSOL_DATA; 8129 else 8130 stp->sd_rput_opt &= ~SR_CONSOL_DATA; 8131 8132 if (flags & SH_SIGALLDATA) 8133 stp->sd_rput_opt |= SR_SIGALLDATA; 8134 else 8135 stp->sd_rput_opt &= ~SR_SIGALLDATA; 8136 8137 if (flags & SH_IGN_ZEROLEN) 8138 stp->sd_rput_opt |= SR_IGN_ZEROLEN; 8139 else 8140 stp->sd_rput_opt &= ~SR_IGN_ZEROLEN; 8141 8142 mutex_exit(&stp->sd_lock); 8143 } 8144 8145 void 8146 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime) 8147 { 8148 struct stdata *stp = vp->v_stream; 8149 8150 mutex_enter(&stp->sd_lock); 8151 stp->sd_closetime = closetime; 8152 8153 if (flags & SH_SIGPIPE) 8154 stp->sd_wput_opt |= SW_SIGPIPE; 8155 else 8156 stp->sd_wput_opt &= ~SW_SIGPIPE; 8157 if (flags & SH_RECHECK_ERR) 8158 stp->sd_wput_opt |= SW_RECHECK_ERR; 8159 else 8160 stp->sd_wput_opt &= ~SW_RECHECK_ERR; 8161 8162 mutex_exit(&stp->sd_lock); 8163 } 8164 8165 void 8166 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc) 8167 { 8168 struct stdata *stp = vp->v_stream; 8169 8170 mutex_enter(&stp->sd_lock); 8171 8172 stp->sd_rputdatafunc = rdatafunc; 8173 stp->sd_wputdatafunc = wdatafunc; 8174 8175 mutex_exit(&stp->sd_lock); 8176 } 8177 8178 /* Used within framework when the queue is already locked */ 8179 void 8180 qenable_locked(queue_t *q) 8181 { 8182 stdata_t *stp = STREAM(q); 8183 8184 ASSERT(MUTEX_HELD(QLOCK(q))); 8185 8186 if (!q->q_qinfo->qi_srvp) 8187 return; 8188 8189 /* 8190 * Do not place on run queue if already enabled or closing. 8191 */ 8192 if (q->q_flag & (QWCLOSE|QENAB)) 8193 return; 8194 8195 /* 8196 * mark queue enabled and place on run list if it is not already being 8197 * serviced. If it is serviced, the runservice() function will detect 8198 * that QENAB is set and call service procedure before clearing 8199 * QINSERVICE flag. 8200 */ 8201 q->q_flag |= QENAB; 8202 if (q->q_flag & QINSERVICE) 8203 return; 8204 8205 /* Record the time of qenable */ 8206 q->q_qtstamp = ddi_get_lbolt(); 8207 8208 /* 8209 * Put the queue in the stp list and schedule it for background 8210 * processing if it is not already scheduled or if stream head does not 8211 * intent to process it in the foreground later by setting 8212 * STRS_WILLSERVICE flag. 8213 */ 8214 mutex_enter(&stp->sd_qlock); 8215 /* 8216 * If there are already something on the list, stp flags should show 8217 * intention to drain it. 8218 */ 8219 IMPLY(STREAM_NEEDSERVICE(stp), 8220 (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))); 8221 8222 ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link); 8223 stp->sd_nqueues++; 8224 8225 /* 8226 * If no one will drain this stream we are the first producer and 8227 * need to schedule it for background thread. 8228 */ 8229 if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) { 8230 /* 8231 * No one will service this stream later, so we have to 8232 * schedule it now. 8233 */ 8234 STRSTAT(stenables); 8235 stp->sd_svcflags |= STRS_SCHEDULED; 8236 stp->sd_servid = (void *)taskq_dispatch(streams_taskq, 8237 (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE); 8238 8239 if (stp->sd_servid == NULL) { 8240 /* 8241 * Task queue failed so fail over to the backup 8242 * servicing thread. 8243 */ 8244 STRSTAT(taskqfails); 8245 /* 8246 * It is safe to clear STRS_SCHEDULED flag because it 8247 * was set by this thread above. 8248 */ 8249 stp->sd_svcflags &= ~STRS_SCHEDULED; 8250 8251 /* 8252 * Failover scheduling is protected by service_queue 8253 * lock. 8254 */ 8255 mutex_enter(&service_queue); 8256 ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q)); 8257 ASSERT(q->q_link == NULL); 8258 /* 8259 * Append the queue to qhead/qtail list. 8260 */ 8261 if (qhead == NULL) 8262 qhead = q; 8263 else 8264 qtail->q_link = q; 8265 qtail = q; 8266 /* 8267 * Clear stp queue list. 8268 */ 8269 stp->sd_qhead = stp->sd_qtail = NULL; 8270 stp->sd_nqueues = 0; 8271 /* 8272 * Wakeup background queue processing thread. 8273 */ 8274 cv_signal(&services_to_run); 8275 mutex_exit(&service_queue); 8276 } 8277 } 8278 mutex_exit(&stp->sd_qlock); 8279 } 8280 8281 static void 8282 queue_service(queue_t *q) 8283 { 8284 /* 8285 * The queue in the list should have 8286 * QENAB flag set and should not have 8287 * QINSERVICE flag set. QINSERVICE is 8288 * set when the queue is dequeued and 8289 * qenable_locked doesn't enqueue a 8290 * queue with QINSERVICE set. 8291 */ 8292 8293 ASSERT(!(q->q_flag & QINSERVICE)); 8294 ASSERT((q->q_flag & QENAB)); 8295 mutex_enter(QLOCK(q)); 8296 q->q_flag &= ~QENAB; 8297 q->q_flag |= QINSERVICE; 8298 mutex_exit(QLOCK(q)); 8299 runservice(q); 8300 } 8301 8302 static void 8303 syncq_service(syncq_t *sq) 8304 { 8305 STRSTAT(syncqservice); 8306 mutex_enter(SQLOCK(sq)); 8307 ASSERT(!(sq->sq_svcflags & SQ_SERVICE)); 8308 ASSERT(sq->sq_servcount != 0); 8309 ASSERT(sq->sq_next == NULL); 8310 8311 /* if we came here from the background thread, clear the flag */ 8312 if (sq->sq_svcflags & SQ_BGTHREAD) 8313 sq->sq_svcflags &= ~SQ_BGTHREAD; 8314 8315 /* let drain_syncq know that it's being called in the background */ 8316 sq->sq_svcflags |= SQ_SERVICE; 8317 drain_syncq(sq); 8318 } 8319 8320 static void 8321 qwriter_outer_service(syncq_t *outer) 8322 { 8323 /* 8324 * Note that SQ_WRITER is used on the outer perimeter 8325 * to signal that a qwriter(OUTER) is either investigating 8326 * running or that it is actually running a function. 8327 */ 8328 outer_enter(outer, SQ_BLOCKED|SQ_WRITER); 8329 8330 /* 8331 * All inner syncq are empty and have SQ_WRITER set 8332 * to block entering the outer perimeter. 8333 * 8334 * We do not need to explicitly call write_now since 8335 * outer_exit does it for us. 8336 */ 8337 outer_exit(outer); 8338 } 8339 8340 static void 8341 mblk_free(mblk_t *mp) 8342 { 8343 dblk_t *dbp = mp->b_datap; 8344 frtn_t *frp = dbp->db_frtnp; 8345 8346 mp->b_next = NULL; 8347 if (dbp->db_fthdr != NULL) 8348 str_ftfree(dbp); 8349 8350 ASSERT(dbp->db_fthdr == NULL); 8351 frp->free_func(frp->free_arg); 8352 ASSERT(dbp->db_mblk == mp); 8353 8354 if (dbp->db_credp != NULL) { 8355 crfree(dbp->db_credp); 8356 dbp->db_credp = NULL; 8357 } 8358 dbp->db_cpid = -1; 8359 dbp->db_struioflag = 0; 8360 dbp->db_struioun.cksum.flags = 0; 8361 8362 kmem_cache_free(dbp->db_cache, dbp); 8363 } 8364 8365 /* 8366 * Background processing of the stream queue list. 8367 */ 8368 static void 8369 stream_service(stdata_t *stp) 8370 { 8371 queue_t *q; 8372 8373 mutex_enter(&stp->sd_qlock); 8374 8375 STR_SERVICE(stp, q); 8376 8377 stp->sd_svcflags &= ~STRS_SCHEDULED; 8378 stp->sd_servid = NULL; 8379 cv_signal(&stp->sd_qcv); 8380 mutex_exit(&stp->sd_qlock); 8381 } 8382 8383 /* 8384 * Foreground processing of the stream queue list. 8385 */ 8386 void 8387 stream_runservice(stdata_t *stp) 8388 { 8389 queue_t *q; 8390 8391 mutex_enter(&stp->sd_qlock); 8392 STRSTAT(rservice); 8393 /* 8394 * We are going to drain this stream queue list, so qenable_locked will 8395 * not schedule it until we finish. 8396 */ 8397 stp->sd_svcflags |= STRS_WILLSERVICE; 8398 8399 STR_SERVICE(stp, q); 8400 8401 stp->sd_svcflags &= ~STRS_WILLSERVICE; 8402 mutex_exit(&stp->sd_qlock); 8403 /* 8404 * Help backup background thread to drain the qhead/qtail list. 8405 */ 8406 while (qhead != NULL) { 8407 STRSTAT(qhelps); 8408 mutex_enter(&service_queue); 8409 DQ(q, qhead, qtail, q_link); 8410 mutex_exit(&service_queue); 8411 if (q != NULL) 8412 queue_service(q); 8413 } 8414 } 8415 8416 void 8417 stream_willservice(stdata_t *stp) 8418 { 8419 mutex_enter(&stp->sd_qlock); 8420 stp->sd_svcflags |= STRS_WILLSERVICE; 8421 mutex_exit(&stp->sd_qlock); 8422 } 8423 8424 /* 8425 * Replace the cred currently in the mblk with a different one. 8426 * Also update db_cpid. 8427 */ 8428 void 8429 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid) 8430 { 8431 dblk_t *dbp = mp->b_datap; 8432 cred_t *ocr = dbp->db_credp; 8433 8434 ASSERT(cr != NULL); 8435 8436 if (cr != ocr) { 8437 crhold(dbp->db_credp = cr); 8438 if (ocr != NULL) 8439 crfree(ocr); 8440 } 8441 /* Don't overwrite with NOPID */ 8442 if (cpid != NOPID) 8443 dbp->db_cpid = cpid; 8444 } 8445 8446 /* 8447 * If the src message has a cred, then replace the cred currently in the mblk 8448 * with it. 8449 * Also update db_cpid. 8450 */ 8451 void 8452 mblk_copycred(mblk_t *mp, const mblk_t *src) 8453 { 8454 dblk_t *dbp = mp->b_datap; 8455 cred_t *cr, *ocr; 8456 pid_t cpid; 8457 8458 cr = msg_getcred(src, &cpid); 8459 if (cr == NULL) 8460 return; 8461 8462 ocr = dbp->db_credp; 8463 if (cr != ocr) { 8464 crhold(dbp->db_credp = cr); 8465 if (ocr != NULL) 8466 crfree(ocr); 8467 } 8468 /* Don't overwrite with NOPID */ 8469 if (cpid != NOPID) 8470 dbp->db_cpid = cpid; 8471 } 8472 8473 int 8474 hcksum_assoc(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8475 uint32_t start, uint32_t stuff, uint32_t end, uint32_t value, 8476 uint32_t flags, int km_flags) 8477 { 8478 int rc = 0; 8479 8480 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8481 if (mp->b_datap->db_type == M_DATA) { 8482 /* Associate values for M_DATA type */ 8483 DB_CKSUMSTART(mp) = (intptr_t)start; 8484 DB_CKSUMSTUFF(mp) = (intptr_t)stuff; 8485 DB_CKSUMEND(mp) = (intptr_t)end; 8486 DB_CKSUMFLAGS(mp) = flags; 8487 DB_CKSUM16(mp) = (uint16_t)value; 8488 8489 } else { 8490 pattrinfo_t pa_info; 8491 8492 ASSERT(mmd != NULL); 8493 8494 pa_info.type = PATTR_HCKSUM; 8495 pa_info.len = sizeof (pattr_hcksum_t); 8496 8497 if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) { 8498 pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf; 8499 8500 hck->hcksum_start_offset = start; 8501 hck->hcksum_stuff_offset = stuff; 8502 hck->hcksum_end_offset = end; 8503 hck->hcksum_cksum_val.inet_cksum = (uint16_t)value; 8504 hck->hcksum_flags = flags; 8505 } else { 8506 rc = -1; 8507 } 8508 } 8509 return (rc); 8510 } 8511 8512 void 8513 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd, 8514 uint32_t *start, uint32_t *stuff, uint32_t *end, 8515 uint32_t *value, uint32_t *flags) 8516 { 8517 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8518 if (mp->b_datap->db_type == M_DATA) { 8519 if (flags != NULL) { 8520 *flags = DB_CKSUMFLAGS(mp) & HCK_FLAGS; 8521 if ((*flags & (HCK_PARTIALCKSUM | 8522 HCK_FULLCKSUM)) != 0) { 8523 if (value != NULL) 8524 *value = (uint32_t)DB_CKSUM16(mp); 8525 if ((*flags & HCK_PARTIALCKSUM) != 0) { 8526 if (start != NULL) 8527 *start = 8528 (uint32_t)DB_CKSUMSTART(mp); 8529 if (stuff != NULL) 8530 *stuff = 8531 (uint32_t)DB_CKSUMSTUFF(mp); 8532 if (end != NULL) 8533 *end = 8534 (uint32_t)DB_CKSUMEND(mp); 8535 } 8536 } 8537 } 8538 } else { 8539 pattrinfo_t hck_attr = {PATTR_HCKSUM}; 8540 8541 ASSERT(mmd != NULL); 8542 8543 /* get hardware checksum attribute */ 8544 if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) { 8545 pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf; 8546 8547 ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t)); 8548 if (flags != NULL) 8549 *flags = hck->hcksum_flags; 8550 if (start != NULL) 8551 *start = hck->hcksum_start_offset; 8552 if (stuff != NULL) 8553 *stuff = hck->hcksum_stuff_offset; 8554 if (end != NULL) 8555 *end = hck->hcksum_end_offset; 8556 if (value != NULL) 8557 *value = (uint32_t) 8558 hck->hcksum_cksum_val.inet_cksum; 8559 } 8560 } 8561 } 8562 8563 void 8564 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags) 8565 { 8566 ASSERT(DB_TYPE(mp) == M_DATA); 8567 ASSERT((flags & ~HW_LSO_FLAGS) == 0); 8568 8569 /* Set the flags */ 8570 DB_LSOFLAGS(mp) |= flags; 8571 DB_LSOMSS(mp) = mss; 8572 } 8573 8574 void 8575 lso_info_cleanup(mblk_t *mp) 8576 { 8577 ASSERT(DB_TYPE(mp) == M_DATA); 8578 8579 /* Clear the flags */ 8580 DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS; 8581 DB_LSOMSS(mp) = 0; 8582 } 8583 8584 /* 8585 * Checksum buffer *bp for len bytes with psum partial checksum, 8586 * or 0 if none, and return the 16 bit partial checksum. 8587 */ 8588 unsigned 8589 bcksum(uchar_t *bp, int len, unsigned int psum) 8590 { 8591 int odd = len & 1; 8592 extern unsigned int ip_ocsum(); 8593 8594 if (((intptr_t)bp & 1) == 0 && !odd) { 8595 /* 8596 * Bp is 16 bit aligned and len is multiple of 16 bit word. 8597 */ 8598 return (ip_ocsum((ushort_t *)bp, len >> 1, psum)); 8599 } 8600 if (((intptr_t)bp & 1) != 0) { 8601 /* 8602 * Bp isn't 16 bit aligned. 8603 */ 8604 unsigned int tsum; 8605 8606 #ifdef _LITTLE_ENDIAN 8607 psum += *bp; 8608 #else 8609 psum += *bp << 8; 8610 #endif 8611 len--; 8612 bp++; 8613 tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0); 8614 psum += (tsum << 8) & 0xffff | (tsum >> 8); 8615 if (len & 1) { 8616 bp += len - 1; 8617 #ifdef _LITTLE_ENDIAN 8618 psum += *bp << 8; 8619 #else 8620 psum += *bp; 8621 #endif 8622 } 8623 } else { 8624 /* 8625 * Bp is 16 bit aligned. 8626 */ 8627 psum = ip_ocsum((ushort_t *)bp, len >> 1, psum); 8628 if (odd) { 8629 bp += len - 1; 8630 #ifdef _LITTLE_ENDIAN 8631 psum += *bp; 8632 #else 8633 psum += *bp << 8; 8634 #endif 8635 } 8636 } 8637 /* 8638 * Normalize psum to 16 bits before returning the new partial 8639 * checksum. The max psum value before normalization is 0x3FDFE. 8640 */ 8641 return ((psum >> 16) + (psum & 0xFFFF)); 8642 } 8643 8644 boolean_t 8645 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd) 8646 { 8647 boolean_t rc; 8648 8649 ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA); 8650 if (DB_TYPE(mp) == M_DATA) { 8651 rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0); 8652 } else { 8653 pattrinfo_t zcopy_attr = {PATTR_ZCOPY}; 8654 8655 ASSERT(mmd != NULL); 8656 rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL); 8657 } 8658 return (rc); 8659 } 8660 8661 void 8662 freemsgchain(mblk_t *mp) 8663 { 8664 mblk_t *next; 8665 8666 while (mp != NULL) { 8667 next = mp->b_next; 8668 mp->b_next = NULL; 8669 8670 freemsg(mp); 8671 mp = next; 8672 } 8673 } 8674 8675 mblk_t * 8676 copymsgchain(mblk_t *mp) 8677 { 8678 mblk_t *nmp = NULL; 8679 mblk_t **nmpp = &nmp; 8680 8681 for (; mp != NULL; mp = mp->b_next) { 8682 if ((*nmpp = copymsg(mp)) == NULL) { 8683 freemsgchain(nmp); 8684 return (NULL); 8685 } 8686 8687 nmpp = &((*nmpp)->b_next); 8688 } 8689 8690 return (nmp); 8691 } 8692 8693 /* NOTE: Do not add code after this point. */ 8694 #undef QLOCK 8695 8696 /* 8697 * Replacement for QLOCK macro for those that can't use it. 8698 */ 8699 kmutex_t * 8700 QLOCK(queue_t *q) 8701 { 8702 return (&(q)->q_lock); 8703 } 8704 8705 /* 8706 * Dummy runqueues/queuerun functions functions for backwards compatibility. 8707 */ 8708 #undef runqueues 8709 void 8710 runqueues(void) 8711 { 8712 } 8713 8714 #undef queuerun 8715 void 8716 queuerun(void) 8717 { 8718 } 8719 8720 /* 8721 * Initialize the STR stack instance, which tracks autopush and persistent 8722 * links. 8723 */ 8724 /* ARGSUSED */ 8725 static void * 8726 str_stack_init(netstackid_t stackid, netstack_t *ns) 8727 { 8728 str_stack_t *ss; 8729 int i; 8730 8731 ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP); 8732 ss->ss_netstack = ns; 8733 8734 /* 8735 * set up autopush 8736 */ 8737 sad_initspace(ss); 8738 8739 /* 8740 * set up mux_node structures. 8741 */ 8742 ss->ss_devcnt = devcnt; /* In case it should change before free */ 8743 ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) * 8744 ss->ss_devcnt), KM_SLEEP); 8745 for (i = 0; i < ss->ss_devcnt; i++) 8746 ss->ss_mux_nodes[i].mn_imaj = i; 8747 return (ss); 8748 } 8749 8750 /* 8751 * Note: run at zone shutdown and not destroy so that the PLINKs are 8752 * gone by the time other cleanup happens from the destroy callbacks. 8753 */ 8754 static void 8755 str_stack_shutdown(netstackid_t stackid, void *arg) 8756 { 8757 str_stack_t *ss = (str_stack_t *)arg; 8758 int i; 8759 cred_t *cr; 8760 8761 cr = zone_get_kcred(netstackid_to_zoneid(stackid)); 8762 ASSERT(cr != NULL); 8763 8764 /* Undo all the I_PLINKs for this zone */ 8765 for (i = 0; i < ss->ss_devcnt; i++) { 8766 struct mux_edge *ep; 8767 ldi_handle_t lh; 8768 ldi_ident_t li; 8769 int ret; 8770 int rval; 8771 dev_t rdev; 8772 8773 ep = ss->ss_mux_nodes[i].mn_outp; 8774 if (ep == NULL) 8775 continue; 8776 ret = ldi_ident_from_major((major_t)i, &li); 8777 if (ret != 0) { 8778 continue; 8779 } 8780 rdev = ep->me_dev; 8781 ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE, 8782 cr, &lh, li); 8783 if (ret != 0) { 8784 ldi_ident_release(li); 8785 continue; 8786 } 8787 8788 ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL, 8789 cr, &rval); 8790 if (ret) { 8791 (void) ldi_close(lh, FREAD|FWRITE, cr); 8792 ldi_ident_release(li); 8793 continue; 8794 } 8795 (void) ldi_close(lh, FREAD|FWRITE, cr); 8796 8797 /* Close layered handles */ 8798 ldi_ident_release(li); 8799 } 8800 crfree(cr); 8801 8802 sad_freespace(ss); 8803 8804 kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt); 8805 ss->ss_mux_nodes = NULL; 8806 } 8807 8808 /* 8809 * Free the structure; str_stack_shutdown did the other cleanup work. 8810 */ 8811 /* ARGSUSED */ 8812 static void 8813 str_stack_fini(netstackid_t stackid, void *arg) 8814 { 8815 str_stack_t *ss = (str_stack_t *)arg; 8816 8817 kmem_free(ss, sizeof (*ss)); 8818 } 8819