xref: /titanic_41/usr/src/uts/common/os/strsubr.c (revision 0d6bb4c6728fd20087fe25f4028a3838250e6e9c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
22 /*	  All Rights Reserved  	*/
23 
24 
25 /*
26  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
27  * Use is subject to license terms.
28  */
29 
30 #include <sys/types.h>
31 #include <sys/sysmacros.h>
32 #include <sys/param.h>
33 #include <sys/errno.h>
34 #include <sys/signal.h>
35 #include <sys/proc.h>
36 #include <sys/conf.h>
37 #include <sys/cred.h>
38 #include <sys/user.h>
39 #include <sys/vnode.h>
40 #include <sys/file.h>
41 #include <sys/session.h>
42 #include <sys/stream.h>
43 #include <sys/strsubr.h>
44 #include <sys/stropts.h>
45 #include <sys/poll.h>
46 #include <sys/systm.h>
47 #include <sys/cpuvar.h>
48 #include <sys/uio.h>
49 #include <sys/cmn_err.h>
50 #include <sys/priocntl.h>
51 #include <sys/procset.h>
52 #include <sys/vmem.h>
53 #include <sys/bitmap.h>
54 #include <sys/kmem.h>
55 #include <sys/siginfo.h>
56 #include <sys/vtrace.h>
57 #include <sys/callb.h>
58 #include <sys/debug.h>
59 #include <sys/modctl.h>
60 #include <sys/vmsystm.h>
61 #include <vm/page.h>
62 #include <sys/atomic.h>
63 #include <sys/suntpi.h>
64 #include <sys/strlog.h>
65 #include <sys/promif.h>
66 #include <sys/project.h>
67 #include <sys/vm.h>
68 #include <sys/taskq.h>
69 #include <sys/sunddi.h>
70 #include <sys/sunldi_impl.h>
71 #include <sys/strsun.h>
72 #include <sys/isa_defs.h>
73 #include <sys/multidata.h>
74 #include <sys/pattr.h>
75 #include <sys/strft.h>
76 #include <sys/fs/snode.h>
77 #include <sys/zone.h>
78 #include <sys/open.h>
79 #include <sys/sunldi.h>
80 #include <sys/sad.h>
81 #include <sys/netstack.h>
82 
83 #define	O_SAMESTR(q)	(((q)->q_next) && \
84 	(((q)->q_flag & QREADR) == ((q)->q_next->q_flag & QREADR)))
85 
86 /*
87  * WARNING:
88  * The variables and routines in this file are private, belonging
89  * to the STREAMS subsystem. These should not be used by modules
90  * or drivers. Compatibility will not be guaranteed.
91  */
92 
93 /*
94  * Id value used to distinguish between different multiplexor links.
95  */
96 static int32_t lnk_id = 0;
97 
98 #define	STREAMS_LOPRI MINCLSYSPRI
99 static pri_t streams_lopri = STREAMS_LOPRI;
100 
101 #define	STRSTAT(x)	(str_statistics.x.value.ui64++)
102 typedef struct str_stat {
103 	kstat_named_t	sqenables;
104 	kstat_named_t	stenables;
105 	kstat_named_t	syncqservice;
106 	kstat_named_t	freebs;
107 	kstat_named_t	qwr_outer;
108 	kstat_named_t	rservice;
109 	kstat_named_t	strwaits;
110 	kstat_named_t	taskqfails;
111 	kstat_named_t	bufcalls;
112 	kstat_named_t	qhelps;
113 	kstat_named_t	qremoved;
114 	kstat_named_t	sqremoved;
115 	kstat_named_t	bcwaits;
116 	kstat_named_t	sqtoomany;
117 } str_stat_t;
118 
119 static str_stat_t str_statistics = {
120 	{ "sqenables",		KSTAT_DATA_UINT64 },
121 	{ "stenables",		KSTAT_DATA_UINT64 },
122 	{ "syncqservice",	KSTAT_DATA_UINT64 },
123 	{ "freebs",		KSTAT_DATA_UINT64 },
124 	{ "qwr_outer",		KSTAT_DATA_UINT64 },
125 	{ "rservice",		KSTAT_DATA_UINT64 },
126 	{ "strwaits",		KSTAT_DATA_UINT64 },
127 	{ "taskqfails",		KSTAT_DATA_UINT64 },
128 	{ "bufcalls",		KSTAT_DATA_UINT64 },
129 	{ "qhelps",		KSTAT_DATA_UINT64 },
130 	{ "qremoved",		KSTAT_DATA_UINT64 },
131 	{ "sqremoved",		KSTAT_DATA_UINT64 },
132 	{ "bcwaits",		KSTAT_DATA_UINT64 },
133 	{ "sqtoomany",		KSTAT_DATA_UINT64 },
134 };
135 
136 static kstat_t *str_kstat;
137 
138 /*
139  * qrunflag was used previously to control background scheduling of queues. It
140  * is not used anymore, but kept here in case some module still wants to access
141  * it via qready() and setqsched macros.
142  */
143 char qrunflag;			/*  Unused */
144 
145 /*
146  * Most of the streams scheduling is done via task queues. Task queues may fail
147  * for non-sleep dispatches, so there are two backup threads servicing failed
148  * requests for queues and syncqs. Both of these threads also service failed
149  * dispatches freebs requests. Queues are put in the list specified by `qhead'
150  * and `qtail' pointers, syncqs use `sqhead' and `sqtail' pointers and freebs
151  * requests are put into `freebs_list' which has no tail pointer. All three
152  * lists are protected by a single `service_queue' lock and use
153  * `services_to_run' condition variable for signaling background threads. Use of
154  * a single lock should not be a problem because it is only used under heavy
155  * loads when task queues start to fail and at that time it may be a good idea
156  * to throttle scheduling requests.
157  *
158  * NOTE: queues and syncqs should be scheduled by two separate threads because
159  * queue servicing may be blocked waiting for a syncq which may be also
160  * scheduled for background execution. This may create a deadlock when only one
161  * thread is used for both.
162  */
163 
164 static taskq_t *streams_taskq;		/* Used for most STREAMS scheduling */
165 
166 static kmutex_t service_queue;		/* protects all of servicing vars */
167 static kcondvar_t services_to_run;	/* wake up background service thread */
168 static kcondvar_t syncqs_to_run;	/* wake up background service thread */
169 
170 /*
171  * List of queues scheduled for background processing due to lack of resources
172  * in the task queues. Protected by service_queue lock;
173  */
174 static struct queue *qhead;
175 static struct queue *qtail;
176 
177 /*
178  * Same list for syncqs
179  */
180 static syncq_t *sqhead;
181 static syncq_t *sqtail;
182 
183 static mblk_t *freebs_list;	/* list of buffers to free */
184 
185 /*
186  * Backup threads for servicing queues and syncqs
187  */
188 kthread_t *streams_qbkgrnd_thread;
189 kthread_t *streams_sqbkgrnd_thread;
190 
191 /*
192  * Bufcalls related variables.
193  */
194 struct bclist	strbcalls;	/* list of waiting bufcalls */
195 kmutex_t	strbcall_lock;	/* protects bufcall list (strbcalls) */
196 kcondvar_t	strbcall_cv;	/* Signaling when a bufcall is added */
197 kmutex_t	bcall_monitor;	/* sleep/wakeup style monitor */
198 kcondvar_t	bcall_cv;	/* wait 'till executing bufcall completes */
199 kthread_t	*bc_bkgrnd_thread; /* Thread to service bufcall requests */
200 
201 kmutex_t	strresources;	/* protects global resources */
202 kmutex_t	muxifier;	/* single-threads multiplexor creation */
203 
204 static void	*str_stack_init(netstackid_t stackid, netstack_t *ns);
205 static void	str_stack_shutdown(netstackid_t stackid, void *arg);
206 static void	str_stack_fini(netstackid_t stackid, void *arg);
207 
208 /*
209  * run_queues is no longer used, but is kept in case some 3rd party
210  * module/driver decides to use it.
211  */
212 int run_queues = 0;
213 
214 /*
215  * sq_max_size is the depth of the syncq (in number of messages) before
216  * qfill_syncq() starts QFULL'ing destination queues. As its primary
217  * consumer - IP is no longer D_MTPERMOD, but there may be other
218  * modules/drivers depend on this syncq flow control, we prefer to
219  * choose a large number as the default value. For potential
220  * performance gain, this value is tunable in /etc/system.
221  */
222 int sq_max_size = 10000;
223 
224 /*
225  * The number of ciputctrl structures per syncq and stream we create when
226  * needed.
227  */
228 int n_ciputctrl;
229 int max_n_ciputctrl = 16;
230 /*
231  * If n_ciputctrl is < min_n_ciputctrl don't even create ciputctrl_cache.
232  */
233 int min_n_ciputctrl = 2;
234 
235 /*
236  * Per-driver/module syncqs
237  * ========================
238  *
239  * For drivers/modules that use PERMOD or outer syncqs we keep a list of
240  * perdm structures, new entries being added (and new syncqs allocated) when
241  * setq() encounters a module/driver with a streamtab that it hasn't seen
242  * before.
243  * The reason for this mechanism is that some modules and drivers share a
244  * common streamtab and it is necessary for those modules and drivers to also
245  * share a common PERMOD syncq.
246  *
247  * perdm_list --> dm_str == streamtab_1
248  *                dm_sq == syncq_1
249  *                dm_ref
250  *                dm_next --> dm_str == streamtab_2
251  *                            dm_sq == syncq_2
252  *                            dm_ref
253  *                            dm_next --> ... NULL
254  *
255  * The dm_ref field is incremented for each new driver/module that takes
256  * a reference to the perdm structure and hence shares the syncq.
257  * References are held in the fmodsw_impl_t structure for each STREAMS module
258  * or the dev_impl array (indexed by device major number) for each driver.
259  *
260  * perdm_list -> [dm_ref == 1] -> [dm_ref == 2] -> [dm_ref == 1] -> NULL
261  *		     ^                 ^ ^               ^
262  *                   |  ______________/  |               |
263  *                   | /                 |               |
264  * dev_impl:     ...|x|y|...          module A	      module B
265  *
266  * When a module/driver is unloaded the reference count is decremented and,
267  * when it falls to zero, the perdm structure is removed from the list and
268  * the syncq is freed (see rele_dm()).
269  */
270 perdm_t *perdm_list = NULL;
271 static krwlock_t perdm_rwlock;
272 cdevsw_impl_t *devimpl;
273 
274 extern struct qinit strdata;
275 extern struct qinit stwdata;
276 
277 static void runservice(queue_t *);
278 static void streams_bufcall_service(void);
279 static void streams_qbkgrnd_service(void);
280 static void streams_sqbkgrnd_service(void);
281 static syncq_t *new_syncq(void);
282 static void free_syncq(syncq_t *);
283 static void outer_insert(syncq_t *, syncq_t *);
284 static void outer_remove(syncq_t *, syncq_t *);
285 static void write_now(syncq_t *);
286 static void clr_qfull(queue_t *);
287 static void runbufcalls(void);
288 static void sqenable(syncq_t *);
289 static void sqfill_events(syncq_t *, queue_t *, mblk_t *, void (*)());
290 static void wait_q_syncq(queue_t *);
291 static void backenable_insertedq(queue_t *);
292 
293 static void queue_service(queue_t *);
294 static void stream_service(stdata_t *);
295 static void syncq_service(syncq_t *);
296 static void qwriter_outer_service(syncq_t *);
297 static void mblk_free(mblk_t *);
298 #ifdef DEBUG
299 static int qprocsareon(queue_t *);
300 #endif
301 
302 static void set_nfsrv_ptr(queue_t *, queue_t *, queue_t *, queue_t *);
303 static void reset_nfsrv_ptr(queue_t *, queue_t *);
304 void set_qfull(queue_t *);
305 
306 static void sq_run_events(syncq_t *);
307 static int propagate_syncq(queue_t *);
308 
309 static void	blocksq(syncq_t *, ushort_t, int);
310 static void	unblocksq(syncq_t *, ushort_t, int);
311 static int	dropsq(syncq_t *, uint16_t);
312 static void	emptysq(syncq_t *);
313 static sqlist_t *sqlist_alloc(struct stdata *, int);
314 static void	sqlist_free(sqlist_t *);
315 static sqlist_t	*sqlist_build(queue_t *, struct stdata *, boolean_t);
316 static void	sqlist_insert(sqlist_t *, syncq_t *);
317 static void	sqlist_insertall(sqlist_t *, queue_t *);
318 
319 static void	strsetuio(stdata_t *);
320 
321 struct kmem_cache *stream_head_cache;
322 struct kmem_cache *queue_cache;
323 struct kmem_cache *syncq_cache;
324 struct kmem_cache *qband_cache;
325 struct kmem_cache *linkinfo_cache;
326 struct kmem_cache *ciputctrl_cache = NULL;
327 
328 static linkinfo_t *linkinfo_list;
329 
330 /* Global esballoc throttling queue */
331 static esb_queue_t system_esbq;
332 
333 /* Array of esballoc throttling queues, of length esbq_nelem */
334 static esb_queue_t *volatile system_esbq_array;
335 static int esbq_nelem;
336 static kmutex_t esbq_lock;
337 static int esbq_log2_cpus_per_q = 0;
338 
339 /* Scale the system_esbq length by setting number of CPUs per queue. */
340 uint_t esbq_cpus_per_q = 1;
341 
342 /*
343  * esballoc tunable parameters.
344  */
345 int		esbq_max_qlen = 0x16;	/* throttled queue length */
346 clock_t		esbq_timeout = 0x8;	/* timeout to process esb queue */
347 
348 /*
349  * Routines to handle esballoc queueing.
350  */
351 static void esballoc_process_queue(esb_queue_t *);
352 static void esballoc_enqueue_mblk(mblk_t *);
353 static void esballoc_timer(void *);
354 static void esballoc_set_timer(esb_queue_t *, clock_t);
355 static void esballoc_mblk_free(mblk_t *);
356 
357 /*
358  *  Qinit structure and Module_info structures
359  *	for passthru read and write queues
360  */
361 
362 static void pass_wput(queue_t *, mblk_t *);
363 static queue_t *link_addpassthru(stdata_t *);
364 static void link_rempassthru(queue_t *);
365 
366 struct  module_info passthru_info = {
367 	0,
368 	"passthru",
369 	0,
370 	INFPSZ,
371 	STRHIGH,
372 	STRLOW
373 };
374 
375 struct  qinit passthru_rinit = {
376 	(int (*)())putnext,
377 	NULL,
378 	NULL,
379 	NULL,
380 	NULL,
381 	&passthru_info,
382 	NULL
383 };
384 
385 struct  qinit passthru_winit = {
386 	(int (*)()) pass_wput,
387 	NULL,
388 	NULL,
389 	NULL,
390 	NULL,
391 	&passthru_info,
392 	NULL
393 };
394 
395 /*
396  * Verify correctness of list head/tail pointers.
397  */
398 #define	LISTCHECK(head, tail, link) {				\
399 	EQUIV(head, tail);					\
400 	IMPLY(tail != NULL, tail->link == NULL);		\
401 }
402 
403 /*
404  * Enqueue a list element `el' in the end of a list denoted by `head' and `tail'
405  * using a `link' field.
406  */
407 #define	ENQUEUE(el, head, tail, link) {				\
408 	ASSERT(el->link == NULL);				\
409 	LISTCHECK(head, tail, link);				\
410 	if (head == NULL)					\
411 		head = el;					\
412 	else							\
413 		tail->link = el;				\
414 	tail = el;						\
415 }
416 
417 /*
418  * Dequeue the first element of the list denoted by `head' and `tail' pointers
419  * using a `link' field and put result into `el'.
420  */
421 #define	DQ(el, head, tail, link) {				\
422 	LISTCHECK(head, tail, link);				\
423 	el = head;						\
424 	if (head != NULL) {					\
425 		head = head->link;				\
426 		if (head == NULL)				\
427 			tail = NULL;				\
428 		el->link = NULL;				\
429 	}							\
430 }
431 
432 /*
433  * Remove `el' from the list using `chase' and `curr' pointers and return result
434  * in `succeed'.
435  */
436 #define	RMQ(el, head, tail, link, chase, curr, succeed) {	\
437 	LISTCHECK(head, tail, link);				\
438 	chase = NULL;						\
439 	succeed = 0;						\
440 	for (curr = head; (curr != el) && (curr != NULL); curr = curr->link) \
441 		chase = curr;					\
442 	if (curr != NULL) {					\
443 		succeed = 1;					\
444 		ASSERT(curr == el);				\
445 		if (chase != NULL)				\
446 			chase->link = curr->link;		\
447 		else						\
448 			head = curr->link;			\
449 		curr->link = NULL;				\
450 		if (curr == tail)				\
451 			tail = chase;				\
452 	}							\
453 	LISTCHECK(head, tail, link);				\
454 }
455 
456 /* Handling of delayed messages on the inner syncq. */
457 
458 /*
459  * DEBUG versions should use function versions (to simplify tracing) and
460  * non-DEBUG kernels should use macro versions.
461  */
462 
463 /*
464  * Put a queue on the syncq list of queues.
465  * Assumes SQLOCK held.
466  */
467 #define	SQPUT_Q(sq, qp)							\
468 {									\
469 	ASSERT(MUTEX_HELD(SQLOCK(sq)));					\
470 	if (!(qp->q_sqflags & Q_SQQUEUED)) {				\
471 		/* The queue should not be linked anywhere */		\
472 		ASSERT((qp->q_sqprev == NULL) && (qp->q_sqnext == NULL)); \
473 		/* Head and tail may only be NULL simultaneously */	\
474 		EQUIV(sq->sq_head, sq->sq_tail);			\
475 		/* Queue may be only enqueued on its syncq */		\
476 		ASSERT(sq == qp->q_syncq);				\
477 		/* Check the correctness of SQ_MESSAGES flag */		\
478 		EQUIV(sq->sq_head, (sq->sq_flags & SQ_MESSAGES));	\
479 		/* Sanity check first/last elements of the list */	\
480 		IMPLY(sq->sq_head != NULL, sq->sq_head->q_sqprev == NULL);\
481 		IMPLY(sq->sq_tail != NULL, sq->sq_tail->q_sqnext == NULL);\
482 		/*							\
483 		 * Sanity check of priority field: empty queue should	\
484 		 * have zero priority					\
485 		 * and nqueues equal to zero.				\
486 		 */							\
487 		IMPLY(sq->sq_head == NULL, sq->sq_pri == 0);		\
488 		/* Sanity check of sq_nqueues field */			\
489 		EQUIV(sq->sq_head, sq->sq_nqueues);			\
490 		if (sq->sq_head == NULL) {				\
491 			sq->sq_head = sq->sq_tail = qp;			\
492 			sq->sq_flags |= SQ_MESSAGES;			\
493 		} else if (qp->q_spri == 0) {				\
494 			qp->q_sqprev = sq->sq_tail;			\
495 			sq->sq_tail->q_sqnext = qp;			\
496 			sq->sq_tail = qp;				\
497 		} else {						\
498 			/*						\
499 			 * Put this queue in priority order: higher	\
500 			 * priority gets closer to the head.		\
501 			 */						\
502 			queue_t **qpp = &sq->sq_tail;			\
503 			queue_t *qnext = NULL;				\
504 									\
505 			while (*qpp != NULL && qp->q_spri > (*qpp)->q_spri) { \
506 				qnext = *qpp;				\
507 				qpp = &(*qpp)->q_sqprev;		\
508 			}						\
509 			qp->q_sqnext = qnext;				\
510 			qp->q_sqprev = *qpp;				\
511 			if (*qpp != NULL) {				\
512 				(*qpp)->q_sqnext = qp;			\
513 			} else {					\
514 				sq->sq_head = qp;			\
515 				sq->sq_pri = sq->sq_head->q_spri;	\
516 			}						\
517 			*qpp = qp;					\
518 		}							\
519 		qp->q_sqflags |= Q_SQQUEUED;				\
520 		qp->q_sqtstamp = ddi_get_lbolt();			\
521 		sq->sq_nqueues++;					\
522 	}								\
523 }
524 
525 /*
526  * Remove a queue from the syncq list
527  * Assumes SQLOCK held.
528  */
529 #define	SQRM_Q(sq, qp)							\
530 	{								\
531 		ASSERT(MUTEX_HELD(SQLOCK(sq)));				\
532 		ASSERT(qp->q_sqflags & Q_SQQUEUED);			\
533 		ASSERT(sq->sq_head != NULL && sq->sq_tail != NULL);	\
534 		ASSERT((sq->sq_flags & SQ_MESSAGES) != 0);		\
535 		/* Check that the queue is actually in the list */	\
536 		ASSERT(qp->q_sqnext != NULL || sq->sq_tail == qp);	\
537 		ASSERT(qp->q_sqprev != NULL || sq->sq_head == qp);	\
538 		ASSERT(sq->sq_nqueues != 0);				\
539 		if (qp->q_sqprev == NULL) {				\
540 			/* First queue on list, make head q_sqnext */	\
541 			sq->sq_head = qp->q_sqnext;			\
542 		} else {						\
543 			/* Make prev->next == next */			\
544 			qp->q_sqprev->q_sqnext = qp->q_sqnext;		\
545 		}							\
546 		if (qp->q_sqnext == NULL) {				\
547 			/* Last queue on list, make tail sqprev */	\
548 			sq->sq_tail = qp->q_sqprev;			\
549 		} else {						\
550 			/* Make next->prev == prev */			\
551 			qp->q_sqnext->q_sqprev = qp->q_sqprev;		\
552 		}							\
553 		/* clear out references on this queue */		\
554 		qp->q_sqprev = qp->q_sqnext = NULL;			\
555 		qp->q_sqflags &= ~Q_SQQUEUED;				\
556 		/* If there is nothing queued, clear SQ_MESSAGES */	\
557 		if (sq->sq_head != NULL) {				\
558 			sq->sq_pri = sq->sq_head->q_spri;		\
559 		} else	{						\
560 			sq->sq_flags &= ~SQ_MESSAGES;			\
561 			sq->sq_pri = 0;					\
562 		}							\
563 		sq->sq_nqueues--;					\
564 		ASSERT(sq->sq_head != NULL || sq->sq_evhead != NULL ||	\
565 		    (sq->sq_flags & SQ_QUEUED) == 0);			\
566 	}
567 
568 /* Hide the definition from the header file. */
569 #ifdef SQPUT_MP
570 #undef SQPUT_MP
571 #endif
572 
573 /*
574  * Put a message on the queue syncq.
575  * Assumes QLOCK held.
576  */
577 #define	SQPUT_MP(qp, mp)						\
578 	{								\
579 		ASSERT(MUTEX_HELD(QLOCK(qp)));				\
580 		ASSERT(qp->q_sqhead == NULL ||				\
581 		    (qp->q_sqtail != NULL &&				\
582 		    qp->q_sqtail->b_next == NULL));			\
583 		qp->q_syncqmsgs++;					\
584 		ASSERT(qp->q_syncqmsgs != 0);	/* Wraparound */	\
585 		if (qp->q_sqhead == NULL) {				\
586 			qp->q_sqhead = qp->q_sqtail = mp;		\
587 		} else {						\
588 			qp->q_sqtail->b_next = mp;			\
589 			qp->q_sqtail = mp;				\
590 		}							\
591 		ASSERT(qp->q_syncqmsgs > 0);				\
592 		set_qfull(qp);						\
593 	}
594 
595 #define	SQ_PUTCOUNT_SETFAST_LOCKED(sq) {				\
596 		ASSERT(MUTEX_HELD(SQLOCK(sq)));				\
597 		if ((sq)->sq_ciputctrl != NULL) {			\
598 			int i;						\
599 			int nlocks = (sq)->sq_nciputctrl;		\
600 			ciputctrl_t *cip = (sq)->sq_ciputctrl;		\
601 			ASSERT((sq)->sq_type & SQ_CIPUT);		\
602 			for (i = 0; i <= nlocks; i++) {			\
603 				ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
604 				cip[i].ciputctrl_count |= SQ_FASTPUT;	\
605 			}						\
606 		}							\
607 	}
608 
609 
610 #define	SQ_PUTCOUNT_CLRFAST_LOCKED(sq) {				\
611 		ASSERT(MUTEX_HELD(SQLOCK(sq)));				\
612 		if ((sq)->sq_ciputctrl != NULL) {			\
613 			int i;						\
614 			int nlocks = (sq)->sq_nciputctrl;		\
615 			ciputctrl_t *cip = (sq)->sq_ciputctrl;		\
616 			ASSERT((sq)->sq_type & SQ_CIPUT);		\
617 			for (i = 0; i <= nlocks; i++) {			\
618 				ASSERT(MUTEX_HELD(&cip[i].ciputctrl_lock)); \
619 				cip[i].ciputctrl_count &= ~SQ_FASTPUT;	\
620 			}						\
621 		}							\
622 	}
623 
624 /*
625  * Run service procedures for all queues in the stream head.
626  */
627 #define	STR_SERVICE(stp, q) {						\
628 	ASSERT(MUTEX_HELD(&stp->sd_qlock));				\
629 	while (stp->sd_qhead != NULL) {					\
630 		DQ(q, stp->sd_qhead, stp->sd_qtail, q_link);		\
631 		ASSERT(stp->sd_nqueues > 0);				\
632 		stp->sd_nqueues--;					\
633 		ASSERT(!(q->q_flag & QINSERVICE));			\
634 		mutex_exit(&stp->sd_qlock);				\
635 		queue_service(q);					\
636 		mutex_enter(&stp->sd_qlock);				\
637 	}								\
638 	ASSERT(stp->sd_nqueues == 0);					\
639 	ASSERT((stp->sd_qhead == NULL) && (stp->sd_qtail == NULL));	\
640 }
641 
642 /*
643  * Constructor/destructor routines for the stream head cache
644  */
645 /* ARGSUSED */
646 static int
stream_head_constructor(void * buf,void * cdrarg,int kmflags)647 stream_head_constructor(void *buf, void *cdrarg, int kmflags)
648 {
649 	stdata_t *stp = buf;
650 
651 	mutex_init(&stp->sd_lock, NULL, MUTEX_DEFAULT, NULL);
652 	mutex_init(&stp->sd_reflock, NULL, MUTEX_DEFAULT, NULL);
653 	mutex_init(&stp->sd_qlock, NULL, MUTEX_DEFAULT, NULL);
654 	cv_init(&stp->sd_monitor, NULL, CV_DEFAULT, NULL);
655 	cv_init(&stp->sd_iocmonitor, NULL, CV_DEFAULT, NULL);
656 	cv_init(&stp->sd_refmonitor, NULL, CV_DEFAULT, NULL);
657 	cv_init(&stp->sd_qcv, NULL, CV_DEFAULT, NULL);
658 	cv_init(&stp->sd_zcopy_wait, NULL, CV_DEFAULT, NULL);
659 	stp->sd_wrq = NULL;
660 
661 	return (0);
662 }
663 
664 /* ARGSUSED */
665 static void
stream_head_destructor(void * buf,void * cdrarg)666 stream_head_destructor(void *buf, void *cdrarg)
667 {
668 	stdata_t *stp = buf;
669 
670 	mutex_destroy(&stp->sd_lock);
671 	mutex_destroy(&stp->sd_reflock);
672 	mutex_destroy(&stp->sd_qlock);
673 	cv_destroy(&stp->sd_monitor);
674 	cv_destroy(&stp->sd_iocmonitor);
675 	cv_destroy(&stp->sd_refmonitor);
676 	cv_destroy(&stp->sd_qcv);
677 	cv_destroy(&stp->sd_zcopy_wait);
678 }
679 
680 /*
681  * Constructor/destructor routines for the queue cache
682  */
683 /* ARGSUSED */
684 static int
queue_constructor(void * buf,void * cdrarg,int kmflags)685 queue_constructor(void *buf, void *cdrarg, int kmflags)
686 {
687 	queinfo_t *qip = buf;
688 	queue_t *qp = &qip->qu_rqueue;
689 	queue_t *wqp = &qip->qu_wqueue;
690 	syncq_t	*sq = &qip->qu_syncq;
691 
692 	qp->q_first = NULL;
693 	qp->q_link = NULL;
694 	qp->q_count = 0;
695 	qp->q_mblkcnt = 0;
696 	qp->q_sqhead = NULL;
697 	qp->q_sqtail = NULL;
698 	qp->q_sqnext = NULL;
699 	qp->q_sqprev = NULL;
700 	qp->q_sqflags = 0;
701 	qp->q_rwcnt = 0;
702 	qp->q_spri = 0;
703 
704 	mutex_init(QLOCK(qp), NULL, MUTEX_DEFAULT, NULL);
705 	cv_init(&qp->q_wait, NULL, CV_DEFAULT, NULL);
706 
707 	wqp->q_first = NULL;
708 	wqp->q_link = NULL;
709 	wqp->q_count = 0;
710 	wqp->q_mblkcnt = 0;
711 	wqp->q_sqhead = NULL;
712 	wqp->q_sqtail = NULL;
713 	wqp->q_sqnext = NULL;
714 	wqp->q_sqprev = NULL;
715 	wqp->q_sqflags = 0;
716 	wqp->q_rwcnt = 0;
717 	wqp->q_spri = 0;
718 
719 	mutex_init(QLOCK(wqp), NULL, MUTEX_DEFAULT, NULL);
720 	cv_init(&wqp->q_wait, NULL, CV_DEFAULT, NULL);
721 
722 	sq->sq_head = NULL;
723 	sq->sq_tail = NULL;
724 	sq->sq_evhead = NULL;
725 	sq->sq_evtail = NULL;
726 	sq->sq_callbpend = NULL;
727 	sq->sq_outer = NULL;
728 	sq->sq_onext = NULL;
729 	sq->sq_oprev = NULL;
730 	sq->sq_next = NULL;
731 	sq->sq_svcflags = 0;
732 	sq->sq_servcount = 0;
733 	sq->sq_needexcl = 0;
734 	sq->sq_nqueues = 0;
735 	sq->sq_pri = 0;
736 
737 	mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
738 	cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
739 	cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
740 
741 	return (0);
742 }
743 
744 /* ARGSUSED */
745 static void
queue_destructor(void * buf,void * cdrarg)746 queue_destructor(void *buf, void *cdrarg)
747 {
748 	queinfo_t *qip = buf;
749 	queue_t *qp = &qip->qu_rqueue;
750 	queue_t *wqp = &qip->qu_wqueue;
751 	syncq_t	*sq = &qip->qu_syncq;
752 
753 	ASSERT(qp->q_sqhead == NULL);
754 	ASSERT(wqp->q_sqhead == NULL);
755 	ASSERT(qp->q_sqnext == NULL);
756 	ASSERT(wqp->q_sqnext == NULL);
757 	ASSERT(qp->q_rwcnt == 0);
758 	ASSERT(wqp->q_rwcnt == 0);
759 
760 	mutex_destroy(&qp->q_lock);
761 	cv_destroy(&qp->q_wait);
762 
763 	mutex_destroy(&wqp->q_lock);
764 	cv_destroy(&wqp->q_wait);
765 
766 	mutex_destroy(&sq->sq_lock);
767 	cv_destroy(&sq->sq_wait);
768 	cv_destroy(&sq->sq_exitwait);
769 }
770 
771 /*
772  * Constructor/destructor routines for the syncq cache
773  */
774 /* ARGSUSED */
775 static int
syncq_constructor(void * buf,void * cdrarg,int kmflags)776 syncq_constructor(void *buf, void *cdrarg, int kmflags)
777 {
778 	syncq_t	*sq = buf;
779 
780 	bzero(buf, sizeof (syncq_t));
781 
782 	mutex_init(&sq->sq_lock, NULL, MUTEX_DEFAULT, NULL);
783 	cv_init(&sq->sq_wait, NULL, CV_DEFAULT, NULL);
784 	cv_init(&sq->sq_exitwait, NULL, CV_DEFAULT, NULL);
785 
786 	return (0);
787 }
788 
789 /* ARGSUSED */
790 static void
syncq_destructor(void * buf,void * cdrarg)791 syncq_destructor(void *buf, void *cdrarg)
792 {
793 	syncq_t	*sq = buf;
794 
795 	ASSERT(sq->sq_head == NULL);
796 	ASSERT(sq->sq_tail == NULL);
797 	ASSERT(sq->sq_evhead == NULL);
798 	ASSERT(sq->sq_evtail == NULL);
799 	ASSERT(sq->sq_callbpend == NULL);
800 	ASSERT(sq->sq_callbflags == 0);
801 	ASSERT(sq->sq_outer == NULL);
802 	ASSERT(sq->sq_onext == NULL);
803 	ASSERT(sq->sq_oprev == NULL);
804 	ASSERT(sq->sq_next == NULL);
805 	ASSERT(sq->sq_needexcl == 0);
806 	ASSERT(sq->sq_svcflags == 0);
807 	ASSERT(sq->sq_servcount == 0);
808 	ASSERT(sq->sq_nqueues == 0);
809 	ASSERT(sq->sq_pri == 0);
810 	ASSERT(sq->sq_count == 0);
811 	ASSERT(sq->sq_rmqcount == 0);
812 	ASSERT(sq->sq_cancelid == 0);
813 	ASSERT(sq->sq_ciputctrl == NULL);
814 	ASSERT(sq->sq_nciputctrl == 0);
815 	ASSERT(sq->sq_type == 0);
816 	ASSERT(sq->sq_flags == 0);
817 
818 	mutex_destroy(&sq->sq_lock);
819 	cv_destroy(&sq->sq_wait);
820 	cv_destroy(&sq->sq_exitwait);
821 }
822 
823 /* ARGSUSED */
824 static int
ciputctrl_constructor(void * buf,void * cdrarg,int kmflags)825 ciputctrl_constructor(void *buf, void *cdrarg, int kmflags)
826 {
827 	ciputctrl_t *cip = buf;
828 	int i;
829 
830 	for (i = 0; i < n_ciputctrl; i++) {
831 		cip[i].ciputctrl_count = SQ_FASTPUT;
832 		mutex_init(&cip[i].ciputctrl_lock, NULL, MUTEX_DEFAULT, NULL);
833 	}
834 
835 	return (0);
836 }
837 
838 /* ARGSUSED */
839 static void
ciputctrl_destructor(void * buf,void * cdrarg)840 ciputctrl_destructor(void *buf, void *cdrarg)
841 {
842 	ciputctrl_t *cip = buf;
843 	int i;
844 
845 	for (i = 0; i < n_ciputctrl; i++) {
846 		ASSERT(cip[i].ciputctrl_count & SQ_FASTPUT);
847 		mutex_destroy(&cip[i].ciputctrl_lock);
848 	}
849 }
850 
851 /*
852  * Init routine run from main at boot time.
853  */
854 void
strinit(void)855 strinit(void)
856 {
857 	int ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus);
858 
859 	stream_head_cache = kmem_cache_create("stream_head_cache",
860 	    sizeof (stdata_t), 0,
861 	    stream_head_constructor, stream_head_destructor, NULL,
862 	    NULL, NULL, 0);
863 
864 	queue_cache = kmem_cache_create("queue_cache", sizeof (queinfo_t), 0,
865 	    queue_constructor, queue_destructor, NULL, NULL, NULL, 0);
866 
867 	syncq_cache = kmem_cache_create("syncq_cache", sizeof (syncq_t), 0,
868 	    syncq_constructor, syncq_destructor, NULL, NULL, NULL, 0);
869 
870 	qband_cache = kmem_cache_create("qband_cache",
871 	    sizeof (qband_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
872 
873 	linkinfo_cache = kmem_cache_create("linkinfo_cache",
874 	    sizeof (linkinfo_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
875 
876 	n_ciputctrl = ncpus;
877 	n_ciputctrl = 1 << highbit(n_ciputctrl - 1);
878 	ASSERT(n_ciputctrl >= 1);
879 	n_ciputctrl = MIN(n_ciputctrl, max_n_ciputctrl);
880 	if (n_ciputctrl >= min_n_ciputctrl) {
881 		ciputctrl_cache = kmem_cache_create("ciputctrl_cache",
882 		    sizeof (ciputctrl_t) * n_ciputctrl,
883 		    sizeof (ciputctrl_t), ciputctrl_constructor,
884 		    ciputctrl_destructor, NULL, NULL, NULL, 0);
885 	}
886 
887 	streams_taskq = system_taskq;
888 
889 	if (streams_taskq == NULL)
890 		panic("strinit: no memory for streams taskq!");
891 
892 	bc_bkgrnd_thread = thread_create(NULL, 0,
893 	    streams_bufcall_service, NULL, 0, &p0, TS_RUN, streams_lopri);
894 
895 	streams_qbkgrnd_thread = thread_create(NULL, 0,
896 	    streams_qbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
897 
898 	streams_sqbkgrnd_thread = thread_create(NULL, 0,
899 	    streams_sqbkgrnd_service, NULL, 0, &p0, TS_RUN, streams_lopri);
900 
901 	/*
902 	 * Create STREAMS kstats.
903 	 */
904 	str_kstat = kstat_create("streams", 0, "strstat",
905 	    "net", KSTAT_TYPE_NAMED,
906 	    sizeof (str_statistics) / sizeof (kstat_named_t),
907 	    KSTAT_FLAG_VIRTUAL);
908 
909 	if (str_kstat != NULL) {
910 		str_kstat->ks_data = &str_statistics;
911 		kstat_install(str_kstat);
912 	}
913 
914 	/*
915 	 * TPI support routine initialisation.
916 	 */
917 	tpi_init();
918 
919 	/*
920 	 * Handle to have autopush and persistent link information per
921 	 * zone.
922 	 * Note: uses shutdown hook instead of destroy hook so that the
923 	 * persistent links can be torn down before the destroy hooks
924 	 * in the TCP/IP stack are called.
925 	 */
926 	netstack_register(NS_STR, str_stack_init, str_stack_shutdown,
927 	    str_stack_fini);
928 }
929 
930 void
str_sendsig(vnode_t * vp,int event,uchar_t band,int error)931 str_sendsig(vnode_t *vp, int event, uchar_t band, int error)
932 {
933 	struct stdata *stp;
934 
935 	ASSERT(vp->v_stream);
936 	stp = vp->v_stream;
937 	/* Have to hold sd_lock to prevent siglist from changing */
938 	mutex_enter(&stp->sd_lock);
939 	if (stp->sd_sigflags & event)
940 		strsendsig(stp->sd_siglist, event, band, error);
941 	mutex_exit(&stp->sd_lock);
942 }
943 
944 /*
945  * Send the "sevent" set of signals to a process.
946  * This might send more than one signal if the process is registered
947  * for multiple events. The caller should pass in an sevent that only
948  * includes the events for which the process has registered.
949  */
950 static void
dosendsig(proc_t * proc,int events,int sevent,k_siginfo_t * info,uchar_t band,int error)951 dosendsig(proc_t *proc, int events, int sevent, k_siginfo_t *info,
952 	uchar_t band, int error)
953 {
954 	ASSERT(MUTEX_HELD(&proc->p_lock));
955 
956 	info->si_band = 0;
957 	info->si_errno = 0;
958 
959 	if (sevent & S_ERROR) {
960 		sevent &= ~S_ERROR;
961 		info->si_code = POLL_ERR;
962 		info->si_errno = error;
963 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
964 		    "strsendsig:proc %p info %p", proc, info);
965 		sigaddq(proc, NULL, info, KM_NOSLEEP);
966 		info->si_errno = 0;
967 	}
968 	if (sevent & S_HANGUP) {
969 		sevent &= ~S_HANGUP;
970 		info->si_code = POLL_HUP;
971 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
972 		    "strsendsig:proc %p info %p", proc, info);
973 		sigaddq(proc, NULL, info, KM_NOSLEEP);
974 	}
975 	if (sevent & S_HIPRI) {
976 		sevent &= ~S_HIPRI;
977 		info->si_code = POLL_PRI;
978 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
979 		    "strsendsig:proc %p info %p", proc, info);
980 		sigaddq(proc, NULL, info, KM_NOSLEEP);
981 	}
982 	if (sevent & S_RDBAND) {
983 		sevent &= ~S_RDBAND;
984 		if (events & S_BANDURG)
985 			sigtoproc(proc, NULL, SIGURG);
986 		else
987 			sigtoproc(proc, NULL, SIGPOLL);
988 	}
989 	if (sevent & S_WRBAND) {
990 		sevent &= ~S_WRBAND;
991 		sigtoproc(proc, NULL, SIGPOLL);
992 	}
993 	if (sevent & S_INPUT) {
994 		sevent &= ~S_INPUT;
995 		info->si_code = POLL_IN;
996 		info->si_band = band;
997 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
998 		    "strsendsig:proc %p info %p", proc, info);
999 		sigaddq(proc, NULL, info, KM_NOSLEEP);
1000 		info->si_band = 0;
1001 	}
1002 	if (sevent & S_OUTPUT) {
1003 		sevent &= ~S_OUTPUT;
1004 		info->si_code = POLL_OUT;
1005 		info->si_band = band;
1006 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1007 		    "strsendsig:proc %p info %p", proc, info);
1008 		sigaddq(proc, NULL, info, KM_NOSLEEP);
1009 		info->si_band = 0;
1010 	}
1011 	if (sevent & S_MSG) {
1012 		sevent &= ~S_MSG;
1013 		info->si_code = POLL_MSG;
1014 		info->si_band = band;
1015 		TRACE_2(TR_FAC_STREAMS_FR, TR_STRSENDSIG,
1016 		    "strsendsig:proc %p info %p", proc, info);
1017 		sigaddq(proc, NULL, info, KM_NOSLEEP);
1018 		info->si_band = 0;
1019 	}
1020 	if (sevent & S_RDNORM) {
1021 		sevent &= ~S_RDNORM;
1022 		sigtoproc(proc, NULL, SIGPOLL);
1023 	}
1024 	if (sevent != 0) {
1025 		panic("strsendsig: unknown event(s) %x", sevent);
1026 	}
1027 }
1028 
1029 /*
1030  * Send SIGPOLL/SIGURG signal to all processes and process groups
1031  * registered on the given signal list that want a signal for at
1032  * least one of the specified events.
1033  *
1034  * Must be called with exclusive access to siglist (caller holding sd_lock).
1035  *
1036  * strioctl(I_SETSIG/I_ESETSIG) will only change siglist when holding
1037  * sd_lock and the ioctl code maintains a PID_HOLD on the pid structure
1038  * while it is in the siglist.
1039  *
1040  * For performance reasons (MP scalability) the code drops pidlock
1041  * when sending signals to a single process.
1042  * When sending to a process group the code holds
1043  * pidlock to prevent the membership in the process group from changing
1044  * while walking the p_pglink list.
1045  */
1046 void
strsendsig(strsig_t * siglist,int event,uchar_t band,int error)1047 strsendsig(strsig_t *siglist, int event, uchar_t band, int error)
1048 {
1049 	strsig_t *ssp;
1050 	k_siginfo_t info;
1051 	struct pid *pidp;
1052 	proc_t  *proc;
1053 
1054 	info.si_signo = SIGPOLL;
1055 	info.si_errno = 0;
1056 	for (ssp = siglist; ssp; ssp = ssp->ss_next) {
1057 		int sevent;
1058 
1059 		sevent = ssp->ss_events & event;
1060 		if (sevent == 0)
1061 			continue;
1062 
1063 		if ((pidp = ssp->ss_pidp) == NULL) {
1064 			/* pid was released but still on event list */
1065 			continue;
1066 		}
1067 
1068 
1069 		if (ssp->ss_pid > 0) {
1070 			/*
1071 			 * XXX This unfortunately still generates
1072 			 * a signal when a fd is closed but
1073 			 * the proc is active.
1074 			 */
1075 			ASSERT(ssp->ss_pid == pidp->pid_id);
1076 
1077 			mutex_enter(&pidlock);
1078 			proc = prfind_zone(pidp->pid_id, ALL_ZONES);
1079 			if (proc == NULL) {
1080 				mutex_exit(&pidlock);
1081 				continue;
1082 			}
1083 			mutex_enter(&proc->p_lock);
1084 			mutex_exit(&pidlock);
1085 			dosendsig(proc, ssp->ss_events, sevent, &info,
1086 			    band, error);
1087 			mutex_exit(&proc->p_lock);
1088 		} else {
1089 			/*
1090 			 * Send to process group. Hold pidlock across
1091 			 * calls to dosendsig().
1092 			 */
1093 			pid_t pgrp = -ssp->ss_pid;
1094 
1095 			mutex_enter(&pidlock);
1096 			proc = pgfind_zone(pgrp, ALL_ZONES);
1097 			while (proc != NULL) {
1098 				mutex_enter(&proc->p_lock);
1099 				dosendsig(proc, ssp->ss_events, sevent,
1100 				    &info, band, error);
1101 				mutex_exit(&proc->p_lock);
1102 				proc = proc->p_pglink;
1103 			}
1104 			mutex_exit(&pidlock);
1105 		}
1106 	}
1107 }
1108 
1109 /*
1110  * Attach a stream device or module.
1111  * qp is a read queue; the new queue goes in so its next
1112  * read ptr is the argument, and the write queue corresponding
1113  * to the argument points to this queue. Return 0 on success,
1114  * or a non-zero errno on failure.
1115  */
1116 int
qattach(queue_t * qp,dev_t * devp,int oflag,cred_t * crp,fmodsw_impl_t * fp,boolean_t is_insert)1117 qattach(queue_t *qp, dev_t *devp, int oflag, cred_t *crp, fmodsw_impl_t *fp,
1118     boolean_t is_insert)
1119 {
1120 	major_t			major;
1121 	cdevsw_impl_t		*dp;
1122 	struct streamtab	*str;
1123 	queue_t			*rq;
1124 	queue_t			*wrq;
1125 	uint32_t		qflag;
1126 	uint32_t		sqtype;
1127 	perdm_t			*dmp;
1128 	int			error;
1129 	int			sflag;
1130 
1131 	rq = allocq();
1132 	wrq = _WR(rq);
1133 	STREAM(rq) = STREAM(wrq) = STREAM(qp);
1134 
1135 	if (fp != NULL) {
1136 		str = fp->f_str;
1137 		qflag = fp->f_qflag;
1138 		sqtype = fp->f_sqtype;
1139 		dmp = fp->f_dmp;
1140 		IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
1141 		sflag = MODOPEN;
1142 
1143 		/*
1144 		 * stash away a pointer to the module structure so we can
1145 		 * unref it in qdetach.
1146 		 */
1147 		rq->q_fp = fp;
1148 	} else {
1149 		ASSERT(!is_insert);
1150 
1151 		major = getmajor(*devp);
1152 		dp = &devimpl[major];
1153 
1154 		str = dp->d_str;
1155 		ASSERT(str == STREAMSTAB(major));
1156 
1157 		qflag = dp->d_qflag;
1158 		ASSERT(qflag & QISDRV);
1159 		sqtype = dp->d_sqtype;
1160 
1161 		/* create perdm_t if needed */
1162 		if (NEED_DM(dp->d_dmp, qflag))
1163 			dp->d_dmp = hold_dm(str, qflag, sqtype);
1164 
1165 		dmp = dp->d_dmp;
1166 		sflag = 0;
1167 	}
1168 
1169 	TRACE_2(TR_FAC_STREAMS_FR, TR_QATTACH_FLAGS,
1170 	    "qattach:qflag == %X(%X)", qflag, *devp);
1171 
1172 	/* setq might sleep in allocator - avoid holding locks. */
1173 	setq(rq, str->st_rdinit, str->st_wrinit, dmp, qflag, sqtype, B_FALSE);
1174 
1175 	/*
1176 	 * Before calling the module's open routine, set up the q_next
1177 	 * pointer for inserting a module in the middle of a stream.
1178 	 *
1179 	 * Note that we can always set _QINSERTING and set up q_next
1180 	 * pointer for both inserting and pushing a module.  Then there
1181 	 * is no need for the is_insert parameter.  In insertq(), called
1182 	 * by qprocson(), assume that q_next of the new module always points
1183 	 * to the correct queue and use it for insertion.  Everything should
1184 	 * work out fine.  But in the first release of _I_INSERT, we
1185 	 * distinguish between inserting and pushing to make sure that
1186 	 * pushing a module follows the same code path as before.
1187 	 */
1188 	if (is_insert) {
1189 		rq->q_flag |= _QINSERTING;
1190 		rq->q_next = qp;
1191 	}
1192 
1193 	/*
1194 	 * If there is an outer perimeter get exclusive access during
1195 	 * the open procedure.  Bump up the reference count on the queue.
1196 	 */
1197 	entersq(rq->q_syncq, SQ_OPENCLOSE);
1198 	error = (*rq->q_qinfo->qi_qopen)(rq, devp, oflag, sflag, crp);
1199 	if (error != 0)
1200 		goto failed;
1201 	leavesq(rq->q_syncq, SQ_OPENCLOSE);
1202 	ASSERT(qprocsareon(rq));
1203 	return (0);
1204 
1205 failed:
1206 	rq->q_flag &= ~_QINSERTING;
1207 	if (backq(wrq) != NULL && backq(wrq)->q_next == wrq)
1208 		qprocsoff(rq);
1209 	leavesq(rq->q_syncq, SQ_OPENCLOSE);
1210 	rq->q_next = wrq->q_next = NULL;
1211 	qdetach(rq, 0, 0, crp, B_FALSE);
1212 	return (error);
1213 }
1214 
1215 /*
1216  * Handle second open of stream. For modules, set the
1217  * last argument to MODOPEN and do not pass any open flags.
1218  * Ignore dummydev since this is not the first open.
1219  */
1220 int
qreopen(queue_t * qp,dev_t * devp,int flag,cred_t * crp)1221 qreopen(queue_t *qp, dev_t *devp, int flag, cred_t *crp)
1222 {
1223 	int	error;
1224 	dev_t dummydev;
1225 	queue_t *wqp = _WR(qp);
1226 
1227 	ASSERT(qp->q_flag & QREADR);
1228 	entersq(qp->q_syncq, SQ_OPENCLOSE);
1229 
1230 	dummydev = *devp;
1231 	if (error = ((*qp->q_qinfo->qi_qopen)(qp, &dummydev,
1232 	    (wqp->q_next ? 0 : flag), (wqp->q_next ? MODOPEN : 0), crp))) {
1233 		leavesq(qp->q_syncq, SQ_OPENCLOSE);
1234 		mutex_enter(&STREAM(qp)->sd_lock);
1235 		qp->q_stream->sd_flag |= STREOPENFAIL;
1236 		mutex_exit(&STREAM(qp)->sd_lock);
1237 		return (error);
1238 	}
1239 	leavesq(qp->q_syncq, SQ_OPENCLOSE);
1240 
1241 	/*
1242 	 * successful open should have done qprocson()
1243 	 */
1244 	ASSERT(qprocsareon(_RD(qp)));
1245 	return (0);
1246 }
1247 
1248 /*
1249  * Detach a stream module or device.
1250  * If clmode == 1 then the module or driver was opened and its
1251  * close routine must be called. If clmode == 0, the module
1252  * or driver was never opened or the open failed, and so its close
1253  * should not be called.
1254  */
1255 void
qdetach(queue_t * qp,int clmode,int flag,cred_t * crp,boolean_t is_remove)1256 qdetach(queue_t *qp, int clmode, int flag, cred_t *crp, boolean_t is_remove)
1257 {
1258 	queue_t *wqp = _WR(qp);
1259 	ASSERT(STREAM(qp)->sd_flag & (STRCLOSE|STWOPEN|STRPLUMB));
1260 
1261 	if (STREAM_NEEDSERVICE(STREAM(qp)))
1262 		stream_runservice(STREAM(qp));
1263 
1264 	if (clmode) {
1265 		/*
1266 		 * Make sure that all the messages on the write side syncq are
1267 		 * processed and nothing is left. Since we are closing, no new
1268 		 * messages may appear there.
1269 		 */
1270 		wait_q_syncq(wqp);
1271 
1272 		entersq(qp->q_syncq, SQ_OPENCLOSE);
1273 		if (is_remove) {
1274 			mutex_enter(QLOCK(qp));
1275 			qp->q_flag |= _QREMOVING;
1276 			mutex_exit(QLOCK(qp));
1277 		}
1278 		(*qp->q_qinfo->qi_qclose)(qp, flag, crp);
1279 		/*
1280 		 * Check that qprocsoff() was actually called.
1281 		 */
1282 		ASSERT((qp->q_flag & QWCLOSE) && (wqp->q_flag & QWCLOSE));
1283 
1284 		leavesq(qp->q_syncq, SQ_OPENCLOSE);
1285 	} else {
1286 		disable_svc(qp);
1287 	}
1288 
1289 	/*
1290 	 * Allow any threads blocked in entersq to proceed and discover
1291 	 * the QWCLOSE is set.
1292 	 * Note: This assumes that all users of entersq check QWCLOSE.
1293 	 * Currently runservice is the only entersq that can happen
1294 	 * after removeq has finished.
1295 	 * Removeq will have discarded all messages destined to the closing
1296 	 * pair of queues from the syncq.
1297 	 * NOTE: Calling a function inside an assert is unconventional.
1298 	 * However, it does not cause any problem since flush_syncq() does
1299 	 * not change any state except when it returns non-zero i.e.
1300 	 * when the assert will trigger.
1301 	 */
1302 	ASSERT(flush_syncq(qp->q_syncq, qp) == 0);
1303 	ASSERT(flush_syncq(wqp->q_syncq, wqp) == 0);
1304 	ASSERT((qp->q_flag & QPERMOD) ||
1305 	    ((qp->q_syncq->sq_head == NULL) &&
1306 	    (wqp->q_syncq->sq_head == NULL)));
1307 
1308 	/* release any fmodsw_impl_t structure held on behalf of the queue */
1309 	ASSERT(qp->q_fp != NULL || qp->q_flag & QISDRV);
1310 	if (qp->q_fp != NULL)
1311 		fmodsw_rele(qp->q_fp);
1312 
1313 	/* freeq removes us from the outer perimeter if any */
1314 	freeq(qp);
1315 }
1316 
1317 /* Prevent service procedures from being called */
1318 void
disable_svc(queue_t * qp)1319 disable_svc(queue_t *qp)
1320 {
1321 	queue_t *wqp = _WR(qp);
1322 
1323 	ASSERT(qp->q_flag & QREADR);
1324 	mutex_enter(QLOCK(qp));
1325 	qp->q_flag |= QWCLOSE;
1326 	mutex_exit(QLOCK(qp));
1327 	mutex_enter(QLOCK(wqp));
1328 	wqp->q_flag |= QWCLOSE;
1329 	mutex_exit(QLOCK(wqp));
1330 }
1331 
1332 /* Allow service procedures to be called again */
1333 void
enable_svc(queue_t * qp)1334 enable_svc(queue_t *qp)
1335 {
1336 	queue_t *wqp = _WR(qp);
1337 
1338 	ASSERT(qp->q_flag & QREADR);
1339 	mutex_enter(QLOCK(qp));
1340 	qp->q_flag &= ~QWCLOSE;
1341 	mutex_exit(QLOCK(qp));
1342 	mutex_enter(QLOCK(wqp));
1343 	wqp->q_flag &= ~QWCLOSE;
1344 	mutex_exit(QLOCK(wqp));
1345 }
1346 
1347 /*
1348  * Remove queue from qhead/qtail if it is enabled.
1349  * Only reset QENAB if the queue was removed from the runlist.
1350  * A queue goes through 3 stages:
1351  *	It is on the service list and QENAB is set.
1352  *	It is removed from the service list but QENAB is still set.
1353  *	QENAB gets changed to QINSERVICE.
1354  *	QINSERVICE is reset (when the service procedure is done)
1355  * Thus we can not reset QENAB unless we actually removed it from the service
1356  * queue.
1357  */
1358 void
remove_runlist(queue_t * qp)1359 remove_runlist(queue_t *qp)
1360 {
1361 	if (qp->q_flag & QENAB && qhead != NULL) {
1362 		queue_t *q_chase;
1363 		queue_t *q_curr;
1364 		int removed;
1365 
1366 		mutex_enter(&service_queue);
1367 		RMQ(qp, qhead, qtail, q_link, q_chase, q_curr, removed);
1368 		mutex_exit(&service_queue);
1369 		if (removed) {
1370 			STRSTAT(qremoved);
1371 			qp->q_flag &= ~QENAB;
1372 		}
1373 	}
1374 }
1375 
1376 
1377 /*
1378  * Wait for any pending service processing to complete.
1379  * The removal of queues from the runlist is not atomic with the
1380  * clearing of the QENABLED flag and setting the INSERVICE flag.
1381  * consequently it is possible for remove_runlist in strclose
1382  * to not find the queue on the runlist but for it to be QENABLED
1383  * and not yet INSERVICE -> hence wait_svc needs to check QENABLED
1384  * as well as INSERVICE.
1385  */
1386 void
wait_svc(queue_t * qp)1387 wait_svc(queue_t *qp)
1388 {
1389 	queue_t *wqp = _WR(qp);
1390 
1391 	ASSERT(qp->q_flag & QREADR);
1392 
1393 	/*
1394 	 * Try to remove queues from qhead/qtail list.
1395 	 */
1396 	if (qhead != NULL) {
1397 		remove_runlist(qp);
1398 		remove_runlist(wqp);
1399 	}
1400 	/*
1401 	 * Wait till the syncqs associated with the queue disappear from the
1402 	 * background processing list.
1403 	 * This only needs to be done for non-PERMOD perimeters since
1404 	 * for PERMOD perimeters the syncq may be shared and will only be freed
1405 	 * when the last module/driver is unloaded.
1406 	 * If for PERMOD perimeters queue was on the syncq list, removeq()
1407 	 * should call propagate_syncq() or drain_syncq() for it. Both of these
1408 	 * functions remove the queue from its syncq list, so sqthread will not
1409 	 * try to access the queue.
1410 	 */
1411 	if (!(qp->q_flag & QPERMOD)) {
1412 		syncq_t *rsq = qp->q_syncq;
1413 		syncq_t *wsq = wqp->q_syncq;
1414 
1415 		/*
1416 		 * Disable rsq and wsq and wait for any background processing of
1417 		 * syncq to complete.
1418 		 */
1419 		wait_sq_svc(rsq);
1420 		if (wsq != rsq)
1421 			wait_sq_svc(wsq);
1422 	}
1423 
1424 	mutex_enter(QLOCK(qp));
1425 	while (qp->q_flag & (QINSERVICE|QENAB))
1426 		cv_wait(&qp->q_wait, QLOCK(qp));
1427 	mutex_exit(QLOCK(qp));
1428 	mutex_enter(QLOCK(wqp));
1429 	while (wqp->q_flag & (QINSERVICE|QENAB))
1430 		cv_wait(&wqp->q_wait, QLOCK(wqp));
1431 	mutex_exit(QLOCK(wqp));
1432 }
1433 
1434 /*
1435  * Put ioctl data from userland buffer `arg' into the mblk chain `bp'.
1436  * `flag' must always contain either K_TO_K or U_TO_K; STR_NOSIG may
1437  * also be set, and is passed through to allocb_cred_wait().
1438  *
1439  * Returns errno on failure, zero on success.
1440  */
1441 int
putiocd(mblk_t * bp,char * arg,int flag,cred_t * cr)1442 putiocd(mblk_t *bp, char *arg, int flag, cred_t *cr)
1443 {
1444 	mblk_t *tmp;
1445 	ssize_t  count;
1446 	int error = 0;
1447 
1448 	ASSERT((flag & (U_TO_K | K_TO_K)) == U_TO_K ||
1449 	    (flag & (U_TO_K | K_TO_K)) == K_TO_K);
1450 
1451 	if (bp->b_datap->db_type == M_IOCTL) {
1452 		count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1453 	} else {
1454 		ASSERT(bp->b_datap->db_type == M_COPYIN);
1455 		count = ((struct copyreq *)bp->b_rptr)->cq_size;
1456 	}
1457 	/*
1458 	 * strdoioctl validates ioc_count, so if this assert fails it
1459 	 * cannot be due to user error.
1460 	 */
1461 	ASSERT(count >= 0);
1462 
1463 	if ((tmp = allocb_cred_wait(count, (flag & STR_NOSIG), &error, cr,
1464 	    curproc->p_pid)) == NULL) {
1465 		return (error);
1466 	}
1467 	error = strcopyin(arg, tmp->b_wptr, count, flag & (U_TO_K|K_TO_K));
1468 	if (error != 0) {
1469 		freeb(tmp);
1470 		return (error);
1471 	}
1472 	DB_CPID(tmp) = curproc->p_pid;
1473 	tmp->b_wptr += count;
1474 	bp->b_cont = tmp;
1475 
1476 	return (0);
1477 }
1478 
1479 /*
1480  * Copy ioctl data to user-land. Return non-zero errno on failure,
1481  * 0 for success.
1482  */
1483 int
getiocd(mblk_t * bp,char * arg,int copymode)1484 getiocd(mblk_t *bp, char *arg, int copymode)
1485 {
1486 	ssize_t count;
1487 	size_t  n;
1488 	int	error;
1489 
1490 	if (bp->b_datap->db_type == M_IOCACK)
1491 		count = ((struct iocblk *)bp->b_rptr)->ioc_count;
1492 	else {
1493 		ASSERT(bp->b_datap->db_type == M_COPYOUT);
1494 		count = ((struct copyreq *)bp->b_rptr)->cq_size;
1495 	}
1496 	ASSERT(count >= 0);
1497 
1498 	for (bp = bp->b_cont; bp && count;
1499 	    count -= n, bp = bp->b_cont, arg += n) {
1500 		n = MIN(count, bp->b_wptr - bp->b_rptr);
1501 		error = strcopyout(bp->b_rptr, arg, n, copymode);
1502 		if (error)
1503 			return (error);
1504 	}
1505 	ASSERT(count == 0);
1506 	return (0);
1507 }
1508 
1509 /*
1510  * Allocate a linkinfo entry given the write queue of the
1511  * bottom module of the top stream and the write queue of the
1512  * stream head of the bottom stream.
1513  */
1514 linkinfo_t *
alloclink(queue_t * qup,queue_t * qdown,file_t * fpdown)1515 alloclink(queue_t *qup, queue_t *qdown, file_t *fpdown)
1516 {
1517 	linkinfo_t *linkp;
1518 
1519 	linkp = kmem_cache_alloc(linkinfo_cache, KM_SLEEP);
1520 
1521 	linkp->li_lblk.l_qtop = qup;
1522 	linkp->li_lblk.l_qbot = qdown;
1523 	linkp->li_fpdown = fpdown;
1524 
1525 	mutex_enter(&strresources);
1526 	linkp->li_next = linkinfo_list;
1527 	linkp->li_prev = NULL;
1528 	if (linkp->li_next)
1529 		linkp->li_next->li_prev = linkp;
1530 	linkinfo_list = linkp;
1531 	linkp->li_lblk.l_index = ++lnk_id;
1532 	ASSERT(lnk_id != 0);	/* this should never wrap in practice */
1533 	mutex_exit(&strresources);
1534 
1535 	return (linkp);
1536 }
1537 
1538 /*
1539  * Free a linkinfo entry.
1540  */
1541 void
lbfree(linkinfo_t * linkp)1542 lbfree(linkinfo_t *linkp)
1543 {
1544 	mutex_enter(&strresources);
1545 	if (linkp->li_next)
1546 		linkp->li_next->li_prev = linkp->li_prev;
1547 	if (linkp->li_prev)
1548 		linkp->li_prev->li_next = linkp->li_next;
1549 	else
1550 		linkinfo_list = linkp->li_next;
1551 	mutex_exit(&strresources);
1552 
1553 	kmem_cache_free(linkinfo_cache, linkp);
1554 }
1555 
1556 /*
1557  * Check for a potential linking cycle.
1558  * Return 1 if a link will result in a cycle,
1559  * and 0 otherwise.
1560  */
1561 int
linkcycle(stdata_t * upstp,stdata_t * lostp,str_stack_t * ss)1562 linkcycle(stdata_t *upstp, stdata_t *lostp, str_stack_t *ss)
1563 {
1564 	struct mux_node *np;
1565 	struct mux_edge *ep;
1566 	int i;
1567 	major_t lomaj;
1568 	major_t upmaj;
1569 	/*
1570 	 * if the lower stream is a pipe/FIFO, return, since link
1571 	 * cycles can not happen on pipes/FIFOs
1572 	 */
1573 	if (lostp->sd_vnode->v_type == VFIFO)
1574 		return (0);
1575 
1576 	for (i = 0; i < ss->ss_devcnt; i++) {
1577 		np = &ss->ss_mux_nodes[i];
1578 		MUX_CLEAR(np);
1579 	}
1580 	lomaj = getmajor(lostp->sd_vnode->v_rdev);
1581 	upmaj = getmajor(upstp->sd_vnode->v_rdev);
1582 	np = &ss->ss_mux_nodes[lomaj];
1583 	for (;;) {
1584 		if (!MUX_DIDVISIT(np)) {
1585 			if (np->mn_imaj == upmaj)
1586 				return (1);
1587 			if (np->mn_outp == NULL) {
1588 				MUX_VISIT(np);
1589 				if (np->mn_originp == NULL)
1590 					return (0);
1591 				np = np->mn_originp;
1592 				continue;
1593 			}
1594 			MUX_VISIT(np);
1595 			np->mn_startp = np->mn_outp;
1596 		} else {
1597 			if (np->mn_startp == NULL) {
1598 				if (np->mn_originp == NULL)
1599 					return (0);
1600 				else {
1601 					np = np->mn_originp;
1602 					continue;
1603 				}
1604 			}
1605 			/*
1606 			 * If ep->me_nodep is a FIFO (me_nodep == NULL),
1607 			 * ignore the edge and move on. ep->me_nodep gets
1608 			 * set to NULL in mux_addedge() if it is a FIFO.
1609 			 *
1610 			 */
1611 			ep = np->mn_startp;
1612 			np->mn_startp = ep->me_nextp;
1613 			if (ep->me_nodep == NULL)
1614 				continue;
1615 			ep->me_nodep->mn_originp = np;
1616 			np = ep->me_nodep;
1617 		}
1618 	}
1619 }
1620 
1621 /*
1622  * Find linkinfo entry corresponding to the parameters.
1623  */
1624 linkinfo_t *
findlinks(stdata_t * stp,int index,int type,str_stack_t * ss)1625 findlinks(stdata_t *stp, int index, int type, str_stack_t *ss)
1626 {
1627 	linkinfo_t *linkp;
1628 	struct mux_edge *mep;
1629 	struct mux_node *mnp;
1630 	queue_t *qup;
1631 
1632 	mutex_enter(&strresources);
1633 	if ((type & LINKTYPEMASK) == LINKNORMAL) {
1634 		qup = getendq(stp->sd_wrq);
1635 		for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1636 			if ((qup == linkp->li_lblk.l_qtop) &&
1637 			    (!index || (index == linkp->li_lblk.l_index))) {
1638 				mutex_exit(&strresources);
1639 				return (linkp);
1640 			}
1641 		}
1642 	} else {
1643 		ASSERT((type & LINKTYPEMASK) == LINKPERSIST);
1644 		mnp = &ss->ss_mux_nodes[getmajor(stp->sd_vnode->v_rdev)];
1645 		mep = mnp->mn_outp;
1646 		while (mep) {
1647 			if ((index == 0) || (index == mep->me_muxid))
1648 				break;
1649 			mep = mep->me_nextp;
1650 		}
1651 		if (!mep) {
1652 			mutex_exit(&strresources);
1653 			return (NULL);
1654 		}
1655 		for (linkp = linkinfo_list; linkp; linkp = linkp->li_next) {
1656 			if ((!linkp->li_lblk.l_qtop) &&
1657 			    (mep->me_muxid == linkp->li_lblk.l_index)) {
1658 				mutex_exit(&strresources);
1659 				return (linkp);
1660 			}
1661 		}
1662 	}
1663 	mutex_exit(&strresources);
1664 	return (NULL);
1665 }
1666 
1667 /*
1668  * Given a queue ptr, follow the chain of q_next pointers until you reach the
1669  * last queue on the chain and return it.
1670  */
1671 queue_t *
getendq(queue_t * q)1672 getendq(queue_t *q)
1673 {
1674 	ASSERT(q != NULL);
1675 	while (_SAMESTR(q))
1676 		q = q->q_next;
1677 	return (q);
1678 }
1679 
1680 /*
1681  * Wait for the syncq count to drop to zero.
1682  * sq could be either outer or inner.
1683  */
1684 
1685 static void
wait_syncq(syncq_t * sq)1686 wait_syncq(syncq_t *sq)
1687 {
1688 	uint16_t count;
1689 
1690 	mutex_enter(SQLOCK(sq));
1691 	count = sq->sq_count;
1692 	SQ_PUTLOCKS_ENTER(sq);
1693 	SUM_SQ_PUTCOUNTS(sq, count);
1694 	while (count != 0) {
1695 		sq->sq_flags |= SQ_WANTWAKEUP;
1696 		SQ_PUTLOCKS_EXIT(sq);
1697 		cv_wait(&sq->sq_wait, SQLOCK(sq));
1698 		count = sq->sq_count;
1699 		SQ_PUTLOCKS_ENTER(sq);
1700 		SUM_SQ_PUTCOUNTS(sq, count);
1701 	}
1702 	SQ_PUTLOCKS_EXIT(sq);
1703 	mutex_exit(SQLOCK(sq));
1704 }
1705 
1706 /*
1707  * Wait while there are any messages for the queue in its syncq.
1708  */
1709 static void
wait_q_syncq(queue_t * q)1710 wait_q_syncq(queue_t *q)
1711 {
1712 	if ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1713 		syncq_t *sq = q->q_syncq;
1714 
1715 		mutex_enter(SQLOCK(sq));
1716 		while ((q->q_sqflags & Q_SQQUEUED) || (q->q_syncqmsgs > 0)) {
1717 			sq->sq_flags |= SQ_WANTWAKEUP;
1718 			cv_wait(&sq->sq_wait, SQLOCK(sq));
1719 		}
1720 		mutex_exit(SQLOCK(sq));
1721 	}
1722 }
1723 
1724 
1725 int
mlink_file(vnode_t * vp,int cmd,struct file * fpdown,cred_t * crp,int * rvalp,int lhlink)1726 mlink_file(vnode_t *vp, int cmd, struct file *fpdown, cred_t *crp, int *rvalp,
1727     int lhlink)
1728 {
1729 	struct stdata *stp;
1730 	struct strioctl strioc;
1731 	struct linkinfo *linkp;
1732 	struct stdata *stpdown;
1733 	struct streamtab *str;
1734 	queue_t *passq;
1735 	syncq_t *passyncq;
1736 	queue_t *rq;
1737 	cdevsw_impl_t *dp;
1738 	uint32_t qflag;
1739 	uint32_t sqtype;
1740 	perdm_t *dmp;
1741 	int error = 0;
1742 	netstack_t *ns;
1743 	str_stack_t *ss;
1744 
1745 	stp = vp->v_stream;
1746 	TRACE_1(TR_FAC_STREAMS_FR,
1747 	    TR_I_LINK, "I_LINK/I_PLINK:stp %p", stp);
1748 	/*
1749 	 * Test for invalid upper stream
1750 	 */
1751 	if (stp->sd_flag & STRHUP) {
1752 		return (ENXIO);
1753 	}
1754 	if (vp->v_type == VFIFO) {
1755 		return (EINVAL);
1756 	}
1757 	if (stp->sd_strtab == NULL) {
1758 		return (EINVAL);
1759 	}
1760 	if (!stp->sd_strtab->st_muxwinit) {
1761 		return (EINVAL);
1762 	}
1763 	if (fpdown == NULL) {
1764 		return (EBADF);
1765 	}
1766 	ns = netstack_find_by_cred(crp);
1767 	ASSERT(ns != NULL);
1768 	ss = ns->netstack_str;
1769 	ASSERT(ss != NULL);
1770 
1771 	if (getmajor(stp->sd_vnode->v_rdev) >= ss->ss_devcnt) {
1772 		netstack_rele(ss->ss_netstack);
1773 		return (EINVAL);
1774 	}
1775 	mutex_enter(&muxifier);
1776 	if (stp->sd_flag & STPLEX) {
1777 		mutex_exit(&muxifier);
1778 		netstack_rele(ss->ss_netstack);
1779 		return (ENXIO);
1780 	}
1781 
1782 	/*
1783 	 * Test for invalid lower stream.
1784 	 * The check for the v_type != VFIFO and having a major
1785 	 * number not >= devcnt is done to avoid problems with
1786 	 * adding mux_node entry past the end of mux_nodes[].
1787 	 * For FIFO's we don't add an entry so this isn't a
1788 	 * problem.
1789 	 */
1790 	if (((stpdown = fpdown->f_vnode->v_stream) == NULL) ||
1791 	    (stpdown == stp) || (stpdown->sd_flag &
1792 	    (STPLEX|STRHUP|STRDERR|STWRERR|IOCWAIT|STRPLUMB)) ||
1793 	    ((stpdown->sd_vnode->v_type != VFIFO) &&
1794 	    (getmajor(stpdown->sd_vnode->v_rdev) >= ss->ss_devcnt)) ||
1795 	    linkcycle(stp, stpdown, ss)) {
1796 		mutex_exit(&muxifier);
1797 		netstack_rele(ss->ss_netstack);
1798 		return (EINVAL);
1799 	}
1800 	TRACE_1(TR_FAC_STREAMS_FR,
1801 	    TR_STPDOWN, "stpdown:%p", stpdown);
1802 	rq = getendq(stp->sd_wrq);
1803 	if (cmd == I_PLINK)
1804 		rq = NULL;
1805 
1806 	linkp = alloclink(rq, stpdown->sd_wrq, fpdown);
1807 
1808 	strioc.ic_cmd = cmd;
1809 	strioc.ic_timout = INFTIM;
1810 	strioc.ic_len = sizeof (struct linkblk);
1811 	strioc.ic_dp = (char *)&linkp->li_lblk;
1812 
1813 	/*
1814 	 * STRPLUMB protects plumbing changes and should be set before
1815 	 * link_addpassthru()/link_rempassthru() are called, so it is set here
1816 	 * and cleared in the end of mlink when passthru queue is removed.
1817 	 * Setting of STRPLUMB prevents reopens of the stream while passthru
1818 	 * queue is in-place (it is not a proper module and doesn't have open
1819 	 * entry point).
1820 	 *
1821 	 * STPLEX prevents any threads from entering the stream from above. It
1822 	 * can't be set before the call to link_addpassthru() because putnext
1823 	 * from below may cause stream head I/O routines to be called and these
1824 	 * routines assert that STPLEX is not set. After link_addpassthru()
1825 	 * nothing may come from below since the pass queue syncq is blocked.
1826 	 * Note also that STPLEX should be cleared before the call to
1827 	 * link_rempassthru() since when messages start flowing to the stream
1828 	 * head (e.g. because of message propagation from the pass queue) stream
1829 	 * head I/O routines may be called with STPLEX flag set.
1830 	 *
1831 	 * When STPLEX is set, nothing may come into the stream from above and
1832 	 * it is safe to do a setq which will change stream head. So, the
1833 	 * correct sequence of actions is:
1834 	 *
1835 	 * 1) Set STRPLUMB
1836 	 * 2) Call link_addpassthru()
1837 	 * 3) Set STPLEX
1838 	 * 4) Call setq and update the stream state
1839 	 * 5) Clear STPLEX
1840 	 * 6) Call link_rempassthru()
1841 	 * 7) Clear STRPLUMB
1842 	 *
1843 	 * The same sequence applies to munlink() code.
1844 	 */
1845 	mutex_enter(&stpdown->sd_lock);
1846 	stpdown->sd_flag |= STRPLUMB;
1847 	mutex_exit(&stpdown->sd_lock);
1848 	/*
1849 	 * Add passthru queue below lower mux. This will block
1850 	 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
1851 	 */
1852 	passq = link_addpassthru(stpdown);
1853 
1854 	mutex_enter(&stpdown->sd_lock);
1855 	stpdown->sd_flag |= STPLEX;
1856 	mutex_exit(&stpdown->sd_lock);
1857 
1858 	rq = _RD(stpdown->sd_wrq);
1859 	/*
1860 	 * There may be messages in the streamhead's syncq due to messages
1861 	 * that arrived before link_addpassthru() was done. To avoid
1862 	 * background processing of the syncq happening simultaneous with
1863 	 * setq processing, we disable the streamhead syncq and wait until
1864 	 * existing background thread finishes working on it.
1865 	 */
1866 	wait_sq_svc(rq->q_syncq);
1867 	passyncq = passq->q_syncq;
1868 	if (!(passyncq->sq_flags & SQ_BLOCKED))
1869 		blocksq(passyncq, SQ_BLOCKED, 0);
1870 
1871 	ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
1872 	ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
1873 	rq->q_ptr = _WR(rq)->q_ptr = NULL;
1874 
1875 	/* setq might sleep in allocator - avoid holding locks. */
1876 	/* Note: we are holding muxifier here. */
1877 
1878 	str = stp->sd_strtab;
1879 	dp = &devimpl[getmajor(vp->v_rdev)];
1880 	ASSERT(dp->d_str == str);
1881 
1882 	qflag = dp->d_qflag;
1883 	sqtype = dp->d_sqtype;
1884 
1885 	/* create perdm_t if needed */
1886 	if (NEED_DM(dp->d_dmp, qflag))
1887 		dp->d_dmp = hold_dm(str, qflag, sqtype);
1888 
1889 	dmp = dp->d_dmp;
1890 
1891 	setq(rq, str->st_muxrinit, str->st_muxwinit, dmp, qflag, sqtype,
1892 	    B_TRUE);
1893 
1894 	/*
1895 	 * XXX Remove any "odd" messages from the queue.
1896 	 * Keep only M_DATA, M_PROTO, M_PCPROTO.
1897 	 */
1898 	error = strdoioctl(stp, &strioc, FNATIVE,
1899 	    K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
1900 	if (error != 0) {
1901 		lbfree(linkp);
1902 
1903 		if (!(passyncq->sq_flags & SQ_BLOCKED))
1904 			blocksq(passyncq, SQ_BLOCKED, 0);
1905 		/*
1906 		 * Restore the stream head queue and then remove
1907 		 * the passq. Turn off STPLEX before we turn on
1908 		 * the stream by removing the passq.
1909 		 */
1910 		rq->q_ptr = _WR(rq)->q_ptr = stpdown;
1911 		setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO,
1912 		    B_TRUE);
1913 
1914 		mutex_enter(&stpdown->sd_lock);
1915 		stpdown->sd_flag &= ~STPLEX;
1916 		mutex_exit(&stpdown->sd_lock);
1917 
1918 		link_rempassthru(passq);
1919 
1920 		mutex_enter(&stpdown->sd_lock);
1921 		stpdown->sd_flag &= ~STRPLUMB;
1922 		/* Wakeup anyone waiting for STRPLUMB to clear. */
1923 		cv_broadcast(&stpdown->sd_monitor);
1924 		mutex_exit(&stpdown->sd_lock);
1925 
1926 		mutex_exit(&muxifier);
1927 		netstack_rele(ss->ss_netstack);
1928 		return (error);
1929 	}
1930 	mutex_enter(&fpdown->f_tlock);
1931 	fpdown->f_count++;
1932 	mutex_exit(&fpdown->f_tlock);
1933 
1934 	/*
1935 	 * if we've made it here the linkage is all set up so we should also
1936 	 * set up the layered driver linkages
1937 	 */
1938 
1939 	ASSERT((cmd == I_LINK) || (cmd == I_PLINK));
1940 	if (cmd == I_LINK) {
1941 		ldi_mlink_fp(stp, fpdown, lhlink, LINKNORMAL);
1942 	} else {
1943 		ldi_mlink_fp(stp, fpdown, lhlink, LINKPERSIST);
1944 	}
1945 
1946 	link_rempassthru(passq);
1947 
1948 	mux_addedge(stp, stpdown, linkp->li_lblk.l_index, ss);
1949 
1950 	/*
1951 	 * Mark the upper stream as having dependent links
1952 	 * so that strclose can clean it up.
1953 	 */
1954 	if (cmd == I_LINK) {
1955 		mutex_enter(&stp->sd_lock);
1956 		stp->sd_flag |= STRHASLINKS;
1957 		mutex_exit(&stp->sd_lock);
1958 	}
1959 	/*
1960 	 * Wake up any other processes that may have been
1961 	 * waiting on the lower stream. These will all
1962 	 * error out.
1963 	 */
1964 	mutex_enter(&stpdown->sd_lock);
1965 	/* The passthru module is removed so we may release STRPLUMB */
1966 	stpdown->sd_flag &= ~STRPLUMB;
1967 	cv_broadcast(&rq->q_wait);
1968 	cv_broadcast(&_WR(rq)->q_wait);
1969 	cv_broadcast(&stpdown->sd_monitor);
1970 	mutex_exit(&stpdown->sd_lock);
1971 	mutex_exit(&muxifier);
1972 	*rvalp = linkp->li_lblk.l_index;
1973 	netstack_rele(ss->ss_netstack);
1974 	return (0);
1975 }
1976 
1977 int
mlink(vnode_t * vp,int cmd,int arg,cred_t * crp,int * rvalp,int lhlink)1978 mlink(vnode_t *vp, int cmd, int arg, cred_t *crp, int *rvalp, int lhlink)
1979 {
1980 	int		ret;
1981 	struct file	*fpdown;
1982 
1983 	fpdown = getf(arg);
1984 	ret = mlink_file(vp, cmd, fpdown, crp, rvalp, lhlink);
1985 	if (fpdown != NULL)
1986 		releasef(arg);
1987 	return (ret);
1988 }
1989 
1990 /*
1991  * Unlink a multiplexor link. Stp is the controlling stream for the
1992  * link, and linkp points to the link's entry in the linkinfo list.
1993  * The muxifier lock must be held on entry and is dropped on exit.
1994  *
1995  * NOTE : Currently it is assumed that mux would process all the messages
1996  * sitting on it's queue before ACKing the UNLINK. It is the responsibility
1997  * of the mux to handle all the messages that arrive before UNLINK.
1998  * If the mux has to send down messages on its lower stream before
1999  * ACKing I_UNLINK, then it *should* know to handle messages even
2000  * after the UNLINK is acked (actually it should be able to handle till we
2001  * re-block the read side of the pass queue here). If the mux does not
2002  * open up the lower stream, any messages that arrive during UNLINK
2003  * will be put in the stream head. In the case of lower stream opening
2004  * up, some messages might land in the stream head depending on when
2005  * the message arrived and when the read side of the pass queue was
2006  * re-blocked.
2007  */
2008 int
munlink(stdata_t * stp,linkinfo_t * linkp,int flag,cred_t * crp,int * rvalp,str_stack_t * ss)2009 munlink(stdata_t *stp, linkinfo_t *linkp, int flag, cred_t *crp, int *rvalp,
2010     str_stack_t *ss)
2011 {
2012 	struct strioctl strioc;
2013 	struct stdata *stpdown;
2014 	queue_t *rq, *wrq;
2015 	queue_t	*passq;
2016 	syncq_t *passyncq;
2017 	int error = 0;
2018 	file_t *fpdown;
2019 
2020 	ASSERT(MUTEX_HELD(&muxifier));
2021 
2022 	stpdown = linkp->li_fpdown->f_vnode->v_stream;
2023 
2024 	/*
2025 	 * See the comment in mlink() concerning STRPLUMB/STPLEX flags.
2026 	 */
2027 	mutex_enter(&stpdown->sd_lock);
2028 	stpdown->sd_flag |= STRPLUMB;
2029 	mutex_exit(&stpdown->sd_lock);
2030 
2031 	/*
2032 	 * Add passthru queue below lower mux. This will block
2033 	 * syncqs of lower muxs read queue during I_LINK/I_UNLINK.
2034 	 */
2035 	passq = link_addpassthru(stpdown);
2036 
2037 	if ((flag & LINKTYPEMASK) == LINKNORMAL)
2038 		strioc.ic_cmd = I_UNLINK;
2039 	else
2040 		strioc.ic_cmd = I_PUNLINK;
2041 	strioc.ic_timout = INFTIM;
2042 	strioc.ic_len = sizeof (struct linkblk);
2043 	strioc.ic_dp = (char *)&linkp->li_lblk;
2044 
2045 	error = strdoioctl(stp, &strioc, FNATIVE,
2046 	    K_TO_K | STR_NOERROR | STR_NOSIG, crp, rvalp);
2047 
2048 	/*
2049 	 * If there was an error and this is not called via strclose,
2050 	 * return to the user. Otherwise, pretend there was no error
2051 	 * and close the link.
2052 	 */
2053 	if (error) {
2054 		if (flag & LINKCLOSE) {
2055 			cmn_err(CE_WARN, "KERNEL: munlink: could not perform "
2056 			    "unlink ioctl, closing anyway (%d)\n", error);
2057 		} else {
2058 			link_rempassthru(passq);
2059 			mutex_enter(&stpdown->sd_lock);
2060 			stpdown->sd_flag &= ~STRPLUMB;
2061 			cv_broadcast(&stpdown->sd_monitor);
2062 			mutex_exit(&stpdown->sd_lock);
2063 			mutex_exit(&muxifier);
2064 			return (error);
2065 		}
2066 	}
2067 
2068 	mux_rmvedge(stp, linkp->li_lblk.l_index, ss);
2069 	fpdown = linkp->li_fpdown;
2070 	lbfree(linkp);
2071 
2072 	/*
2073 	 * We go ahead and drop muxifier here--it's a nasty global lock that
2074 	 * can slow others down. It's okay to since attempts to mlink() this
2075 	 * stream will be stopped because STPLEX is still set in the stdata
2076 	 * structure, and munlink() is stopped because mux_rmvedge() and
2077 	 * lbfree() have removed it from mux_nodes[] and linkinfo_list,
2078 	 * respectively.  Note that we defer the closef() of fpdown until
2079 	 * after we drop muxifier since strclose() can call munlinkall().
2080 	 */
2081 	mutex_exit(&muxifier);
2082 
2083 	wrq = stpdown->sd_wrq;
2084 	rq = _RD(wrq);
2085 
2086 	/*
2087 	 * Get rid of outstanding service procedure runs, before we make
2088 	 * it a stream head, since a stream head doesn't have any service
2089 	 * procedure.
2090 	 */
2091 	disable_svc(rq);
2092 	wait_svc(rq);
2093 
2094 	/*
2095 	 * Since we don't disable the syncq for QPERMOD, we wait for whatever
2096 	 * is queued up to be finished. mux should take care that nothing is
2097 	 * send down to this queue. We should do it now as we're going to block
2098 	 * passyncq if it was unblocked.
2099 	 */
2100 	if (wrq->q_flag & QPERMOD) {
2101 		syncq_t	*sq = wrq->q_syncq;
2102 
2103 		mutex_enter(SQLOCK(sq));
2104 		while (wrq->q_sqflags & Q_SQQUEUED) {
2105 			sq->sq_flags |= SQ_WANTWAKEUP;
2106 			cv_wait(&sq->sq_wait, SQLOCK(sq));
2107 		}
2108 		mutex_exit(SQLOCK(sq));
2109 	}
2110 	passyncq = passq->q_syncq;
2111 	if (!(passyncq->sq_flags & SQ_BLOCKED)) {
2112 
2113 		syncq_t *sq, *outer;
2114 
2115 		/*
2116 		 * Messages could be flowing from underneath. We will
2117 		 * block the read side of the passq. This would be
2118 		 * sufficient for QPAIR and QPERQ muxes to ensure
2119 		 * that no data is flowing up into this queue
2120 		 * and hence no thread active in this instance of
2121 		 * lower mux. But for QPERMOD and QMTOUTPERIM there
2122 		 * could be messages on the inner and outer/inner
2123 		 * syncqs respectively. We will wait for them to drain.
2124 		 * Because passq is blocked messages end up in the syncq
2125 		 * And qfill_syncq could possibly end up setting QFULL
2126 		 * which will access the rq->q_flag. Hence, we have to
2127 		 * acquire the QLOCK in setq.
2128 		 *
2129 		 * XXX Messages can also flow from top into this
2130 		 * queue though the unlink is over (Ex. some instance
2131 		 * in putnext() called from top that has still not
2132 		 * accessed this queue. And also putq(lowerq) ?).
2133 		 * Solution : How about blocking the l_qtop queue ?
2134 		 * Do we really care about such pure D_MP muxes ?
2135 		 */
2136 
2137 		blocksq(passyncq, SQ_BLOCKED, 0);
2138 
2139 		sq = rq->q_syncq;
2140 		if ((outer = sq->sq_outer) != NULL) {
2141 
2142 			/*
2143 			 * We have to just wait for the outer sq_count
2144 			 * drop to zero. As this does not prevent new
2145 			 * messages to enter the outer perimeter, this
2146 			 * is subject to starvation.
2147 			 *
2148 			 * NOTE :Because of blocksq above, messages could
2149 			 * be in the inner syncq only because of some
2150 			 * thread holding the outer perimeter exclusively.
2151 			 * Hence it would be sufficient to wait for the
2152 			 * exclusive holder of the outer perimeter to drain
2153 			 * the inner and outer syncqs. But we will not depend
2154 			 * on this feature and hence check the inner syncqs
2155 			 * separately.
2156 			 */
2157 			wait_syncq(outer);
2158 		}
2159 
2160 
2161 		/*
2162 		 * There could be messages destined for
2163 		 * this queue. Let the exclusive holder
2164 		 * drain it.
2165 		 */
2166 
2167 		wait_syncq(sq);
2168 		ASSERT((rq->q_flag & QPERMOD) ||
2169 		    ((rq->q_syncq->sq_head == NULL) &&
2170 		    (_WR(rq)->q_syncq->sq_head == NULL)));
2171 	}
2172 
2173 	/*
2174 	 * We haven't taken care of QPERMOD case yet. QPERMOD is a special
2175 	 * case as we don't disable its syncq or remove it off the syncq
2176 	 * service list.
2177 	 */
2178 	if (rq->q_flag & QPERMOD) {
2179 		syncq_t	*sq = rq->q_syncq;
2180 
2181 		mutex_enter(SQLOCK(sq));
2182 		while (rq->q_sqflags & Q_SQQUEUED) {
2183 			sq->sq_flags |= SQ_WANTWAKEUP;
2184 			cv_wait(&sq->sq_wait, SQLOCK(sq));
2185 		}
2186 		mutex_exit(SQLOCK(sq));
2187 	}
2188 
2189 	/*
2190 	 * flush_syncq changes states only when there are some messages to
2191 	 * free, i.e. when it returns non-zero value to return.
2192 	 */
2193 	ASSERT(flush_syncq(rq->q_syncq, rq) == 0);
2194 	ASSERT(flush_syncq(wrq->q_syncq, wrq) == 0);
2195 
2196 	/*
2197 	 * Nobody else should know about this queue now.
2198 	 * If the mux did not process the messages before
2199 	 * acking the I_UNLINK, free them now.
2200 	 */
2201 
2202 	flushq(rq, FLUSHALL);
2203 	flushq(_WR(rq), FLUSHALL);
2204 
2205 	/*
2206 	 * Convert the mux lower queue into a stream head queue.
2207 	 * Turn off STPLEX before we turn on the stream by removing the passq.
2208 	 */
2209 	rq->q_ptr = wrq->q_ptr = stpdown;
2210 	setq(rq, &strdata, &stwdata, NULL, QMTSAFE, SQ_CI|SQ_CO, B_TRUE);
2211 
2212 	ASSERT((rq->q_flag & QMT_TYPEMASK) == QMTSAFE);
2213 	ASSERT(rq->q_syncq == SQ(rq) && _WR(rq)->q_syncq == SQ(rq));
2214 
2215 	enable_svc(rq);
2216 
2217 	/*
2218 	 * Now it is a proper stream, so STPLEX is cleared. But STRPLUMB still
2219 	 * needs to be set to prevent reopen() of the stream - such reopen may
2220 	 * try to call non-existent pass queue open routine and panic.
2221 	 */
2222 	mutex_enter(&stpdown->sd_lock);
2223 	stpdown->sd_flag &= ~STPLEX;
2224 	mutex_exit(&stpdown->sd_lock);
2225 
2226 	ASSERT(((flag & LINKTYPEMASK) == LINKNORMAL) ||
2227 	    ((flag & LINKTYPEMASK) == LINKPERSIST));
2228 
2229 	/* clean up the layered driver linkages */
2230 	if ((flag & LINKTYPEMASK) == LINKNORMAL) {
2231 		ldi_munlink_fp(stp, fpdown, LINKNORMAL);
2232 	} else {
2233 		ldi_munlink_fp(stp, fpdown, LINKPERSIST);
2234 	}
2235 
2236 	link_rempassthru(passq);
2237 
2238 	/*
2239 	 * Now all plumbing changes are finished and STRPLUMB is no
2240 	 * longer needed.
2241 	 */
2242 	mutex_enter(&stpdown->sd_lock);
2243 	stpdown->sd_flag &= ~STRPLUMB;
2244 	cv_broadcast(&stpdown->sd_monitor);
2245 	mutex_exit(&stpdown->sd_lock);
2246 
2247 	(void) closef(fpdown);
2248 	return (0);
2249 }
2250 
2251 /*
2252  * Unlink all multiplexor links for which stp is the controlling stream.
2253  * Return 0, or a non-zero errno on failure.
2254  */
2255 int
munlinkall(stdata_t * stp,int flag,cred_t * crp,int * rvalp,str_stack_t * ss)2256 munlinkall(stdata_t *stp, int flag, cred_t *crp, int *rvalp, str_stack_t *ss)
2257 {
2258 	linkinfo_t *linkp;
2259 	int error = 0;
2260 
2261 	mutex_enter(&muxifier);
2262 	while (linkp = findlinks(stp, 0, flag, ss)) {
2263 		/*
2264 		 * munlink() releases the muxifier lock.
2265 		 */
2266 		if (error = munlink(stp, linkp, flag, crp, rvalp, ss))
2267 			return (error);
2268 		mutex_enter(&muxifier);
2269 	}
2270 	mutex_exit(&muxifier);
2271 	return (0);
2272 }
2273 
2274 /*
2275  * A multiplexor link has been made. Add an
2276  * edge to the directed graph.
2277  */
2278 void
mux_addedge(stdata_t * upstp,stdata_t * lostp,int muxid,str_stack_t * ss)2279 mux_addedge(stdata_t *upstp, stdata_t *lostp, int muxid, str_stack_t *ss)
2280 {
2281 	struct mux_node *np;
2282 	struct mux_edge *ep;
2283 	major_t upmaj;
2284 	major_t lomaj;
2285 
2286 	upmaj = getmajor(upstp->sd_vnode->v_rdev);
2287 	lomaj = getmajor(lostp->sd_vnode->v_rdev);
2288 	np = &ss->ss_mux_nodes[upmaj];
2289 	if (np->mn_outp) {
2290 		ep = np->mn_outp;
2291 		while (ep->me_nextp)
2292 			ep = ep->me_nextp;
2293 		ep->me_nextp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2294 		ep = ep->me_nextp;
2295 	} else {
2296 		np->mn_outp = kmem_alloc(sizeof (struct mux_edge), KM_SLEEP);
2297 		ep = np->mn_outp;
2298 	}
2299 	ep->me_nextp = NULL;
2300 	ep->me_muxid = muxid;
2301 	/*
2302 	 * Save the dev_t for the purposes of str_stack_shutdown.
2303 	 * str_stack_shutdown assumes that the device allows reopen, since
2304 	 * this dev_t is the one after any cloning by xx_open().
2305 	 * Would prefer finding the dev_t from before any cloning,
2306 	 * but specfs doesn't retain that.
2307 	 */
2308 	ep->me_dev = upstp->sd_vnode->v_rdev;
2309 	if (lostp->sd_vnode->v_type == VFIFO)
2310 		ep->me_nodep = NULL;
2311 	else
2312 		ep->me_nodep = &ss->ss_mux_nodes[lomaj];
2313 }
2314 
2315 /*
2316  * A multiplexor link has been removed. Remove the
2317  * edge in the directed graph.
2318  */
2319 void
mux_rmvedge(stdata_t * upstp,int muxid,str_stack_t * ss)2320 mux_rmvedge(stdata_t *upstp, int muxid, str_stack_t *ss)
2321 {
2322 	struct mux_node *np;
2323 	struct mux_edge *ep;
2324 	struct mux_edge *pep = NULL;
2325 	major_t upmaj;
2326 
2327 	upmaj = getmajor(upstp->sd_vnode->v_rdev);
2328 	np = &ss->ss_mux_nodes[upmaj];
2329 	ASSERT(np->mn_outp != NULL);
2330 	ep = np->mn_outp;
2331 	while (ep) {
2332 		if (ep->me_muxid == muxid) {
2333 			if (pep)
2334 				pep->me_nextp = ep->me_nextp;
2335 			else
2336 				np->mn_outp = ep->me_nextp;
2337 			kmem_free(ep, sizeof (struct mux_edge));
2338 			return;
2339 		}
2340 		pep = ep;
2341 		ep = ep->me_nextp;
2342 	}
2343 	ASSERT(0);	/* should not reach here */
2344 }
2345 
2346 /*
2347  * Translate the device flags (from conf.h) to the corresponding
2348  * qflag and sq_flag (type) values.
2349  */
2350 int
devflg_to_qflag(struct streamtab * stp,uint32_t devflag,uint32_t * qflagp,uint32_t * sqtypep)2351 devflg_to_qflag(struct streamtab *stp, uint32_t devflag, uint32_t *qflagp,
2352 	uint32_t *sqtypep)
2353 {
2354 	uint32_t qflag = 0;
2355 	uint32_t sqtype = 0;
2356 
2357 	if (devflag & _D_OLD)
2358 		goto bad;
2359 
2360 	/* Inner perimeter presence and scope */
2361 	switch (devflag & D_MTINNER_MASK) {
2362 	case D_MP:
2363 		qflag |= QMTSAFE;
2364 		sqtype |= SQ_CI;
2365 		break;
2366 	case D_MTPERQ|D_MP:
2367 		qflag |= QPERQ;
2368 		break;
2369 	case D_MTQPAIR|D_MP:
2370 		qflag |= QPAIR;
2371 		break;
2372 	case D_MTPERMOD|D_MP:
2373 		qflag |= QPERMOD;
2374 		break;
2375 	default:
2376 		goto bad;
2377 	}
2378 
2379 	/* Outer perimeter */
2380 	if (devflag & D_MTOUTPERIM) {
2381 		switch (devflag & D_MTINNER_MASK) {
2382 		case D_MP:
2383 		case D_MTPERQ|D_MP:
2384 		case D_MTQPAIR|D_MP:
2385 			break;
2386 		default:
2387 			goto bad;
2388 		}
2389 		qflag |= QMTOUTPERIM;
2390 	}
2391 
2392 	/* Inner perimeter modifiers */
2393 	if (devflag & D_MTINNER_MOD) {
2394 		switch (devflag & D_MTINNER_MASK) {
2395 		case D_MP:
2396 			goto bad;
2397 		default:
2398 			break;
2399 		}
2400 		if (devflag & D_MTPUTSHARED)
2401 			sqtype |= SQ_CIPUT;
2402 		if (devflag & _D_MTOCSHARED) {
2403 			/*
2404 			 * The code in putnext assumes that it has the
2405 			 * highest concurrency by not checking sq_count.
2406 			 * Thus _D_MTOCSHARED can only be supported when
2407 			 * D_MTPUTSHARED is set.
2408 			 */
2409 			if (!(devflag & D_MTPUTSHARED))
2410 				goto bad;
2411 			sqtype |= SQ_CIOC;
2412 		}
2413 		if (devflag & _D_MTCBSHARED) {
2414 			/*
2415 			 * The code in putnext assumes that it has the
2416 			 * highest concurrency by not checking sq_count.
2417 			 * Thus _D_MTCBSHARED can only be supported when
2418 			 * D_MTPUTSHARED is set.
2419 			 */
2420 			if (!(devflag & D_MTPUTSHARED))
2421 				goto bad;
2422 			sqtype |= SQ_CICB;
2423 		}
2424 		if (devflag & _D_MTSVCSHARED) {
2425 			/*
2426 			 * The code in putnext assumes that it has the
2427 			 * highest concurrency by not checking sq_count.
2428 			 * Thus _D_MTSVCSHARED can only be supported when
2429 			 * D_MTPUTSHARED is set. Also _D_MTSVCSHARED is
2430 			 * supported only for QPERMOD.
2431 			 */
2432 			if (!(devflag & D_MTPUTSHARED) || !(qflag & QPERMOD))
2433 				goto bad;
2434 			sqtype |= SQ_CISVC;
2435 		}
2436 	}
2437 
2438 	/* Default outer perimeter concurrency */
2439 	sqtype |= SQ_CO;
2440 
2441 	/* Outer perimeter modifiers */
2442 	if (devflag & D_MTOCEXCL) {
2443 		if (!(devflag & D_MTOUTPERIM)) {
2444 			/* No outer perimeter */
2445 			goto bad;
2446 		}
2447 		sqtype &= ~SQ_COOC;
2448 	}
2449 
2450 	/* Synchronous Streams extended qinit structure */
2451 	if (devflag & D_SYNCSTR)
2452 		qflag |= QSYNCSTR;
2453 
2454 	/*
2455 	 * Private flag used by a transport module to indicate
2456 	 * to sockfs that it supports direct-access mode without
2457 	 * having to go through STREAMS.
2458 	 */
2459 	if (devflag & _D_DIRECT) {
2460 		/* Reject unless the module is fully-MT (no perimeter) */
2461 		if ((qflag & QMT_TYPEMASK) != QMTSAFE)
2462 			goto bad;
2463 		qflag |= _QDIRECT;
2464 	}
2465 
2466 	*qflagp = qflag;
2467 	*sqtypep = sqtype;
2468 	return (0);
2469 
2470 bad:
2471 	cmn_err(CE_WARN,
2472 	    "stropen: bad MT flags (0x%x) in driver '%s'",
2473 	    (int)(qflag & D_MTSAFETY_MASK),
2474 	    stp->st_rdinit->qi_minfo->mi_idname);
2475 
2476 	return (EINVAL);
2477 }
2478 
2479 /*
2480  * Set the interface values for a pair of queues (qinit structure,
2481  * packet sizes, water marks).
2482  * setq assumes that the caller does not have a claim (entersq or claimq)
2483  * on the queue.
2484  */
2485 void
setq(queue_t * rq,struct qinit * rinit,struct qinit * winit,perdm_t * dmp,uint32_t qflag,uint32_t sqtype,boolean_t lock_needed)2486 setq(queue_t *rq, struct qinit *rinit, struct qinit *winit,
2487     perdm_t *dmp, uint32_t qflag, uint32_t sqtype, boolean_t lock_needed)
2488 {
2489 	queue_t *wq;
2490 	syncq_t	*sq, *outer;
2491 
2492 	ASSERT(rq->q_flag & QREADR);
2493 	ASSERT((qflag & QMT_TYPEMASK) != 0);
2494 	IMPLY((qflag & (QPERMOD | QMTOUTPERIM)), dmp != NULL);
2495 
2496 	wq = _WR(rq);
2497 	rq->q_qinfo = rinit;
2498 	rq->q_hiwat = rinit->qi_minfo->mi_hiwat;
2499 	rq->q_lowat = rinit->qi_minfo->mi_lowat;
2500 	rq->q_minpsz = rinit->qi_minfo->mi_minpsz;
2501 	rq->q_maxpsz = rinit->qi_minfo->mi_maxpsz;
2502 	wq->q_qinfo = winit;
2503 	wq->q_hiwat = winit->qi_minfo->mi_hiwat;
2504 	wq->q_lowat = winit->qi_minfo->mi_lowat;
2505 	wq->q_minpsz = winit->qi_minfo->mi_minpsz;
2506 	wq->q_maxpsz = winit->qi_minfo->mi_maxpsz;
2507 
2508 	/* Remove old syncqs */
2509 	sq = rq->q_syncq;
2510 	outer = sq->sq_outer;
2511 	if (outer != NULL) {
2512 		ASSERT(wq->q_syncq->sq_outer == outer);
2513 		outer_remove(outer, rq->q_syncq);
2514 		if (wq->q_syncq != rq->q_syncq)
2515 			outer_remove(outer, wq->q_syncq);
2516 	}
2517 	ASSERT(sq->sq_outer == NULL);
2518 	ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2519 
2520 	if (sq != SQ(rq)) {
2521 		if (!(rq->q_flag & QPERMOD))
2522 			free_syncq(sq);
2523 		if (wq->q_syncq == rq->q_syncq)
2524 			wq->q_syncq = NULL;
2525 		rq->q_syncq = NULL;
2526 	}
2527 	if (wq->q_syncq != NULL && wq->q_syncq != sq &&
2528 	    wq->q_syncq != SQ(rq)) {
2529 		free_syncq(wq->q_syncq);
2530 		wq->q_syncq = NULL;
2531 	}
2532 	ASSERT(rq->q_syncq == NULL || (rq->q_syncq->sq_head == NULL &&
2533 	    rq->q_syncq->sq_tail == NULL));
2534 	ASSERT(wq->q_syncq == NULL || (wq->q_syncq->sq_head == NULL &&
2535 	    wq->q_syncq->sq_tail == NULL));
2536 
2537 	if (!(rq->q_flag & QPERMOD) &&
2538 	    rq->q_syncq != NULL && rq->q_syncq->sq_ciputctrl != NULL) {
2539 		ASSERT(rq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2540 		SUMCHECK_CIPUTCTRL_COUNTS(rq->q_syncq->sq_ciputctrl,
2541 		    rq->q_syncq->sq_nciputctrl, 0);
2542 		ASSERT(ciputctrl_cache != NULL);
2543 		kmem_cache_free(ciputctrl_cache, rq->q_syncq->sq_ciputctrl);
2544 		rq->q_syncq->sq_ciputctrl = NULL;
2545 		rq->q_syncq->sq_nciputctrl = 0;
2546 	}
2547 
2548 	if (!(wq->q_flag & QPERMOD) &&
2549 	    wq->q_syncq != NULL && wq->q_syncq->sq_ciputctrl != NULL) {
2550 		ASSERT(wq->q_syncq->sq_nciputctrl == n_ciputctrl - 1);
2551 		SUMCHECK_CIPUTCTRL_COUNTS(wq->q_syncq->sq_ciputctrl,
2552 		    wq->q_syncq->sq_nciputctrl, 0);
2553 		ASSERT(ciputctrl_cache != NULL);
2554 		kmem_cache_free(ciputctrl_cache, wq->q_syncq->sq_ciputctrl);
2555 		wq->q_syncq->sq_ciputctrl = NULL;
2556 		wq->q_syncq->sq_nciputctrl = 0;
2557 	}
2558 
2559 	sq = SQ(rq);
2560 	ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
2561 	ASSERT(sq->sq_outer == NULL);
2562 	ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
2563 
2564 	/*
2565 	 * Create syncqs based on qflag and sqtype. Set the SQ_TYPES_IN_FLAGS
2566 	 * bits in sq_flag based on the sqtype.
2567 	 */
2568 	ASSERT((sq->sq_flags & ~SQ_TYPES_IN_FLAGS) == 0);
2569 
2570 	rq->q_syncq = wq->q_syncq = sq;
2571 	sq->sq_type = sqtype;
2572 	sq->sq_flags = (sqtype & SQ_TYPES_IN_FLAGS);
2573 
2574 	/*
2575 	 *  We are making sq_svcflags zero,
2576 	 *  resetting SQ_DISABLED in case it was set by
2577 	 *  wait_svc() in the munlink path.
2578 	 *
2579 	 */
2580 	ASSERT((sq->sq_svcflags & SQ_SERVICE) == 0);
2581 	sq->sq_svcflags = 0;
2582 
2583 	/*
2584 	 * We need to acquire the lock here for the mlink and munlink case,
2585 	 * where canputnext, backenable, etc can access the q_flag.
2586 	 */
2587 	if (lock_needed) {
2588 		mutex_enter(QLOCK(rq));
2589 		rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2590 		mutex_exit(QLOCK(rq));
2591 		mutex_enter(QLOCK(wq));
2592 		wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2593 		mutex_exit(QLOCK(wq));
2594 	} else {
2595 		rq->q_flag = (rq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2596 		wq->q_flag = (wq->q_flag & ~QMT_TYPEMASK) | QWANTR | qflag;
2597 	}
2598 
2599 	if (qflag & QPERQ) {
2600 		/* Allocate a separate syncq for the write side */
2601 		sq = new_syncq();
2602 		sq->sq_type = rq->q_syncq->sq_type;
2603 		sq->sq_flags = rq->q_syncq->sq_flags;
2604 		ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2605 		    sq->sq_oprev == NULL);
2606 		wq->q_syncq = sq;
2607 	}
2608 	if (qflag & QPERMOD) {
2609 		sq = dmp->dm_sq;
2610 
2611 		/*
2612 		 * Assert that we do have an inner perimeter syncq and that it
2613 		 * does not have an outer perimeter associated with it.
2614 		 */
2615 		ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
2616 		    sq->sq_oprev == NULL);
2617 		rq->q_syncq = wq->q_syncq = sq;
2618 	}
2619 	if (qflag & QMTOUTPERIM) {
2620 		outer = dmp->dm_sq;
2621 
2622 		ASSERT(outer->sq_outer == NULL);
2623 		outer_insert(outer, rq->q_syncq);
2624 		if (wq->q_syncq != rq->q_syncq)
2625 			outer_insert(outer, wq->q_syncq);
2626 	}
2627 	ASSERT((rq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2628 	    (rq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2629 	ASSERT((wq->q_syncq->sq_flags & SQ_TYPES_IN_FLAGS) ==
2630 	    (wq->q_syncq->sq_type & SQ_TYPES_IN_FLAGS));
2631 	ASSERT((rq->q_flag & QMT_TYPEMASK) == (qflag & QMT_TYPEMASK));
2632 
2633 	/*
2634 	 * Initialize struio() types.
2635 	 */
2636 	rq->q_struiot =
2637 	    (rq->q_flag & QSYNCSTR) ? rinit->qi_struiot : STRUIOT_NONE;
2638 	wq->q_struiot =
2639 	    (wq->q_flag & QSYNCSTR) ? winit->qi_struiot : STRUIOT_NONE;
2640 }
2641 
2642 perdm_t *
hold_dm(struct streamtab * str,uint32_t qflag,uint32_t sqtype)2643 hold_dm(struct streamtab *str, uint32_t qflag, uint32_t sqtype)
2644 {
2645 	syncq_t	*sq;
2646 	perdm_t	**pp;
2647 	perdm_t	*p;
2648 	perdm_t	*dmp;
2649 
2650 	ASSERT(str != NULL);
2651 	ASSERT(qflag & (QPERMOD | QMTOUTPERIM));
2652 
2653 	rw_enter(&perdm_rwlock, RW_READER);
2654 	for (p = perdm_list; p != NULL; p = p->dm_next) {
2655 		if (p->dm_str == str) {	/* found one */
2656 			atomic_inc_32(&(p->dm_ref));
2657 			rw_exit(&perdm_rwlock);
2658 			return (p);
2659 		}
2660 	}
2661 	rw_exit(&perdm_rwlock);
2662 
2663 	sq = new_syncq();
2664 	if (qflag & QPERMOD) {
2665 		sq->sq_type = sqtype | SQ_PERMOD;
2666 		sq->sq_flags = sqtype & SQ_TYPES_IN_FLAGS;
2667 	} else {
2668 		ASSERT(qflag & QMTOUTPERIM);
2669 		sq->sq_onext = sq->sq_oprev = sq;
2670 	}
2671 
2672 	dmp = kmem_alloc(sizeof (perdm_t), KM_SLEEP);
2673 	dmp->dm_sq = sq;
2674 	dmp->dm_str = str;
2675 	dmp->dm_ref = 1;
2676 	dmp->dm_next = NULL;
2677 
2678 	rw_enter(&perdm_rwlock, RW_WRITER);
2679 	for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next)) {
2680 		if (p->dm_str == str) {	/* already present */
2681 			p->dm_ref++;
2682 			rw_exit(&perdm_rwlock);
2683 			free_syncq(sq);
2684 			kmem_free(dmp, sizeof (perdm_t));
2685 			return (p);
2686 		}
2687 	}
2688 
2689 	*pp = dmp;
2690 	rw_exit(&perdm_rwlock);
2691 	return (dmp);
2692 }
2693 
2694 void
rele_dm(perdm_t * dmp)2695 rele_dm(perdm_t *dmp)
2696 {
2697 	perdm_t **pp;
2698 	perdm_t *p;
2699 
2700 	rw_enter(&perdm_rwlock, RW_WRITER);
2701 	ASSERT(dmp->dm_ref > 0);
2702 
2703 	if (--dmp->dm_ref > 0) {
2704 		rw_exit(&perdm_rwlock);
2705 		return;
2706 	}
2707 
2708 	for (pp = &perdm_list; (p = *pp) != NULL; pp = &(p->dm_next))
2709 		if (p == dmp)
2710 			break;
2711 	ASSERT(p == dmp);
2712 	*pp = p->dm_next;
2713 	rw_exit(&perdm_rwlock);
2714 
2715 	/*
2716 	 * Wait for any background processing that relies on the
2717 	 * syncq to complete before it is freed.
2718 	 */
2719 	wait_sq_svc(p->dm_sq);
2720 	free_syncq(p->dm_sq);
2721 	kmem_free(p, sizeof (perdm_t));
2722 }
2723 
2724 /*
2725  * Make a protocol message given control and data buffers.
2726  * n.b., this can block; be careful of what locks you hold when calling it.
2727  *
2728  * If sd_maxblk is less than *iosize this routine can fail part way through
2729  * (due to an allocation failure). In this case on return *iosize will contain
2730  * the amount that was consumed. Otherwise *iosize will not be modified
2731  * i.e. it will contain the amount that was consumed.
2732  */
2733 int
strmakemsg(struct strbuf * mctl,ssize_t * iosize,struct uio * uiop,stdata_t * stp,int32_t flag,mblk_t ** mpp)2734 strmakemsg(
2735 	struct strbuf *mctl,
2736 	ssize_t *iosize,
2737 	struct uio *uiop,
2738 	stdata_t *stp,
2739 	int32_t flag,
2740 	mblk_t **mpp)
2741 {
2742 	mblk_t *mpctl = NULL;
2743 	mblk_t *mpdata = NULL;
2744 	int error;
2745 
2746 	ASSERT(uiop != NULL);
2747 
2748 	*mpp = NULL;
2749 	/* Create control part, if any */
2750 	if ((mctl != NULL) && (mctl->len >= 0)) {
2751 		error = strmakectl(mctl, flag, uiop->uio_fmode, &mpctl);
2752 		if (error)
2753 			return (error);
2754 	}
2755 	/* Create data part, if any */
2756 	if (*iosize >= 0) {
2757 		error = strmakedata(iosize, uiop, stp, flag, &mpdata);
2758 		if (error) {
2759 			freemsg(mpctl);
2760 			return (error);
2761 		}
2762 	}
2763 	if (mpctl != NULL) {
2764 		if (mpdata != NULL)
2765 			linkb(mpctl, mpdata);
2766 		*mpp = mpctl;
2767 	} else {
2768 		*mpp = mpdata;
2769 	}
2770 	return (0);
2771 }
2772 
2773 /*
2774  * Make the control part of a protocol message given a control buffer.
2775  * n.b., this can block; be careful of what locks you hold when calling it.
2776  */
2777 int
strmakectl(struct strbuf * mctl,int32_t flag,int32_t fflag,mblk_t ** mpp)2778 strmakectl(
2779 	struct strbuf *mctl,
2780 	int32_t flag,
2781 	int32_t fflag,
2782 	mblk_t **mpp)
2783 {
2784 	mblk_t *bp = NULL;
2785 	unsigned char msgtype;
2786 	int error = 0;
2787 	cred_t *cr = CRED();
2788 
2789 	/* We do not support interrupt threads using the stream head to send */
2790 	ASSERT(cr != NULL);
2791 
2792 	*mpp = NULL;
2793 	/*
2794 	 * Create control part of message, if any.
2795 	 */
2796 	if ((mctl != NULL) && (mctl->len >= 0)) {
2797 		caddr_t base;
2798 		int ctlcount;
2799 		int allocsz;
2800 
2801 		if (flag & RS_HIPRI)
2802 			msgtype = M_PCPROTO;
2803 		else
2804 			msgtype = M_PROTO;
2805 
2806 		ctlcount = mctl->len;
2807 		base = mctl->buf;
2808 
2809 		/*
2810 		 * Give modules a better chance to reuse M_PROTO/M_PCPROTO
2811 		 * blocks by increasing the size to something more usable.
2812 		 */
2813 		allocsz = MAX(ctlcount, 64);
2814 
2815 		/*
2816 		 * Range checking has already been done; simply try
2817 		 * to allocate a message block for the ctl part.
2818 		 */
2819 		while ((bp = allocb_cred(allocsz, cr,
2820 		    curproc->p_pid)) == NULL) {
2821 			if (fflag & (FNDELAY|FNONBLOCK))
2822 				return (EAGAIN);
2823 			if (error = strwaitbuf(allocsz, BPRI_MED))
2824 				return (error);
2825 		}
2826 
2827 		bp->b_datap->db_type = msgtype;
2828 		if (copyin(base, bp->b_wptr, ctlcount)) {
2829 			freeb(bp);
2830 			return (EFAULT);
2831 		}
2832 		bp->b_wptr += ctlcount;
2833 	}
2834 	*mpp = bp;
2835 	return (0);
2836 }
2837 
2838 /*
2839  * Make a protocol message given data buffers.
2840  * n.b., this can block; be careful of what locks you hold when calling it.
2841  *
2842  * If sd_maxblk is less than *iosize this routine can fail part way through
2843  * (due to an allocation failure). In this case on return *iosize will contain
2844  * the amount that was consumed. Otherwise *iosize will not be modified
2845  * i.e. it will contain the amount that was consumed.
2846  */
2847 int
strmakedata(ssize_t * iosize,struct uio * uiop,stdata_t * stp,int32_t flag,mblk_t ** mpp)2848 strmakedata(
2849 	ssize_t   *iosize,
2850 	struct uio *uiop,
2851 	stdata_t *stp,
2852 	int32_t flag,
2853 	mblk_t **mpp)
2854 {
2855 	mblk_t *mp = NULL;
2856 	mblk_t *bp;
2857 	int wroff = (int)stp->sd_wroff;
2858 	int tail_len = (int)stp->sd_tail;
2859 	int extra = wroff + tail_len;
2860 	int error = 0;
2861 	ssize_t maxblk;
2862 	ssize_t count = *iosize;
2863 	cred_t *cr;
2864 
2865 	*mpp = NULL;
2866 	if (count < 0)
2867 		return (0);
2868 
2869 	/* We do not support interrupt threads using the stream head to send */
2870 	cr = CRED();
2871 	ASSERT(cr != NULL);
2872 
2873 	maxblk = stp->sd_maxblk;
2874 	if (maxblk == INFPSZ)
2875 		maxblk = count;
2876 
2877 	/*
2878 	 * Create data part of message, if any.
2879 	 */
2880 	do {
2881 		ssize_t size;
2882 		dblk_t  *dp;
2883 
2884 		ASSERT(uiop);
2885 
2886 		size = MIN(count, maxblk);
2887 
2888 		while ((bp = allocb_cred(size + extra, cr,
2889 		    curproc->p_pid)) == NULL) {
2890 			error = EAGAIN;
2891 			if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) ||
2892 			    (error = strwaitbuf(size + extra, BPRI_MED)) != 0) {
2893 				if (count == *iosize) {
2894 					freemsg(mp);
2895 					return (error);
2896 				} else {
2897 					*iosize -= count;
2898 					*mpp = mp;
2899 					return (0);
2900 				}
2901 			}
2902 		}
2903 		dp = bp->b_datap;
2904 		dp->db_cpid = curproc->p_pid;
2905 		ASSERT(wroff <= dp->db_lim - bp->b_wptr);
2906 		bp->b_wptr = bp->b_rptr = bp->b_rptr + wroff;
2907 
2908 		if (flag & STRUIO_POSTPONE) {
2909 			/*
2910 			 * Setup the stream uio portion of the
2911 			 * dblk for subsequent use by struioget().
2912 			 */
2913 			dp->db_struioflag = STRUIO_SPEC;
2914 			dp->db_cksumstart = 0;
2915 			dp->db_cksumstuff = 0;
2916 			dp->db_cksumend = size;
2917 			*(long long *)dp->db_struioun.data = 0ll;
2918 			bp->b_wptr += size;
2919 		} else {
2920 			if (stp->sd_copyflag & STRCOPYCACHED)
2921 				uiop->uio_extflg |= UIO_COPY_CACHED;
2922 
2923 			if (size != 0) {
2924 				error = uiomove(bp->b_wptr, size, UIO_WRITE,
2925 				    uiop);
2926 				if (error != 0) {
2927 					freeb(bp);
2928 					freemsg(mp);
2929 					return (error);
2930 				}
2931 			}
2932 			bp->b_wptr += size;
2933 
2934 			if (stp->sd_wputdatafunc != NULL) {
2935 				mblk_t *newbp;
2936 
2937 				newbp = (stp->sd_wputdatafunc)(stp->sd_vnode,
2938 				    bp, NULL, NULL, NULL, NULL);
2939 				if (newbp == NULL) {
2940 					freeb(bp);
2941 					freemsg(mp);
2942 					return (ECOMM);
2943 				}
2944 				bp = newbp;
2945 			}
2946 		}
2947 
2948 		count -= size;
2949 
2950 		if (mp == NULL)
2951 			mp = bp;
2952 		else
2953 			linkb(mp, bp);
2954 	} while (count > 0);
2955 
2956 	*mpp = mp;
2957 	return (0);
2958 }
2959 
2960 /*
2961  * Wait for a buffer to become available. Return non-zero errno
2962  * if not able to wait, 0 if buffer is probably there.
2963  */
2964 int
strwaitbuf(size_t size,int pri)2965 strwaitbuf(size_t size, int pri)
2966 {
2967 	bufcall_id_t id;
2968 
2969 	mutex_enter(&bcall_monitor);
2970 	if ((id = bufcall(size, pri, (void (*)(void *))cv_broadcast,
2971 	    &ttoproc(curthread)->p_flag_cv)) == 0) {
2972 		mutex_exit(&bcall_monitor);
2973 		return (ENOSR);
2974 	}
2975 	if (!cv_wait_sig(&(ttoproc(curthread)->p_flag_cv), &bcall_monitor)) {
2976 		unbufcall(id);
2977 		mutex_exit(&bcall_monitor);
2978 		return (EINTR);
2979 	}
2980 	unbufcall(id);
2981 	mutex_exit(&bcall_monitor);
2982 	return (0);
2983 }
2984 
2985 /*
2986  * This function waits for a read or write event to happen on a stream.
2987  * fmode can specify FNDELAY and/or FNONBLOCK.
2988  * The timeout is in ms with -1 meaning infinite.
2989  * The flag values work as follows:
2990  *	READWAIT	Check for read side errors, send M_READ
2991  *	GETWAIT		Check for read side errors, no M_READ
2992  *	WRITEWAIT	Check for write side errors.
2993  *	NOINTR		Do not return error if nonblocking or timeout.
2994  * 	STR_NOERROR	Ignore all errors except STPLEX.
2995  *	STR_NOSIG	Ignore/hold signals during the duration of the call.
2996  *	STR_PEEK	Pass through the strgeterr().
2997  */
2998 int
strwaitq(stdata_t * stp,int flag,ssize_t count,int fmode,clock_t timout,int * done)2999 strwaitq(stdata_t *stp, int flag, ssize_t count, int fmode, clock_t timout,
3000     int *done)
3001 {
3002 	int slpflg, errs;
3003 	int error;
3004 	kcondvar_t *sleepon;
3005 	mblk_t *mp;
3006 	ssize_t *rd_count;
3007 	clock_t rval;
3008 
3009 	ASSERT(MUTEX_HELD(&stp->sd_lock));
3010 	if ((flag & READWAIT) || (flag & GETWAIT)) {
3011 		slpflg = RSLEEP;
3012 		sleepon = &_RD(stp->sd_wrq)->q_wait;
3013 		errs = STRDERR|STPLEX;
3014 	} else {
3015 		slpflg = WSLEEP;
3016 		sleepon = &stp->sd_wrq->q_wait;
3017 		errs = STWRERR|STRHUP|STPLEX;
3018 	}
3019 	if (flag & STR_NOERROR)
3020 		errs = STPLEX;
3021 
3022 	if (stp->sd_wakeq & slpflg) {
3023 		/*
3024 		 * A strwakeq() is pending, no need to sleep.
3025 		 */
3026 		stp->sd_wakeq &= ~slpflg;
3027 		*done = 0;
3028 		return (0);
3029 	}
3030 
3031 	if (stp->sd_flag & errs) {
3032 		/*
3033 		 * Check for errors before going to sleep since the
3034 		 * caller might not have checked this while holding
3035 		 * sd_lock.
3036 		 */
3037 		error = strgeterr(stp, errs, (flag & STR_PEEK));
3038 		if (error != 0) {
3039 			*done = 1;
3040 			return (error);
3041 		}
3042 	}
3043 
3044 	/*
3045 	 * If any module downstream has requested read notification
3046 	 * by setting SNDMREAD flag using M_SETOPTS, send a message
3047 	 * down stream.
3048 	 */
3049 	if ((flag & READWAIT) && (stp->sd_flag & SNDMREAD)) {
3050 		mutex_exit(&stp->sd_lock);
3051 		if (!(mp = allocb_wait(sizeof (ssize_t), BPRI_MED,
3052 		    (flag & STR_NOSIG), &error))) {
3053 			mutex_enter(&stp->sd_lock);
3054 			*done = 1;
3055 			return (error);
3056 		}
3057 		mp->b_datap->db_type = M_READ;
3058 		rd_count = (ssize_t *)mp->b_wptr;
3059 		*rd_count = count;
3060 		mp->b_wptr += sizeof (ssize_t);
3061 		/*
3062 		 * Send the number of bytes requested by the
3063 		 * read as the argument to M_READ.
3064 		 */
3065 		stream_willservice(stp);
3066 		putnext(stp->sd_wrq, mp);
3067 		stream_runservice(stp);
3068 		mutex_enter(&stp->sd_lock);
3069 
3070 		/*
3071 		 * If any data arrived due to inline processing
3072 		 * of putnext(), don't sleep.
3073 		 */
3074 		if (_RD(stp->sd_wrq)->q_first != NULL) {
3075 			*done = 0;
3076 			return (0);
3077 		}
3078 	}
3079 
3080 	if (fmode & (FNDELAY|FNONBLOCK)) {
3081 		if (!(flag & NOINTR))
3082 			error = EAGAIN;
3083 		else
3084 			error = 0;
3085 		*done = 1;
3086 		return (error);
3087 	}
3088 
3089 	stp->sd_flag |= slpflg;
3090 	TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAIT2,
3091 	    "strwaitq sleeps (2):%p, %X, %lX, %X, %p",
3092 	    stp, flag, count, fmode, done);
3093 
3094 	rval = str_cv_wait(sleepon, &stp->sd_lock, timout, flag & STR_NOSIG);
3095 	if (rval > 0) {
3096 		/* EMPTY */
3097 		TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_WAKE2,
3098 		    "strwaitq awakes(2):%X, %X, %X, %X, %X",
3099 		    stp, flag, count, fmode, done);
3100 	} else if (rval == 0) {
3101 		TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_INTR2,
3102 		    "strwaitq interrupt #2:%p, %X, %lX, %X, %p",
3103 		    stp, flag, count, fmode, done);
3104 		stp->sd_flag &= ~slpflg;
3105 		cv_broadcast(sleepon);
3106 		if (!(flag & NOINTR))
3107 			error = EINTR;
3108 		else
3109 			error = 0;
3110 		*done = 1;
3111 		return (error);
3112 	} else {
3113 		/* timeout */
3114 		TRACE_5(TR_FAC_STREAMS_FR, TR_STRWAITQ_TIME,
3115 		    "strwaitq timeout:%p, %X, %lX, %X, %p",
3116 		    stp, flag, count, fmode, done);
3117 		*done = 1;
3118 		if (!(flag & NOINTR))
3119 			return (ETIME);
3120 		else
3121 			return (0);
3122 	}
3123 	/*
3124 	 * If the caller implements delayed errors (i.e. queued after data)
3125 	 * we can not check for errors here since data as well as an
3126 	 * error might have arrived at the stream head. We return to
3127 	 * have the caller check the read queue before checking for errors.
3128 	 */
3129 	if ((stp->sd_flag & errs) && !(flag & STR_DELAYERR)) {
3130 		error = strgeterr(stp, errs, (flag & STR_PEEK));
3131 		if (error != 0) {
3132 			*done = 1;
3133 			return (error);
3134 		}
3135 	}
3136 	*done = 0;
3137 	return (0);
3138 }
3139 
3140 /*
3141  * Perform job control discipline access checks.
3142  * Return 0 for success and the errno for failure.
3143  */
3144 
3145 #define	cantsend(p, t, sig) \
3146 	(sigismember(&(p)->p_ignore, sig) || signal_is_blocked((t), sig))
3147 
3148 int
straccess(struct stdata * stp,enum jcaccess mode)3149 straccess(struct stdata *stp, enum jcaccess mode)
3150 {
3151 	extern kcondvar_t lbolt_cv;	/* XXX: should be in a header file */
3152 	kthread_t *t = curthread;
3153 	proc_t *p = ttoproc(t);
3154 	sess_t *sp;
3155 
3156 	ASSERT(mutex_owned(&stp->sd_lock));
3157 
3158 	if (stp->sd_sidp == NULL || stp->sd_vnode->v_type == VFIFO)
3159 		return (0);
3160 
3161 	mutex_enter(&p->p_lock);		/* protects p_pgidp */
3162 
3163 	for (;;) {
3164 		mutex_enter(&p->p_splock);	/* protects p->p_sessp */
3165 		sp = p->p_sessp;
3166 		mutex_enter(&sp->s_lock);	/* protects sp->* */
3167 
3168 		/*
3169 		 * If this is not the calling process's controlling terminal
3170 		 * or if the calling process is already in the foreground
3171 		 * then allow access.
3172 		 */
3173 		if (sp->s_dev != stp->sd_vnode->v_rdev ||
3174 		    p->p_pgidp == stp->sd_pgidp) {
3175 			mutex_exit(&sp->s_lock);
3176 			mutex_exit(&p->p_splock);
3177 			mutex_exit(&p->p_lock);
3178 			return (0);
3179 		}
3180 
3181 		/*
3182 		 * Check to see if controlling terminal has been deallocated.
3183 		 */
3184 		if (sp->s_vp == NULL) {
3185 			if (!cantsend(p, t, SIGHUP))
3186 				sigtoproc(p, t, SIGHUP);
3187 			mutex_exit(&sp->s_lock);
3188 			mutex_exit(&p->p_splock);
3189 			mutex_exit(&p->p_lock);
3190 			return (EIO);
3191 		}
3192 
3193 		mutex_exit(&sp->s_lock);
3194 		mutex_exit(&p->p_splock);
3195 
3196 		if (mode == JCGETP) {
3197 			mutex_exit(&p->p_lock);
3198 			return (0);
3199 		}
3200 
3201 		if (mode == JCREAD) {
3202 			if (p->p_detached || cantsend(p, t, SIGTTIN)) {
3203 				mutex_exit(&p->p_lock);
3204 				return (EIO);
3205 			}
3206 			mutex_exit(&p->p_lock);
3207 			mutex_exit(&stp->sd_lock);
3208 			pgsignal(p->p_pgidp, SIGTTIN);
3209 			mutex_enter(&stp->sd_lock);
3210 			mutex_enter(&p->p_lock);
3211 		} else {  /* mode == JCWRITE or JCSETP */
3212 			if ((mode == JCWRITE && !(stp->sd_flag & STRTOSTOP)) ||
3213 			    cantsend(p, t, SIGTTOU)) {
3214 				mutex_exit(&p->p_lock);
3215 				return (0);
3216 			}
3217 			if (p->p_detached) {
3218 				mutex_exit(&p->p_lock);
3219 				return (EIO);
3220 			}
3221 			mutex_exit(&p->p_lock);
3222 			mutex_exit(&stp->sd_lock);
3223 			pgsignal(p->p_pgidp, SIGTTOU);
3224 			mutex_enter(&stp->sd_lock);
3225 			mutex_enter(&p->p_lock);
3226 		}
3227 
3228 		/*
3229 		 * We call cv_wait_sig_swap() to cause the appropriate
3230 		 * action for the jobcontrol signal to take place.
3231 		 * If the signal is being caught, we will take the
3232 		 * EINTR error return.  Otherwise, the default action
3233 		 * of causing the process to stop will take place.
3234 		 * In this case, we rely on the periodic cv_broadcast() on
3235 		 * &lbolt_cv to wake us up to loop around and test again.
3236 		 * We can't get here if the signal is ignored or
3237 		 * if the current thread is blocking the signal.
3238 		 */
3239 		mutex_exit(&stp->sd_lock);
3240 		if (!cv_wait_sig_swap(&lbolt_cv, &p->p_lock)) {
3241 			mutex_exit(&p->p_lock);
3242 			mutex_enter(&stp->sd_lock);
3243 			return (EINTR);
3244 		}
3245 		mutex_exit(&p->p_lock);
3246 		mutex_enter(&stp->sd_lock);
3247 		mutex_enter(&p->p_lock);
3248 	}
3249 }
3250 
3251 /*
3252  * Return size of message of block type (bp->b_datap->db_type)
3253  */
3254 size_t
xmsgsize(mblk_t * bp)3255 xmsgsize(mblk_t *bp)
3256 {
3257 	unsigned char type;
3258 	size_t count = 0;
3259 
3260 	type = bp->b_datap->db_type;
3261 
3262 	for (; bp; bp = bp->b_cont) {
3263 		if (type != bp->b_datap->db_type)
3264 			break;
3265 		ASSERT(bp->b_wptr >= bp->b_rptr);
3266 		count += bp->b_wptr - bp->b_rptr;
3267 	}
3268 	return (count);
3269 }
3270 
3271 /*
3272  * Allocate a stream head.
3273  */
3274 struct stdata *
shalloc(queue_t * qp)3275 shalloc(queue_t *qp)
3276 {
3277 	stdata_t *stp;
3278 
3279 	stp = kmem_cache_alloc(stream_head_cache, KM_SLEEP);
3280 
3281 	stp->sd_wrq = _WR(qp);
3282 	stp->sd_strtab = NULL;
3283 	stp->sd_iocid = 0;
3284 	stp->sd_mate = NULL;
3285 	stp->sd_freezer = NULL;
3286 	stp->sd_refcnt = 0;
3287 	stp->sd_wakeq = 0;
3288 	stp->sd_anchor = 0;
3289 	stp->sd_struiowrq = NULL;
3290 	stp->sd_struiordq = NULL;
3291 	stp->sd_struiodnak = 0;
3292 	stp->sd_struionak = NULL;
3293 	stp->sd_t_audit_data = NULL;
3294 	stp->sd_rput_opt = 0;
3295 	stp->sd_wput_opt = 0;
3296 	stp->sd_read_opt = 0;
3297 	stp->sd_rprotofunc = strrput_proto;
3298 	stp->sd_rmiscfunc = strrput_misc;
3299 	stp->sd_rderrfunc = stp->sd_wrerrfunc = NULL;
3300 	stp->sd_rputdatafunc = stp->sd_wputdatafunc = NULL;
3301 	stp->sd_ciputctrl = NULL;
3302 	stp->sd_nciputctrl = 0;
3303 	stp->sd_qhead = NULL;
3304 	stp->sd_qtail = NULL;
3305 	stp->sd_servid = NULL;
3306 	stp->sd_nqueues = 0;
3307 	stp->sd_svcflags = 0;
3308 	stp->sd_copyflag = 0;
3309 
3310 	return (stp);
3311 }
3312 
3313 /*
3314  * Free a stream head.
3315  */
3316 void
shfree(stdata_t * stp)3317 shfree(stdata_t *stp)
3318 {
3319 	ASSERT(MUTEX_NOT_HELD(&stp->sd_lock));
3320 
3321 	stp->sd_wrq = NULL;
3322 
3323 	mutex_enter(&stp->sd_qlock);
3324 	while (stp->sd_svcflags & STRS_SCHEDULED) {
3325 		STRSTAT(strwaits);
3326 		cv_wait(&stp->sd_qcv, &stp->sd_qlock);
3327 	}
3328 	mutex_exit(&stp->sd_qlock);
3329 
3330 	if (stp->sd_ciputctrl != NULL) {
3331 		ASSERT(stp->sd_nciputctrl == n_ciputctrl - 1);
3332 		SUMCHECK_CIPUTCTRL_COUNTS(stp->sd_ciputctrl,
3333 		    stp->sd_nciputctrl, 0);
3334 		ASSERT(ciputctrl_cache != NULL);
3335 		kmem_cache_free(ciputctrl_cache, stp->sd_ciputctrl);
3336 		stp->sd_ciputctrl = NULL;
3337 		stp->sd_nciputctrl = 0;
3338 	}
3339 	ASSERT(stp->sd_qhead == NULL);
3340 	ASSERT(stp->sd_qtail == NULL);
3341 	ASSERT(stp->sd_nqueues == 0);
3342 	kmem_cache_free(stream_head_cache, stp);
3343 }
3344 
3345 /*
3346  * Allocate a pair of queues and a syncq for the pair
3347  */
3348 queue_t *
allocq(void)3349 allocq(void)
3350 {
3351 	queinfo_t *qip;
3352 	queue_t *qp, *wqp;
3353 	syncq_t	*sq;
3354 
3355 	qip = kmem_cache_alloc(queue_cache, KM_SLEEP);
3356 
3357 	qp = &qip->qu_rqueue;
3358 	wqp = &qip->qu_wqueue;
3359 	sq = &qip->qu_syncq;
3360 
3361 	qp->q_last	= NULL;
3362 	qp->q_next	= NULL;
3363 	qp->q_ptr	= NULL;
3364 	qp->q_flag	= QUSE | QREADR;
3365 	qp->q_bandp	= NULL;
3366 	qp->q_stream	= NULL;
3367 	qp->q_syncq	= sq;
3368 	qp->q_nband	= 0;
3369 	qp->q_nfsrv	= NULL;
3370 	qp->q_draining	= 0;
3371 	qp->q_syncqmsgs	= 0;
3372 	qp->q_spri	= 0;
3373 	qp->q_qtstamp	= 0;
3374 	qp->q_sqtstamp	= 0;
3375 	qp->q_fp	= NULL;
3376 
3377 	wqp->q_last	= NULL;
3378 	wqp->q_next	= NULL;
3379 	wqp->q_ptr	= NULL;
3380 	wqp->q_flag	= QUSE;
3381 	wqp->q_bandp	= NULL;
3382 	wqp->q_stream	= NULL;
3383 	wqp->q_syncq	= sq;
3384 	wqp->q_nband	= 0;
3385 	wqp->q_nfsrv	= NULL;
3386 	wqp->q_draining	= 0;
3387 	wqp->q_syncqmsgs = 0;
3388 	wqp->q_qtstamp	= 0;
3389 	wqp->q_sqtstamp	= 0;
3390 	wqp->q_spri	= 0;
3391 
3392 	sq->sq_count	= 0;
3393 	sq->sq_rmqcount	= 0;
3394 	sq->sq_flags	= 0;
3395 	sq->sq_type	= 0;
3396 	sq->sq_callbflags = 0;
3397 	sq->sq_cancelid	= 0;
3398 	sq->sq_ciputctrl = NULL;
3399 	sq->sq_nciputctrl = 0;
3400 	sq->sq_needexcl = 0;
3401 	sq->sq_svcflags = 0;
3402 
3403 	return (qp);
3404 }
3405 
3406 /*
3407  * Free a pair of queues and the "attached" syncq.
3408  * Discard any messages left on the syncq(s), remove the syncq(s) from the
3409  * outer perimeter, and free the syncq(s) if they are not the "attached" syncq.
3410  */
3411 void
freeq(queue_t * qp)3412 freeq(queue_t *qp)
3413 {
3414 	qband_t *qbp, *nqbp;
3415 	syncq_t *sq, *outer;
3416 	queue_t *wqp = _WR(qp);
3417 
3418 	ASSERT(qp->q_flag & QREADR);
3419 
3420 	/*
3421 	 * If a previously dispatched taskq job is scheduled to run
3422 	 * sync_service() or a service routine is scheduled for the
3423 	 * queues about to be freed, wait here until all service is
3424 	 * done on the queue and all associated queues and syncqs.
3425 	 */
3426 	wait_svc(qp);
3427 
3428 	(void) flush_syncq(qp->q_syncq, qp);
3429 	(void) flush_syncq(wqp->q_syncq, wqp);
3430 	ASSERT(qp->q_syncqmsgs == 0 && wqp->q_syncqmsgs == 0);
3431 
3432 	/*
3433 	 * Flush the queues before q_next is set to NULL This is needed
3434 	 * in order to backenable any downstream queue before we go away.
3435 	 * Note: we are already removed from the stream so that the
3436 	 * backenabling will not cause any messages to be delivered to our
3437 	 * put procedures.
3438 	 */
3439 	flushq(qp, FLUSHALL);
3440 	flushq(wqp, FLUSHALL);
3441 
3442 	/* Tidy up - removeq only does a half-remove from stream */
3443 	qp->q_next = wqp->q_next = NULL;
3444 	ASSERT(!(qp->q_flag & QENAB));
3445 	ASSERT(!(wqp->q_flag & QENAB));
3446 
3447 	outer = qp->q_syncq->sq_outer;
3448 	if (outer != NULL) {
3449 		outer_remove(outer, qp->q_syncq);
3450 		if (wqp->q_syncq != qp->q_syncq)
3451 			outer_remove(outer, wqp->q_syncq);
3452 	}
3453 	/*
3454 	 * Free any syncqs that are outside what allocq returned.
3455 	 */
3456 	if (qp->q_syncq != SQ(qp) && !(qp->q_flag & QPERMOD))
3457 		free_syncq(qp->q_syncq);
3458 	if (qp->q_syncq != wqp->q_syncq && wqp->q_syncq != SQ(qp))
3459 		free_syncq(wqp->q_syncq);
3460 
3461 	ASSERT((qp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3462 	ASSERT((wqp->q_sqflags & (Q_SQQUEUED | Q_SQDRAINING)) == 0);
3463 	ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
3464 	ASSERT(MUTEX_NOT_HELD(QLOCK(wqp)));
3465 	sq = SQ(qp);
3466 	ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
3467 	ASSERT(sq->sq_head == NULL && sq->sq_tail == NULL);
3468 	ASSERT(sq->sq_outer == NULL);
3469 	ASSERT(sq->sq_onext == NULL && sq->sq_oprev == NULL);
3470 	ASSERT(sq->sq_callbpend == NULL);
3471 	ASSERT(sq->sq_needexcl == 0);
3472 
3473 	if (sq->sq_ciputctrl != NULL) {
3474 		ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
3475 		SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
3476 		    sq->sq_nciputctrl, 0);
3477 		ASSERT(ciputctrl_cache != NULL);
3478 		kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
3479 		sq->sq_ciputctrl = NULL;
3480 		sq->sq_nciputctrl = 0;
3481 	}
3482 
3483 	ASSERT(qp->q_first == NULL && wqp->q_first == NULL);
3484 	ASSERT(qp->q_count == 0 && wqp->q_count == 0);
3485 	ASSERT(qp->q_mblkcnt == 0 && wqp->q_mblkcnt == 0);
3486 
3487 	qp->q_flag &= ~QUSE;
3488 	wqp->q_flag &= ~QUSE;
3489 
3490 	/* NOTE: Uncomment the assert below once bugid 1159635 is fixed. */
3491 	/* ASSERT((qp->q_flag & QWANTW) == 0 && (wqp->q_flag & QWANTW) == 0); */
3492 
3493 	qbp = qp->q_bandp;
3494 	while (qbp) {
3495 		nqbp = qbp->qb_next;
3496 		freeband(qbp);
3497 		qbp = nqbp;
3498 	}
3499 	qbp = wqp->q_bandp;
3500 	while (qbp) {
3501 		nqbp = qbp->qb_next;
3502 		freeband(qbp);
3503 		qbp = nqbp;
3504 	}
3505 	kmem_cache_free(queue_cache, qp);
3506 }
3507 
3508 /*
3509  * Allocate a qband structure.
3510  */
3511 qband_t *
allocband(void)3512 allocband(void)
3513 {
3514 	qband_t *qbp;
3515 
3516 	qbp = kmem_cache_alloc(qband_cache, KM_NOSLEEP);
3517 	if (qbp == NULL)
3518 		return (NULL);
3519 
3520 	qbp->qb_next	= NULL;
3521 	qbp->qb_count	= 0;
3522 	qbp->qb_mblkcnt	= 0;
3523 	qbp->qb_first	= NULL;
3524 	qbp->qb_last	= NULL;
3525 	qbp->qb_flag	= 0;
3526 
3527 	return (qbp);
3528 }
3529 
3530 /*
3531  * Free a qband structure.
3532  */
3533 void
freeband(qband_t * qbp)3534 freeband(qband_t *qbp)
3535 {
3536 	kmem_cache_free(qband_cache, qbp);
3537 }
3538 
3539 /*
3540  * Just like putnextctl(9F), except that allocb_wait() is used.
3541  *
3542  * Consolidation Private, and of course only callable from the stream head or
3543  * routines that may block.
3544  */
3545 int
putnextctl_wait(queue_t * q,int type)3546 putnextctl_wait(queue_t *q, int type)
3547 {
3548 	mblk_t *bp;
3549 	int error;
3550 
3551 	if ((datamsg(type) && (type != M_DELAY)) ||
3552 	    (bp = allocb_wait(0, BPRI_HI, 0, &error)) == NULL)
3553 		return (0);
3554 
3555 	bp->b_datap->db_type = (unsigned char)type;
3556 	putnext(q, bp);
3557 	return (1);
3558 }
3559 
3560 /*
3561  * Run any possible bufcalls.
3562  */
3563 void
runbufcalls(void)3564 runbufcalls(void)
3565 {
3566 	strbufcall_t *bcp;
3567 
3568 	mutex_enter(&bcall_monitor);
3569 	mutex_enter(&strbcall_lock);
3570 
3571 	if (strbcalls.bc_head) {
3572 		size_t count;
3573 		int nevent;
3574 
3575 		/*
3576 		 * count how many events are on the list
3577 		 * now so we can check to avoid looping
3578 		 * in low memory situations
3579 		 */
3580 		nevent = 0;
3581 		for (bcp = strbcalls.bc_head; bcp; bcp = bcp->bc_next)
3582 			nevent++;
3583 
3584 		/*
3585 		 * get estimate of available memory from kmem_avail().
3586 		 * awake all bufcall functions waiting for
3587 		 * memory whose request could be satisfied
3588 		 * by 'count' memory and let 'em fight for it.
3589 		 */
3590 		count = kmem_avail();
3591 		while ((bcp = strbcalls.bc_head) != NULL && nevent) {
3592 			STRSTAT(bufcalls);
3593 			--nevent;
3594 			if (bcp->bc_size <= count) {
3595 				bcp->bc_executor = curthread;
3596 				mutex_exit(&strbcall_lock);
3597 				(*bcp->bc_func)(bcp->bc_arg);
3598 				mutex_enter(&strbcall_lock);
3599 				bcp->bc_executor = NULL;
3600 				cv_broadcast(&bcall_cv);
3601 				strbcalls.bc_head = bcp->bc_next;
3602 				kmem_free(bcp, sizeof (strbufcall_t));
3603 			} else {
3604 				/*
3605 				 * too big, try again later - note
3606 				 * that nevent was decremented above
3607 				 * so we won't retry this one on this
3608 				 * iteration of the loop
3609 				 */
3610 				if (bcp->bc_next != NULL) {
3611 					strbcalls.bc_head = bcp->bc_next;
3612 					bcp->bc_next = NULL;
3613 					strbcalls.bc_tail->bc_next = bcp;
3614 					strbcalls.bc_tail = bcp;
3615 				}
3616 			}
3617 		}
3618 		if (strbcalls.bc_head == NULL)
3619 			strbcalls.bc_tail = NULL;
3620 	}
3621 
3622 	mutex_exit(&strbcall_lock);
3623 	mutex_exit(&bcall_monitor);
3624 }
3625 
3626 
3627 /*
3628  * Actually run queue's service routine.
3629  */
3630 static void
runservice(queue_t * q)3631 runservice(queue_t *q)
3632 {
3633 	qband_t *qbp;
3634 
3635 	ASSERT(q->q_qinfo->qi_srvp);
3636 again:
3637 	entersq(q->q_syncq, SQ_SVC);
3638 	TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_START,
3639 	    "runservice starts:%p", q);
3640 
3641 	if (!(q->q_flag & QWCLOSE))
3642 		(*q->q_qinfo->qi_srvp)(q);
3643 
3644 	TRACE_1(TR_FAC_STREAMS_FR, TR_QRUNSERVICE_END,
3645 	    "runservice ends:(%p)", q);
3646 
3647 	leavesq(q->q_syncq, SQ_SVC);
3648 
3649 	mutex_enter(QLOCK(q));
3650 	if (q->q_flag & QENAB) {
3651 		q->q_flag &= ~QENAB;
3652 		mutex_exit(QLOCK(q));
3653 		goto again;
3654 	}
3655 	q->q_flag &= ~QINSERVICE;
3656 	q->q_flag &= ~QBACK;
3657 	for (qbp = q->q_bandp; qbp; qbp = qbp->qb_next)
3658 		qbp->qb_flag &= ~QB_BACK;
3659 	/*
3660 	 * Wakeup thread waiting for the service procedure
3661 	 * to be run (strclose and qdetach).
3662 	 */
3663 	cv_broadcast(&q->q_wait);
3664 
3665 	mutex_exit(QLOCK(q));
3666 }
3667 
3668 /*
3669  * Background processing of bufcalls.
3670  */
3671 void
streams_bufcall_service(void)3672 streams_bufcall_service(void)
3673 {
3674 	callb_cpr_t	cprinfo;
3675 
3676 	CALLB_CPR_INIT(&cprinfo, &strbcall_lock, callb_generic_cpr,
3677 	    "streams_bufcall_service");
3678 
3679 	mutex_enter(&strbcall_lock);
3680 
3681 	for (;;) {
3682 		if (strbcalls.bc_head != NULL && kmem_avail() > 0) {
3683 			mutex_exit(&strbcall_lock);
3684 			runbufcalls();
3685 			mutex_enter(&strbcall_lock);
3686 		}
3687 		if (strbcalls.bc_head != NULL) {
3688 			STRSTAT(bcwaits);
3689 			/* Wait for memory to become available */
3690 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
3691 			(void) cv_reltimedwait(&memavail_cv, &strbcall_lock,
3692 			    SEC_TO_TICK(60), TR_CLOCK_TICK);
3693 			CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3694 		}
3695 
3696 		/* Wait for new work to arrive */
3697 		if (strbcalls.bc_head == NULL) {
3698 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
3699 			cv_wait(&strbcall_cv, &strbcall_lock);
3700 			CALLB_CPR_SAFE_END(&cprinfo, &strbcall_lock);
3701 		}
3702 	}
3703 }
3704 
3705 /*
3706  * Background processing of streams background tasks which failed
3707  * taskq_dispatch.
3708  */
3709 static void
streams_qbkgrnd_service(void)3710 streams_qbkgrnd_service(void)
3711 {
3712 	callb_cpr_t cprinfo;
3713 	queue_t *q;
3714 
3715 	CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3716 	    "streams_bkgrnd_service");
3717 
3718 	mutex_enter(&service_queue);
3719 
3720 	for (;;) {
3721 		/*
3722 		 * Wait for work to arrive.
3723 		 */
3724 		while ((freebs_list == NULL) && (qhead == NULL)) {
3725 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
3726 			cv_wait(&services_to_run, &service_queue);
3727 			CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3728 		}
3729 		/*
3730 		 * Handle all pending freebs requests to free memory.
3731 		 */
3732 		while (freebs_list != NULL) {
3733 			mblk_t *mp = freebs_list;
3734 			freebs_list = mp->b_next;
3735 			mutex_exit(&service_queue);
3736 			mblk_free(mp);
3737 			mutex_enter(&service_queue);
3738 		}
3739 		/*
3740 		 * Run pending queues.
3741 		 */
3742 		while (qhead != NULL) {
3743 			DQ(q, qhead, qtail, q_link);
3744 			ASSERT(q != NULL);
3745 			mutex_exit(&service_queue);
3746 			queue_service(q);
3747 			mutex_enter(&service_queue);
3748 		}
3749 		ASSERT(qhead == NULL && qtail == NULL);
3750 	}
3751 }
3752 
3753 /*
3754  * Background processing of streams background tasks which failed
3755  * taskq_dispatch.
3756  */
3757 static void
streams_sqbkgrnd_service(void)3758 streams_sqbkgrnd_service(void)
3759 {
3760 	callb_cpr_t cprinfo;
3761 	syncq_t *sq;
3762 
3763 	CALLB_CPR_INIT(&cprinfo, &service_queue, callb_generic_cpr,
3764 	    "streams_sqbkgrnd_service");
3765 
3766 	mutex_enter(&service_queue);
3767 
3768 	for (;;) {
3769 		/*
3770 		 * Wait for work to arrive.
3771 		 */
3772 		while (sqhead == NULL) {
3773 			CALLB_CPR_SAFE_BEGIN(&cprinfo);
3774 			cv_wait(&syncqs_to_run, &service_queue);
3775 			CALLB_CPR_SAFE_END(&cprinfo, &service_queue);
3776 		}
3777 
3778 		/*
3779 		 * Run pending syncqs.
3780 		 */
3781 		while (sqhead != NULL) {
3782 			DQ(sq, sqhead, sqtail, sq_next);
3783 			ASSERT(sq != NULL);
3784 			ASSERT(sq->sq_svcflags & SQ_BGTHREAD);
3785 			mutex_exit(&service_queue);
3786 			syncq_service(sq);
3787 			mutex_enter(&service_queue);
3788 		}
3789 	}
3790 }
3791 
3792 /*
3793  * Disable the syncq and wait for background syncq processing to complete.
3794  * If the syncq is placed on the sqhead/sqtail queue, try to remove it from the
3795  * list.
3796  */
3797 void
wait_sq_svc(syncq_t * sq)3798 wait_sq_svc(syncq_t *sq)
3799 {
3800 	mutex_enter(SQLOCK(sq));
3801 	sq->sq_svcflags |= SQ_DISABLED;
3802 	if (sq->sq_svcflags & SQ_BGTHREAD) {
3803 		syncq_t *sq_chase;
3804 		syncq_t *sq_curr;
3805 		int removed;
3806 
3807 		ASSERT(sq->sq_servcount == 1);
3808 		mutex_enter(&service_queue);
3809 		RMQ(sq, sqhead, sqtail, sq_next, sq_chase, sq_curr, removed);
3810 		mutex_exit(&service_queue);
3811 		if (removed) {
3812 			sq->sq_svcflags &= ~SQ_BGTHREAD;
3813 			sq->sq_servcount = 0;
3814 			STRSTAT(sqremoved);
3815 			goto done;
3816 		}
3817 	}
3818 	while (sq->sq_servcount != 0) {
3819 		sq->sq_flags |= SQ_WANTWAKEUP;
3820 		cv_wait(&sq->sq_wait, SQLOCK(sq));
3821 	}
3822 done:
3823 	mutex_exit(SQLOCK(sq));
3824 }
3825 
3826 /*
3827  * Put a syncq on the list of syncq's to be serviced by the sqthread.
3828  * Add the argument to the end of the sqhead list and set the flag
3829  * indicating this syncq has been enabled.  If it has already been
3830  * enabled, don't do anything.
3831  * This routine assumes that SQLOCK is held.
3832  * NOTE that the lock order is to have the SQLOCK first,
3833  * so if the service_syncq lock is held, we need to release it
3834  * before acquiring the SQLOCK (mostly relevant for the background
3835  * thread, and this seems to be common among the STREAMS global locks).
3836  * Note that the sq_svcflags are protected by the SQLOCK.
3837  */
3838 void
sqenable(syncq_t * sq)3839 sqenable(syncq_t *sq)
3840 {
3841 	/*
3842 	 * This is probably not important except for where I believe it
3843 	 * is being called.  At that point, it should be held (and it
3844 	 * is a pain to release it just for this routine, so don't do
3845 	 * it).
3846 	 */
3847 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
3848 
3849 	IMPLY(sq->sq_servcount == 0, sq->sq_next == NULL);
3850 	IMPLY(sq->sq_next != NULL, sq->sq_svcflags & SQ_BGTHREAD);
3851 
3852 	/*
3853 	 * Do not put on list if background thread is scheduled or
3854 	 * syncq is disabled.
3855 	 */
3856 	if (sq->sq_svcflags & (SQ_DISABLED | SQ_BGTHREAD))
3857 		return;
3858 
3859 	/*
3860 	 * Check whether we should enable sq at all.
3861 	 * Non PERMOD syncqs may be drained by at most one thread.
3862 	 * PERMOD syncqs may be drained by several threads but we limit the
3863 	 * total amount to the lesser of
3864 	 *	Number of queues on the squeue and
3865 	 *	Number of CPUs.
3866 	 */
3867 	if (sq->sq_servcount != 0) {
3868 		if (((sq->sq_type & SQ_PERMOD) == 0) ||
3869 		    (sq->sq_servcount >= MIN(sq->sq_nqueues, ncpus_online))) {
3870 			STRSTAT(sqtoomany);
3871 			return;
3872 		}
3873 	}
3874 
3875 	sq->sq_tstamp = ddi_get_lbolt();
3876 	STRSTAT(sqenables);
3877 
3878 	/* Attempt a taskq dispatch */
3879 	sq->sq_servid = (void *)taskq_dispatch(streams_taskq,
3880 	    (task_func_t *)syncq_service, sq, TQ_NOSLEEP | TQ_NOQUEUE);
3881 	if (sq->sq_servid != NULL) {
3882 		sq->sq_servcount++;
3883 		return;
3884 	}
3885 
3886 	/*
3887 	 * This taskq dispatch failed, but a previous one may have succeeded.
3888 	 * Don't try to schedule on the background thread whilst there is
3889 	 * outstanding taskq processing.
3890 	 */
3891 	if (sq->sq_servcount != 0)
3892 		return;
3893 
3894 	/*
3895 	 * System is low on resources and can't perform a non-sleeping
3896 	 * dispatch. Schedule the syncq for a background thread and mark the
3897 	 * syncq to avoid any further taskq dispatch attempts.
3898 	 */
3899 	mutex_enter(&service_queue);
3900 	STRSTAT(taskqfails);
3901 	ENQUEUE(sq, sqhead, sqtail, sq_next);
3902 	sq->sq_svcflags |= SQ_BGTHREAD;
3903 	sq->sq_servcount = 1;
3904 	cv_signal(&syncqs_to_run);
3905 	mutex_exit(&service_queue);
3906 }
3907 
3908 /*
3909  * Note: fifo_close() depends on the mblk_t on the queue being freed
3910  * asynchronously. The asynchronous freeing of messages breaks the
3911  * recursive call chain of fifo_close() while there are I_SENDFD type of
3912  * messages referring to other file pointers on the queue. Then when
3913  * closing pipes it can avoid stack overflow in case of daisy-chained
3914  * pipes, and also avoid deadlock in case of fifonode_t pairs (which
3915  * share the same fifolock_t).
3916  *
3917  * No need to kpreempt_disable to access cpu_seqid.  If we migrate and
3918  * the esb queue does not match the new CPU, that is OK.
3919  */
3920 void
freebs_enqueue(mblk_t * mp,dblk_t * dbp)3921 freebs_enqueue(mblk_t *mp, dblk_t *dbp)
3922 {
3923 	int qindex = CPU->cpu_seqid >> esbq_log2_cpus_per_q;
3924 	esb_queue_t *eqp;
3925 
3926 	ASSERT(dbp->db_mblk == mp);
3927 	ASSERT(qindex < esbq_nelem);
3928 
3929 	eqp = system_esbq_array;
3930 	if (eqp != NULL) {
3931 		eqp += qindex;
3932 	} else {
3933 		mutex_enter(&esbq_lock);
3934 		if (kmem_ready && system_esbq_array == NULL)
3935 			system_esbq_array = (esb_queue_t *)kmem_zalloc(
3936 			    esbq_nelem * sizeof (esb_queue_t), KM_NOSLEEP);
3937 		mutex_exit(&esbq_lock);
3938 		eqp = system_esbq_array;
3939 		if (eqp != NULL)
3940 			eqp += qindex;
3941 		else
3942 			eqp = &system_esbq;
3943 	}
3944 
3945 	/*
3946 	 * Check data sanity. The dblock should have non-empty free function.
3947 	 * It is better to panic here then later when the dblock is freed
3948 	 * asynchronously when the context is lost.
3949 	 */
3950 	if (dbp->db_frtnp->free_func == NULL) {
3951 		panic("freebs_enqueue: dblock %p has a NULL free callback",
3952 		    (void *)dbp);
3953 	}
3954 
3955 	mutex_enter(&eqp->eq_lock);
3956 	/* queue the new mblk on the esballoc queue */
3957 	if (eqp->eq_head == NULL) {
3958 		eqp->eq_head = eqp->eq_tail = mp;
3959 	} else {
3960 		eqp->eq_tail->b_next = mp;
3961 		eqp->eq_tail = mp;
3962 	}
3963 	eqp->eq_len++;
3964 
3965 	/* If we're the first thread to reach the threshold, process */
3966 	if (eqp->eq_len >= esbq_max_qlen &&
3967 	    !(eqp->eq_flags & ESBQ_PROCESSING))
3968 		esballoc_process_queue(eqp);
3969 
3970 	esballoc_set_timer(eqp, esbq_timeout);
3971 	mutex_exit(&eqp->eq_lock);
3972 }
3973 
3974 static void
esballoc_process_queue(esb_queue_t * eqp)3975 esballoc_process_queue(esb_queue_t *eqp)
3976 {
3977 	mblk_t	*mp;
3978 
3979 	ASSERT(MUTEX_HELD(&eqp->eq_lock));
3980 
3981 	eqp->eq_flags |= ESBQ_PROCESSING;
3982 
3983 	do {
3984 		/*
3985 		 * Detach the message chain for processing.
3986 		 */
3987 		mp = eqp->eq_head;
3988 		eqp->eq_tail->b_next = NULL;
3989 		eqp->eq_head = eqp->eq_tail = NULL;
3990 		eqp->eq_len = 0;
3991 		mutex_exit(&eqp->eq_lock);
3992 
3993 		/*
3994 		 * Process the message chain.
3995 		 */
3996 		esballoc_enqueue_mblk(mp);
3997 		mutex_enter(&eqp->eq_lock);
3998 	} while ((eqp->eq_len >= esbq_max_qlen) && (eqp->eq_len > 0));
3999 
4000 	eqp->eq_flags &= ~ESBQ_PROCESSING;
4001 }
4002 
4003 /*
4004  * taskq callback routine to free esballoced mblk's
4005  */
4006 static void
esballoc_mblk_free(mblk_t * mp)4007 esballoc_mblk_free(mblk_t *mp)
4008 {
4009 	mblk_t	*nextmp;
4010 
4011 	for (; mp != NULL; mp = nextmp) {
4012 		nextmp = mp->b_next;
4013 		mp->b_next = NULL;
4014 		mblk_free(mp);
4015 	}
4016 }
4017 
4018 static void
esballoc_enqueue_mblk(mblk_t * mp)4019 esballoc_enqueue_mblk(mblk_t *mp)
4020 {
4021 
4022 	if (taskq_dispatch(system_taskq, (task_func_t *)esballoc_mblk_free, mp,
4023 	    TQ_NOSLEEP) == NULL) {
4024 		mblk_t *first_mp = mp;
4025 		/*
4026 		 * System is low on resources and can't perform a non-sleeping
4027 		 * dispatch. Schedule for a background thread.
4028 		 */
4029 		mutex_enter(&service_queue);
4030 		STRSTAT(taskqfails);
4031 
4032 		while (mp->b_next != NULL)
4033 			mp = mp->b_next;
4034 
4035 		mp->b_next = freebs_list;
4036 		freebs_list = first_mp;
4037 		cv_signal(&services_to_run);
4038 		mutex_exit(&service_queue);
4039 	}
4040 }
4041 
4042 static void
esballoc_timer(void * arg)4043 esballoc_timer(void *arg)
4044 {
4045 	esb_queue_t *eqp = arg;
4046 
4047 	mutex_enter(&eqp->eq_lock);
4048 	eqp->eq_flags &= ~ESBQ_TIMER;
4049 
4050 	if (!(eqp->eq_flags & ESBQ_PROCESSING) &&
4051 	    eqp->eq_len > 0)
4052 		esballoc_process_queue(eqp);
4053 
4054 	esballoc_set_timer(eqp, esbq_timeout);
4055 	mutex_exit(&eqp->eq_lock);
4056 }
4057 
4058 static void
esballoc_set_timer(esb_queue_t * eqp,clock_t eq_timeout)4059 esballoc_set_timer(esb_queue_t *eqp, clock_t eq_timeout)
4060 {
4061 	ASSERT(MUTEX_HELD(&eqp->eq_lock));
4062 
4063 	if (eqp->eq_len > 0 && !(eqp->eq_flags & ESBQ_TIMER)) {
4064 		(void) timeout(esballoc_timer, eqp, eq_timeout);
4065 		eqp->eq_flags |= ESBQ_TIMER;
4066 	}
4067 }
4068 
4069 /*
4070  * Setup esbq array length based upon NCPU scaled by CPUs per
4071  * queue. Use static system_esbq until kmem_ready and we can
4072  * create an array in freebs_enqueue().
4073  */
4074 void
esballoc_queue_init(void)4075 esballoc_queue_init(void)
4076 {
4077 	esbq_log2_cpus_per_q = highbit(esbq_cpus_per_q - 1);
4078 	esbq_cpus_per_q = 1 << esbq_log2_cpus_per_q;
4079 	esbq_nelem = howmany(NCPU, esbq_cpus_per_q);
4080 	system_esbq.eq_len = 0;
4081 	system_esbq.eq_head = system_esbq.eq_tail = NULL;
4082 	system_esbq.eq_flags = 0;
4083 }
4084 
4085 /*
4086  * Set the QBACK or QB_BACK flag in the given queue for
4087  * the given priority band.
4088  */
4089 void
setqback(queue_t * q,unsigned char pri)4090 setqback(queue_t *q, unsigned char pri)
4091 {
4092 	int i;
4093 	qband_t *qbp;
4094 	qband_t **qbpp;
4095 
4096 	ASSERT(MUTEX_HELD(QLOCK(q)));
4097 	if (pri != 0) {
4098 		if (pri > q->q_nband) {
4099 			qbpp = &q->q_bandp;
4100 			while (*qbpp)
4101 				qbpp = &(*qbpp)->qb_next;
4102 			while (pri > q->q_nband) {
4103 				if ((*qbpp = allocband()) == NULL) {
4104 					cmn_err(CE_WARN,
4105 					    "setqback: can't allocate qband\n");
4106 					return;
4107 				}
4108 				(*qbpp)->qb_hiwat = q->q_hiwat;
4109 				(*qbpp)->qb_lowat = q->q_lowat;
4110 				q->q_nband++;
4111 				qbpp = &(*qbpp)->qb_next;
4112 			}
4113 		}
4114 		qbp = q->q_bandp;
4115 		i = pri;
4116 		while (--i)
4117 			qbp = qbp->qb_next;
4118 		qbp->qb_flag |= QB_BACK;
4119 	} else {
4120 		q->q_flag |= QBACK;
4121 	}
4122 }
4123 
4124 int
strcopyin(void * from,void * to,size_t len,int copyflag)4125 strcopyin(void *from, void *to, size_t len, int copyflag)
4126 {
4127 	if (copyflag & U_TO_K) {
4128 		ASSERT((copyflag & K_TO_K) == 0);
4129 		if (copyin(from, to, len))
4130 			return (EFAULT);
4131 	} else {
4132 		ASSERT(copyflag & K_TO_K);
4133 		bcopy(from, to, len);
4134 	}
4135 	return (0);
4136 }
4137 
4138 int
strcopyout(void * from,void * to,size_t len,int copyflag)4139 strcopyout(void *from, void *to, size_t len, int copyflag)
4140 {
4141 	if (copyflag & U_TO_K) {
4142 		if (copyout(from, to, len))
4143 			return (EFAULT);
4144 	} else {
4145 		ASSERT(copyflag & K_TO_K);
4146 		bcopy(from, to, len);
4147 	}
4148 	return (0);
4149 }
4150 
4151 /*
4152  * strsignal_nolock() posts a signal to the process(es) at the stream head.
4153  * It assumes that the stream head lock is already held, whereas strsignal()
4154  * acquires the lock first.  This routine was created because a few callers
4155  * release the stream head lock before calling only to re-acquire it after
4156  * it returns.
4157  */
4158 void
strsignal_nolock(stdata_t * stp,int sig,uchar_t band)4159 strsignal_nolock(stdata_t *stp, int sig, uchar_t band)
4160 {
4161 	ASSERT(MUTEX_HELD(&stp->sd_lock));
4162 	switch (sig) {
4163 	case SIGPOLL:
4164 		if (stp->sd_sigflags & S_MSG)
4165 			strsendsig(stp->sd_siglist, S_MSG, band, 0);
4166 		break;
4167 	default:
4168 		if (stp->sd_pgidp)
4169 			pgsignal(stp->sd_pgidp, sig);
4170 		break;
4171 	}
4172 }
4173 
4174 void
strsignal(stdata_t * stp,int sig,int32_t band)4175 strsignal(stdata_t *stp, int sig, int32_t band)
4176 {
4177 	TRACE_3(TR_FAC_STREAMS_FR, TR_SENDSIG,
4178 	    "strsignal:%p, %X, %X", stp, sig, band);
4179 
4180 	mutex_enter(&stp->sd_lock);
4181 	switch (sig) {
4182 	case SIGPOLL:
4183 		if (stp->sd_sigflags & S_MSG)
4184 			strsendsig(stp->sd_siglist, S_MSG, (uchar_t)band, 0);
4185 		break;
4186 
4187 	default:
4188 		if (stp->sd_pgidp) {
4189 			pgsignal(stp->sd_pgidp, sig);
4190 		}
4191 		break;
4192 	}
4193 	mutex_exit(&stp->sd_lock);
4194 }
4195 
4196 void
strhup(stdata_t * stp)4197 strhup(stdata_t *stp)
4198 {
4199 	ASSERT(mutex_owned(&stp->sd_lock));
4200 	pollwakeup(&stp->sd_pollist, POLLHUP);
4201 	if (stp->sd_sigflags & S_HANGUP)
4202 		strsendsig(stp->sd_siglist, S_HANGUP, 0, 0);
4203 }
4204 
4205 /*
4206  * Backenable the first queue upstream from `q' with a service procedure.
4207  */
4208 void
backenable(queue_t * q,uchar_t pri)4209 backenable(queue_t *q, uchar_t pri)
4210 {
4211 	queue_t	*nq;
4212 
4213 	/*
4214 	 * Our presence might not prevent other modules in our own
4215 	 * stream from popping/pushing since the caller of getq might not
4216 	 * have a claim on the queue (some drivers do a getq on somebody
4217 	 * else's queue - they know that the queue itself is not going away
4218 	 * but the framework has to guarantee q_next in that stream).
4219 	 */
4220 	claimstr(q);
4221 
4222 	/* Find nearest back queue with service proc */
4223 	for (nq = backq(q); nq && !nq->q_qinfo->qi_srvp; nq = backq(nq)) {
4224 		ASSERT(STRMATED(q->q_stream) || STREAM(q) == STREAM(nq));
4225 	}
4226 
4227 	if (nq) {
4228 		kthread_t *freezer;
4229 		/*
4230 		 * backenable can be called either with no locks held
4231 		 * or with the stream frozen (the latter occurs when a module
4232 		 * calls rmvq with the stream frozen). If the stream is frozen
4233 		 * by the caller the caller will hold all qlocks in the stream.
4234 		 * Note that a frozen stream doesn't freeze a mated stream,
4235 		 * so we explicitly check for that.
4236 		 */
4237 		freezer = STREAM(q)->sd_freezer;
4238 		if (freezer != curthread || STREAM(q) != STREAM(nq)) {
4239 			mutex_enter(QLOCK(nq));
4240 		}
4241 #ifdef DEBUG
4242 		else {
4243 			ASSERT(frozenstr(q));
4244 			ASSERT(MUTEX_HELD(QLOCK(q)));
4245 			ASSERT(MUTEX_HELD(QLOCK(nq)));
4246 		}
4247 #endif
4248 		setqback(nq, pri);
4249 		qenable_locked(nq);
4250 		if (freezer != curthread || STREAM(q) != STREAM(nq))
4251 			mutex_exit(QLOCK(nq));
4252 	}
4253 	releasestr(q);
4254 }
4255 
4256 /*
4257  * Return the appropriate errno when one of flags_to_check is set
4258  * in sd_flags. Uses the exported error routines if they are set.
4259  * Will return 0 if non error is set (or if the exported error routines
4260  * do not return an error).
4261  *
4262  * If there is both a read and write error to check, we prefer the read error.
4263  * Also, give preference to recorded errno's over the error functions.
4264  * The flags that are handled are:
4265  *	STPLEX		return EINVAL
4266  *	STRDERR		return sd_rerror (and clear if STRDERRNONPERSIST)
4267  *	STWRERR		return sd_werror (and clear if STWRERRNONPERSIST)
4268  *	STRHUP		return sd_werror
4269  *
4270  * If the caller indicates that the operation is a peek, a nonpersistent error
4271  * is not cleared.
4272  */
4273 int
strgeterr(stdata_t * stp,int32_t flags_to_check,int ispeek)4274 strgeterr(stdata_t *stp, int32_t flags_to_check, int ispeek)
4275 {
4276 	int32_t sd_flag = stp->sd_flag & flags_to_check;
4277 	int error = 0;
4278 
4279 	ASSERT(MUTEX_HELD(&stp->sd_lock));
4280 	ASSERT((flags_to_check & ~(STRDERR|STWRERR|STRHUP|STPLEX)) == 0);
4281 	if (sd_flag & STPLEX)
4282 		error = EINVAL;
4283 	else if (sd_flag & STRDERR) {
4284 		error = stp->sd_rerror;
4285 		if ((stp->sd_flag & STRDERRNONPERSIST) && !ispeek) {
4286 			/*
4287 			 * Read errors are non-persistent i.e. discarded once
4288 			 * returned to a non-peeking caller,
4289 			 */
4290 			stp->sd_rerror = 0;
4291 			stp->sd_flag &= ~STRDERR;
4292 		}
4293 		if (error == 0 && stp->sd_rderrfunc != NULL) {
4294 			int clearerr = 0;
4295 
4296 			error = (*stp->sd_rderrfunc)(stp->sd_vnode, ispeek,
4297 			    &clearerr);
4298 			if (clearerr) {
4299 				stp->sd_flag &= ~STRDERR;
4300 				stp->sd_rderrfunc = NULL;
4301 			}
4302 		}
4303 	} else if (sd_flag & STWRERR) {
4304 		error = stp->sd_werror;
4305 		if ((stp->sd_flag & STWRERRNONPERSIST) && !ispeek) {
4306 			/*
4307 			 * Write errors are non-persistent i.e. discarded once
4308 			 * returned to a non-peeking caller,
4309 			 */
4310 			stp->sd_werror = 0;
4311 			stp->sd_flag &= ~STWRERR;
4312 		}
4313 		if (error == 0 && stp->sd_wrerrfunc != NULL) {
4314 			int clearerr = 0;
4315 
4316 			error = (*stp->sd_wrerrfunc)(stp->sd_vnode, ispeek,
4317 			    &clearerr);
4318 			if (clearerr) {
4319 				stp->sd_flag &= ~STWRERR;
4320 				stp->sd_wrerrfunc = NULL;
4321 			}
4322 		}
4323 	} else if (sd_flag & STRHUP) {
4324 		/* sd_werror set when STRHUP */
4325 		error = stp->sd_werror;
4326 	}
4327 	return (error);
4328 }
4329 
4330 
4331 /*
4332  * Single-thread open/close/push/pop
4333  * for twisted streams also
4334  */
4335 int
strstartplumb(stdata_t * stp,int flag,int cmd)4336 strstartplumb(stdata_t *stp, int flag, int cmd)
4337 {
4338 	int waited = 1;
4339 	int error = 0;
4340 
4341 	if (STRMATED(stp)) {
4342 		struct stdata *stmatep = stp->sd_mate;
4343 
4344 		STRLOCKMATES(stp);
4345 		while (waited) {
4346 			waited = 0;
4347 			while (stmatep->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4348 				if ((cmd == I_POP) &&
4349 				    (flag & (FNDELAY|FNONBLOCK))) {
4350 					STRUNLOCKMATES(stp);
4351 					return (EAGAIN);
4352 				}
4353 				waited = 1;
4354 				mutex_exit(&stp->sd_lock);
4355 				if (!cv_wait_sig(&stmatep->sd_monitor,
4356 				    &stmatep->sd_lock)) {
4357 					mutex_exit(&stmatep->sd_lock);
4358 					return (EINTR);
4359 				}
4360 				mutex_exit(&stmatep->sd_lock);
4361 				STRLOCKMATES(stp);
4362 			}
4363 			while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4364 				if ((cmd == I_POP) &&
4365 				    (flag & (FNDELAY|FNONBLOCK))) {
4366 					STRUNLOCKMATES(stp);
4367 					return (EAGAIN);
4368 				}
4369 				waited = 1;
4370 				mutex_exit(&stmatep->sd_lock);
4371 				if (!cv_wait_sig(&stp->sd_monitor,
4372 				    &stp->sd_lock)) {
4373 					mutex_exit(&stp->sd_lock);
4374 					return (EINTR);
4375 				}
4376 				mutex_exit(&stp->sd_lock);
4377 				STRLOCKMATES(stp);
4378 			}
4379 			if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4380 				error = strgeterr(stp,
4381 				    STRDERR|STWRERR|STRHUP|STPLEX, 0);
4382 				if (error != 0) {
4383 					STRUNLOCKMATES(stp);
4384 					return (error);
4385 				}
4386 			}
4387 		}
4388 		stp->sd_flag |= STRPLUMB;
4389 		STRUNLOCKMATES(stp);
4390 	} else {
4391 		mutex_enter(&stp->sd_lock);
4392 		while (stp->sd_flag & (STWOPEN|STRCLOSE|STRPLUMB)) {
4393 			if (((cmd == I_POP) || (cmd == _I_REMOVE)) &&
4394 			    (flag & (FNDELAY|FNONBLOCK))) {
4395 				mutex_exit(&stp->sd_lock);
4396 				return (EAGAIN);
4397 			}
4398 			if (!cv_wait_sig(&stp->sd_monitor, &stp->sd_lock)) {
4399 				mutex_exit(&stp->sd_lock);
4400 				return (EINTR);
4401 			}
4402 			if (stp->sd_flag & (STRDERR|STWRERR|STRHUP|STPLEX)) {
4403 				error = strgeterr(stp,
4404 				    STRDERR|STWRERR|STRHUP|STPLEX, 0);
4405 				if (error != 0) {
4406 					mutex_exit(&stp->sd_lock);
4407 					return (error);
4408 				}
4409 			}
4410 		}
4411 		stp->sd_flag |= STRPLUMB;
4412 		mutex_exit(&stp->sd_lock);
4413 	}
4414 	return (0);
4415 }
4416 
4417 /*
4418  * Complete the plumbing operation associated with stream `stp'.
4419  */
4420 void
strendplumb(stdata_t * stp)4421 strendplumb(stdata_t *stp)
4422 {
4423 	ASSERT(MUTEX_HELD(&stp->sd_lock));
4424 	ASSERT(stp->sd_flag & STRPLUMB);
4425 	stp->sd_flag &= ~STRPLUMB;
4426 	cv_broadcast(&stp->sd_monitor);
4427 }
4428 
4429 /*
4430  * This describes how the STREAMS framework handles synchronization
4431  * during open/push and close/pop.
4432  * The key interfaces for open and close are qprocson and qprocsoff,
4433  * respectively. While the close case in general is harder both open
4434  * have close have significant similarities.
4435  *
4436  * During close the STREAMS framework has to both ensure that there
4437  * are no stale references to the queue pair (and syncq) that
4438  * are being closed and also provide the guarantees that are documented
4439  * in qprocsoff(9F).
4440  * If there are stale references to the queue that is closing it can
4441  * result in kernel memory corruption or kernel panics.
4442  *
4443  * Note that is it up to the module/driver to ensure that it itself
4444  * does not have any stale references to the closing queues once its close
4445  * routine returns. This includes:
4446  *  - Cancelling any timeout/bufcall/qtimeout/qbufcall callback routines
4447  *    associated with the queues. For timeout and bufcall callbacks the
4448  *    module/driver also has to ensure (or wait for) any callbacks that
4449  *    are in progress.
4450  *  - If the module/driver is using esballoc it has to ensure that any
4451  *    esballoc free functions do not refer to a queue that has closed.
4452  *    (Note that in general the close routine can not wait for the esballoc'ed
4453  *    messages to be freed since that can cause a deadlock.)
4454  *  - Cancelling any interrupts that refer to the closing queues and
4455  *    also ensuring that there are no interrupts in progress that will
4456  *    refer to the closing queues once the close routine returns.
4457  *  - For multiplexors removing any driver global state that refers to
4458  *    the closing queue and also ensuring that there are no threads in
4459  *    the multiplexor that has picked up a queue pointer but not yet
4460  *    finished using it.
4461  *
4462  * In addition, a driver/module can only reference the q_next pointer
4463  * in its open, close, put, or service procedures or in a
4464  * qtimeout/qbufcall callback procedure executing "on" the correct
4465  * stream. Thus it can not reference the q_next pointer in an interrupt
4466  * routine or a timeout, bufcall or esballoc callback routine. Likewise
4467  * it can not reference q_next of a different queue e.g. in a mux that
4468  * passes messages from one queues put/service procedure to another queue.
4469  * In all the cases when the driver/module can not access the q_next
4470  * field it must use the *next* versions e.g. canputnext instead of
4471  * canput(q->q_next) and putnextctl instead of putctl(q->q_next, ...).
4472  *
4473  *
4474  * Assuming that the driver/module conforms to the above constraints
4475  * the STREAMS framework has to avoid stale references to q_next for all
4476  * the framework internal cases which include (but are not limited to):
4477  *  - Threads in canput/canputnext/backenable and elsewhere that are
4478  *    walking q_next.
4479  *  - Messages on a syncq that have a reference to the queue through b_queue.
4480  *  - Messages on an outer perimeter (syncq) that have a reference to the
4481  *    queue through b_queue.
4482  *  - Threads that use q_nfsrv (e.g. canput) to find a queue.
4483  *    Note that only canput and bcanput use q_nfsrv without any locking.
4484  *
4485  * The STREAMS framework providing the qprocsoff(9F) guarantees means that
4486  * after qprocsoff returns, the framework has to ensure that no threads can
4487  * enter the put or service routines for the closing read or write-side queue.
4488  * In addition to preventing "direct" entry into the put procedures
4489  * the framework also has to prevent messages being drained from
4490  * the syncq or the outer perimeter.
4491  * XXX Note that currently qdetach does relies on D_MTOCEXCL as the only
4492  * mechanism to prevent qwriter(PERIM_OUTER) from running after
4493  * qprocsoff has returned.
4494  * Note that if a module/driver uses put(9F) on one of its own queues
4495  * it is up to the module/driver to ensure that the put() doesn't
4496  * get called when the queue is closing.
4497  *
4498  *
4499  * The framework aspects of the above "contract" is implemented by
4500  * qprocsoff, removeq, and strlock:
4501  *  - qprocsoff (disable_svc) sets QWCLOSE to prevent runservice from
4502  *    entering the service procedures.
4503  *  - strlock acquires the sd_lock and sd_reflock to prevent putnext,
4504  *    canputnext, backenable etc from dereferencing the q_next that will
4505  *    soon change.
4506  *  - strlock waits for sd_refcnt to be zero to wait for e.g. any canputnext
4507  *    or other q_next walker that uses claimstr/releasestr to finish.
4508  *  - optionally for every syncq in the stream strlock acquires all the
4509  *    sq_lock's and waits for all sq_counts to drop to a value that indicates
4510  *    that no thread executes in the put or service procedures and that no
4511  *    thread is draining into the module/driver. This ensures that no
4512  *    open, close, put, service, or qtimeout/qbufcall callback procedure is
4513  *    currently executing hence no such thread can end up with the old stale
4514  *    q_next value and no canput/backenable can have the old stale
4515  *    q_nfsrv/q_next.
4516  *  - qdetach (wait_svc) makes sure that any scheduled or running threads
4517  *    have either finished or observed the QWCLOSE flag and gone away.
4518  */
4519 
4520 
4521 /*
4522  * Get all the locks necessary to change q_next.
4523  *
4524  * Wait for sd_refcnt to reach 0 and, if sqlist is present, wait for the
4525  * sq_count of each syncq in the list to drop to sq_rmqcount, indicating that
4526  * the only threads inside the syncq are threads currently calling removeq().
4527  * Since threads calling removeq() are in the process of removing their queues
4528  * from the stream, we do not need to worry about them accessing a stale q_next
4529  * pointer and thus we do not need to wait for them to exit (in fact, waiting
4530  * for them can cause deadlock).
4531  *
4532  * This routine is subject to starvation since it does not set any flag to
4533  * prevent threads from entering a module in the stream (i.e. sq_count can
4534  * increase on some syncq while it is waiting on some other syncq).
4535  *
4536  * Assumes that only one thread attempts to call strlock for a given
4537  * stream. If this is not the case the two threads would deadlock.
4538  * This assumption is guaranteed since strlock is only called by insertq
4539  * and removeq and streams plumbing changes are single-threaded for
4540  * a given stream using the STWOPEN, STRCLOSE, and STRPLUMB flags.
4541  *
4542  * For pipes, it is not difficult to atomically designate a pair of streams
4543  * to be mated. Once mated atomically by the framework the twisted pair remain
4544  * configured that way until dismantled atomically by the framework.
4545  * When plumbing takes place on a twisted stream it is necessary to ensure that
4546  * this operation is done exclusively on the twisted stream since two such
4547  * operations, each initiated on different ends of the pipe will deadlock
4548  * waiting for each other to complete.
4549  *
4550  * On entry, no locks should be held.
4551  * The locks acquired and held by strlock depends on a few factors.
4552  * - If sqlist is non-NULL all the syncq locks in the sqlist will be acquired
4553  *   and held on exit and all sq_count are at an acceptable level.
4554  * - In all cases, sd_lock and sd_reflock are acquired and held on exit with
4555  *   sd_refcnt being zero.
4556  */
4557 
4558 static void
strlock(struct stdata * stp,sqlist_t * sqlist)4559 strlock(struct stdata *stp, sqlist_t *sqlist)
4560 {
4561 	syncql_t *sql, *sql2;
4562 retry:
4563 	/*
4564 	 * Wait for any claimstr to go away.
4565 	 */
4566 	if (STRMATED(stp)) {
4567 		struct stdata *stp1, *stp2;
4568 
4569 		STRLOCKMATES(stp);
4570 		/*
4571 		 * Note that the selection of locking order is not
4572 		 * important, just that they are always acquired in
4573 		 * the same order.  To assure this, we choose this
4574 		 * order based on the value of the pointer, and since
4575 		 * the pointer will not change for the life of this
4576 		 * pair, we will always grab the locks in the same
4577 		 * order (and hence, prevent deadlocks).
4578 		 */
4579 		if (&(stp->sd_lock) > &((stp->sd_mate)->sd_lock)) {
4580 			stp1 = stp;
4581 			stp2 = stp->sd_mate;
4582 		} else {
4583 			stp2 = stp;
4584 			stp1 = stp->sd_mate;
4585 		}
4586 		mutex_enter(&stp1->sd_reflock);
4587 		if (stp1->sd_refcnt > 0) {
4588 			STRUNLOCKMATES(stp);
4589 			cv_wait(&stp1->sd_refmonitor, &stp1->sd_reflock);
4590 			mutex_exit(&stp1->sd_reflock);
4591 			goto retry;
4592 		}
4593 		mutex_enter(&stp2->sd_reflock);
4594 		if (stp2->sd_refcnt > 0) {
4595 			STRUNLOCKMATES(stp);
4596 			mutex_exit(&stp1->sd_reflock);
4597 			cv_wait(&stp2->sd_refmonitor, &stp2->sd_reflock);
4598 			mutex_exit(&stp2->sd_reflock);
4599 			goto retry;
4600 		}
4601 		STREAM_PUTLOCKS_ENTER(stp1);
4602 		STREAM_PUTLOCKS_ENTER(stp2);
4603 	} else {
4604 		mutex_enter(&stp->sd_lock);
4605 		mutex_enter(&stp->sd_reflock);
4606 		while (stp->sd_refcnt > 0) {
4607 			mutex_exit(&stp->sd_lock);
4608 			cv_wait(&stp->sd_refmonitor, &stp->sd_reflock);
4609 			if (mutex_tryenter(&stp->sd_lock) == 0) {
4610 				mutex_exit(&stp->sd_reflock);
4611 				mutex_enter(&stp->sd_lock);
4612 				mutex_enter(&stp->sd_reflock);
4613 			}
4614 		}
4615 		STREAM_PUTLOCKS_ENTER(stp);
4616 	}
4617 
4618 	if (sqlist == NULL)
4619 		return;
4620 
4621 	for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4622 		syncq_t *sq = sql->sql_sq;
4623 		uint16_t count;
4624 
4625 		mutex_enter(SQLOCK(sq));
4626 		count = sq->sq_count;
4627 		ASSERT(sq->sq_rmqcount <= count);
4628 		SQ_PUTLOCKS_ENTER(sq);
4629 		SUM_SQ_PUTCOUNTS(sq, count);
4630 		if (count == sq->sq_rmqcount)
4631 			continue;
4632 
4633 		/* Failed - drop all locks that we have acquired so far */
4634 		if (STRMATED(stp)) {
4635 			STREAM_PUTLOCKS_EXIT(stp);
4636 			STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4637 			STRUNLOCKMATES(stp);
4638 			mutex_exit(&stp->sd_reflock);
4639 			mutex_exit(&stp->sd_mate->sd_reflock);
4640 		} else {
4641 			STREAM_PUTLOCKS_EXIT(stp);
4642 			mutex_exit(&stp->sd_lock);
4643 			mutex_exit(&stp->sd_reflock);
4644 		}
4645 		for (sql2 = sqlist->sqlist_head; sql2 != sql;
4646 		    sql2 = sql2->sql_next) {
4647 			SQ_PUTLOCKS_EXIT(sql2->sql_sq);
4648 			mutex_exit(SQLOCK(sql2->sql_sq));
4649 		}
4650 
4651 		/*
4652 		 * The wait loop below may starve when there are many threads
4653 		 * claiming the syncq. This is especially a problem with permod
4654 		 * syncqs (IP). To lessen the impact of the problem we increment
4655 		 * sq_needexcl and clear fastbits so that putnexts will slow
4656 		 * down and call sqenable instead of draining right away.
4657 		 */
4658 		sq->sq_needexcl++;
4659 		SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
4660 		while (count > sq->sq_rmqcount) {
4661 			sq->sq_flags |= SQ_WANTWAKEUP;
4662 			SQ_PUTLOCKS_EXIT(sq);
4663 			cv_wait(&sq->sq_wait, SQLOCK(sq));
4664 			count = sq->sq_count;
4665 			SQ_PUTLOCKS_ENTER(sq);
4666 			SUM_SQ_PUTCOUNTS(sq, count);
4667 		}
4668 		sq->sq_needexcl--;
4669 		if (sq->sq_needexcl == 0)
4670 			SQ_PUTCOUNT_SETFAST_LOCKED(sq);
4671 		SQ_PUTLOCKS_EXIT(sq);
4672 		ASSERT(count == sq->sq_rmqcount);
4673 		mutex_exit(SQLOCK(sq));
4674 		goto retry;
4675 	}
4676 }
4677 
4678 /*
4679  * Drop all the locks that strlock acquired.
4680  */
4681 static void
strunlock(struct stdata * stp,sqlist_t * sqlist)4682 strunlock(struct stdata *stp, sqlist_t *sqlist)
4683 {
4684 	syncql_t *sql;
4685 
4686 	if (STRMATED(stp)) {
4687 		STREAM_PUTLOCKS_EXIT(stp);
4688 		STREAM_PUTLOCKS_EXIT(stp->sd_mate);
4689 		STRUNLOCKMATES(stp);
4690 		mutex_exit(&stp->sd_reflock);
4691 		mutex_exit(&stp->sd_mate->sd_reflock);
4692 	} else {
4693 		STREAM_PUTLOCKS_EXIT(stp);
4694 		mutex_exit(&stp->sd_lock);
4695 		mutex_exit(&stp->sd_reflock);
4696 	}
4697 
4698 	if (sqlist == NULL)
4699 		return;
4700 
4701 	for (sql = sqlist->sqlist_head; sql; sql = sql->sql_next) {
4702 		SQ_PUTLOCKS_EXIT(sql->sql_sq);
4703 		mutex_exit(SQLOCK(sql->sql_sq));
4704 	}
4705 }
4706 
4707 /*
4708  * When the module has service procedure, we need check if the next
4709  * module which has service procedure is in flow control to trigger
4710  * the backenable.
4711  */
4712 static void
backenable_insertedq(queue_t * q)4713 backenable_insertedq(queue_t *q)
4714 {
4715 	qband_t	*qbp;
4716 
4717 	claimstr(q);
4718 	if (q->q_qinfo->qi_srvp != NULL && q->q_next != NULL) {
4719 		if (q->q_next->q_nfsrv->q_flag & QWANTW)
4720 			backenable(q, 0);
4721 
4722 		qbp = q->q_next->q_nfsrv->q_bandp;
4723 		for (; qbp != NULL; qbp = qbp->qb_next)
4724 			if ((qbp->qb_flag & QB_WANTW) && qbp->qb_first != NULL)
4725 				backenable(q, qbp->qb_first->b_band);
4726 	}
4727 	releasestr(q);
4728 }
4729 
4730 /*
4731  * Given two read queues, insert a new single one after another.
4732  *
4733  * This routine acquires all the necessary locks in order to change
4734  * q_next and related pointer using strlock().
4735  * It depends on the stream head ensuring that there are no concurrent
4736  * insertq or removeq on the same stream. The stream head ensures this
4737  * using the flags STWOPEN, STRCLOSE, and STRPLUMB.
4738  *
4739  * Note that no syncq locks are held during the q_next change. This is
4740  * applied to all streams since, unlike removeq, there is no problem of stale
4741  * pointers when adding a module to the stream. Thus drivers/modules that do a
4742  * canput(rq->q_next) would never get a closed/freed queue pointer even if we
4743  * applied this optimization to all streams.
4744  */
4745 void
insertq(struct stdata * stp,queue_t * new)4746 insertq(struct stdata *stp, queue_t *new)
4747 {
4748 	queue_t	*after;
4749 	queue_t *wafter;
4750 	queue_t *wnew = _WR(new);
4751 	boolean_t have_fifo = B_FALSE;
4752 
4753 	if (new->q_flag & _QINSERTING) {
4754 		ASSERT(stp->sd_vnode->v_type != VFIFO);
4755 		after = new->q_next;
4756 		wafter = _WR(new->q_next);
4757 	} else {
4758 		after = _RD(stp->sd_wrq);
4759 		wafter = stp->sd_wrq;
4760 	}
4761 
4762 	TRACE_2(TR_FAC_STREAMS_FR, TR_INSERTQ,
4763 	    "insertq:%p, %p", after, new);
4764 	ASSERT(after->q_flag & QREADR);
4765 	ASSERT(new->q_flag & QREADR);
4766 
4767 	strlock(stp, NULL);
4768 
4769 	/* Do we have a FIFO? */
4770 	if (wafter->q_next == after) {
4771 		have_fifo = B_TRUE;
4772 		wnew->q_next = new;
4773 	} else {
4774 		wnew->q_next = wafter->q_next;
4775 	}
4776 	new->q_next = after;
4777 
4778 	set_nfsrv_ptr(new, wnew, after, wafter);
4779 	/*
4780 	 * set_nfsrv_ptr() needs to know if this is an insertion or not,
4781 	 * so only reset this flag after calling it.
4782 	 */
4783 	new->q_flag &= ~_QINSERTING;
4784 
4785 	if (have_fifo) {
4786 		wafter->q_next = wnew;
4787 	} else {
4788 		if (wafter->q_next)
4789 			_OTHERQ(wafter->q_next)->q_next = new;
4790 		wafter->q_next = wnew;
4791 	}
4792 
4793 	set_qend(new);
4794 	/* The QEND flag might have to be updated for the upstream guy */
4795 	set_qend(after);
4796 
4797 	ASSERT(_SAMESTR(new) == O_SAMESTR(new));
4798 	ASSERT(_SAMESTR(wnew) == O_SAMESTR(wnew));
4799 	ASSERT(_SAMESTR(after) == O_SAMESTR(after));
4800 	ASSERT(_SAMESTR(wafter) == O_SAMESTR(wafter));
4801 	strsetuio(stp);
4802 
4803 	/*
4804 	 * If this was a module insertion, bump the push count.
4805 	 */
4806 	if (!(new->q_flag & QISDRV))
4807 		stp->sd_pushcnt++;
4808 
4809 	strunlock(stp, NULL);
4810 
4811 	/* check if the write Q needs backenable */
4812 	backenable_insertedq(wnew);
4813 
4814 	/* check if the read Q needs backenable */
4815 	backenable_insertedq(new);
4816 }
4817 
4818 /*
4819  * Given a read queue, unlink it from any neighbors.
4820  *
4821  * This routine acquires all the necessary locks in order to
4822  * change q_next and related pointers and also guard against
4823  * stale references (e.g. through q_next) to the queue that
4824  * is being removed. It also plays part of the role in ensuring
4825  * that the module's/driver's put procedure doesn't get called
4826  * after qprocsoff returns.
4827  *
4828  * Removeq depends on the stream head ensuring that there are
4829  * no concurrent insertq or removeq on the same stream. The
4830  * stream head ensures this using the flags STWOPEN, STRCLOSE and
4831  * STRPLUMB.
4832  *
4833  * The set of locks needed to remove the queue is different in
4834  * different cases:
4835  *
4836  * Acquire sd_lock, sd_reflock, and all the syncq locks in the stream after
4837  * waiting for the syncq reference count to drop to 0 indicating that no
4838  * non-close threads are present anywhere in the stream. This ensures that any
4839  * module/driver can reference q_next in its open, close, put, or service
4840  * procedures.
4841  *
4842  * The sq_rmqcount counter tracks the number of threads inside removeq().
4843  * strlock() ensures that there is either no threads executing inside perimeter
4844  * or there is only a thread calling qprocsoff().
4845  *
4846  * strlock() compares the value of sq_count with the number of threads inside
4847  * removeq() and waits until sq_count is equal to sq_rmqcount. We need to wakeup
4848  * any threads waiting in strlock() when the sq_rmqcount increases.
4849  */
4850 
4851 void
removeq(queue_t * qp)4852 removeq(queue_t *qp)
4853 {
4854 	queue_t *wqp = _WR(qp);
4855 	struct stdata *stp = STREAM(qp);
4856 	sqlist_t *sqlist = NULL;
4857 	boolean_t isdriver;
4858 	int moved;
4859 	syncq_t *sq = qp->q_syncq;
4860 	syncq_t *wsq = wqp->q_syncq;
4861 
4862 	ASSERT(stp);
4863 
4864 	TRACE_2(TR_FAC_STREAMS_FR, TR_REMOVEQ,
4865 	    "removeq:%p %p", qp, wqp);
4866 	ASSERT(qp->q_flag&QREADR);
4867 
4868 	/*
4869 	 * For queues using Synchronous streams, we must wait for all threads in
4870 	 * rwnext() to drain out before proceeding.
4871 	 */
4872 	if (qp->q_flag & QSYNCSTR) {
4873 		/* First, we need wakeup any threads blocked in rwnext() */
4874 		mutex_enter(SQLOCK(sq));
4875 		if (sq->sq_flags & SQ_WANTWAKEUP) {
4876 			sq->sq_flags &= ~SQ_WANTWAKEUP;
4877 			cv_broadcast(&sq->sq_wait);
4878 		}
4879 		mutex_exit(SQLOCK(sq));
4880 
4881 		if (wsq != sq) {
4882 			mutex_enter(SQLOCK(wsq));
4883 			if (wsq->sq_flags & SQ_WANTWAKEUP) {
4884 				wsq->sq_flags &= ~SQ_WANTWAKEUP;
4885 				cv_broadcast(&wsq->sq_wait);
4886 			}
4887 			mutex_exit(SQLOCK(wsq));
4888 		}
4889 
4890 		mutex_enter(QLOCK(qp));
4891 		while (qp->q_rwcnt > 0) {
4892 			qp->q_flag |= QWANTRMQSYNC;
4893 			cv_wait(&qp->q_wait, QLOCK(qp));
4894 		}
4895 		mutex_exit(QLOCK(qp));
4896 
4897 		mutex_enter(QLOCK(wqp));
4898 		while (wqp->q_rwcnt > 0) {
4899 			wqp->q_flag |= QWANTRMQSYNC;
4900 			cv_wait(&wqp->q_wait, QLOCK(wqp));
4901 		}
4902 		mutex_exit(QLOCK(wqp));
4903 	}
4904 
4905 	mutex_enter(SQLOCK(sq));
4906 	sq->sq_rmqcount++;
4907 	if (sq->sq_flags & SQ_WANTWAKEUP) {
4908 		sq->sq_flags &= ~SQ_WANTWAKEUP;
4909 		cv_broadcast(&sq->sq_wait);
4910 	}
4911 	mutex_exit(SQLOCK(sq));
4912 
4913 	isdriver = (qp->q_flag & QISDRV);
4914 
4915 	sqlist = sqlist_build(qp, stp, STRMATED(stp));
4916 	strlock(stp, sqlist);
4917 
4918 	reset_nfsrv_ptr(qp, wqp);
4919 
4920 	ASSERT(wqp->q_next == NULL || backq(qp)->q_next == qp);
4921 	ASSERT(qp->q_next == NULL || backq(wqp)->q_next == wqp);
4922 	/* Do we have a FIFO? */
4923 	if (wqp->q_next == qp) {
4924 		stp->sd_wrq->q_next = _RD(stp->sd_wrq);
4925 	} else {
4926 		if (wqp->q_next)
4927 			backq(qp)->q_next = qp->q_next;
4928 		if (qp->q_next)
4929 			backq(wqp)->q_next = wqp->q_next;
4930 	}
4931 
4932 	/* The QEND flag might have to be updated for the upstream guy */
4933 	if (qp->q_next)
4934 		set_qend(qp->q_next);
4935 
4936 	ASSERT(_SAMESTR(stp->sd_wrq) == O_SAMESTR(stp->sd_wrq));
4937 	ASSERT(_SAMESTR(_RD(stp->sd_wrq)) == O_SAMESTR(_RD(stp->sd_wrq)));
4938 
4939 	/*
4940 	 * Move any messages destined for the put procedures to the next
4941 	 * syncq in line. Otherwise free them.
4942 	 */
4943 	moved = 0;
4944 	/*
4945 	 * Quick check to see whether there are any messages or events.
4946 	 */
4947 	if (qp->q_syncqmsgs != 0 || (qp->q_syncq->sq_flags & SQ_EVENTS))
4948 		moved += propagate_syncq(qp);
4949 	if (wqp->q_syncqmsgs != 0 ||
4950 	    (wqp->q_syncq->sq_flags & SQ_EVENTS))
4951 		moved += propagate_syncq(wqp);
4952 
4953 	strsetuio(stp);
4954 
4955 	/*
4956 	 * If this was a module removal, decrement the push count.
4957 	 */
4958 	if (!isdriver)
4959 		stp->sd_pushcnt--;
4960 
4961 	strunlock(stp, sqlist);
4962 	sqlist_free(sqlist);
4963 
4964 	/*
4965 	 * Make sure any messages that were propagated are drained.
4966 	 * Also clear any QFULL bit caused by messages that were propagated.
4967 	 */
4968 
4969 	if (qp->q_next != NULL) {
4970 		clr_qfull(qp);
4971 		/*
4972 		 * For the driver calling qprocsoff, propagate_syncq
4973 		 * frees all the messages instead of putting it in
4974 		 * the stream head
4975 		 */
4976 		if (!isdriver && (moved > 0))
4977 			emptysq(qp->q_next->q_syncq);
4978 	}
4979 	if (wqp->q_next != NULL) {
4980 		clr_qfull(wqp);
4981 		/*
4982 		 * We come here for any pop of a module except for the
4983 		 * case of driver being removed. We don't call emptysq
4984 		 * if we did not move any messages. This will avoid holding
4985 		 * PERMOD syncq locks in emptysq
4986 		 */
4987 		if (moved > 0)
4988 			emptysq(wqp->q_next->q_syncq);
4989 	}
4990 
4991 	mutex_enter(SQLOCK(sq));
4992 	sq->sq_rmqcount--;
4993 	mutex_exit(SQLOCK(sq));
4994 }
4995 
4996 /*
4997  * Prevent further entry by setting a flag (like SQ_FROZEN, SQ_BLOCKED or
4998  * SQ_WRITER) on a syncq.
4999  * If maxcnt is not -1 it assumes that caller has "maxcnt" claim(s) on the
5000  * sync queue and waits until sq_count reaches maxcnt.
5001  *
5002  * If maxcnt is -1 there's no need to grab sq_putlocks since the caller
5003  * does not care about putnext threads that are in the middle of calling put
5004  * entry points.
5005  *
5006  * This routine is used for both inner and outer syncqs.
5007  */
5008 static void
blocksq(syncq_t * sq,ushort_t flag,int maxcnt)5009 blocksq(syncq_t *sq, ushort_t flag, int maxcnt)
5010 {
5011 	uint16_t count = 0;
5012 
5013 	mutex_enter(SQLOCK(sq));
5014 	/*
5015 	 * Wait for SQ_FROZEN/SQ_BLOCKED to be reset.
5016 	 * SQ_FROZEN will be set if there is a frozen stream that has a
5017 	 * queue which also refers to this "shared" syncq.
5018 	 * SQ_BLOCKED will be set if there is "off" queue which also
5019 	 * refers to this "shared" syncq.
5020 	 */
5021 	if (maxcnt != -1) {
5022 		count = sq->sq_count;
5023 		SQ_PUTLOCKS_ENTER(sq);
5024 		SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5025 		SUM_SQ_PUTCOUNTS(sq, count);
5026 	}
5027 	sq->sq_needexcl++;
5028 	ASSERT(sq->sq_needexcl != 0);	/* wraparound */
5029 
5030 	while ((sq->sq_flags & flag) ||
5031 	    (maxcnt != -1 && count > (unsigned)maxcnt)) {
5032 		sq->sq_flags |= SQ_WANTWAKEUP;
5033 		if (maxcnt != -1) {
5034 			SQ_PUTLOCKS_EXIT(sq);
5035 		}
5036 		cv_wait(&sq->sq_wait, SQLOCK(sq));
5037 		if (maxcnt != -1) {
5038 			count = sq->sq_count;
5039 			SQ_PUTLOCKS_ENTER(sq);
5040 			SUM_SQ_PUTCOUNTS(sq, count);
5041 		}
5042 	}
5043 	sq->sq_needexcl--;
5044 	sq->sq_flags |= flag;
5045 	ASSERT(maxcnt == -1 || count == maxcnt);
5046 	if (maxcnt != -1) {
5047 		if (sq->sq_needexcl == 0) {
5048 			SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5049 		}
5050 		SQ_PUTLOCKS_EXIT(sq);
5051 	} else if (sq->sq_needexcl == 0) {
5052 		SQ_PUTCOUNT_SETFAST(sq);
5053 	}
5054 
5055 	mutex_exit(SQLOCK(sq));
5056 }
5057 
5058 /*
5059  * Reset a flag that was set with blocksq.
5060  *
5061  * Can not use this routine to reset SQ_WRITER.
5062  *
5063  * If "isouter" is set then the syncq is assumed to be an outer perimeter
5064  * and drain_syncq is not called. Instead we rely on the qwriter_outer thread
5065  * to handle the queued qwriter operations.
5066  *
5067  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5068  * sq_putlocks are used.
5069  */
5070 static void
unblocksq(syncq_t * sq,uint16_t resetflag,int isouter)5071 unblocksq(syncq_t *sq, uint16_t resetflag, int isouter)
5072 {
5073 	uint16_t flags;
5074 
5075 	mutex_enter(SQLOCK(sq));
5076 	ASSERT(resetflag != SQ_WRITER);
5077 	ASSERT(sq->sq_flags & resetflag);
5078 	flags = sq->sq_flags & ~resetflag;
5079 	sq->sq_flags = flags;
5080 	if (flags & (SQ_QUEUED | SQ_WANTWAKEUP)) {
5081 		if (flags & SQ_WANTWAKEUP) {
5082 			flags &= ~SQ_WANTWAKEUP;
5083 			cv_broadcast(&sq->sq_wait);
5084 		}
5085 		sq->sq_flags = flags;
5086 		if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5087 			if (!isouter) {
5088 				/* drain_syncq drops SQLOCK */
5089 				drain_syncq(sq);
5090 				return;
5091 			}
5092 		}
5093 	}
5094 	mutex_exit(SQLOCK(sq));
5095 }
5096 
5097 /*
5098  * Reset a flag that was set with blocksq.
5099  * Does not drain the syncq. Use emptysq() for that.
5100  * Returns 1 if SQ_QUEUED is set. Otherwise 0.
5101  *
5102  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5103  * sq_putlocks are used.
5104  */
5105 static int
dropsq(syncq_t * sq,uint16_t resetflag)5106 dropsq(syncq_t *sq, uint16_t resetflag)
5107 {
5108 	uint16_t flags;
5109 
5110 	mutex_enter(SQLOCK(sq));
5111 	ASSERT(sq->sq_flags & resetflag);
5112 	flags = sq->sq_flags & ~resetflag;
5113 	if (flags & SQ_WANTWAKEUP) {
5114 		flags &= ~SQ_WANTWAKEUP;
5115 		cv_broadcast(&sq->sq_wait);
5116 	}
5117 	sq->sq_flags = flags;
5118 	mutex_exit(SQLOCK(sq));
5119 	if (flags & SQ_QUEUED)
5120 		return (1);
5121 	return (0);
5122 }
5123 
5124 /*
5125  * Empty all the messages on a syncq.
5126  *
5127  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5128  * sq_putlocks are used.
5129  */
5130 static void
emptysq(syncq_t * sq)5131 emptysq(syncq_t *sq)
5132 {
5133 	uint16_t flags;
5134 
5135 	mutex_enter(SQLOCK(sq));
5136 	flags = sq->sq_flags;
5137 	if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5138 		/*
5139 		 * To prevent potential recursive invocation of drain_syncq we
5140 		 * do not call drain_syncq if count is non-zero.
5141 		 */
5142 		if (sq->sq_count == 0) {
5143 			/* drain_syncq() drops SQLOCK */
5144 			drain_syncq(sq);
5145 			return;
5146 		} else
5147 			sqenable(sq);
5148 	}
5149 	mutex_exit(SQLOCK(sq));
5150 }
5151 
5152 /*
5153  * Ordered insert while removing duplicates.
5154  */
5155 static void
sqlist_insert(sqlist_t * sqlist,syncq_t * sqp)5156 sqlist_insert(sqlist_t *sqlist, syncq_t *sqp)
5157 {
5158 	syncql_t *sqlp, **prev_sqlpp, *new_sqlp;
5159 
5160 	prev_sqlpp = &sqlist->sqlist_head;
5161 	while ((sqlp = *prev_sqlpp) != NULL) {
5162 		if (sqlp->sql_sq >= sqp) {
5163 			if (sqlp->sql_sq == sqp)	/* duplicate */
5164 				return;
5165 			break;
5166 		}
5167 		prev_sqlpp = &sqlp->sql_next;
5168 	}
5169 	new_sqlp = &sqlist->sqlist_array[sqlist->sqlist_index++];
5170 	ASSERT((char *)new_sqlp < (char *)sqlist + sqlist->sqlist_size);
5171 	new_sqlp->sql_next = sqlp;
5172 	new_sqlp->sql_sq = sqp;
5173 	*prev_sqlpp = new_sqlp;
5174 }
5175 
5176 /*
5177  * Walk the write side queues until we hit either the driver
5178  * or a twist in the stream (_SAMESTR will return false in both
5179  * these cases) then turn around and walk the read side queues
5180  * back up to the stream head.
5181  */
5182 static void
sqlist_insertall(sqlist_t * sqlist,queue_t * q)5183 sqlist_insertall(sqlist_t *sqlist, queue_t *q)
5184 {
5185 	while (q != NULL) {
5186 		sqlist_insert(sqlist, q->q_syncq);
5187 
5188 		if (_SAMESTR(q))
5189 			q = q->q_next;
5190 		else if (!(q->q_flag & QREADR))
5191 			q = _RD(q);
5192 		else
5193 			q = NULL;
5194 	}
5195 }
5196 
5197 /*
5198  * Allocate and build a list of all syncqs in a stream and the syncq(s)
5199  * associated with the "q" parameter. The resulting list is sorted in a
5200  * canonical order and is free of duplicates.
5201  * Assumes the passed queue is a _RD(q).
5202  */
5203 static sqlist_t *
sqlist_build(queue_t * q,struct stdata * stp,boolean_t do_twist)5204 sqlist_build(queue_t *q, struct stdata *stp, boolean_t do_twist)
5205 {
5206 	sqlist_t *sqlist = sqlist_alloc(stp, KM_SLEEP);
5207 
5208 	/*
5209 	 * start with the current queue/qpair
5210 	 */
5211 	ASSERT(q->q_flag & QREADR);
5212 
5213 	sqlist_insert(sqlist, q->q_syncq);
5214 	sqlist_insert(sqlist, _WR(q)->q_syncq);
5215 
5216 	sqlist_insertall(sqlist, stp->sd_wrq);
5217 	if (do_twist)
5218 		sqlist_insertall(sqlist, stp->sd_mate->sd_wrq);
5219 
5220 	return (sqlist);
5221 }
5222 
5223 static sqlist_t *
sqlist_alloc(struct stdata * stp,int kmflag)5224 sqlist_alloc(struct stdata *stp, int kmflag)
5225 {
5226 	size_t sqlist_size;
5227 	sqlist_t *sqlist;
5228 
5229 	/*
5230 	 * Allocate 2 syncql_t's for each pushed module. Note that
5231 	 * the sqlist_t structure already has 4 syncql_t's built in:
5232 	 * 2 for the stream head, and 2 for the driver/other stream head.
5233 	 */
5234 	sqlist_size = 2 * sizeof (syncql_t) * stp->sd_pushcnt +
5235 	    sizeof (sqlist_t);
5236 	if (STRMATED(stp))
5237 		sqlist_size += 2 * sizeof (syncql_t) * stp->sd_mate->sd_pushcnt;
5238 	sqlist = kmem_alloc(sqlist_size, kmflag);
5239 
5240 	sqlist->sqlist_head = NULL;
5241 	sqlist->sqlist_size = sqlist_size;
5242 	sqlist->sqlist_index = 0;
5243 
5244 	return (sqlist);
5245 }
5246 
5247 /*
5248  * Free the list created by sqlist_alloc()
5249  */
5250 static void
sqlist_free(sqlist_t * sqlist)5251 sqlist_free(sqlist_t *sqlist)
5252 {
5253 	kmem_free(sqlist, sqlist->sqlist_size);
5254 }
5255 
5256 /*
5257  * Prevent any new entries into any syncq in this stream.
5258  * Used by freezestr.
5259  */
5260 void
strblock(queue_t * q)5261 strblock(queue_t *q)
5262 {
5263 	struct stdata	*stp;
5264 	syncql_t	*sql;
5265 	sqlist_t	*sqlist;
5266 
5267 	q = _RD(q);
5268 
5269 	stp = STREAM(q);
5270 	ASSERT(stp != NULL);
5271 
5272 	/*
5273 	 * Get a sorted list with all the duplicates removed containing
5274 	 * all the syncqs referenced by this stream.
5275 	 */
5276 	sqlist = sqlist_build(q, stp, B_FALSE);
5277 	for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5278 		blocksq(sql->sql_sq, SQ_FROZEN, -1);
5279 	sqlist_free(sqlist);
5280 }
5281 
5282 /*
5283  * Release the block on new entries into this stream
5284  */
5285 void
strunblock(queue_t * q)5286 strunblock(queue_t *q)
5287 {
5288 	struct stdata	*stp;
5289 	syncql_t	*sql;
5290 	sqlist_t	*sqlist;
5291 	int		drain_needed;
5292 
5293 	q = _RD(q);
5294 
5295 	/*
5296 	 * Get a sorted list with all the duplicates removed containing
5297 	 * all the syncqs referenced by this stream.
5298 	 * Have to drop the SQ_FROZEN flag on all the syncqs before
5299 	 * starting to drain them; otherwise the draining might
5300 	 * cause a freezestr in some module on the stream (which
5301 	 * would deadlock).
5302 	 */
5303 	stp = STREAM(q);
5304 	ASSERT(stp != NULL);
5305 	sqlist = sqlist_build(q, stp, B_FALSE);
5306 	drain_needed = 0;
5307 	for (sql = sqlist->sqlist_head; sql != NULL; sql = sql->sql_next)
5308 		drain_needed += dropsq(sql->sql_sq, SQ_FROZEN);
5309 	if (drain_needed) {
5310 		for (sql = sqlist->sqlist_head; sql != NULL;
5311 		    sql = sql->sql_next)
5312 			emptysq(sql->sql_sq);
5313 	}
5314 	sqlist_free(sqlist);
5315 }
5316 
5317 #ifdef DEBUG
5318 static int
qprocsareon(queue_t * rq)5319 qprocsareon(queue_t *rq)
5320 {
5321 	if (rq->q_next == NULL)
5322 		return (0);
5323 	return (_WR(rq->q_next)->q_next == _WR(rq));
5324 }
5325 
5326 int
qclaimed(queue_t * q)5327 qclaimed(queue_t *q)
5328 {
5329 	uint_t count;
5330 
5331 	count = q->q_syncq->sq_count;
5332 	SUM_SQ_PUTCOUNTS(q->q_syncq, count);
5333 	return (count != 0);
5334 }
5335 
5336 /*
5337  * Check if anyone has frozen this stream with freezestr
5338  */
5339 int
frozenstr(queue_t * q)5340 frozenstr(queue_t *q)
5341 {
5342 	return ((q->q_syncq->sq_flags & SQ_FROZEN) != 0);
5343 }
5344 #endif /* DEBUG */
5345 
5346 /*
5347  * Enter a queue.
5348  * Obsoleted interface. Should not be used.
5349  */
5350 void
enterq(queue_t * q)5351 enterq(queue_t *q)
5352 {
5353 	entersq(q->q_syncq, SQ_CALLBACK);
5354 }
5355 
5356 void
leaveq(queue_t * q)5357 leaveq(queue_t *q)
5358 {
5359 	leavesq(q->q_syncq, SQ_CALLBACK);
5360 }
5361 
5362 /*
5363  * Enter a perimeter. c_inner and c_outer specifies which concurrency bits
5364  * to check.
5365  * Wait if SQ_QUEUED is set to preserve ordering between messages and qwriter
5366  * calls and the running of open, close and service procedures.
5367  *
5368  * If c_inner bit is set no need to grab sq_putlocks since we don't care
5369  * if other threads have entered or are entering put entry point.
5370  *
5371  * If c_inner bit is set it might have been possible to use
5372  * sq_putlocks/sq_putcounts instead of SQLOCK/sq_count (e.g. to optimize
5373  * open/close path for IP) but since the count may need to be decremented in
5374  * qwait() we wouldn't know which counter to decrement. Currently counter is
5375  * selected by current cpu_seqid and current CPU can change at any moment. XXX
5376  * in the future we might use curthread id bits to select the counter and this
5377  * would stay constant across routine calls.
5378  */
5379 void
entersq(syncq_t * sq,int entrypoint)5380 entersq(syncq_t *sq, int entrypoint)
5381 {
5382 	uint16_t	count = 0;
5383 	uint16_t	flags;
5384 	uint16_t	waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
5385 	uint16_t	type;
5386 	uint_t		c_inner = entrypoint & SQ_CI;
5387 	uint_t		c_outer = entrypoint & SQ_CO;
5388 
5389 	/*
5390 	 * Increment ref count to keep closes out of this queue.
5391 	 */
5392 	ASSERT(sq);
5393 	ASSERT(c_inner && c_outer);
5394 	mutex_enter(SQLOCK(sq));
5395 	flags = sq->sq_flags;
5396 	type = sq->sq_type;
5397 	if (!(type & c_inner)) {
5398 		/* Make sure all putcounts now use slowlock. */
5399 		count = sq->sq_count;
5400 		SQ_PUTLOCKS_ENTER(sq);
5401 		SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
5402 		SUM_SQ_PUTCOUNTS(sq, count);
5403 		sq->sq_needexcl++;
5404 		ASSERT(sq->sq_needexcl != 0);	/* wraparound */
5405 		waitflags |= SQ_MESSAGES;
5406 	}
5407 	/*
5408 	 * Wait until we can enter the inner perimeter.
5409 	 * If we want exclusive access we wait until sq_count is 0.
5410 	 * We have to do this before entering the outer perimeter in order
5411 	 * to preserve put/close message ordering.
5412 	 */
5413 	while ((flags & waitflags) || (!(type & c_inner) && count != 0)) {
5414 		sq->sq_flags = flags | SQ_WANTWAKEUP;
5415 		if (!(type & c_inner)) {
5416 			SQ_PUTLOCKS_EXIT(sq);
5417 		}
5418 		cv_wait(&sq->sq_wait, SQLOCK(sq));
5419 		if (!(type & c_inner)) {
5420 			count = sq->sq_count;
5421 			SQ_PUTLOCKS_ENTER(sq);
5422 			SUM_SQ_PUTCOUNTS(sq, count);
5423 		}
5424 		flags = sq->sq_flags;
5425 	}
5426 
5427 	if (!(type & c_inner)) {
5428 		ASSERT(sq->sq_needexcl > 0);
5429 		sq->sq_needexcl--;
5430 		if (sq->sq_needexcl == 0) {
5431 			SQ_PUTCOUNT_SETFAST_LOCKED(sq);
5432 		}
5433 	}
5434 
5435 	/* Check if we need to enter the outer perimeter */
5436 	if (!(type & c_outer)) {
5437 		/*
5438 		 * We have to enter the outer perimeter exclusively before
5439 		 * we can increment sq_count to avoid deadlock. This implies
5440 		 * that we have to re-check sq_flags and sq_count.
5441 		 *
5442 		 * is it possible to have c_inner set when c_outer is not set?
5443 		 */
5444 		if (!(type & c_inner)) {
5445 			SQ_PUTLOCKS_EXIT(sq);
5446 		}
5447 		mutex_exit(SQLOCK(sq));
5448 		outer_enter(sq->sq_outer, SQ_GOAWAY);
5449 		mutex_enter(SQLOCK(sq));
5450 		flags = sq->sq_flags;
5451 		/*
5452 		 * there should be no need to recheck sq_putcounts
5453 		 * because outer_enter() has already waited for them to clear
5454 		 * after setting SQ_WRITER.
5455 		 */
5456 		count = sq->sq_count;
5457 #ifdef DEBUG
5458 		/*
5459 		 * SUMCHECK_SQ_PUTCOUNTS should return the sum instead
5460 		 * of doing an ASSERT internally. Others should do
5461 		 * something like
5462 		 *	 ASSERT(SUMCHECK_SQ_PUTCOUNTS(sq) == 0);
5463 		 * without the need to #ifdef DEBUG it.
5464 		 */
5465 		SUMCHECK_SQ_PUTCOUNTS(sq, 0);
5466 #endif
5467 		while ((flags & (SQ_EXCL|SQ_BLOCKED|SQ_FROZEN)) ||
5468 		    (!(type & c_inner) && count != 0)) {
5469 			sq->sq_flags = flags | SQ_WANTWAKEUP;
5470 			cv_wait(&sq->sq_wait, SQLOCK(sq));
5471 			count = sq->sq_count;
5472 			flags = sq->sq_flags;
5473 		}
5474 	}
5475 
5476 	sq->sq_count++;
5477 	ASSERT(sq->sq_count != 0);	/* Wraparound */
5478 	if (!(type & c_inner)) {
5479 		/* Exclusive entry */
5480 		ASSERT(sq->sq_count == 1);
5481 		sq->sq_flags |= SQ_EXCL;
5482 		if (type & c_outer) {
5483 			SQ_PUTLOCKS_EXIT(sq);
5484 		}
5485 	}
5486 	mutex_exit(SQLOCK(sq));
5487 }
5488 
5489 /*
5490  * Leave a syncq. Announce to framework that closes may proceed.
5491  * c_inner and c_outer specify which concurrency bits to check.
5492  *
5493  * Must never be called from driver or module put entry point.
5494  *
5495  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5496  * sq_putlocks are used.
5497  */
5498 void
leavesq(syncq_t * sq,int entrypoint)5499 leavesq(syncq_t *sq, int entrypoint)
5500 {
5501 	uint16_t	flags;
5502 	uint16_t	type;
5503 	uint_t		c_outer = entrypoint & SQ_CO;
5504 #ifdef DEBUG
5505 	uint_t		c_inner = entrypoint & SQ_CI;
5506 #endif
5507 
5508 	/*
5509 	 * Decrement ref count, drain the syncq if possible, and wake up
5510 	 * any waiting close.
5511 	 */
5512 	ASSERT(sq);
5513 	ASSERT(c_inner && c_outer);
5514 	mutex_enter(SQLOCK(sq));
5515 	flags = sq->sq_flags;
5516 	type = sq->sq_type;
5517 	if (flags & (SQ_QUEUED|SQ_WANTWAKEUP|SQ_WANTEXWAKEUP)) {
5518 
5519 		if (flags & SQ_WANTWAKEUP) {
5520 			flags &= ~SQ_WANTWAKEUP;
5521 			cv_broadcast(&sq->sq_wait);
5522 		}
5523 		if (flags & SQ_WANTEXWAKEUP) {
5524 			flags &= ~SQ_WANTEXWAKEUP;
5525 			cv_broadcast(&sq->sq_exitwait);
5526 		}
5527 
5528 		if ((flags & SQ_QUEUED) && !(flags & SQ_STAYAWAY)) {
5529 			/*
5530 			 * The syncq needs to be drained. "Exit" the syncq
5531 			 * before calling drain_syncq.
5532 			 */
5533 			ASSERT(sq->sq_count != 0);
5534 			sq->sq_count--;
5535 			ASSERT((flags & SQ_EXCL) || (type & c_inner));
5536 			sq->sq_flags = flags & ~SQ_EXCL;
5537 			drain_syncq(sq);
5538 			ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
5539 			/* Check if we need to exit the outer perimeter */
5540 			/* XXX will this ever be true? */
5541 			if (!(type & c_outer))
5542 				outer_exit(sq->sq_outer);
5543 			return;
5544 		}
5545 	}
5546 	ASSERT(sq->sq_count != 0);
5547 	sq->sq_count--;
5548 	ASSERT((flags & SQ_EXCL) || (type & c_inner));
5549 	sq->sq_flags = flags & ~SQ_EXCL;
5550 	mutex_exit(SQLOCK(sq));
5551 
5552 	/* Check if we need to exit the outer perimeter */
5553 	if (!(sq->sq_type & c_outer))
5554 		outer_exit(sq->sq_outer);
5555 }
5556 
5557 /*
5558  * Prevent q_next from changing in this stream by incrementing sq_count.
5559  *
5560  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5561  * sq_putlocks are used.
5562  */
5563 void
claimq(queue_t * qp)5564 claimq(queue_t *qp)
5565 {
5566 	syncq_t	*sq = qp->q_syncq;
5567 
5568 	mutex_enter(SQLOCK(sq));
5569 	sq->sq_count++;
5570 	ASSERT(sq->sq_count != 0);	/* Wraparound */
5571 	mutex_exit(SQLOCK(sq));
5572 }
5573 
5574 /*
5575  * Undo claimq.
5576  *
5577  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
5578  * sq_putlocks are used.
5579  */
5580 void
releaseq(queue_t * qp)5581 releaseq(queue_t *qp)
5582 {
5583 	syncq_t	*sq = qp->q_syncq;
5584 	uint16_t flags;
5585 
5586 	mutex_enter(SQLOCK(sq));
5587 	ASSERT(sq->sq_count > 0);
5588 	sq->sq_count--;
5589 
5590 	flags = sq->sq_flags;
5591 	if (flags & (SQ_WANTWAKEUP|SQ_QUEUED)) {
5592 		if (flags & SQ_WANTWAKEUP) {
5593 			flags &= ~SQ_WANTWAKEUP;
5594 			cv_broadcast(&sq->sq_wait);
5595 		}
5596 		sq->sq_flags = flags;
5597 		if ((flags & SQ_QUEUED) && !(flags & (SQ_STAYAWAY|SQ_EXCL))) {
5598 			/*
5599 			 * To prevent potential recursive invocation of
5600 			 * drain_syncq we do not call drain_syncq if count is
5601 			 * non-zero.
5602 			 */
5603 			if (sq->sq_count == 0) {
5604 				drain_syncq(sq);
5605 				return;
5606 			} else
5607 				sqenable(sq);
5608 		}
5609 	}
5610 	mutex_exit(SQLOCK(sq));
5611 }
5612 
5613 /*
5614  * Prevent q_next from changing in this stream by incrementing sd_refcnt.
5615  */
5616 void
claimstr(queue_t * qp)5617 claimstr(queue_t *qp)
5618 {
5619 	struct stdata *stp = STREAM(qp);
5620 
5621 	mutex_enter(&stp->sd_reflock);
5622 	stp->sd_refcnt++;
5623 	ASSERT(stp->sd_refcnt != 0);	/* Wraparound */
5624 	mutex_exit(&stp->sd_reflock);
5625 }
5626 
5627 /*
5628  * Undo claimstr.
5629  */
5630 void
releasestr(queue_t * qp)5631 releasestr(queue_t *qp)
5632 {
5633 	struct stdata *stp = STREAM(qp);
5634 
5635 	mutex_enter(&stp->sd_reflock);
5636 	ASSERT(stp->sd_refcnt != 0);
5637 	if (--stp->sd_refcnt == 0)
5638 		cv_broadcast(&stp->sd_refmonitor);
5639 	mutex_exit(&stp->sd_reflock);
5640 }
5641 
5642 static syncq_t *
new_syncq(void)5643 new_syncq(void)
5644 {
5645 	return (kmem_cache_alloc(syncq_cache, KM_SLEEP));
5646 }
5647 
5648 static void
free_syncq(syncq_t * sq)5649 free_syncq(syncq_t *sq)
5650 {
5651 	ASSERT(sq->sq_head == NULL);
5652 	ASSERT(sq->sq_outer == NULL);
5653 	ASSERT(sq->sq_callbpend == NULL);
5654 	ASSERT((sq->sq_onext == NULL && sq->sq_oprev == NULL) ||
5655 	    (sq->sq_onext == sq && sq->sq_oprev == sq));
5656 
5657 	if (sq->sq_ciputctrl != NULL) {
5658 		ASSERT(sq->sq_nciputctrl == n_ciputctrl - 1);
5659 		SUMCHECK_CIPUTCTRL_COUNTS(sq->sq_ciputctrl,
5660 		    sq->sq_nciputctrl, 0);
5661 		ASSERT(ciputctrl_cache != NULL);
5662 		kmem_cache_free(ciputctrl_cache, sq->sq_ciputctrl);
5663 	}
5664 
5665 	sq->sq_tail = NULL;
5666 	sq->sq_evhead = NULL;
5667 	sq->sq_evtail = NULL;
5668 	sq->sq_ciputctrl = NULL;
5669 	sq->sq_nciputctrl = 0;
5670 	sq->sq_count = 0;
5671 	sq->sq_rmqcount = 0;
5672 	sq->sq_callbflags = 0;
5673 	sq->sq_cancelid = 0;
5674 	sq->sq_next = NULL;
5675 	sq->sq_needexcl = 0;
5676 	sq->sq_svcflags = 0;
5677 	sq->sq_nqueues = 0;
5678 	sq->sq_pri = 0;
5679 	sq->sq_onext = NULL;
5680 	sq->sq_oprev = NULL;
5681 	sq->sq_flags = 0;
5682 	sq->sq_type = 0;
5683 	sq->sq_servcount = 0;
5684 
5685 	kmem_cache_free(syncq_cache, sq);
5686 }
5687 
5688 /* Outer perimeter code */
5689 
5690 /*
5691  * The outer syncq uses the fields and flags in the syncq slightly
5692  * differently from the inner syncqs.
5693  *	sq_count	Incremented when there are pending or running
5694  *			writers at the outer perimeter to prevent the set of
5695  *			inner syncqs that belong to the outer perimeter from
5696  *			changing.
5697  *	sq_head/tail	List of deferred qwriter(OUTER) operations.
5698  *
5699  *	SQ_BLOCKED	Set to prevent traversing of sq_next,sq_prev while
5700  *			inner syncqs are added to or removed from the
5701  *			outer perimeter.
5702  *	SQ_QUEUED	sq_head/tail has messages or events queued.
5703  *
5704  *	SQ_WRITER	A thread is currently traversing all the inner syncqs
5705  *			setting the SQ_WRITER flag.
5706  */
5707 
5708 /*
5709  * Get write access at the outer perimeter.
5710  * Note that read access is done by entersq, putnext, and put by simply
5711  * incrementing sq_count in the inner syncq.
5712  *
5713  * Waits until "flags" is no longer set in the outer to prevent multiple
5714  * threads from having write access at the same time. SQ_WRITER has to be part
5715  * of "flags".
5716  *
5717  * Increases sq_count on the outer syncq to keep away outer_insert/remove
5718  * until the outer_exit is finished.
5719  *
5720  * outer_enter is vulnerable to starvation since it does not prevent new
5721  * threads from entering the inner syncqs while it is waiting for sq_count to
5722  * go to zero.
5723  */
5724 void
outer_enter(syncq_t * outer,uint16_t flags)5725 outer_enter(syncq_t *outer, uint16_t flags)
5726 {
5727 	syncq_t	*sq;
5728 	int	wait_needed;
5729 	uint16_t	count;
5730 
5731 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5732 	    outer->sq_oprev != NULL);
5733 	ASSERT(flags & SQ_WRITER);
5734 
5735 retry:
5736 	mutex_enter(SQLOCK(outer));
5737 	while (outer->sq_flags & flags) {
5738 		outer->sq_flags |= SQ_WANTWAKEUP;
5739 		cv_wait(&outer->sq_wait, SQLOCK(outer));
5740 	}
5741 
5742 	ASSERT(!(outer->sq_flags & SQ_WRITER));
5743 	outer->sq_flags |= SQ_WRITER;
5744 	outer->sq_count++;
5745 	ASSERT(outer->sq_count != 0);	/* wraparound */
5746 	wait_needed = 0;
5747 	/*
5748 	 * Set SQ_WRITER on all the inner syncqs while holding
5749 	 * the SQLOCK on the outer syncq. This ensures that the changing
5750 	 * of SQ_WRITER is atomic under the outer SQLOCK.
5751 	 */
5752 	for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5753 		mutex_enter(SQLOCK(sq));
5754 		count = sq->sq_count;
5755 		SQ_PUTLOCKS_ENTER(sq);
5756 		sq->sq_flags |= SQ_WRITER;
5757 		SUM_SQ_PUTCOUNTS(sq, count);
5758 		if (count != 0)
5759 			wait_needed = 1;
5760 		SQ_PUTLOCKS_EXIT(sq);
5761 		mutex_exit(SQLOCK(sq));
5762 	}
5763 	mutex_exit(SQLOCK(outer));
5764 
5765 	/*
5766 	 * Get everybody out of the syncqs sequentially.
5767 	 * Note that we don't actually need to acquire the PUTLOCKS, since
5768 	 * we have already cleared the fastbit, and set QWRITER.  By
5769 	 * definition, the count can not increase since putnext will
5770 	 * take the slowlock path (and the purpose of acquiring the
5771 	 * putlocks was to make sure it didn't increase while we were
5772 	 * waiting).
5773 	 *
5774 	 * Note that we still acquire the PUTLOCKS to be safe.
5775 	 */
5776 	if (wait_needed) {
5777 		for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
5778 			mutex_enter(SQLOCK(sq));
5779 			count = sq->sq_count;
5780 			SQ_PUTLOCKS_ENTER(sq);
5781 			SUM_SQ_PUTCOUNTS(sq, count);
5782 			while (count != 0) {
5783 				sq->sq_flags |= SQ_WANTWAKEUP;
5784 				SQ_PUTLOCKS_EXIT(sq);
5785 				cv_wait(&sq->sq_wait, SQLOCK(sq));
5786 				count = sq->sq_count;
5787 				SQ_PUTLOCKS_ENTER(sq);
5788 				SUM_SQ_PUTCOUNTS(sq, count);
5789 			}
5790 			SQ_PUTLOCKS_EXIT(sq);
5791 			mutex_exit(SQLOCK(sq));
5792 		}
5793 		/*
5794 		 * Verify that none of the flags got set while we
5795 		 * were waiting for the sq_counts to drop.
5796 		 * If this happens we exit and retry entering the
5797 		 * outer perimeter.
5798 		 */
5799 		mutex_enter(SQLOCK(outer));
5800 		if (outer->sq_flags & (flags & ~SQ_WRITER)) {
5801 			mutex_exit(SQLOCK(outer));
5802 			outer_exit(outer);
5803 			goto retry;
5804 		}
5805 		mutex_exit(SQLOCK(outer));
5806 	}
5807 }
5808 
5809 /*
5810  * Drop the write access at the outer perimeter.
5811  * Read access is dropped implicitly (by putnext, put, and leavesq) by
5812  * decrementing sq_count.
5813  */
5814 void
outer_exit(syncq_t * outer)5815 outer_exit(syncq_t *outer)
5816 {
5817 	syncq_t	*sq;
5818 	int	 drain_needed;
5819 	uint16_t flags;
5820 
5821 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5822 	    outer->sq_oprev != NULL);
5823 	ASSERT(MUTEX_NOT_HELD(SQLOCK(outer)));
5824 
5825 	/*
5826 	 * Atomically (from the perspective of threads calling become_writer)
5827 	 * drop the write access at the outer perimeter by holding
5828 	 * SQLOCK(outer) across all the dropsq calls and the resetting of
5829 	 * SQ_WRITER.
5830 	 * This defines a locking order between the outer perimeter
5831 	 * SQLOCK and the inner perimeter SQLOCKs.
5832 	 */
5833 	mutex_enter(SQLOCK(outer));
5834 	flags = outer->sq_flags;
5835 	ASSERT(outer->sq_flags & SQ_WRITER);
5836 	if (flags & SQ_QUEUED) {
5837 		write_now(outer);
5838 		flags = outer->sq_flags;
5839 	}
5840 
5841 	/*
5842 	 * sq_onext is stable since sq_count has not yet been decreased.
5843 	 * Reset the SQ_WRITER flags in all syncqs.
5844 	 * After dropping SQ_WRITER on the outer syncq we empty all the
5845 	 * inner syncqs.
5846 	 */
5847 	drain_needed = 0;
5848 	for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5849 		drain_needed += dropsq(sq, SQ_WRITER);
5850 	ASSERT(!(outer->sq_flags & SQ_QUEUED));
5851 	flags &= ~SQ_WRITER;
5852 	if (drain_needed) {
5853 		outer->sq_flags = flags;
5854 		mutex_exit(SQLOCK(outer));
5855 		for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext)
5856 			emptysq(sq);
5857 		mutex_enter(SQLOCK(outer));
5858 		flags = outer->sq_flags;
5859 	}
5860 	if (flags & SQ_WANTWAKEUP) {
5861 		flags &= ~SQ_WANTWAKEUP;
5862 		cv_broadcast(&outer->sq_wait);
5863 	}
5864 	outer->sq_flags = flags;
5865 	ASSERT(outer->sq_count > 0);
5866 	outer->sq_count--;
5867 	mutex_exit(SQLOCK(outer));
5868 }
5869 
5870 /*
5871  * Add another syncq to an outer perimeter.
5872  * Block out all other access to the outer perimeter while it is being
5873  * changed using blocksq.
5874  * Assumes that the caller has *not* done an outer_enter.
5875  *
5876  * Vulnerable to starvation in blocksq.
5877  */
5878 static void
outer_insert(syncq_t * outer,syncq_t * sq)5879 outer_insert(syncq_t *outer, syncq_t *sq)
5880 {
5881 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5882 	    outer->sq_oprev != NULL);
5883 	ASSERT(sq->sq_outer == NULL && sq->sq_onext == NULL &&
5884 	    sq->sq_oprev == NULL);	/* Can't be in an outer perimeter */
5885 
5886 	/* Get exclusive access to the outer perimeter list */
5887 	blocksq(outer, SQ_BLOCKED, 0);
5888 	ASSERT(outer->sq_flags & SQ_BLOCKED);
5889 	ASSERT(!(outer->sq_flags & SQ_WRITER));
5890 
5891 	mutex_enter(SQLOCK(sq));
5892 	sq->sq_outer = outer;
5893 	outer->sq_onext->sq_oprev = sq;
5894 	sq->sq_onext = outer->sq_onext;
5895 	outer->sq_onext = sq;
5896 	sq->sq_oprev = outer;
5897 	mutex_exit(SQLOCK(sq));
5898 	unblocksq(outer, SQ_BLOCKED, 1);
5899 }
5900 
5901 /*
5902  * Remove a syncq from an outer perimeter.
5903  * Block out all other access to the outer perimeter while it is being
5904  * changed using blocksq.
5905  * Assumes that the caller has *not* done an outer_enter.
5906  *
5907  * Vulnerable to starvation in blocksq.
5908  */
5909 static void
outer_remove(syncq_t * outer,syncq_t * sq)5910 outer_remove(syncq_t *outer, syncq_t *sq)
5911 {
5912 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5913 	    outer->sq_oprev != NULL);
5914 	ASSERT(sq->sq_outer == outer);
5915 
5916 	/* Get exclusive access to the outer perimeter list */
5917 	blocksq(outer, SQ_BLOCKED, 0);
5918 	ASSERT(outer->sq_flags & SQ_BLOCKED);
5919 	ASSERT(!(outer->sq_flags & SQ_WRITER));
5920 
5921 	mutex_enter(SQLOCK(sq));
5922 	sq->sq_outer = NULL;
5923 	sq->sq_onext->sq_oprev = sq->sq_oprev;
5924 	sq->sq_oprev->sq_onext = sq->sq_onext;
5925 	sq->sq_oprev = sq->sq_onext = NULL;
5926 	mutex_exit(SQLOCK(sq));
5927 	unblocksq(outer, SQ_BLOCKED, 1);
5928 }
5929 
5930 /*
5931  * Queue a deferred qwriter(OUTER) callback for this outer perimeter.
5932  * If this is the first callback for this outer perimeter then add
5933  * this outer perimeter to the list of outer perimeters that
5934  * the qwriter_outer_thread will process.
5935  *
5936  * Increments sq_count in the outer syncq to prevent the membership
5937  * of the outer perimeter (in terms of inner syncqs) to change while
5938  * the callback is pending.
5939  */
5940 static void
queue_writer(syncq_t * outer,void (* func)(),queue_t * q,mblk_t * mp)5941 queue_writer(syncq_t *outer, void (*func)(), queue_t *q, mblk_t *mp)
5942 {
5943 	ASSERT(MUTEX_HELD(SQLOCK(outer)));
5944 
5945 	mp->b_prev = (mblk_t *)func;
5946 	mp->b_queue = q;
5947 	mp->b_next = NULL;
5948 	outer->sq_count++;	/* Decremented when dequeued */
5949 	ASSERT(outer->sq_count != 0);	/* Wraparound */
5950 	if (outer->sq_evhead == NULL) {
5951 		/* First message. */
5952 		outer->sq_evhead = outer->sq_evtail = mp;
5953 		outer->sq_flags |= SQ_EVENTS;
5954 		mutex_exit(SQLOCK(outer));
5955 		STRSTAT(qwr_outer);
5956 		(void) taskq_dispatch(streams_taskq,
5957 		    (task_func_t *)qwriter_outer_service, outer, TQ_SLEEP);
5958 	} else {
5959 		ASSERT(outer->sq_flags & SQ_EVENTS);
5960 		outer->sq_evtail->b_next = mp;
5961 		outer->sq_evtail = mp;
5962 		mutex_exit(SQLOCK(outer));
5963 	}
5964 }
5965 
5966 /*
5967  * Try and upgrade to write access at the outer perimeter. If this can
5968  * not be done without blocking then queue the callback to be done
5969  * by the qwriter_outer_thread.
5970  *
5971  * This routine can only be called from put or service procedures plus
5972  * asynchronous callback routines that have properly entered the queue (with
5973  * entersq). Thus qwriter(OUTER) assumes the caller has one claim on the syncq
5974  * associated with q.
5975  */
5976 void
qwriter_outer(queue_t * q,mblk_t * mp,void (* func)())5977 qwriter_outer(queue_t *q, mblk_t *mp, void (*func)())
5978 {
5979 	syncq_t	*osq, *sq, *outer;
5980 	int	failed;
5981 	uint16_t flags;
5982 
5983 	osq = q->q_syncq;
5984 	outer = osq->sq_outer;
5985 	if (outer == NULL)
5986 		panic("qwriter(PERIM_OUTER): no outer perimeter");
5987 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
5988 	    outer->sq_oprev != NULL);
5989 
5990 	mutex_enter(SQLOCK(outer));
5991 	flags = outer->sq_flags;
5992 	/*
5993 	 * If some thread is traversing sq_next, or if we are blocked by
5994 	 * outer_insert or outer_remove, or if the we already have queued
5995 	 * callbacks, then queue this callback for later processing.
5996 	 *
5997 	 * Also queue the qwriter for an interrupt thread in order
5998 	 * to reduce the time spent running at high IPL.
5999 	 * to identify there are events.
6000 	 */
6001 	if ((flags & SQ_GOAWAY) || (curthread->t_pri >= kpreemptpri)) {
6002 		/*
6003 		 * Queue the become_writer request.
6004 		 * The queueing is atomic under SQLOCK(outer) in order
6005 		 * to synchronize with outer_exit.
6006 		 * queue_writer will drop the outer SQLOCK
6007 		 */
6008 		if (flags & SQ_BLOCKED) {
6009 			/* Must set SQ_WRITER on inner perimeter */
6010 			mutex_enter(SQLOCK(osq));
6011 			osq->sq_flags |= SQ_WRITER;
6012 			mutex_exit(SQLOCK(osq));
6013 		} else {
6014 			if (!(flags & SQ_WRITER)) {
6015 				/*
6016 				 * The outer could have been SQ_BLOCKED thus
6017 				 * SQ_WRITER might not be set on the inner.
6018 				 */
6019 				mutex_enter(SQLOCK(osq));
6020 				osq->sq_flags |= SQ_WRITER;
6021 				mutex_exit(SQLOCK(osq));
6022 			}
6023 			ASSERT(osq->sq_flags & SQ_WRITER);
6024 		}
6025 		queue_writer(outer, func, q, mp);
6026 		return;
6027 	}
6028 	/*
6029 	 * We are half-way to exclusive access to the outer perimeter.
6030 	 * Prevent any outer_enter, qwriter(OUTER), or outer_insert/remove
6031 	 * while the inner syncqs are traversed.
6032 	 */
6033 	outer->sq_count++;
6034 	ASSERT(outer->sq_count != 0);	/* wraparound */
6035 	flags |= SQ_WRITER;
6036 	/*
6037 	 * Check if we can run the function immediately. Mark all
6038 	 * syncqs with the writer flag to prevent new entries into
6039 	 * put and service procedures.
6040 	 *
6041 	 * Set SQ_WRITER on all the inner syncqs while holding
6042 	 * the SQLOCK on the outer syncq. This ensures that the changing
6043 	 * of SQ_WRITER is atomic under the outer SQLOCK.
6044 	 */
6045 	failed = 0;
6046 	for (sq = outer->sq_onext; sq != outer; sq = sq->sq_onext) {
6047 		uint16_t count;
6048 		uint_t	maxcnt = (sq == osq) ? 1 : 0;
6049 
6050 		mutex_enter(SQLOCK(sq));
6051 		count = sq->sq_count;
6052 		SQ_PUTLOCKS_ENTER(sq);
6053 		SUM_SQ_PUTCOUNTS(sq, count);
6054 		if (sq->sq_count > maxcnt)
6055 			failed = 1;
6056 		sq->sq_flags |= SQ_WRITER;
6057 		SQ_PUTLOCKS_EXIT(sq);
6058 		mutex_exit(SQLOCK(sq));
6059 	}
6060 	if (failed) {
6061 		/*
6062 		 * Some other thread has a read claim on the outer perimeter.
6063 		 * Queue the callback for deferred processing.
6064 		 *
6065 		 * queue_writer will set SQ_QUEUED before we drop SQ_WRITER
6066 		 * so that other qwriter(OUTER) calls will queue their
6067 		 * callbacks as well. queue_writer increments sq_count so we
6068 		 * decrement to compensate for the our increment.
6069 		 *
6070 		 * Dropping SQ_WRITER enables the writer thread to work
6071 		 * on this outer perimeter.
6072 		 */
6073 		outer->sq_flags = flags;
6074 		queue_writer(outer, func, q, mp);
6075 		/* queue_writer dropper the lock */
6076 		mutex_enter(SQLOCK(outer));
6077 		ASSERT(outer->sq_count > 0);
6078 		outer->sq_count--;
6079 		ASSERT(outer->sq_flags & SQ_WRITER);
6080 		flags = outer->sq_flags;
6081 		flags &= ~SQ_WRITER;
6082 		if (flags & SQ_WANTWAKEUP) {
6083 			flags &= ~SQ_WANTWAKEUP;
6084 			cv_broadcast(&outer->sq_wait);
6085 		}
6086 		outer->sq_flags = flags;
6087 		mutex_exit(SQLOCK(outer));
6088 		return;
6089 	} else {
6090 		outer->sq_flags = flags;
6091 		mutex_exit(SQLOCK(outer));
6092 	}
6093 
6094 	/* Can run it immediately */
6095 	(*func)(q, mp);
6096 
6097 	outer_exit(outer);
6098 }
6099 
6100 /*
6101  * Dequeue all writer callbacks from the outer perimeter and run them.
6102  */
6103 static void
write_now(syncq_t * outer)6104 write_now(syncq_t *outer)
6105 {
6106 	mblk_t		*mp;
6107 	queue_t		*q;
6108 	void	(*func)();
6109 
6110 	ASSERT(MUTEX_HELD(SQLOCK(outer)));
6111 	ASSERT(outer->sq_outer == NULL && outer->sq_onext != NULL &&
6112 	    outer->sq_oprev != NULL);
6113 	while ((mp = outer->sq_evhead) != NULL) {
6114 		/*
6115 		 * queues cannot be placed on the queuelist on the outer
6116 		 * perimeter.
6117 		 */
6118 		ASSERT(!(outer->sq_flags & SQ_MESSAGES));
6119 		ASSERT((outer->sq_flags & SQ_EVENTS));
6120 
6121 		outer->sq_evhead = mp->b_next;
6122 		if (outer->sq_evhead == NULL) {
6123 			outer->sq_evtail = NULL;
6124 			outer->sq_flags &= ~SQ_EVENTS;
6125 		}
6126 		ASSERT(outer->sq_count != 0);
6127 		outer->sq_count--;	/* Incremented when enqueued. */
6128 		mutex_exit(SQLOCK(outer));
6129 		/*
6130 		 * Drop the message if the queue is closing.
6131 		 * Make sure that the queue is "claimed" when the callback
6132 		 * is run in order to satisfy various ASSERTs.
6133 		 */
6134 		q = mp->b_queue;
6135 		func = (void (*)())mp->b_prev;
6136 		ASSERT(func != NULL);
6137 		mp->b_next = mp->b_prev = NULL;
6138 		if (q->q_flag & QWCLOSE) {
6139 			freemsg(mp);
6140 		} else {
6141 			claimq(q);
6142 			(*func)(q, mp);
6143 			releaseq(q);
6144 		}
6145 		mutex_enter(SQLOCK(outer));
6146 	}
6147 	ASSERT(MUTEX_HELD(SQLOCK(outer)));
6148 }
6149 
6150 /*
6151  * The list of messages on the inner syncq is effectively hashed
6152  * by destination queue.  These destination queues are doubly
6153  * linked lists (hopefully) in priority order.  Messages are then
6154  * put on the queue referenced by the q_sqhead/q_sqtail elements.
6155  * Additional messages are linked together by the b_next/b_prev
6156  * elements in the mblk, with (similar to putq()) the first message
6157  * having a NULL b_prev and the last message having a NULL b_next.
6158  *
6159  * Events, such as qwriter callbacks, are put onto a list in FIFO
6160  * order referenced by sq_evhead, and sq_evtail.  This is a singly
6161  * linked list, and messages here MUST be processed in the order queued.
6162  */
6163 
6164 /*
6165  * Run the events on the syncq event list (sq_evhead).
6166  * Assumes there is only one claim on the syncq, it is
6167  * already exclusive (SQ_EXCL set), and the SQLOCK held.
6168  * Messages here are processed in order, with the SQ_EXCL bit
6169  * held all the way through till the last message is processed.
6170  */
6171 void
sq_run_events(syncq_t * sq)6172 sq_run_events(syncq_t *sq)
6173 {
6174 	mblk_t		*bp;
6175 	queue_t		*qp;
6176 	uint16_t	flags = sq->sq_flags;
6177 	void		(*func)();
6178 
6179 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
6180 	ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6181 	    sq->sq_oprev == NULL) ||
6182 	    (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6183 	    sq->sq_oprev != NULL));
6184 
6185 	ASSERT(flags & SQ_EXCL);
6186 	ASSERT(sq->sq_count == 1);
6187 
6188 	/*
6189 	 * We need to process all of the events on this list.  It
6190 	 * is possible that new events will be added while we are
6191 	 * away processing a callback, so on every loop, we start
6192 	 * back at the beginning of the list.
6193 	 */
6194 	/*
6195 	 * We have to reaccess sq_evhead since there is a
6196 	 * possibility of a new entry while we were running
6197 	 * the callback.
6198 	 */
6199 	for (bp = sq->sq_evhead; bp != NULL; bp = sq->sq_evhead) {
6200 		ASSERT(bp->b_queue->q_syncq == sq);
6201 		ASSERT(sq->sq_flags & SQ_EVENTS);
6202 
6203 		qp = bp->b_queue;
6204 		func = (void (*)())bp->b_prev;
6205 		ASSERT(func != NULL);
6206 
6207 		/*
6208 		 * Messages from the event queue must be taken off in
6209 		 * FIFO order.
6210 		 */
6211 		ASSERT(sq->sq_evhead == bp);
6212 		sq->sq_evhead = bp->b_next;
6213 
6214 		if (bp->b_next == NULL) {
6215 			/* Deleting last */
6216 			ASSERT(sq->sq_evtail == bp);
6217 			sq->sq_evtail = NULL;
6218 			sq->sq_flags &= ~SQ_EVENTS;
6219 		}
6220 		bp->b_prev = bp->b_next = NULL;
6221 		ASSERT(bp->b_datap->db_ref != 0);
6222 
6223 		mutex_exit(SQLOCK(sq));
6224 
6225 		(*func)(qp, bp);
6226 
6227 		mutex_enter(SQLOCK(sq));
6228 		/*
6229 		 * re-read the flags, since they could have changed.
6230 		 */
6231 		flags = sq->sq_flags;
6232 		ASSERT(flags & SQ_EXCL);
6233 	}
6234 	ASSERT(sq->sq_evhead == NULL && sq->sq_evtail == NULL);
6235 	ASSERT(!(sq->sq_flags & SQ_EVENTS));
6236 
6237 	if (flags & SQ_WANTWAKEUP) {
6238 		flags &= ~SQ_WANTWAKEUP;
6239 		cv_broadcast(&sq->sq_wait);
6240 	}
6241 	if (flags & SQ_WANTEXWAKEUP) {
6242 		flags &= ~SQ_WANTEXWAKEUP;
6243 		cv_broadcast(&sq->sq_exitwait);
6244 	}
6245 	sq->sq_flags = flags;
6246 }
6247 
6248 /*
6249  * Put messages on the event list.
6250  * If we can go exclusive now, do so and process the event list, otherwise
6251  * let the last claim service this list (or wake the sqthread).
6252  * This procedure assumes SQLOCK is held.  To run the event list, it
6253  * must be called with no claims.
6254  */
6255 static void
sqfill_events(syncq_t * sq,queue_t * q,mblk_t * mp,void (* func)())6256 sqfill_events(syncq_t *sq, queue_t *q, mblk_t *mp, void (*func)())
6257 {
6258 	uint16_t count;
6259 
6260 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
6261 	ASSERT(func != NULL);
6262 
6263 	/*
6264 	 * This is a callback.  Add it to the list of callbacks
6265 	 * and see about upgrading.
6266 	 */
6267 	mp->b_prev = (mblk_t *)func;
6268 	mp->b_queue = q;
6269 	mp->b_next = NULL;
6270 	if (sq->sq_evhead == NULL) {
6271 		sq->sq_evhead = sq->sq_evtail = mp;
6272 		sq->sq_flags |= SQ_EVENTS;
6273 	} else {
6274 		ASSERT(sq->sq_evtail != NULL);
6275 		ASSERT(sq->sq_evtail->b_next == NULL);
6276 		ASSERT(sq->sq_flags & SQ_EVENTS);
6277 		sq->sq_evtail->b_next = mp;
6278 		sq->sq_evtail = mp;
6279 	}
6280 	/*
6281 	 * We have set SQ_EVENTS, so threads will have to
6282 	 * unwind out of the perimeter, and new entries will
6283 	 * not grab a putlock.  But we still need to know
6284 	 * how many threads have already made a claim to the
6285 	 * syncq, so grab the putlocks, and sum the counts.
6286 	 * If there are no claims on the syncq, we can upgrade
6287 	 * to exclusive, and run the event list.
6288 	 * NOTE: We hold the SQLOCK, so we can just grab the
6289 	 * putlocks.
6290 	 */
6291 	count = sq->sq_count;
6292 	SQ_PUTLOCKS_ENTER(sq);
6293 	SUM_SQ_PUTCOUNTS(sq, count);
6294 	/*
6295 	 * We have no claim, so we need to check if there
6296 	 * are no others, then we can upgrade.
6297 	 */
6298 	/*
6299 	 * There are currently no claims on
6300 	 * the syncq by this thread (at least on this entry). The thread who has
6301 	 * the claim should drain syncq.
6302 	 */
6303 	if (count > 0) {
6304 		/*
6305 		 * Can't upgrade - other threads inside.
6306 		 */
6307 		SQ_PUTLOCKS_EXIT(sq);
6308 		mutex_exit(SQLOCK(sq));
6309 		return;
6310 	}
6311 	/*
6312 	 * Need to set SQ_EXCL and make a claim on the syncq.
6313 	 */
6314 	ASSERT((sq->sq_flags & SQ_EXCL) == 0);
6315 	sq->sq_flags |= SQ_EXCL;
6316 	ASSERT(sq->sq_count == 0);
6317 	sq->sq_count++;
6318 	SQ_PUTLOCKS_EXIT(sq);
6319 
6320 	/* Process the events list */
6321 	sq_run_events(sq);
6322 
6323 	/*
6324 	 * Release our claim...
6325 	 */
6326 	sq->sq_count--;
6327 
6328 	/*
6329 	 * And release SQ_EXCL.
6330 	 * We don't need to acquire the putlocks to release
6331 	 * SQ_EXCL, since we are exclusive, and hold the SQLOCK.
6332 	 */
6333 	sq->sq_flags &= ~SQ_EXCL;
6334 
6335 	/*
6336 	 * sq_run_events should have released SQ_EXCL
6337 	 */
6338 	ASSERT(!(sq->sq_flags & SQ_EXCL));
6339 
6340 	/*
6341 	 * If anything happened while we were running the
6342 	 * events (or was there before), we need to process
6343 	 * them now.  We shouldn't be exclusive sine we
6344 	 * released the perimeter above (plus, we asserted
6345 	 * for it).
6346 	 */
6347 	if (!(sq->sq_flags & SQ_STAYAWAY) && (sq->sq_flags & SQ_QUEUED))
6348 		drain_syncq(sq);
6349 	else
6350 		mutex_exit(SQLOCK(sq));
6351 }
6352 
6353 /*
6354  * Perform delayed processing. The caller has to make sure that it is safe
6355  * to enter the syncq (e.g. by checking that none of the SQ_STAYAWAY bits are
6356  * set).
6357  *
6358  * Assume that the caller has NO claims on the syncq.  However, a claim
6359  * on the syncq does not indicate that a thread is draining the syncq.
6360  * There may be more claims on the syncq than there are threads draining
6361  * (i.e.  #_threads_draining <= sq_count)
6362  *
6363  * drain_syncq has to terminate when one of the SQ_STAYAWAY bits gets set
6364  * in order to preserve qwriter(OUTER) ordering constraints.
6365  *
6366  * sq_putcount only needs to be checked when dispatching the queued
6367  * writer call for CIPUT sync queue, but this is handled in sq_run_events.
6368  */
6369 void
drain_syncq(syncq_t * sq)6370 drain_syncq(syncq_t *sq)
6371 {
6372 	queue_t		*qp;
6373 	uint16_t	count;
6374 	uint16_t	type = sq->sq_type;
6375 	uint16_t	flags = sq->sq_flags;
6376 	boolean_t	bg_service = sq->sq_svcflags & SQ_SERVICE;
6377 
6378 	TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6379 	    "drain_syncq start:%p", sq);
6380 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
6381 	ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6382 	    sq->sq_oprev == NULL) ||
6383 	    (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6384 	    sq->sq_oprev != NULL));
6385 
6386 	/*
6387 	 * Drop SQ_SERVICE flag.
6388 	 */
6389 	if (bg_service)
6390 		sq->sq_svcflags &= ~SQ_SERVICE;
6391 
6392 	/*
6393 	 * If SQ_EXCL is set, someone else is processing this syncq - let him
6394 	 * finish the job.
6395 	 */
6396 	if (flags & SQ_EXCL) {
6397 		if (bg_service) {
6398 			ASSERT(sq->sq_servcount != 0);
6399 			sq->sq_servcount--;
6400 		}
6401 		mutex_exit(SQLOCK(sq));
6402 		return;
6403 	}
6404 
6405 	/*
6406 	 * This routine can be called by a background thread if
6407 	 * it was scheduled by a hi-priority thread.  SO, if there are
6408 	 * NOT messages queued, return (remember, we have the SQLOCK,
6409 	 * and it cannot change until we release it). Wakeup any waiters also.
6410 	 */
6411 	if (!(flags & SQ_QUEUED)) {
6412 		if (flags & SQ_WANTWAKEUP) {
6413 			flags &= ~SQ_WANTWAKEUP;
6414 			cv_broadcast(&sq->sq_wait);
6415 		}
6416 		if (flags & SQ_WANTEXWAKEUP) {
6417 			flags &= ~SQ_WANTEXWAKEUP;
6418 			cv_broadcast(&sq->sq_exitwait);
6419 		}
6420 		sq->sq_flags = flags;
6421 		if (bg_service) {
6422 			ASSERT(sq->sq_servcount != 0);
6423 			sq->sq_servcount--;
6424 		}
6425 		mutex_exit(SQLOCK(sq));
6426 		return;
6427 	}
6428 
6429 	/*
6430 	 * If this is not a concurrent put perimeter, we need to
6431 	 * become exclusive to drain.  Also, if not CIPUT, we would
6432 	 * not have acquired a putlock, so we don't need to check
6433 	 * the putcounts.  If not entering with a claim, we test
6434 	 * for sq_count == 0.
6435 	 */
6436 	type = sq->sq_type;
6437 	if (!(type & SQ_CIPUT)) {
6438 		if (sq->sq_count > 1) {
6439 			if (bg_service) {
6440 				ASSERT(sq->sq_servcount != 0);
6441 				sq->sq_servcount--;
6442 			}
6443 			mutex_exit(SQLOCK(sq));
6444 			return;
6445 		}
6446 		sq->sq_flags |= SQ_EXCL;
6447 	}
6448 
6449 	/*
6450 	 * This is where we make a claim to the syncq.
6451 	 * This can either be done by incrementing a putlock, or
6452 	 * the sq_count.  But since we already have the SQLOCK
6453 	 * here, we just bump the sq_count.
6454 	 *
6455 	 * Note that after we make a claim, we need to let the code
6456 	 * fall through to the end of this routine to clean itself
6457 	 * up.  A return in the while loop will put the syncq in a
6458 	 * very bad state.
6459 	 */
6460 	sq->sq_count++;
6461 	ASSERT(sq->sq_count != 0);	/* wraparound */
6462 
6463 	while ((flags = sq->sq_flags) & SQ_QUEUED) {
6464 		/*
6465 		 * If we are told to stayaway or went exclusive,
6466 		 * we are done.
6467 		 */
6468 		if (flags & (SQ_STAYAWAY)) {
6469 			break;
6470 		}
6471 
6472 		/*
6473 		 * If there are events to run, do so.
6474 		 * We have one claim to the syncq, so if there are
6475 		 * more than one, other threads are running.
6476 		 */
6477 		if (sq->sq_evhead != NULL) {
6478 			ASSERT(sq->sq_flags & SQ_EVENTS);
6479 
6480 			count = sq->sq_count;
6481 			SQ_PUTLOCKS_ENTER(sq);
6482 			SUM_SQ_PUTCOUNTS(sq, count);
6483 			if (count > 1) {
6484 				SQ_PUTLOCKS_EXIT(sq);
6485 				/* Can't upgrade - other threads inside */
6486 				break;
6487 			}
6488 			ASSERT((flags & SQ_EXCL) == 0);
6489 			sq->sq_flags = flags | SQ_EXCL;
6490 			SQ_PUTLOCKS_EXIT(sq);
6491 			/*
6492 			 * we have the only claim, run the events,
6493 			 * sq_run_events will clear the SQ_EXCL flag.
6494 			 */
6495 			sq_run_events(sq);
6496 
6497 			/*
6498 			 * If this is a CIPUT perimeter, we need
6499 			 * to drop the SQ_EXCL flag so we can properly
6500 			 * continue draining the syncq.
6501 			 */
6502 			if (type & SQ_CIPUT) {
6503 				ASSERT(sq->sq_flags & SQ_EXCL);
6504 				sq->sq_flags &= ~SQ_EXCL;
6505 			}
6506 
6507 			/*
6508 			 * And go back to the beginning just in case
6509 			 * anything changed while we were away.
6510 			 */
6511 			ASSERT((sq->sq_flags & SQ_EXCL) || (type & SQ_CIPUT));
6512 			continue;
6513 		}
6514 
6515 		ASSERT(sq->sq_evhead == NULL);
6516 		ASSERT(!(sq->sq_flags & SQ_EVENTS));
6517 
6518 		/*
6519 		 * Find the queue that is not draining.
6520 		 *
6521 		 * q_draining is protected by QLOCK which we do not hold.
6522 		 * But if it was set, then a thread was draining, and if it gets
6523 		 * cleared, then it was because the thread has successfully
6524 		 * drained the syncq, or a GOAWAY state occurred. For the GOAWAY
6525 		 * state to happen, a thread needs the SQLOCK which we hold, and
6526 		 * if there was such a flag, we would have already seen it.
6527 		 */
6528 
6529 		for (qp = sq->sq_head;
6530 		    qp != NULL && (qp->q_draining ||
6531 		    (qp->q_sqflags & Q_SQDRAINING));
6532 		    qp = qp->q_sqnext)
6533 			;
6534 
6535 		if (qp == NULL)
6536 			break;
6537 
6538 		/*
6539 		 * We have a queue to work on, and we hold the
6540 		 * SQLOCK and one claim, call qdrain_syncq.
6541 		 * This means we need to release the SQLOCK and
6542 		 * acquire the QLOCK (OK since we have a claim).
6543 		 * Note that qdrain_syncq will actually dequeue
6544 		 * this queue from the sq_head list when it is
6545 		 * convinced all the work is done and release
6546 		 * the QLOCK before returning.
6547 		 */
6548 		qp->q_sqflags |= Q_SQDRAINING;
6549 		mutex_exit(SQLOCK(sq));
6550 		mutex_enter(QLOCK(qp));
6551 		qdrain_syncq(sq, qp);
6552 		mutex_enter(SQLOCK(sq));
6553 
6554 		/* The queue is drained */
6555 		ASSERT(qp->q_sqflags & Q_SQDRAINING);
6556 		qp->q_sqflags &= ~Q_SQDRAINING;
6557 		/*
6558 		 * NOTE: After this point qp should not be used since it may be
6559 		 * closed.
6560 		 */
6561 	}
6562 
6563 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
6564 	flags = sq->sq_flags;
6565 
6566 	/*
6567 	 * sq->sq_head cannot change because we hold the
6568 	 * sqlock. However, a thread CAN decide that it is no longer
6569 	 * going to drain that queue.  However, this should be due to
6570 	 * a GOAWAY state, and we should see that here.
6571 	 *
6572 	 * This loop is not very efficient. One solution may be adding a second
6573 	 * pointer to the "draining" queue, but it is difficult to do when
6574 	 * queues are inserted in the middle due to priority ordering. Another
6575 	 * possibility is to yank the queue out of the sq list and put it onto
6576 	 * the "draining list" and then put it back if it can't be drained.
6577 	 */
6578 
6579 	ASSERT((sq->sq_head == NULL) || (flags & SQ_GOAWAY) ||
6580 	    (type & SQ_CI) || sq->sq_head->q_draining);
6581 
6582 	/* Drop SQ_EXCL for non-CIPUT perimeters */
6583 	if (!(type & SQ_CIPUT))
6584 		flags &= ~SQ_EXCL;
6585 	ASSERT((flags & SQ_EXCL) == 0);
6586 
6587 	/* Wake up any waiters. */
6588 	if (flags & SQ_WANTWAKEUP) {
6589 		flags &= ~SQ_WANTWAKEUP;
6590 		cv_broadcast(&sq->sq_wait);
6591 	}
6592 	if (flags & SQ_WANTEXWAKEUP) {
6593 		flags &= ~SQ_WANTEXWAKEUP;
6594 		cv_broadcast(&sq->sq_exitwait);
6595 	}
6596 	sq->sq_flags = flags;
6597 
6598 	ASSERT(sq->sq_count != 0);
6599 	/* Release our claim. */
6600 	sq->sq_count--;
6601 
6602 	if (bg_service) {
6603 		ASSERT(sq->sq_servcount != 0);
6604 		sq->sq_servcount--;
6605 	}
6606 
6607 	mutex_exit(SQLOCK(sq));
6608 
6609 	TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6610 	    "drain_syncq end:%p", sq);
6611 }
6612 
6613 
6614 /*
6615  *
6616  * qdrain_syncq can be called (currently) from only one of two places:
6617  *	drain_syncq
6618  * 	putnext  (or some variation of it).
6619  * and eventually
6620  * 	qwait(_sig)
6621  *
6622  * If called from drain_syncq, we found it in the list of queues needing
6623  * service, so there is work to be done (or it wouldn't be in the list).
6624  *
6625  * If called from some putnext variation, it was because the
6626  * perimeter is open, but messages are blocking a putnext and
6627  * there is not a thread working on it.  Now a thread could start
6628  * working on it while we are getting ready to do so ourself, but
6629  * the thread would set the q_draining flag, and we can spin out.
6630  *
6631  * As for qwait(_sig), I think I shall let it continue to call
6632  * drain_syncq directly (after all, it will get here eventually).
6633  *
6634  * qdrain_syncq has to terminate when:
6635  * - one of the SQ_STAYAWAY bits gets set to preserve qwriter(OUTER) ordering
6636  * - SQ_EVENTS gets set to preserve qwriter(INNER) ordering
6637  *
6638  * ASSUMES:
6639  *	One claim
6640  * 	QLOCK held
6641  * 	SQLOCK not held
6642  *	Will release QLOCK before returning
6643  */
6644 void
qdrain_syncq(syncq_t * sq,queue_t * q)6645 qdrain_syncq(syncq_t *sq, queue_t *q)
6646 {
6647 	mblk_t		*bp;
6648 #ifdef DEBUG
6649 	uint16_t	count;
6650 #endif
6651 
6652 	TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_START,
6653 	    "drain_syncq start:%p", sq);
6654 	ASSERT(q->q_syncq == sq);
6655 	ASSERT(MUTEX_HELD(QLOCK(q)));
6656 	ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6657 	/*
6658 	 * For non-CIPUT perimeters, we should be called with the exclusive bit
6659 	 * set already. For CIPUT perimeters, we will be doing a concurrent
6660 	 * drain, so it better not be set.
6661 	 */
6662 	ASSERT((sq->sq_flags & (SQ_EXCL|SQ_CIPUT)));
6663 	ASSERT(!((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)));
6664 	ASSERT((sq->sq_type & SQ_CIPUT) || (sq->sq_flags & SQ_EXCL));
6665 	/*
6666 	 * All outer pointers are set, or none of them are
6667 	 */
6668 	ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6669 	    sq->sq_oprev == NULL) ||
6670 	    (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6671 	    sq->sq_oprev != NULL));
6672 #ifdef DEBUG
6673 	count = sq->sq_count;
6674 	/*
6675 	 * This is OK without the putlocks, because we have one
6676 	 * claim either from the sq_count, or a putcount.  We could
6677 	 * get an erroneous value from other counts, but ours won't
6678 	 * change, so one way or another, we will have at least a
6679 	 * value of one.
6680 	 */
6681 	SUM_SQ_PUTCOUNTS(sq, count);
6682 	ASSERT(count >= 1);
6683 #endif /* DEBUG */
6684 
6685 	/*
6686 	 * The first thing to do is find out if a thread is already draining
6687 	 * this queue. If so, we are done, just return.
6688 	 */
6689 	if (q->q_draining) {
6690 		mutex_exit(QLOCK(q));
6691 		return;
6692 	}
6693 
6694 	/*
6695 	 * If the perimeter is exclusive, there is nothing we can do right now,
6696 	 * go away. Note that there is nothing to prevent this case from
6697 	 * changing right after this check, but the spin-out will catch it.
6698 	 */
6699 
6700 	/* Tell other threads that we are draining this queue */
6701 	q->q_draining = 1;	/* Protected by QLOCK */
6702 
6703 	/*
6704 	 * If there is nothing to do, clear QFULL as necessary. This caters for
6705 	 * the case where an empty queue was enqueued onto the syncq.
6706 	 */
6707 	if (q->q_sqhead == NULL) {
6708 		ASSERT(q->q_syncqmsgs == 0);
6709 		mutex_exit(QLOCK(q));
6710 		clr_qfull(q);
6711 		mutex_enter(QLOCK(q));
6712 	}
6713 
6714 	/*
6715 	 * Note that q_sqhead must be re-checked here in case another message
6716 	 * was enqueued whilst QLOCK was dropped during the call to clr_qfull.
6717 	 */
6718 	for (bp = q->q_sqhead; bp != NULL; bp = q->q_sqhead) {
6719 		/*
6720 		 * Because we can enter this routine just because a putnext is
6721 		 * blocked, we need to spin out if the perimeter wants to go
6722 		 * exclusive as well as just blocked. We need to spin out also
6723 		 * if events are queued on the syncq.
6724 		 * Don't check for SQ_EXCL, because non-CIPUT perimeters would
6725 		 * set it, and it can't become exclusive while we hold a claim.
6726 		 */
6727 		if (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)) {
6728 			break;
6729 		}
6730 
6731 #ifdef DEBUG
6732 		/*
6733 		 * Since we are in qdrain_syncq, we already know the queue,
6734 		 * but for sanity, we want to check this against the qp that
6735 		 * was passed in by bp->b_queue.
6736 		 */
6737 
6738 		ASSERT(bp->b_queue == q);
6739 		ASSERT(bp->b_queue->q_syncq == sq);
6740 		bp->b_queue = NULL;
6741 
6742 		/*
6743 		 * We would have the following check in the DEBUG code:
6744 		 *
6745 		 * if (bp->b_prev != NULL)  {
6746 		 *	ASSERT(bp->b_prev == (void (*)())q->q_qinfo->qi_putp);
6747 		 * }
6748 		 *
6749 		 * This can't be done, however, since IP modifies qinfo
6750 		 * structure at run-time (switching between IPv4 qinfo and IPv6
6751 		 * qinfo), invalidating the check.
6752 		 * So the assignment to func is left here, but the ASSERT itself
6753 		 * is removed until the whole issue is resolved.
6754 		 */
6755 #endif
6756 		ASSERT(q->q_sqhead == bp);
6757 		q->q_sqhead = bp->b_next;
6758 		bp->b_prev = bp->b_next = NULL;
6759 		ASSERT(q->q_syncqmsgs > 0);
6760 		mutex_exit(QLOCK(q));
6761 
6762 		ASSERT(bp->b_datap->db_ref != 0);
6763 
6764 		(void) (*q->q_qinfo->qi_putp)(q, bp);
6765 
6766 		mutex_enter(QLOCK(q));
6767 
6768 		/*
6769 		 * q_syncqmsgs should only be decremented after executing the
6770 		 * put procedure to avoid message re-ordering. This is due to an
6771 		 * optimisation in putnext() which can call the put procedure
6772 		 * directly if it sees q_syncqmsgs == 0 (despite Q_SQQUEUED
6773 		 * being set).
6774 		 *
6775 		 * We also need to clear QFULL in the next service procedure
6776 		 * queue if this is the last message destined for that queue.
6777 		 *
6778 		 * It would make better sense to have some sort of tunable for
6779 		 * the low water mark, but these semantics are not yet defined.
6780 		 * So, alas, we use a constant.
6781 		 */
6782 		if (--q->q_syncqmsgs == 0) {
6783 			mutex_exit(QLOCK(q));
6784 			clr_qfull(q);
6785 			mutex_enter(QLOCK(q));
6786 		}
6787 
6788 		/*
6789 		 * Always clear SQ_EXCL when CIPUT in order to handle
6790 		 * qwriter(INNER). The putp() can call qwriter and get exclusive
6791 		 * access IFF this is the only claim. So, we need to test for
6792 		 * this possibility, acquire the mutex and clear the bit.
6793 		 */
6794 		if ((sq->sq_type & SQ_CIPUT) && (sq->sq_flags & SQ_EXCL)) {
6795 			mutex_enter(SQLOCK(sq));
6796 			sq->sq_flags &= ~SQ_EXCL;
6797 			mutex_exit(SQLOCK(sq));
6798 		}
6799 	}
6800 
6801 	/*
6802 	 * We should either have no messages on this queue, or we were told to
6803 	 * goaway by a waiter (which we will wake up at the end of this
6804 	 * function).
6805 	 */
6806 	ASSERT((q->q_sqhead == NULL) ||
6807 	    (sq->sq_flags & (SQ_STAYAWAY | SQ_EVENTS)));
6808 
6809 	ASSERT(MUTEX_HELD(QLOCK(q)));
6810 	ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6811 
6812 	/* Remove the q from the syncq list if all the messages are drained. */
6813 	if (q->q_sqhead == NULL) {
6814 		ASSERT(q->q_syncqmsgs == 0);
6815 		mutex_enter(SQLOCK(sq));
6816 		if (q->q_sqflags & Q_SQQUEUED)
6817 			SQRM_Q(sq, q);
6818 		mutex_exit(SQLOCK(sq));
6819 		/*
6820 		 * Since the queue is removed from the list, reset its priority.
6821 		 */
6822 		q->q_spri = 0;
6823 	}
6824 
6825 	/*
6826 	 * Remember, the q_draining flag is used to let another thread know
6827 	 * that there is a thread currently draining the messages for a queue.
6828 	 * Since we are now done with this queue (even if there may be messages
6829 	 * still there), we need to clear this flag so some thread will work on
6830 	 * it if needed.
6831 	 */
6832 	ASSERT(q->q_draining);
6833 	q->q_draining = 0;
6834 
6835 	/* Called with a claim, so OK to drop all locks. */
6836 	mutex_exit(QLOCK(q));
6837 
6838 	TRACE_1(TR_FAC_STREAMS_FR, TR_DRAIN_SYNCQ_END,
6839 	    "drain_syncq end:%p", sq);
6840 }
6841 /* END OF QDRAIN_SYNCQ  */
6842 
6843 
6844 /*
6845  * This is the mate to qdrain_syncq, except that it is putting the message onto
6846  * the queue instead of draining. Since the message is destined for the queue
6847  * that is selected, there is no need to identify the function because the
6848  * message is intended for the put routine for the queue. For debug kernels,
6849  * this routine will do it anyway just in case.
6850  *
6851  * After the message is enqueued on the syncq, it calls putnext_tail()
6852  * which will schedule a background thread to actually process the message.
6853  *
6854  * Assumes that there is a claim on the syncq (sq->sq_count > 0) and
6855  * SQLOCK(sq) and QLOCK(q) are not held.
6856  */
6857 void
qfill_syncq(syncq_t * sq,queue_t * q,mblk_t * mp)6858 qfill_syncq(syncq_t *sq, queue_t *q, mblk_t *mp)
6859 {
6860 	ASSERT(MUTEX_NOT_HELD(SQLOCK(sq)));
6861 	ASSERT(MUTEX_NOT_HELD(QLOCK(q)));
6862 	ASSERT(sq->sq_count > 0);
6863 	ASSERT(q->q_syncq == sq);
6864 	ASSERT((sq->sq_outer == NULL && sq->sq_onext == NULL &&
6865 	    sq->sq_oprev == NULL) ||
6866 	    (sq->sq_outer != NULL && sq->sq_onext != NULL &&
6867 	    sq->sq_oprev != NULL));
6868 
6869 	mutex_enter(QLOCK(q));
6870 
6871 #ifdef DEBUG
6872 	/*
6873 	 * This is used for debug in the qfill_syncq/qdrain_syncq case
6874 	 * to trace the queue that the message is intended for.  Note
6875 	 * that the original use was to identify the queue and function
6876 	 * to call on the drain.  In the new syncq, we have the context
6877 	 * of the queue that we are draining, so call it's putproc and
6878 	 * don't rely on the saved values.  But for debug this is still
6879 	 * useful information.
6880 	 */
6881 	mp->b_prev = (mblk_t *)q->q_qinfo->qi_putp;
6882 	mp->b_queue = q;
6883 	mp->b_next = NULL;
6884 #endif
6885 	ASSERT(q->q_syncq == sq);
6886 	/*
6887 	 * Enqueue the message on the list.
6888 	 * SQPUT_MP() accesses q_syncqmsgs.  We are already holding QLOCK to
6889 	 * protect it.  So it's ok to acquire SQLOCK after SQPUT_MP().
6890 	 */
6891 	SQPUT_MP(q, mp);
6892 	mutex_enter(SQLOCK(sq));
6893 
6894 	/*
6895 	 * And queue on syncq for scheduling, if not already queued.
6896 	 * Note that we need the SQLOCK for this, and for testing flags
6897 	 * at the end to see if we will drain.  So grab it now, and
6898 	 * release it before we call qdrain_syncq or return.
6899 	 */
6900 	if (!(q->q_sqflags & Q_SQQUEUED)) {
6901 		q->q_spri = curthread->t_pri;
6902 		SQPUT_Q(sq, q);
6903 	}
6904 #ifdef DEBUG
6905 	else {
6906 		/*
6907 		 * All of these conditions MUST be true!
6908 		 */
6909 		ASSERT(sq->sq_tail != NULL);
6910 		if (sq->sq_tail == sq->sq_head) {
6911 			ASSERT((q->q_sqprev == NULL) &&
6912 			    (q->q_sqnext == NULL));
6913 		} else {
6914 			ASSERT((q->q_sqprev != NULL) ||
6915 			    (q->q_sqnext != NULL));
6916 		}
6917 		ASSERT(sq->sq_flags & SQ_QUEUED);
6918 		ASSERT(q->q_syncqmsgs != 0);
6919 		ASSERT(q->q_sqflags & Q_SQQUEUED);
6920 	}
6921 #endif
6922 	mutex_exit(QLOCK(q));
6923 	/*
6924 	 * SQLOCK is still held, so sq_count can be safely decremented.
6925 	 */
6926 	sq->sq_count--;
6927 
6928 	putnext_tail(sq, q, 0);
6929 	/* Should not reference sq or q after this point. */
6930 }
6931 
6932 /*  End of qfill_syncq  */
6933 
6934 /*
6935  * Remove all messages from a syncq (if qp is NULL) or remove all messages
6936  * that would be put into qp by drain_syncq.
6937  * Used when deleting the syncq (qp == NULL) or when detaching
6938  * a queue (qp != NULL).
6939  * Return non-zero if one or more messages were freed.
6940  *
6941  * No need to grab sq_putlocks here. See comment in strsubr.h that explains when
6942  * sq_putlocks are used.
6943  *
6944  * NOTE: This function assumes that it is called from the close() context and
6945  * that all the queues in the syncq are going away. For this reason it doesn't
6946  * acquire QLOCK for modifying q_sqhead/q_sqtail fields. This assumption is
6947  * currently valid, but it is useful to rethink this function to behave properly
6948  * in other cases.
6949  */
6950 int
flush_syncq(syncq_t * sq,queue_t * qp)6951 flush_syncq(syncq_t *sq, queue_t *qp)
6952 {
6953 	mblk_t		*bp, *mp_head, *mp_next, *mp_prev;
6954 	queue_t		*q;
6955 	int		ret = 0;
6956 
6957 	mutex_enter(SQLOCK(sq));
6958 
6959 	/*
6960 	 * Before we leave, we need to make sure there are no
6961 	 * events listed for this queue.  All events for this queue
6962 	 * will just be freed.
6963 	 */
6964 	if (qp != NULL && sq->sq_evhead != NULL) {
6965 		ASSERT(sq->sq_flags & SQ_EVENTS);
6966 
6967 		mp_prev = NULL;
6968 		for (bp = sq->sq_evhead; bp != NULL; bp = mp_next) {
6969 			mp_next = bp->b_next;
6970 			if (bp->b_queue == qp) {
6971 				/* Delete this message */
6972 				if (mp_prev != NULL) {
6973 					mp_prev->b_next = mp_next;
6974 					/*
6975 					 * Update sq_evtail if the last element
6976 					 * is removed.
6977 					 */
6978 					if (bp == sq->sq_evtail) {
6979 						ASSERT(mp_next == NULL);
6980 						sq->sq_evtail = mp_prev;
6981 					}
6982 				} else
6983 					sq->sq_evhead = mp_next;
6984 				if (sq->sq_evhead == NULL)
6985 					sq->sq_flags &= ~SQ_EVENTS;
6986 				bp->b_prev = bp->b_next = NULL;
6987 				freemsg(bp);
6988 				ret++;
6989 			} else {
6990 				mp_prev = bp;
6991 			}
6992 		}
6993 	}
6994 
6995 	/*
6996 	 * Walk sq_head and:
6997 	 *	- match qp if qp is set, remove it's messages
6998 	 *	- all if qp is not set
6999 	 */
7000 	q = sq->sq_head;
7001 	while (q != NULL) {
7002 		ASSERT(q->q_syncq == sq);
7003 		if ((qp == NULL) || (qp == q)) {
7004 			/*
7005 			 * Yank the messages as a list off the queue
7006 			 */
7007 			mp_head = q->q_sqhead;
7008 			/*
7009 			 * We do not have QLOCK(q) here (which is safe due to
7010 			 * assumptions mentioned above). To obtain the lock we
7011 			 * need to release SQLOCK which may allow lots of things
7012 			 * to change upon us. This place requires more analysis.
7013 			 */
7014 			q->q_sqhead = q->q_sqtail = NULL;
7015 			ASSERT(mp_head->b_queue &&
7016 			    mp_head->b_queue->q_syncq == sq);
7017 
7018 			/*
7019 			 * Free each of the messages.
7020 			 */
7021 			for (bp = mp_head; bp != NULL; bp = mp_next) {
7022 				mp_next = bp->b_next;
7023 				bp->b_prev = bp->b_next = NULL;
7024 				freemsg(bp);
7025 				ret++;
7026 			}
7027 			/*
7028 			 * Now remove the queue from the syncq.
7029 			 */
7030 			ASSERT(q->q_sqflags & Q_SQQUEUED);
7031 			SQRM_Q(sq, q);
7032 			q->q_spri = 0;
7033 			q->q_syncqmsgs = 0;
7034 
7035 			/*
7036 			 * If qp was specified, we are done with it and are
7037 			 * going to drop SQLOCK(sq) and return. We wakeup syncq
7038 			 * waiters while we still have the SQLOCK.
7039 			 */
7040 			if ((qp != NULL) && (sq->sq_flags & SQ_WANTWAKEUP)) {
7041 				sq->sq_flags &= ~SQ_WANTWAKEUP;
7042 				cv_broadcast(&sq->sq_wait);
7043 			}
7044 			/* Drop SQLOCK across clr_qfull */
7045 			mutex_exit(SQLOCK(sq));
7046 
7047 			/*
7048 			 * We avoid doing the test that drain_syncq does and
7049 			 * unconditionally clear qfull for every flushed
7050 			 * message. Since flush_syncq is only called during
7051 			 * close this should not be a problem.
7052 			 */
7053 			clr_qfull(q);
7054 			if (qp != NULL) {
7055 				return (ret);
7056 			} else {
7057 				mutex_enter(SQLOCK(sq));
7058 				/*
7059 				 * The head was removed by SQRM_Q above.
7060 				 * reread the new head and flush it.
7061 				 */
7062 				q = sq->sq_head;
7063 			}
7064 		} else {
7065 			q = q->q_sqnext;
7066 		}
7067 		ASSERT(MUTEX_HELD(SQLOCK(sq)));
7068 	}
7069 
7070 	if (sq->sq_flags & SQ_WANTWAKEUP) {
7071 		sq->sq_flags &= ~SQ_WANTWAKEUP;
7072 		cv_broadcast(&sq->sq_wait);
7073 	}
7074 
7075 	mutex_exit(SQLOCK(sq));
7076 	return (ret);
7077 }
7078 
7079 /*
7080  * Propagate all messages from a syncq to the next syncq that are associated
7081  * with the specified queue. If the queue is attached to a driver or if the
7082  * messages have been added due to a qwriter(PERIM_INNER), free the messages.
7083  *
7084  * Assumes that the stream is strlock()'ed. We don't come here if there
7085  * are no messages to propagate.
7086  *
7087  * NOTE : If the queue is attached to a driver, all the messages are freed
7088  * as there is no point in propagating the messages from the driver syncq
7089  * to the closing stream head which will in turn get freed later.
7090  */
7091 static int
propagate_syncq(queue_t * qp)7092 propagate_syncq(queue_t *qp)
7093 {
7094 	mblk_t		*bp, *head, *tail, *prev, *next;
7095 	syncq_t 	*sq;
7096 	queue_t		*nqp;
7097 	syncq_t		*nsq;
7098 	boolean_t	isdriver;
7099 	int 		moved = 0;
7100 	uint16_t	flags;
7101 	pri_t		priority = curthread->t_pri;
7102 #ifdef DEBUG
7103 	void		(*func)();
7104 #endif
7105 
7106 	sq = qp->q_syncq;
7107 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
7108 	/* debug macro */
7109 	SQ_PUTLOCKS_HELD(sq);
7110 	/*
7111 	 * As entersq() does not increment the sq_count for
7112 	 * the write side, check sq_count for non-QPERQ
7113 	 * perimeters alone.
7114 	 */
7115 	ASSERT((qp->q_flag & QPERQ) || (sq->sq_count >= 1));
7116 
7117 	/*
7118 	 * propagate_syncq() can be called because of either messages on the
7119 	 * queue syncq or because on events on the queue syncq. Do actual
7120 	 * message propagations if there are any messages.
7121 	 */
7122 	if (qp->q_syncqmsgs) {
7123 		isdriver = (qp->q_flag & QISDRV);
7124 
7125 		if (!isdriver) {
7126 			nqp = qp->q_next;
7127 			nsq = nqp->q_syncq;
7128 			ASSERT(MUTEX_HELD(SQLOCK(nsq)));
7129 			/* debug macro */
7130 			SQ_PUTLOCKS_HELD(nsq);
7131 #ifdef DEBUG
7132 			func = (void (*)())nqp->q_qinfo->qi_putp;
7133 #endif
7134 		}
7135 
7136 		SQRM_Q(sq, qp);
7137 		priority = MAX(qp->q_spri, priority);
7138 		qp->q_spri = 0;
7139 		head = qp->q_sqhead;
7140 		tail = qp->q_sqtail;
7141 		qp->q_sqhead = qp->q_sqtail = NULL;
7142 		qp->q_syncqmsgs = 0;
7143 
7144 		/*
7145 		 * Walk the list of messages, and free them if this is a driver,
7146 		 * otherwise reset the b_prev and b_queue value to the new putp.
7147 		 * Afterward, we will just add the head to the end of the next
7148 		 * syncq, and point the tail to the end of this one.
7149 		 */
7150 
7151 		for (bp = head; bp != NULL; bp = next) {
7152 			next = bp->b_next;
7153 			if (isdriver) {
7154 				bp->b_prev = bp->b_next = NULL;
7155 				freemsg(bp);
7156 				continue;
7157 			}
7158 			/* Change the q values for this message */
7159 			bp->b_queue = nqp;
7160 #ifdef DEBUG
7161 			bp->b_prev = (mblk_t *)func;
7162 #endif
7163 			moved++;
7164 		}
7165 		/*
7166 		 * Attach list of messages to the end of the new queue (if there
7167 		 * is a list of messages).
7168 		 */
7169 
7170 		if (!isdriver && head != NULL) {
7171 			ASSERT(tail != NULL);
7172 			if (nqp->q_sqhead == NULL) {
7173 				nqp->q_sqhead = head;
7174 			} else {
7175 				ASSERT(nqp->q_sqtail != NULL);
7176 				nqp->q_sqtail->b_next = head;
7177 			}
7178 			nqp->q_sqtail = tail;
7179 			/*
7180 			 * When messages are moved from high priority queue to
7181 			 * another queue, the destination queue priority is
7182 			 * upgraded.
7183 			 */
7184 
7185 			if (priority > nqp->q_spri)
7186 				nqp->q_spri = priority;
7187 
7188 			SQPUT_Q(nsq, nqp);
7189 
7190 			nqp->q_syncqmsgs += moved;
7191 			ASSERT(nqp->q_syncqmsgs != 0);
7192 		}
7193 	}
7194 
7195 	/*
7196 	 * Before we leave, we need to make sure there are no
7197 	 * events listed for this queue.  All events for this queue
7198 	 * will just be freed.
7199 	 */
7200 	if (sq->sq_evhead != NULL) {
7201 		ASSERT(sq->sq_flags & SQ_EVENTS);
7202 		prev = NULL;
7203 		for (bp = sq->sq_evhead; bp != NULL; bp = next) {
7204 			next = bp->b_next;
7205 			if (bp->b_queue == qp) {
7206 				/* Delete this message */
7207 				if (prev != NULL) {
7208 					prev->b_next = next;
7209 					/*
7210 					 * Update sq_evtail if the last element
7211 					 * is removed.
7212 					 */
7213 					if (bp == sq->sq_evtail) {
7214 						ASSERT(next == NULL);
7215 						sq->sq_evtail = prev;
7216 					}
7217 				} else
7218 					sq->sq_evhead = next;
7219 				if (sq->sq_evhead == NULL)
7220 					sq->sq_flags &= ~SQ_EVENTS;
7221 				bp->b_prev = bp->b_next = NULL;
7222 				freemsg(bp);
7223 			} else {
7224 				prev = bp;
7225 			}
7226 		}
7227 	}
7228 
7229 	flags = sq->sq_flags;
7230 
7231 	/* Wake up any waiter before leaving. */
7232 	if (flags & SQ_WANTWAKEUP) {
7233 		flags &= ~SQ_WANTWAKEUP;
7234 		cv_broadcast(&sq->sq_wait);
7235 	}
7236 	sq->sq_flags = flags;
7237 
7238 	return (moved);
7239 }
7240 
7241 /*
7242  * Try and upgrade to exclusive access at the inner perimeter. If this can
7243  * not be done without blocking then request will be queued on the syncq
7244  * and drain_syncq will run it later.
7245  *
7246  * This routine can only be called from put or service procedures plus
7247  * asynchronous callback routines that have properly entered the queue (with
7248  * entersq). Thus qwriter_inner assumes the caller has one claim on the syncq
7249  * associated with q.
7250  */
7251 void
qwriter_inner(queue_t * q,mblk_t * mp,void (* func)())7252 qwriter_inner(queue_t *q, mblk_t *mp, void (*func)())
7253 {
7254 	syncq_t	*sq = q->q_syncq;
7255 	uint16_t count;
7256 
7257 	mutex_enter(SQLOCK(sq));
7258 	count = sq->sq_count;
7259 	SQ_PUTLOCKS_ENTER(sq);
7260 	SUM_SQ_PUTCOUNTS(sq, count);
7261 	ASSERT(count >= 1);
7262 	ASSERT(sq->sq_type & (SQ_CIPUT|SQ_CISVC));
7263 
7264 	if (count == 1) {
7265 		/*
7266 		 * Can upgrade. This case also handles nested qwriter calls
7267 		 * (when the qwriter callback function calls qwriter). In that
7268 		 * case SQ_EXCL is already set.
7269 		 */
7270 		sq->sq_flags |= SQ_EXCL;
7271 		SQ_PUTLOCKS_EXIT(sq);
7272 		mutex_exit(SQLOCK(sq));
7273 		(*func)(q, mp);
7274 		/*
7275 		 * Assumes that leavesq, putnext, and drain_syncq will reset
7276 		 * SQ_EXCL for SQ_CIPUT/SQ_CISVC queues. We leave SQ_EXCL on
7277 		 * until putnext, leavesq, or drain_syncq drops it.
7278 		 * That way we handle nested qwriter(INNER) without dropping
7279 		 * SQ_EXCL until the outermost qwriter callback routine is
7280 		 * done.
7281 		 */
7282 		return;
7283 	}
7284 	SQ_PUTLOCKS_EXIT(sq);
7285 	sqfill_events(sq, q, mp, func);
7286 }
7287 
7288 /*
7289  * Synchronous callback support functions
7290  */
7291 
7292 /*
7293  * Allocate a callback parameter structure.
7294  * Assumes that caller initializes the flags and the id.
7295  * Acquires SQLOCK(sq) if non-NULL is returned.
7296  */
7297 callbparams_t *
callbparams_alloc(syncq_t * sq,void (* func)(void *),void * arg,int kmflags)7298 callbparams_alloc(syncq_t *sq, void (*func)(void *), void *arg, int kmflags)
7299 {
7300 	callbparams_t *cbp;
7301 	size_t size = sizeof (callbparams_t);
7302 
7303 	cbp = kmem_alloc(size, kmflags & ~KM_PANIC);
7304 
7305 	/*
7306 	 * Only try tryhard allocation if the caller is ready to panic.
7307 	 * Otherwise just fail.
7308 	 */
7309 	if (cbp == NULL) {
7310 		if (kmflags & KM_PANIC)
7311 			cbp = kmem_alloc_tryhard(sizeof (callbparams_t),
7312 			    &size, kmflags);
7313 		else
7314 			return (NULL);
7315 	}
7316 
7317 	ASSERT(size >= sizeof (callbparams_t));
7318 	cbp->cbp_size = size;
7319 	cbp->cbp_sq = sq;
7320 	cbp->cbp_func = func;
7321 	cbp->cbp_arg = arg;
7322 	mutex_enter(SQLOCK(sq));
7323 	cbp->cbp_next = sq->sq_callbpend;
7324 	sq->sq_callbpend = cbp;
7325 	return (cbp);
7326 }
7327 
7328 void
callbparams_free(syncq_t * sq,callbparams_t * cbp)7329 callbparams_free(syncq_t *sq, callbparams_t *cbp)
7330 {
7331 	callbparams_t **pp, *p;
7332 
7333 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
7334 
7335 	for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7336 		if (p == cbp) {
7337 			*pp = p->cbp_next;
7338 			kmem_free(p, p->cbp_size);
7339 			return;
7340 		}
7341 	}
7342 	(void) (STRLOG(0, 0, 0, SL_CONSOLE,
7343 	    "callbparams_free: not found\n"));
7344 }
7345 
7346 void
callbparams_free_id(syncq_t * sq,callbparams_id_t id,int32_t flag)7347 callbparams_free_id(syncq_t *sq, callbparams_id_t id, int32_t flag)
7348 {
7349 	callbparams_t **pp, *p;
7350 
7351 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
7352 
7353 	for (pp = &sq->sq_callbpend; (p = *pp) != NULL; pp = &p->cbp_next) {
7354 		if (p->cbp_id == id && p->cbp_flags == flag) {
7355 			*pp = p->cbp_next;
7356 			kmem_free(p, p->cbp_size);
7357 			return;
7358 		}
7359 	}
7360 	(void) (STRLOG(0, 0, 0, SL_CONSOLE,
7361 	    "callbparams_free_id: not found\n"));
7362 }
7363 
7364 /*
7365  * Callback wrapper function used by once-only callbacks that can be
7366  * cancelled (qtimeout and qbufcall)
7367  * Contains inline version of entersq(sq, SQ_CALLBACK) that can be
7368  * cancelled by the qun* functions.
7369  */
7370 void
qcallbwrapper(void * arg)7371 qcallbwrapper(void *arg)
7372 {
7373 	callbparams_t *cbp = arg;
7374 	syncq_t	*sq;
7375 	uint16_t count = 0;
7376 	uint16_t waitflags = SQ_STAYAWAY | SQ_EVENTS | SQ_EXCL;
7377 	uint16_t type;
7378 
7379 	sq = cbp->cbp_sq;
7380 	mutex_enter(SQLOCK(sq));
7381 	type = sq->sq_type;
7382 	if (!(type & SQ_CICB)) {
7383 		count = sq->sq_count;
7384 		SQ_PUTLOCKS_ENTER(sq);
7385 		SQ_PUTCOUNT_CLRFAST_LOCKED(sq);
7386 		SUM_SQ_PUTCOUNTS(sq, count);
7387 		sq->sq_needexcl++;
7388 		ASSERT(sq->sq_needexcl != 0);	/* wraparound */
7389 		waitflags |= SQ_MESSAGES;
7390 	}
7391 	/* Can not handle exclusive entry at outer perimeter */
7392 	ASSERT(type & SQ_COCB);
7393 
7394 	while ((sq->sq_flags & waitflags) || (!(type & SQ_CICB) &&count != 0)) {
7395 		if ((sq->sq_callbflags & cbp->cbp_flags) &&
7396 		    (sq->sq_cancelid == cbp->cbp_id)) {
7397 			/* timeout has been cancelled */
7398 			sq->sq_callbflags |= SQ_CALLB_BYPASSED;
7399 			callbparams_free(sq, cbp);
7400 			if (!(type & SQ_CICB)) {
7401 				ASSERT(sq->sq_needexcl > 0);
7402 				sq->sq_needexcl--;
7403 				if (sq->sq_needexcl == 0) {
7404 					SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7405 				}
7406 				SQ_PUTLOCKS_EXIT(sq);
7407 			}
7408 			mutex_exit(SQLOCK(sq));
7409 			return;
7410 		}
7411 		sq->sq_flags |= SQ_WANTWAKEUP;
7412 		if (!(type & SQ_CICB)) {
7413 			SQ_PUTLOCKS_EXIT(sq);
7414 		}
7415 		cv_wait(&sq->sq_wait, SQLOCK(sq));
7416 		if (!(type & SQ_CICB)) {
7417 			count = sq->sq_count;
7418 			SQ_PUTLOCKS_ENTER(sq);
7419 			SUM_SQ_PUTCOUNTS(sq, count);
7420 		}
7421 	}
7422 
7423 	sq->sq_count++;
7424 	ASSERT(sq->sq_count != 0);	/* Wraparound */
7425 	if (!(type & SQ_CICB)) {
7426 		ASSERT(count == 0);
7427 		sq->sq_flags |= SQ_EXCL;
7428 		ASSERT(sq->sq_needexcl > 0);
7429 		sq->sq_needexcl--;
7430 		if (sq->sq_needexcl == 0) {
7431 			SQ_PUTCOUNT_SETFAST_LOCKED(sq);
7432 		}
7433 		SQ_PUTLOCKS_EXIT(sq);
7434 	}
7435 
7436 	mutex_exit(SQLOCK(sq));
7437 
7438 	cbp->cbp_func(cbp->cbp_arg);
7439 
7440 	/*
7441 	 * We drop the lock only for leavesq to re-acquire it.
7442 	 * Possible optimization is inline of leavesq.
7443 	 */
7444 	mutex_enter(SQLOCK(sq));
7445 	callbparams_free(sq, cbp);
7446 	mutex_exit(SQLOCK(sq));
7447 	leavesq(sq, SQ_CALLBACK);
7448 }
7449 
7450 /*
7451  * No need to grab sq_putlocks here. See comment in strsubr.h that
7452  * explains when sq_putlocks are used.
7453  *
7454  * sq_count (or one of the sq_putcounts) has already been
7455  * decremented by the caller, and if SQ_QUEUED, we need to call
7456  * drain_syncq (the global syncq drain).
7457  * If putnext_tail is called with the SQ_EXCL bit set, we are in
7458  * one of two states, non-CIPUT perimeter, and we need to clear
7459  * it, or we went exclusive in the put procedure.  In any case,
7460  * we want to clear the bit now, and it is probably easier to do
7461  * this at the beginning of this function (remember, we hold
7462  * the SQLOCK).  Lastly, if there are other messages queued
7463  * on the syncq (and not for our destination), enable the syncq
7464  * for background work.
7465  */
7466 
7467 /* ARGSUSED */
7468 void
putnext_tail(syncq_t * sq,queue_t * qp,uint32_t passflags)7469 putnext_tail(syncq_t *sq, queue_t *qp, uint32_t passflags)
7470 {
7471 	uint16_t	flags = sq->sq_flags;
7472 
7473 	ASSERT(MUTEX_HELD(SQLOCK(sq)));
7474 	ASSERT(MUTEX_NOT_HELD(QLOCK(qp)));
7475 
7476 	/* Clear SQ_EXCL if set in passflags */
7477 	if (passflags & SQ_EXCL) {
7478 		flags &= ~SQ_EXCL;
7479 	}
7480 	if (flags & SQ_WANTWAKEUP) {
7481 		flags &= ~SQ_WANTWAKEUP;
7482 		cv_broadcast(&sq->sq_wait);
7483 	}
7484 	if (flags & SQ_WANTEXWAKEUP) {
7485 		flags &= ~SQ_WANTEXWAKEUP;
7486 		cv_broadcast(&sq->sq_exitwait);
7487 	}
7488 	sq->sq_flags = flags;
7489 
7490 	/*
7491 	 * We have cleared SQ_EXCL if we were asked to, and started
7492 	 * the wakeup process for waiters.  If there are no writers
7493 	 * then we need to drain the syncq if we were told to, or
7494 	 * enable the background thread to do it.
7495 	 */
7496 	if (!(flags & (SQ_STAYAWAY|SQ_EXCL))) {
7497 		if ((passflags & SQ_QUEUED) ||
7498 		    (sq->sq_svcflags & SQ_DISABLED)) {
7499 			/* drain_syncq will take care of events in the list */
7500 			drain_syncq(sq);
7501 			return;
7502 		} else if (flags & SQ_QUEUED) {
7503 			sqenable(sq);
7504 		}
7505 	}
7506 	/* Drop the SQLOCK on exit */
7507 	mutex_exit(SQLOCK(sq));
7508 	TRACE_3(TR_FAC_STREAMS_FR, TR_PUTNEXT_END,
7509 	    "putnext_end:(%p, %p, %p) done", NULL, qp, sq);
7510 }
7511 
7512 void
set_qend(queue_t * q)7513 set_qend(queue_t *q)
7514 {
7515 	mutex_enter(QLOCK(q));
7516 	if (!O_SAMESTR(q))
7517 		q->q_flag |= QEND;
7518 	else
7519 		q->q_flag &= ~QEND;
7520 	mutex_exit(QLOCK(q));
7521 	q = _OTHERQ(q);
7522 	mutex_enter(QLOCK(q));
7523 	if (!O_SAMESTR(q))
7524 		q->q_flag |= QEND;
7525 	else
7526 		q->q_flag &= ~QEND;
7527 	mutex_exit(QLOCK(q));
7528 }
7529 
7530 /*
7531  * Set QFULL in next service procedure queue (that cares) if not already
7532  * set and if there are already more messages on the syncq than
7533  * sq_max_size.  If sq_max_size is 0, no flow control will be asserted on
7534  * any syncq.
7535  *
7536  * The fq here is the next queue with a service procedure.  This is where
7537  * we would fail canputnext, so this is where we need to set QFULL.
7538  * In the case when fq != q we need to take QLOCK(fq) to set QFULL flag.
7539  *
7540  * We already have QLOCK at this point. To avoid cross-locks with
7541  * freezestr() which grabs all QLOCKs and with strlock() which grabs both
7542  * SQLOCK and sd_reflock, we need to drop respective locks first.
7543  */
7544 void
set_qfull(queue_t * q)7545 set_qfull(queue_t *q)
7546 {
7547 	queue_t		*fq = NULL;
7548 
7549 	ASSERT(MUTEX_HELD(QLOCK(q)));
7550 	if ((sq_max_size != 0) && (!(q->q_nfsrv->q_flag & QFULL)) &&
7551 	    (q->q_syncqmsgs > sq_max_size)) {
7552 		if ((fq = q->q_nfsrv) == q) {
7553 			fq->q_flag |= QFULL;
7554 		} else {
7555 			mutex_exit(QLOCK(q));
7556 			mutex_enter(QLOCK(fq));
7557 			fq->q_flag |= QFULL;
7558 			mutex_exit(QLOCK(fq));
7559 			mutex_enter(QLOCK(q));
7560 		}
7561 	}
7562 }
7563 
7564 void
clr_qfull(queue_t * q)7565 clr_qfull(queue_t *q)
7566 {
7567 	queue_t	*oq = q;
7568 
7569 	q = q->q_nfsrv;
7570 	/* Fast check if there is any work to do before getting the lock. */
7571 	if ((q->q_flag & (QFULL|QWANTW)) == 0) {
7572 		return;
7573 	}
7574 
7575 	/*
7576 	 * Do not reset QFULL (and backenable) if the q_count is the reason
7577 	 * for QFULL being set.
7578 	 */
7579 	mutex_enter(QLOCK(q));
7580 	/*
7581 	 * If queue is empty i.e q_mblkcnt is zero, queue can not be full.
7582 	 * Hence clear the QFULL.
7583 	 * If both q_count and q_mblkcnt are less than the hiwat mark,
7584 	 * clear the QFULL.
7585 	 */
7586 	if (q->q_mblkcnt == 0 || ((q->q_count < q->q_hiwat) &&
7587 	    (q->q_mblkcnt < q->q_hiwat))) {
7588 		q->q_flag &= ~QFULL;
7589 		/*
7590 		 * A little more confusing, how about this way:
7591 		 * if someone wants to write,
7592 		 * AND
7593 		 *    both counts are less than the lowat mark
7594 		 *    OR
7595 		 *    the lowat mark is zero
7596 		 * THEN
7597 		 * backenable
7598 		 */
7599 		if ((q->q_flag & QWANTW) &&
7600 		    (((q->q_count < q->q_lowat) &&
7601 		    (q->q_mblkcnt < q->q_lowat)) || q->q_lowat == 0)) {
7602 			q->q_flag &= ~QWANTW;
7603 			mutex_exit(QLOCK(q));
7604 			backenable(oq, 0);
7605 		} else
7606 			mutex_exit(QLOCK(q));
7607 	} else
7608 		mutex_exit(QLOCK(q));
7609 }
7610 
7611 /*
7612  * Set the forward service procedure pointer.
7613  *
7614  * Called at insert-time to cache a queue's next forward service procedure in
7615  * q_nfsrv; used by canput() and canputnext().  If the queue to be inserted
7616  * has a service procedure then q_nfsrv points to itself.  If the queue to be
7617  * inserted does not have a service procedure, then q_nfsrv points to the next
7618  * queue forward that has a service procedure.  If the queue is at the logical
7619  * end of the stream (driver for write side, stream head for the read side)
7620  * and does not have a service procedure, then q_nfsrv also points to itself.
7621  */
7622 void
set_nfsrv_ptr(queue_t * rnew,queue_t * wnew,queue_t * prev_rq,queue_t * prev_wq)7623 set_nfsrv_ptr(
7624 	queue_t  *rnew,		/* read queue pointer to new module */
7625 	queue_t  *wnew,		/* write queue pointer to new module */
7626 	queue_t  *prev_rq,	/* read queue pointer to the module above */
7627 	queue_t  *prev_wq)	/* write queue pointer to the module above */
7628 {
7629 	queue_t *qp;
7630 
7631 	if (prev_wq->q_next == NULL) {
7632 		/*
7633 		 * Insert the driver, initialize the driver and stream head.
7634 		 * In this case, prev_rq/prev_wq should be the stream head.
7635 		 * _I_INSERT does not allow inserting a driver.  Make sure
7636 		 * that it is not an insertion.
7637 		 */
7638 		ASSERT(!(rnew->q_flag & _QINSERTING));
7639 		wnew->q_nfsrv = wnew;
7640 		if (rnew->q_qinfo->qi_srvp)
7641 			rnew->q_nfsrv = rnew;
7642 		else
7643 			rnew->q_nfsrv = prev_rq;
7644 		prev_rq->q_nfsrv = prev_rq;
7645 		prev_wq->q_nfsrv = prev_wq;
7646 	} else {
7647 		/*
7648 		 * set up read side q_nfsrv pointer.  This MUST be done
7649 		 * before setting the write side, because the setting of
7650 		 * the write side for a fifo may depend on it.
7651 		 *
7652 		 * Suppose we have a fifo that only has pipemod pushed.
7653 		 * pipemod has no read or write service procedures, so
7654 		 * nfsrv for both pipemod queues points to prev_rq (the
7655 		 * stream read head).  Now push bufmod (which has only a
7656 		 * read service procedure).  Doing the write side first,
7657 		 * wnew->q_nfsrv is set to pipemod's writeq nfsrv, which
7658 		 * is WRONG; the next queue forward from wnew with a
7659 		 * service procedure will be rnew, not the stream read head.
7660 		 * Since the downstream queue (which in the case of a fifo
7661 		 * is the read queue rnew) can affect upstream queues, it
7662 		 * needs to be done first.  Setting up the read side first
7663 		 * sets nfsrv for both pipemod queues to rnew and then
7664 		 * when the write side is set up, wnew-q_nfsrv will also
7665 		 * point to rnew.
7666 		 */
7667 		if (rnew->q_qinfo->qi_srvp) {
7668 			/*
7669 			 * use _OTHERQ() because, if this is a pipe, next
7670 			 * module may have been pushed from other end and
7671 			 * q_next could be a read queue.
7672 			 */
7673 			qp = _OTHERQ(prev_wq->q_next);
7674 			while (qp && qp->q_nfsrv != qp) {
7675 				qp->q_nfsrv = rnew;
7676 				qp = backq(qp);
7677 			}
7678 			rnew->q_nfsrv = rnew;
7679 		} else
7680 			rnew->q_nfsrv = prev_rq->q_nfsrv;
7681 
7682 		/* set up write side q_nfsrv pointer */
7683 		if (wnew->q_qinfo->qi_srvp) {
7684 			wnew->q_nfsrv = wnew;
7685 
7686 			/*
7687 			 * For insertion, need to update nfsrv of the modules
7688 			 * above which do not have a service routine.
7689 			 */
7690 			if (rnew->q_flag & _QINSERTING) {
7691 				for (qp = prev_wq;
7692 				    qp != NULL && qp->q_nfsrv != qp;
7693 				    qp = backq(qp)) {
7694 					qp->q_nfsrv = wnew->q_nfsrv;
7695 				}
7696 			}
7697 		} else {
7698 			if (prev_wq->q_next == prev_rq)
7699 				/*
7700 				 * Since prev_wq/prev_rq are the middle of a
7701 				 * fifo, wnew/rnew will also be the middle of
7702 				 * a fifo and wnew's nfsrv is same as rnew's.
7703 				 */
7704 				wnew->q_nfsrv = rnew->q_nfsrv;
7705 			else
7706 				wnew->q_nfsrv = prev_wq->q_next->q_nfsrv;
7707 		}
7708 	}
7709 }
7710 
7711 /*
7712  * Reset the forward service procedure pointer; called at remove-time.
7713  */
7714 void
reset_nfsrv_ptr(queue_t * rqp,queue_t * wqp)7715 reset_nfsrv_ptr(queue_t *rqp, queue_t *wqp)
7716 {
7717 	queue_t *tmp_qp;
7718 
7719 	/* Reset the write side q_nfsrv pointer for _I_REMOVE */
7720 	if ((rqp->q_flag & _QREMOVING) && (wqp->q_qinfo->qi_srvp != NULL)) {
7721 		for (tmp_qp = backq(wqp);
7722 		    tmp_qp != NULL && tmp_qp->q_nfsrv == wqp;
7723 		    tmp_qp = backq(tmp_qp)) {
7724 			tmp_qp->q_nfsrv = wqp->q_nfsrv;
7725 		}
7726 	}
7727 
7728 	/* reset the read side q_nfsrv pointer */
7729 	if (rqp->q_qinfo->qi_srvp) {
7730 		if (wqp->q_next) {	/* non-driver case */
7731 			tmp_qp = _OTHERQ(wqp->q_next);
7732 			while (tmp_qp && tmp_qp->q_nfsrv == rqp) {
7733 				/* Note that rqp->q_next cannot be NULL */
7734 				ASSERT(rqp->q_next != NULL);
7735 				tmp_qp->q_nfsrv = rqp->q_next->q_nfsrv;
7736 				tmp_qp = backq(tmp_qp);
7737 			}
7738 		}
7739 	}
7740 }
7741 
7742 /*
7743  * This routine should be called after all stream geometry changes to update
7744  * the stream head cached struio() rd/wr queue pointers. Note must be called
7745  * with the streamlock()ed.
7746  *
7747  * Note: only enables Synchronous STREAMS for a side of a Stream which has
7748  *	 an explicit synchronous barrier module queue. That is, a queue that
7749  *	 has specified a struio() type.
7750  */
7751 static void
strsetuio(stdata_t * stp)7752 strsetuio(stdata_t *stp)
7753 {
7754 	queue_t *wrq;
7755 
7756 	if (stp->sd_flag & STPLEX) {
7757 		/*
7758 		 * Not streamhead, but a mux, so no Synchronous STREAMS.
7759 		 */
7760 		stp->sd_struiowrq = NULL;
7761 		stp->sd_struiordq = NULL;
7762 		return;
7763 	}
7764 	/*
7765 	 * Scan the write queue(s) while synchronous
7766 	 * until we find a qinfo uio type specified.
7767 	 */
7768 	wrq = stp->sd_wrq->q_next;
7769 	while (wrq) {
7770 		if (wrq->q_struiot == STRUIOT_NONE) {
7771 			wrq = 0;
7772 			break;
7773 		}
7774 		if (wrq->q_struiot != STRUIOT_DONTCARE)
7775 			break;
7776 		if (! _SAMESTR(wrq)) {
7777 			wrq = 0;
7778 			break;
7779 		}
7780 		wrq = wrq->q_next;
7781 	}
7782 	stp->sd_struiowrq = wrq;
7783 	/*
7784 	 * Scan the read queue(s) while synchronous
7785 	 * until we find a qinfo uio type specified.
7786 	 */
7787 	wrq = stp->sd_wrq->q_next;
7788 	while (wrq) {
7789 		if (_RD(wrq)->q_struiot == STRUIOT_NONE) {
7790 			wrq = 0;
7791 			break;
7792 		}
7793 		if (_RD(wrq)->q_struiot != STRUIOT_DONTCARE)
7794 			break;
7795 		if (! _SAMESTR(wrq)) {
7796 			wrq = 0;
7797 			break;
7798 		}
7799 		wrq = wrq->q_next;
7800 	}
7801 	stp->sd_struiordq = wrq ? _RD(wrq) : 0;
7802 }
7803 
7804 /*
7805  * pass_wput, unblocks the passthru queues, so that
7806  * messages can arrive at muxs lower read queue, before
7807  * I_LINK/I_UNLINK is acked/nacked.
7808  */
7809 static void
pass_wput(queue_t * q,mblk_t * mp)7810 pass_wput(queue_t *q, mblk_t *mp)
7811 {
7812 	syncq_t *sq;
7813 
7814 	sq = _RD(q)->q_syncq;
7815 	if (sq->sq_flags & SQ_BLOCKED)
7816 		unblocksq(sq, SQ_BLOCKED, 0);
7817 	putnext(q, mp);
7818 }
7819 
7820 /*
7821  * Set up queues for the link/unlink.
7822  * Create a new queue and block it and then insert it
7823  * below the stream head on the lower stream.
7824  * This prevents any messages from arriving during the setq
7825  * as well as while the mux is processing the LINK/I_UNLINK.
7826  * The blocked passq is unblocked once the LINK/I_UNLINK has
7827  * been acked or nacked or if a message is generated and sent
7828  * down muxs write put procedure.
7829  * See pass_wput().
7830  *
7831  * After the new queue is inserted, all messages coming from below are
7832  * blocked. The call to strlock will ensure that all activity in the stream head
7833  * read queue syncq is stopped (sq_count drops to zero).
7834  */
7835 static queue_t *
link_addpassthru(stdata_t * stpdown)7836 link_addpassthru(stdata_t *stpdown)
7837 {
7838 	queue_t *passq;
7839 	sqlist_t sqlist;
7840 
7841 	passq = allocq();
7842 	STREAM(passq) = STREAM(_WR(passq)) = stpdown;
7843 	/* setq might sleep in allocator - avoid holding locks. */
7844 	setq(passq, &passthru_rinit, &passthru_winit, NULL, QPERQ,
7845 	    SQ_CI|SQ_CO, B_FALSE);
7846 	claimq(passq);
7847 	blocksq(passq->q_syncq, SQ_BLOCKED, 1);
7848 	insertq(STREAM(passq), passq);
7849 
7850 	/*
7851 	 * Use strlock() to wait for the stream head sq_count to drop to zero
7852 	 * since we are going to change q_ptr in the stream head.  Note that
7853 	 * insertq() doesn't wait for any syncq counts to drop to zero.
7854 	 */
7855 	sqlist.sqlist_head = NULL;
7856 	sqlist.sqlist_index = 0;
7857 	sqlist.sqlist_size = sizeof (sqlist_t);
7858 	sqlist_insert(&sqlist, _RD(stpdown->sd_wrq)->q_syncq);
7859 	strlock(stpdown, &sqlist);
7860 	strunlock(stpdown, &sqlist);
7861 
7862 	releaseq(passq);
7863 	return (passq);
7864 }
7865 
7866 /*
7867  * Let messages flow up into the mux by removing
7868  * the passq.
7869  */
7870 static void
link_rempassthru(queue_t * passq)7871 link_rempassthru(queue_t *passq)
7872 {
7873 	claimq(passq);
7874 	removeq(passq);
7875 	releaseq(passq);
7876 	freeq(passq);
7877 }
7878 
7879 /*
7880  * Wait for the condition variable pointed to by `cvp' to be signaled,
7881  * or for `tim' milliseconds to elapse, whichever comes first.  If `tim'
7882  * is negative, then there is no time limit.  If `nosigs' is non-zero,
7883  * then the wait will be non-interruptible.
7884  *
7885  * Returns >0 if signaled, 0 if interrupted, or -1 upon timeout.
7886  */
7887 clock_t
str_cv_wait(kcondvar_t * cvp,kmutex_t * mp,clock_t tim,int nosigs)7888 str_cv_wait(kcondvar_t *cvp, kmutex_t *mp, clock_t tim, int nosigs)
7889 {
7890 	clock_t ret;
7891 
7892 	if (tim < 0) {
7893 		if (nosigs) {
7894 			cv_wait(cvp, mp);
7895 			ret = 1;
7896 		} else {
7897 			ret = cv_wait_sig(cvp, mp);
7898 		}
7899 	} else if (tim > 0) {
7900 		/*
7901 		 * convert milliseconds to clock ticks
7902 		 */
7903 		if (nosigs) {
7904 			ret = cv_reltimedwait(cvp, mp,
7905 			    MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7906 		} else {
7907 			ret = cv_reltimedwait_sig(cvp, mp,
7908 			    MSEC_TO_TICK_ROUNDUP(tim), TR_CLOCK_TICK);
7909 		}
7910 	} else {
7911 		ret = -1;
7912 	}
7913 	return (ret);
7914 }
7915 
7916 /*
7917  * Wait until the stream head can determine if it is at the mark but
7918  * don't wait forever to prevent a race condition between the "mark" state
7919  * in the stream head and any mark state in the caller/user of this routine.
7920  *
7921  * This is used by sockets and for a socket it would be incorrect
7922  * to return a failure for SIOCATMARK when there is no data in the receive
7923  * queue and the marked urgent data is traveling up the stream.
7924  *
7925  * This routine waits until the mark is known by waiting for one of these
7926  * three events:
7927  *	The stream head read queue becoming non-empty (including an EOF).
7928  *	The STRATMARK flag being set (due to a MSGMARKNEXT message).
7929  *	The STRNOTATMARK flag being set (which indicates that the transport
7930  *	has sent a MSGNOTMARKNEXT message to indicate that it is not at
7931  *	the mark).
7932  *
7933  * The routine returns 1 if the stream is at the mark; 0 if it can
7934  * be determined that the stream is not at the mark.
7935  * If the wait times out and it can't determine
7936  * whether or not the stream might be at the mark the routine will return -1.
7937  *
7938  * Note: This routine should only be used when a mark is pending i.e.,
7939  * in the socket case the SIGURG has been posted.
7940  * Note2: This can not wakeup just because synchronous streams indicate
7941  * that data is available since it is not possible to use the synchronous
7942  * streams interfaces to determine the b_flag value for the data queued below
7943  * the stream head.
7944  */
7945 int
strwaitmark(vnode_t * vp)7946 strwaitmark(vnode_t *vp)
7947 {
7948 	struct stdata *stp = vp->v_stream;
7949 	queue_t *rq = _RD(stp->sd_wrq);
7950 	int mark;
7951 
7952 	mutex_enter(&stp->sd_lock);
7953 	while (rq->q_first == NULL &&
7954 	    !(stp->sd_flag & (STRATMARK|STRNOTATMARK|STREOF))) {
7955 		stp->sd_flag |= RSLEEP;
7956 
7957 		/* Wait for 100 milliseconds for any state change. */
7958 		if (str_cv_wait(&rq->q_wait, &stp->sd_lock, 100, 1) == -1) {
7959 			mutex_exit(&stp->sd_lock);
7960 			return (-1);
7961 		}
7962 	}
7963 	if (stp->sd_flag & STRATMARK)
7964 		mark = 1;
7965 	else if (rq->q_first != NULL && (rq->q_first->b_flag & MSGMARK))
7966 		mark = 1;
7967 	else
7968 		mark = 0;
7969 
7970 	mutex_exit(&stp->sd_lock);
7971 	return (mark);
7972 }
7973 
7974 /*
7975  * Set a read side error. If persist is set change the socket error
7976  * to persistent. If errfunc is set install the function as the exported
7977  * error handler.
7978  */
7979 void
strsetrerror(vnode_t * vp,int error,int persist,errfunc_t errfunc)7980 strsetrerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
7981 {
7982 	struct stdata *stp = vp->v_stream;
7983 
7984 	mutex_enter(&stp->sd_lock);
7985 	stp->sd_rerror = error;
7986 	if (error == 0 && errfunc == NULL)
7987 		stp->sd_flag &= ~STRDERR;
7988 	else
7989 		stp->sd_flag |= STRDERR;
7990 	if (persist) {
7991 		stp->sd_flag &= ~STRDERRNONPERSIST;
7992 	} else {
7993 		stp->sd_flag |= STRDERRNONPERSIST;
7994 	}
7995 	stp->sd_rderrfunc = errfunc;
7996 	if (error != 0 || errfunc != NULL) {
7997 		cv_broadcast(&_RD(stp->sd_wrq)->q_wait);	/* readers */
7998 		cv_broadcast(&stp->sd_wrq->q_wait);		/* writers */
7999 		cv_broadcast(&stp->sd_monitor);			/* ioctllers */
8000 
8001 		mutex_exit(&stp->sd_lock);
8002 		pollwakeup(&stp->sd_pollist, POLLERR);
8003 		mutex_enter(&stp->sd_lock);
8004 
8005 		if (stp->sd_sigflags & S_ERROR)
8006 			strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8007 	}
8008 	mutex_exit(&stp->sd_lock);
8009 }
8010 
8011 /*
8012  * Set a write side error. If persist is set change the socket error
8013  * to persistent.
8014  */
8015 void
strsetwerror(vnode_t * vp,int error,int persist,errfunc_t errfunc)8016 strsetwerror(vnode_t *vp, int error, int persist, errfunc_t errfunc)
8017 {
8018 	struct stdata *stp = vp->v_stream;
8019 
8020 	mutex_enter(&stp->sd_lock);
8021 	stp->sd_werror = error;
8022 	if (error == 0 && errfunc == NULL)
8023 		stp->sd_flag &= ~STWRERR;
8024 	else
8025 		stp->sd_flag |= STWRERR;
8026 	if (persist) {
8027 		stp->sd_flag &= ~STWRERRNONPERSIST;
8028 	} else {
8029 		stp->sd_flag |= STWRERRNONPERSIST;
8030 	}
8031 	stp->sd_wrerrfunc = errfunc;
8032 	if (error != 0 || errfunc != NULL) {
8033 		cv_broadcast(&_RD(stp->sd_wrq)->q_wait);	/* readers */
8034 		cv_broadcast(&stp->sd_wrq->q_wait);		/* writers */
8035 		cv_broadcast(&stp->sd_monitor);			/* ioctllers */
8036 
8037 		mutex_exit(&stp->sd_lock);
8038 		pollwakeup(&stp->sd_pollist, POLLERR);
8039 		mutex_enter(&stp->sd_lock);
8040 
8041 		if (stp->sd_sigflags & S_ERROR)
8042 			strsendsig(stp->sd_siglist, S_ERROR, 0, error);
8043 	}
8044 	mutex_exit(&stp->sd_lock);
8045 }
8046 
8047 /*
8048  * Make the stream return 0 (EOF) when all data has been read.
8049  * No effect on write side.
8050  */
8051 void
strseteof(vnode_t * vp,int eof)8052 strseteof(vnode_t *vp, int eof)
8053 {
8054 	struct stdata *stp = vp->v_stream;
8055 
8056 	mutex_enter(&stp->sd_lock);
8057 	if (!eof) {
8058 		stp->sd_flag &= ~STREOF;
8059 		mutex_exit(&stp->sd_lock);
8060 		return;
8061 	}
8062 	stp->sd_flag |= STREOF;
8063 	if (stp->sd_flag & RSLEEP) {
8064 		stp->sd_flag &= ~RSLEEP;
8065 		cv_broadcast(&_RD(stp->sd_wrq)->q_wait);
8066 	}
8067 
8068 	mutex_exit(&stp->sd_lock);
8069 	pollwakeup(&stp->sd_pollist, POLLIN|POLLRDNORM);
8070 	mutex_enter(&stp->sd_lock);
8071 
8072 	if (stp->sd_sigflags & (S_INPUT|S_RDNORM))
8073 		strsendsig(stp->sd_siglist, S_INPUT|S_RDNORM, 0, 0);
8074 	mutex_exit(&stp->sd_lock);
8075 }
8076 
8077 void
strflushrq(vnode_t * vp,int flag)8078 strflushrq(vnode_t *vp, int flag)
8079 {
8080 	struct stdata *stp = vp->v_stream;
8081 
8082 	mutex_enter(&stp->sd_lock);
8083 	flushq(_RD(stp->sd_wrq), flag);
8084 	mutex_exit(&stp->sd_lock);
8085 }
8086 
8087 void
strsetrputhooks(vnode_t * vp,uint_t flags,msgfunc_t protofunc,msgfunc_t miscfunc)8088 strsetrputhooks(vnode_t *vp, uint_t flags,
8089 		msgfunc_t protofunc, msgfunc_t miscfunc)
8090 {
8091 	struct stdata *stp = vp->v_stream;
8092 
8093 	mutex_enter(&stp->sd_lock);
8094 
8095 	if (protofunc == NULL)
8096 		stp->sd_rprotofunc = strrput_proto;
8097 	else
8098 		stp->sd_rprotofunc = protofunc;
8099 
8100 	if (miscfunc == NULL)
8101 		stp->sd_rmiscfunc = strrput_misc;
8102 	else
8103 		stp->sd_rmiscfunc = miscfunc;
8104 
8105 	if (flags & SH_CONSOL_DATA)
8106 		stp->sd_rput_opt |= SR_CONSOL_DATA;
8107 	else
8108 		stp->sd_rput_opt &= ~SR_CONSOL_DATA;
8109 
8110 	if (flags & SH_SIGALLDATA)
8111 		stp->sd_rput_opt |= SR_SIGALLDATA;
8112 	else
8113 		stp->sd_rput_opt &= ~SR_SIGALLDATA;
8114 
8115 	if (flags & SH_IGN_ZEROLEN)
8116 		stp->sd_rput_opt |= SR_IGN_ZEROLEN;
8117 	else
8118 		stp->sd_rput_opt &= ~SR_IGN_ZEROLEN;
8119 
8120 	mutex_exit(&stp->sd_lock);
8121 }
8122 
8123 void
strsetwputhooks(vnode_t * vp,uint_t flags,clock_t closetime)8124 strsetwputhooks(vnode_t *vp, uint_t flags, clock_t closetime)
8125 {
8126 	struct stdata *stp = vp->v_stream;
8127 
8128 	mutex_enter(&stp->sd_lock);
8129 	stp->sd_closetime = closetime;
8130 
8131 	if (flags & SH_SIGPIPE)
8132 		stp->sd_wput_opt |= SW_SIGPIPE;
8133 	else
8134 		stp->sd_wput_opt &= ~SW_SIGPIPE;
8135 	if (flags & SH_RECHECK_ERR)
8136 		stp->sd_wput_opt |= SW_RECHECK_ERR;
8137 	else
8138 		stp->sd_wput_opt &= ~SW_RECHECK_ERR;
8139 
8140 	mutex_exit(&stp->sd_lock);
8141 }
8142 
8143 void
strsetrwputdatahooks(vnode_t * vp,msgfunc_t rdatafunc,msgfunc_t wdatafunc)8144 strsetrwputdatahooks(vnode_t *vp, msgfunc_t rdatafunc, msgfunc_t wdatafunc)
8145 {
8146 	struct stdata *stp = vp->v_stream;
8147 
8148 	mutex_enter(&stp->sd_lock);
8149 
8150 	stp->sd_rputdatafunc = rdatafunc;
8151 	stp->sd_wputdatafunc = wdatafunc;
8152 
8153 	mutex_exit(&stp->sd_lock);
8154 }
8155 
8156 /* Used within framework when the queue is already locked */
8157 void
qenable_locked(queue_t * q)8158 qenable_locked(queue_t *q)
8159 {
8160 	stdata_t *stp = STREAM(q);
8161 
8162 	ASSERT(MUTEX_HELD(QLOCK(q)));
8163 
8164 	if (!q->q_qinfo->qi_srvp)
8165 		return;
8166 
8167 	/*
8168 	 * Do not place on run queue if already enabled or closing.
8169 	 */
8170 	if (q->q_flag & (QWCLOSE|QENAB))
8171 		return;
8172 
8173 	/*
8174 	 * mark queue enabled and place on run list if it is not already being
8175 	 * serviced. If it is serviced, the runservice() function will detect
8176 	 * that QENAB is set and call service procedure before clearing
8177 	 * QINSERVICE flag.
8178 	 */
8179 	q->q_flag |= QENAB;
8180 	if (q->q_flag & QINSERVICE)
8181 		return;
8182 
8183 	/* Record the time of qenable */
8184 	q->q_qtstamp = ddi_get_lbolt();
8185 
8186 	/*
8187 	 * Put the queue in the stp list and schedule it for background
8188 	 * processing if it is not already scheduled or if stream head does not
8189 	 * intent to process it in the foreground later by setting
8190 	 * STRS_WILLSERVICE flag.
8191 	 */
8192 	mutex_enter(&stp->sd_qlock);
8193 	/*
8194 	 * If there are already something on the list, stp flags should show
8195 	 * intention to drain it.
8196 	 */
8197 	IMPLY(STREAM_NEEDSERVICE(stp),
8198 	    (stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED)));
8199 
8200 	ENQUEUE(q, stp->sd_qhead, stp->sd_qtail, q_link);
8201 	stp->sd_nqueues++;
8202 
8203 	/*
8204 	 * If no one will drain this stream we are the first producer and
8205 	 * need to schedule it for background thread.
8206 	 */
8207 	if (!(stp->sd_svcflags & (STRS_WILLSERVICE | STRS_SCHEDULED))) {
8208 		/*
8209 		 * No one will service this stream later, so we have to
8210 		 * schedule it now.
8211 		 */
8212 		STRSTAT(stenables);
8213 		stp->sd_svcflags |= STRS_SCHEDULED;
8214 		stp->sd_servid = (void *)taskq_dispatch(streams_taskq,
8215 		    (task_func_t *)stream_service, stp, TQ_NOSLEEP|TQ_NOQUEUE);
8216 
8217 		if (stp->sd_servid == NULL) {
8218 			/*
8219 			 * Task queue failed so fail over to the backup
8220 			 * servicing thread.
8221 			 */
8222 			STRSTAT(taskqfails);
8223 			/*
8224 			 * It is safe to clear STRS_SCHEDULED flag because it
8225 			 * was set by this thread above.
8226 			 */
8227 			stp->sd_svcflags &= ~STRS_SCHEDULED;
8228 
8229 			/*
8230 			 * Failover scheduling is protected by service_queue
8231 			 * lock.
8232 			 */
8233 			mutex_enter(&service_queue);
8234 			ASSERT((stp->sd_qhead == q) && (stp->sd_qtail == q));
8235 			ASSERT(q->q_link == NULL);
8236 			/*
8237 			 * Append the queue to qhead/qtail list.
8238 			 */
8239 			if (qhead == NULL)
8240 				qhead = q;
8241 			else
8242 				qtail->q_link = q;
8243 			qtail = q;
8244 			/*
8245 			 * Clear stp queue list.
8246 			 */
8247 			stp->sd_qhead = stp->sd_qtail = NULL;
8248 			stp->sd_nqueues = 0;
8249 			/*
8250 			 * Wakeup background queue processing thread.
8251 			 */
8252 			cv_signal(&services_to_run);
8253 			mutex_exit(&service_queue);
8254 		}
8255 	}
8256 	mutex_exit(&stp->sd_qlock);
8257 }
8258 
8259 static void
queue_service(queue_t * q)8260 queue_service(queue_t *q)
8261 {
8262 	/*
8263 	 * The queue in the list should have
8264 	 * QENAB flag set and should not have
8265 	 * QINSERVICE flag set. QINSERVICE is
8266 	 * set when the queue is dequeued and
8267 	 * qenable_locked doesn't enqueue a
8268 	 * queue with QINSERVICE set.
8269 	 */
8270 
8271 	ASSERT(!(q->q_flag & QINSERVICE));
8272 	ASSERT((q->q_flag & QENAB));
8273 	mutex_enter(QLOCK(q));
8274 	q->q_flag &= ~QENAB;
8275 	q->q_flag |= QINSERVICE;
8276 	mutex_exit(QLOCK(q));
8277 	runservice(q);
8278 }
8279 
8280 static void
syncq_service(syncq_t * sq)8281 syncq_service(syncq_t *sq)
8282 {
8283 	STRSTAT(syncqservice);
8284 	mutex_enter(SQLOCK(sq));
8285 	ASSERT(!(sq->sq_svcflags & SQ_SERVICE));
8286 	ASSERT(sq->sq_servcount != 0);
8287 	ASSERT(sq->sq_next == NULL);
8288 
8289 	/* if we came here from the background thread, clear the flag */
8290 	if (sq->sq_svcflags & SQ_BGTHREAD)
8291 		sq->sq_svcflags &= ~SQ_BGTHREAD;
8292 
8293 	/* let drain_syncq know that it's being called in the background */
8294 	sq->sq_svcflags |= SQ_SERVICE;
8295 	drain_syncq(sq);
8296 }
8297 
8298 static void
qwriter_outer_service(syncq_t * outer)8299 qwriter_outer_service(syncq_t *outer)
8300 {
8301 	/*
8302 	 * Note that SQ_WRITER is used on the outer perimeter
8303 	 * to signal that a qwriter(OUTER) is either investigating
8304 	 * running or that it is actually running a function.
8305 	 */
8306 	outer_enter(outer, SQ_BLOCKED|SQ_WRITER);
8307 
8308 	/*
8309 	 * All inner syncq are empty and have SQ_WRITER set
8310 	 * to block entering the outer perimeter.
8311 	 *
8312 	 * We do not need to explicitly call write_now since
8313 	 * outer_exit does it for us.
8314 	 */
8315 	outer_exit(outer);
8316 }
8317 
8318 static void
mblk_free(mblk_t * mp)8319 mblk_free(mblk_t *mp)
8320 {
8321 	dblk_t *dbp = mp->b_datap;
8322 	frtn_t *frp = dbp->db_frtnp;
8323 
8324 	mp->b_next = NULL;
8325 	if (dbp->db_fthdr != NULL)
8326 		str_ftfree(dbp);
8327 
8328 	ASSERT(dbp->db_fthdr == NULL);
8329 	frp->free_func(frp->free_arg);
8330 	ASSERT(dbp->db_mblk == mp);
8331 
8332 	if (dbp->db_credp != NULL) {
8333 		crfree(dbp->db_credp);
8334 		dbp->db_credp = NULL;
8335 	}
8336 	dbp->db_cpid = -1;
8337 	dbp->db_struioflag = 0;
8338 	dbp->db_struioun.cksum.flags = 0;
8339 
8340 	kmem_cache_free(dbp->db_cache, dbp);
8341 }
8342 
8343 /*
8344  * Background processing of the stream queue list.
8345  */
8346 static void
stream_service(stdata_t * stp)8347 stream_service(stdata_t *stp)
8348 {
8349 	queue_t *q;
8350 
8351 	mutex_enter(&stp->sd_qlock);
8352 
8353 	STR_SERVICE(stp, q);
8354 
8355 	stp->sd_svcflags &= ~STRS_SCHEDULED;
8356 	stp->sd_servid = NULL;
8357 	cv_signal(&stp->sd_qcv);
8358 	mutex_exit(&stp->sd_qlock);
8359 }
8360 
8361 /*
8362  * Foreground processing of the stream queue list.
8363  */
8364 void
stream_runservice(stdata_t * stp)8365 stream_runservice(stdata_t *stp)
8366 {
8367 	queue_t *q;
8368 
8369 	mutex_enter(&stp->sd_qlock);
8370 	STRSTAT(rservice);
8371 	/*
8372 	 * We are going to drain this stream queue list, so qenable_locked will
8373 	 * not schedule it until we finish.
8374 	 */
8375 	stp->sd_svcflags |= STRS_WILLSERVICE;
8376 
8377 	STR_SERVICE(stp, q);
8378 
8379 	stp->sd_svcflags &= ~STRS_WILLSERVICE;
8380 	mutex_exit(&stp->sd_qlock);
8381 	/*
8382 	 * Help backup background thread to drain the qhead/qtail list.
8383 	 */
8384 	while (qhead != NULL) {
8385 		STRSTAT(qhelps);
8386 		mutex_enter(&service_queue);
8387 		DQ(q, qhead, qtail, q_link);
8388 		mutex_exit(&service_queue);
8389 		if (q != NULL)
8390 			queue_service(q);
8391 	}
8392 }
8393 
8394 void
stream_willservice(stdata_t * stp)8395 stream_willservice(stdata_t *stp)
8396 {
8397 	mutex_enter(&stp->sd_qlock);
8398 	stp->sd_svcflags |= STRS_WILLSERVICE;
8399 	mutex_exit(&stp->sd_qlock);
8400 }
8401 
8402 /*
8403  * Replace the cred currently in the mblk with a different one.
8404  * Also update db_cpid.
8405  */
8406 void
mblk_setcred(mblk_t * mp,cred_t * cr,pid_t cpid)8407 mblk_setcred(mblk_t *mp, cred_t *cr, pid_t cpid)
8408 {
8409 	dblk_t *dbp = mp->b_datap;
8410 	cred_t *ocr = dbp->db_credp;
8411 
8412 	ASSERT(cr != NULL);
8413 
8414 	if (cr != ocr) {
8415 		crhold(dbp->db_credp = cr);
8416 		if (ocr != NULL)
8417 			crfree(ocr);
8418 	}
8419 	/* Don't overwrite with NOPID */
8420 	if (cpid != NOPID)
8421 		dbp->db_cpid = cpid;
8422 }
8423 
8424 /*
8425  * If the src message has a cred, then replace the cred currently in the mblk
8426  * with it.
8427  * Also update db_cpid.
8428  */
8429 void
mblk_copycred(mblk_t * mp,const mblk_t * src)8430 mblk_copycred(mblk_t *mp, const mblk_t *src)
8431 {
8432 	dblk_t *dbp = mp->b_datap;
8433 	cred_t *cr, *ocr;
8434 	pid_t cpid;
8435 
8436 	cr = msg_getcred(src, &cpid);
8437 	if (cr == NULL)
8438 		return;
8439 
8440 	ocr = dbp->db_credp;
8441 	if (cr != ocr) {
8442 		crhold(dbp->db_credp = cr);
8443 		if (ocr != NULL)
8444 			crfree(ocr);
8445 	}
8446 	/* Don't overwrite with NOPID */
8447 	if (cpid != NOPID)
8448 		dbp->db_cpid = cpid;
8449 }
8450 
8451 int
hcksum_assoc(mblk_t * mp,multidata_t * mmd,pdesc_t * pd,uint32_t start,uint32_t stuff,uint32_t end,uint32_t value,uint32_t flags,int km_flags)8452 hcksum_assoc(mblk_t *mp,  multidata_t *mmd, pdesc_t *pd,
8453     uint32_t start, uint32_t stuff, uint32_t end, uint32_t value,
8454     uint32_t flags, int km_flags)
8455 {
8456 	int rc = 0;
8457 
8458 	ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8459 	if (mp->b_datap->db_type == M_DATA) {
8460 		/* Associate values for M_DATA type */
8461 		DB_CKSUMSTART(mp) = (intptr_t)start;
8462 		DB_CKSUMSTUFF(mp) = (intptr_t)stuff;
8463 		DB_CKSUMEND(mp) = (intptr_t)end;
8464 		DB_CKSUMFLAGS(mp) = flags;
8465 		DB_CKSUM16(mp) = (uint16_t)value;
8466 
8467 	} else {
8468 		pattrinfo_t pa_info;
8469 
8470 		ASSERT(mmd != NULL);
8471 
8472 		pa_info.type = PATTR_HCKSUM;
8473 		pa_info.len = sizeof (pattr_hcksum_t);
8474 
8475 		if (mmd_addpattr(mmd, pd, &pa_info, B_TRUE, km_flags) != NULL) {
8476 			pattr_hcksum_t *hck = (pattr_hcksum_t *)pa_info.buf;
8477 
8478 			hck->hcksum_start_offset = start;
8479 			hck->hcksum_stuff_offset = stuff;
8480 			hck->hcksum_end_offset = end;
8481 			hck->hcksum_cksum_val.inet_cksum = (uint16_t)value;
8482 			hck->hcksum_flags = flags;
8483 		} else {
8484 			rc = -1;
8485 		}
8486 	}
8487 	return (rc);
8488 }
8489 
8490 void
hcksum_retrieve(mblk_t * mp,multidata_t * mmd,pdesc_t * pd,uint32_t * start,uint32_t * stuff,uint32_t * end,uint32_t * value,uint32_t * flags)8491 hcksum_retrieve(mblk_t *mp, multidata_t *mmd, pdesc_t *pd,
8492     uint32_t *start, uint32_t *stuff, uint32_t *end,
8493     uint32_t *value, uint32_t *flags)
8494 {
8495 	ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8496 	if (mp->b_datap->db_type == M_DATA) {
8497 		if (flags != NULL) {
8498 			*flags = DB_CKSUMFLAGS(mp) & HCK_FLAGS;
8499 			if ((*flags & (HCK_PARTIALCKSUM |
8500 			    HCK_FULLCKSUM)) != 0) {
8501 				if (value != NULL)
8502 					*value = (uint32_t)DB_CKSUM16(mp);
8503 				if ((*flags & HCK_PARTIALCKSUM) != 0) {
8504 					if (start != NULL)
8505 						*start =
8506 						    (uint32_t)DB_CKSUMSTART(mp);
8507 					if (stuff != NULL)
8508 						*stuff =
8509 						    (uint32_t)DB_CKSUMSTUFF(mp);
8510 					if (end != NULL)
8511 						*end =
8512 						    (uint32_t)DB_CKSUMEND(mp);
8513 				}
8514 			}
8515 		}
8516 	} else {
8517 		pattrinfo_t hck_attr = {PATTR_HCKSUM};
8518 
8519 		ASSERT(mmd != NULL);
8520 
8521 		/* get hardware checksum attribute */
8522 		if (mmd_getpattr(mmd, pd, &hck_attr) != NULL) {
8523 			pattr_hcksum_t *hck = (pattr_hcksum_t *)hck_attr.buf;
8524 
8525 			ASSERT(hck_attr.len >= sizeof (pattr_hcksum_t));
8526 			if (flags != NULL)
8527 				*flags = hck->hcksum_flags;
8528 			if (start != NULL)
8529 				*start = hck->hcksum_start_offset;
8530 			if (stuff != NULL)
8531 				*stuff = hck->hcksum_stuff_offset;
8532 			if (end != NULL)
8533 				*end = hck->hcksum_end_offset;
8534 			if (value != NULL)
8535 				*value = (uint32_t)
8536 				    hck->hcksum_cksum_val.inet_cksum;
8537 		}
8538 	}
8539 }
8540 
8541 void
lso_info_set(mblk_t * mp,uint32_t mss,uint32_t flags)8542 lso_info_set(mblk_t *mp, uint32_t mss, uint32_t flags)
8543 {
8544 	ASSERT(DB_TYPE(mp) == M_DATA);
8545 	ASSERT((flags & ~HW_LSO_FLAGS) == 0);
8546 
8547 	/* Set the flags */
8548 	DB_LSOFLAGS(mp) |= flags;
8549 	DB_LSOMSS(mp) = mss;
8550 }
8551 
8552 void
lso_info_cleanup(mblk_t * mp)8553 lso_info_cleanup(mblk_t *mp)
8554 {
8555 	ASSERT(DB_TYPE(mp) == M_DATA);
8556 
8557 	/* Clear the flags */
8558 	DB_LSOFLAGS(mp) &= ~HW_LSO_FLAGS;
8559 	DB_LSOMSS(mp) = 0;
8560 }
8561 
8562 /*
8563  * Checksum buffer *bp for len bytes with psum partial checksum,
8564  * or 0 if none, and return the 16 bit partial checksum.
8565  */
8566 unsigned
bcksum(uchar_t * bp,int len,unsigned int psum)8567 bcksum(uchar_t *bp, int len, unsigned int psum)
8568 {
8569 	int odd = len & 1;
8570 	extern unsigned int ip_ocsum();
8571 
8572 	if (((intptr_t)bp & 1) == 0 && !odd) {
8573 		/*
8574 		 * Bp is 16 bit aligned and len is multiple of 16 bit word.
8575 		 */
8576 		return (ip_ocsum((ushort_t *)bp, len >> 1, psum));
8577 	}
8578 	if (((intptr_t)bp & 1) != 0) {
8579 		/*
8580 		 * Bp isn't 16 bit aligned.
8581 		 */
8582 		unsigned int tsum;
8583 
8584 #ifdef _LITTLE_ENDIAN
8585 		psum += *bp;
8586 #else
8587 		psum += *bp << 8;
8588 #endif
8589 		len--;
8590 		bp++;
8591 		tsum = ip_ocsum((ushort_t *)bp, len >> 1, 0);
8592 		psum += (tsum << 8) & 0xffff | (tsum >> 8);
8593 		if (len & 1) {
8594 			bp += len - 1;
8595 #ifdef _LITTLE_ENDIAN
8596 			psum += *bp << 8;
8597 #else
8598 			psum += *bp;
8599 #endif
8600 		}
8601 	} else {
8602 		/*
8603 		 * Bp is 16 bit aligned.
8604 		 */
8605 		psum = ip_ocsum((ushort_t *)bp, len >> 1, psum);
8606 		if (odd) {
8607 			bp += len - 1;
8608 #ifdef _LITTLE_ENDIAN
8609 			psum += *bp;
8610 #else
8611 			psum += *bp << 8;
8612 #endif
8613 		}
8614 	}
8615 	/*
8616 	 * Normalize psum to 16 bits before returning the new partial
8617 	 * checksum. The max psum value before normalization is 0x3FDFE.
8618 	 */
8619 	return ((psum >> 16) + (psum & 0xFFFF));
8620 }
8621 
8622 boolean_t
is_vmloaned_mblk(mblk_t * mp,multidata_t * mmd,pdesc_t * pd)8623 is_vmloaned_mblk(mblk_t *mp, multidata_t *mmd, pdesc_t *pd)
8624 {
8625 	boolean_t rc;
8626 
8627 	ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_MULTIDATA);
8628 	if (DB_TYPE(mp) == M_DATA) {
8629 		rc = (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0);
8630 	} else {
8631 		pattrinfo_t zcopy_attr = {PATTR_ZCOPY};
8632 
8633 		ASSERT(mmd != NULL);
8634 		rc = (mmd_getpattr(mmd, pd, &zcopy_attr) != NULL);
8635 	}
8636 	return (rc);
8637 }
8638 
8639 void
freemsgchain(mblk_t * mp)8640 freemsgchain(mblk_t *mp)
8641 {
8642 	mblk_t	*next;
8643 
8644 	while (mp != NULL) {
8645 		next = mp->b_next;
8646 		mp->b_next = NULL;
8647 
8648 		freemsg(mp);
8649 		mp = next;
8650 	}
8651 }
8652 
8653 mblk_t *
copymsgchain(mblk_t * mp)8654 copymsgchain(mblk_t *mp)
8655 {
8656 	mblk_t	*nmp = NULL;
8657 	mblk_t	**nmpp = &nmp;
8658 
8659 	for (; mp != NULL; mp = mp->b_next) {
8660 		if ((*nmpp = copymsg(mp)) == NULL) {
8661 			freemsgchain(nmp);
8662 			return (NULL);
8663 		}
8664 
8665 		nmpp = &((*nmpp)->b_next);
8666 	}
8667 
8668 	return (nmp);
8669 }
8670 
8671 /* NOTE: Do not add code after this point. */
8672 #undef QLOCK
8673 
8674 /*
8675  * Replacement for QLOCK macro for those that can't use it.
8676  */
8677 kmutex_t *
QLOCK(queue_t * q)8678 QLOCK(queue_t *q)
8679 {
8680 	return (&(q)->q_lock);
8681 }
8682 
8683 /*
8684  * Dummy runqueues/queuerun functions functions for backwards compatibility.
8685  */
8686 #undef runqueues
8687 void
runqueues(void)8688 runqueues(void)
8689 {
8690 }
8691 
8692 #undef queuerun
8693 void
queuerun(void)8694 queuerun(void)
8695 {
8696 }
8697 
8698 /*
8699  * Initialize the STR stack instance, which tracks autopush and persistent
8700  * links.
8701  */
8702 /* ARGSUSED */
8703 static void *
str_stack_init(netstackid_t stackid,netstack_t * ns)8704 str_stack_init(netstackid_t stackid, netstack_t *ns)
8705 {
8706 	str_stack_t	*ss;
8707 	int i;
8708 
8709 	ss = (str_stack_t *)kmem_zalloc(sizeof (*ss), KM_SLEEP);
8710 	ss->ss_netstack = ns;
8711 
8712 	/*
8713 	 * set up autopush
8714 	 */
8715 	sad_initspace(ss);
8716 
8717 	/*
8718 	 * set up mux_node structures.
8719 	 */
8720 	ss->ss_devcnt = devcnt;	/* In case it should change before free */
8721 	ss->ss_mux_nodes = kmem_zalloc((sizeof (struct mux_node) *
8722 	    ss->ss_devcnt), KM_SLEEP);
8723 	for (i = 0; i < ss->ss_devcnt; i++)
8724 		ss->ss_mux_nodes[i].mn_imaj = i;
8725 	return (ss);
8726 }
8727 
8728 /*
8729  * Note: run at zone shutdown and not destroy so that the PLINKs are
8730  * gone by the time other cleanup happens from the destroy callbacks.
8731  */
8732 static void
str_stack_shutdown(netstackid_t stackid,void * arg)8733 str_stack_shutdown(netstackid_t stackid, void *arg)
8734 {
8735 	str_stack_t *ss = (str_stack_t *)arg;
8736 	int i;
8737 	cred_t *cr;
8738 
8739 	cr = zone_get_kcred(netstackid_to_zoneid(stackid));
8740 	ASSERT(cr != NULL);
8741 
8742 	/* Undo all the I_PLINKs for this zone */
8743 	for (i = 0; i < ss->ss_devcnt; i++) {
8744 		struct mux_edge		*ep;
8745 		ldi_handle_t		lh;
8746 		ldi_ident_t		li;
8747 		int			ret;
8748 		int			rval;
8749 		dev_t			rdev;
8750 
8751 		ep = ss->ss_mux_nodes[i].mn_outp;
8752 		if (ep == NULL)
8753 			continue;
8754 		ret = ldi_ident_from_major((major_t)i, &li);
8755 		if (ret != 0) {
8756 			continue;
8757 		}
8758 		rdev = ep->me_dev;
8759 		ret = ldi_open_by_dev(&rdev, OTYP_CHR, FREAD|FWRITE,
8760 		    cr, &lh, li);
8761 		if (ret != 0) {
8762 			ldi_ident_release(li);
8763 			continue;
8764 		}
8765 
8766 		ret = ldi_ioctl(lh, I_PUNLINK, (intptr_t)MUXID_ALL, FKIOCTL,
8767 		    cr, &rval);
8768 		if (ret) {
8769 			(void) ldi_close(lh, FREAD|FWRITE, cr);
8770 			ldi_ident_release(li);
8771 			continue;
8772 		}
8773 		(void) ldi_close(lh, FREAD|FWRITE, cr);
8774 
8775 		/* Close layered handles */
8776 		ldi_ident_release(li);
8777 	}
8778 	crfree(cr);
8779 
8780 	sad_freespace(ss);
8781 
8782 	kmem_free(ss->ss_mux_nodes, sizeof (struct mux_node) * ss->ss_devcnt);
8783 	ss->ss_mux_nodes = NULL;
8784 }
8785 
8786 /*
8787  * Free the structure; str_stack_shutdown did the other cleanup work.
8788  */
8789 /* ARGSUSED */
8790 static void
str_stack_fini(netstackid_t stackid,void * arg)8791 str_stack_fini(netstackid_t stackid, void *arg)
8792 {
8793 	str_stack_t	*ss = (str_stack_t *)arg;
8794 
8795 	kmem_free(ss, sizeof (*ss));
8796 }
8797