xref: /titanic_41/usr/src/uts/common/inet/tcp/tcp_output.c (revision a1ea75925995e7f6b126c8614418e050f52900be)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2014 by Delphix. All rights reserved.
25  */
26 
27 /* This file contains all TCP output processing functions. */
28 
29 #include <sys/types.h>
30 #include <sys/stream.h>
31 #include <sys/strsun.h>
32 #include <sys/strsubr.h>
33 #include <sys/stropts.h>
34 #include <sys/strlog.h>
35 #define	_SUN_TPI_VERSION 2
36 #include <sys/tihdr.h>
37 #include <sys/suntpi.h>
38 #include <sys/xti_inet.h>
39 #include <sys/timod.h>
40 #include <sys/pattr.h>
41 #include <sys/squeue_impl.h>
42 #include <sys/squeue.h>
43 #include <sys/sockio.h>
44 #include <sys/tsol/tnet.h>
45 
46 #include <inet/common.h>
47 #include <inet/ip.h>
48 #include <inet/tcp.h>
49 #include <inet/tcp_impl.h>
50 #include <inet/snmpcom.h>
51 #include <inet/proto_set.h>
52 #include <inet/ipsec_impl.h>
53 #include <inet/ip_ndp.h>
54 
55 static mblk_t	*tcp_get_seg_mp(tcp_t *, uint32_t, int32_t *);
56 static void	tcp_wput_cmdblk(queue_t *, mblk_t *);
57 static void	tcp_wput_flush(tcp_t *, mblk_t *);
58 static void	tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp);
59 static int	tcp_xmit_end(tcp_t *);
60 static int	tcp_send(tcp_t *, const int, const int, const int,
61 		    const int, int *, uint_t *, int *, mblk_t **, mblk_t *);
62 static void	tcp_xmit_early_reset(char *, mblk_t *, uint32_t, uint32_t,
63 		    int, ip_recv_attr_t *, ip_stack_t *, conn_t *);
64 static boolean_t	tcp_send_rst_chk(tcp_stack_t *);
65 static void	tcp_process_shrunk_swnd(tcp_t *, uint32_t);
66 static void	tcp_fill_header(tcp_t *, uchar_t *, clock_t, int);
67 
68 /*
69  * Functions called directly via squeue having a prototype of edesc_t.
70  */
71 static void	tcp_wput_nondata(void *, mblk_t *, void *, ip_recv_attr_t *);
72 static void	tcp_wput_ioctl(void *, mblk_t *, void *, ip_recv_attr_t *);
73 static void	tcp_wput_proto(void *, mblk_t *, void *, ip_recv_attr_t *);
74 
75 /*
76  * This controls how tiny a write must be before we try to copy it
77  * into the mblk on the tail of the transmit queue.  Not much
78  * speedup is observed for values larger than sixteen.  Zero will
79  * disable the optimisation.
80  */
81 static int tcp_tx_pull_len = 16;
82 
83 void
tcp_wput(queue_t * q,mblk_t * mp)84 tcp_wput(queue_t *q, mblk_t *mp)
85 {
86 	conn_t	*connp = Q_TO_CONN(q);
87 	tcp_t	*tcp;
88 	void (*output_proc)();
89 	t_scalar_t type;
90 	uchar_t *rptr;
91 	struct iocblk	*iocp;
92 	size_t size;
93 
94 	ASSERT(connp->conn_ref >= 2);
95 
96 	switch (DB_TYPE(mp)) {
97 	case M_DATA:
98 		tcp = connp->conn_tcp;
99 		ASSERT(tcp != NULL);
100 
101 		size = msgdsize(mp);
102 
103 		mutex_enter(&tcp->tcp_non_sq_lock);
104 		tcp->tcp_squeue_bytes += size;
105 		if (TCP_UNSENT_BYTES(tcp) > connp->conn_sndbuf) {
106 			tcp_setqfull(tcp);
107 		}
108 		mutex_exit(&tcp->tcp_non_sq_lock);
109 
110 		CONN_INC_REF(connp);
111 		SQUEUE_ENTER_ONE(connp->conn_sqp, mp, tcp_output, connp,
112 		    NULL, tcp_squeue_flag, SQTAG_TCP_OUTPUT);
113 		return;
114 
115 	case M_CMD:
116 		tcp_wput_cmdblk(q, mp);
117 		return;
118 
119 	case M_PROTO:
120 	case M_PCPROTO:
121 		/*
122 		 * if it is a snmp message, don't get behind the squeue
123 		 */
124 		tcp = connp->conn_tcp;
125 		rptr = mp->b_rptr;
126 		if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) {
127 			type = ((union T_primitives *)rptr)->type;
128 		} else {
129 			if (connp->conn_debug) {
130 				(void) strlog(TCP_MOD_ID, 0, 1,
131 				    SL_ERROR|SL_TRACE,
132 				    "tcp_wput_proto, dropping one...");
133 			}
134 			freemsg(mp);
135 			return;
136 		}
137 		if (type == T_SVR4_OPTMGMT_REQ) {
138 			/*
139 			 * All Solaris components should pass a db_credp
140 			 * for this TPI message, hence we ASSERT.
141 			 * But in case there is some other M_PROTO that looks
142 			 * like a TPI message sent by some other kernel
143 			 * component, we check and return an error.
144 			 */
145 			cred_t	*cr = msg_getcred(mp, NULL);
146 
147 			ASSERT(cr != NULL);
148 			if (cr == NULL) {
149 				tcp_err_ack(tcp, mp, TSYSERR, EINVAL);
150 				return;
151 			}
152 			if (snmpcom_req(q, mp, tcp_snmp_set, ip_snmp_get,
153 			    cr)) {
154 				/*
155 				 * This was a SNMP request
156 				 */
157 				return;
158 			} else {
159 				output_proc = tcp_wput_proto;
160 			}
161 		} else {
162 			output_proc = tcp_wput_proto;
163 		}
164 		break;
165 	case M_IOCTL:
166 		/*
167 		 * Most ioctls can be processed right away without going via
168 		 * squeues - process them right here. Those that do require
169 		 * squeue (currently _SIOCSOCKFALLBACK)
170 		 * are processed by tcp_wput_ioctl().
171 		 */
172 		iocp = (struct iocblk *)mp->b_rptr;
173 		tcp = connp->conn_tcp;
174 
175 		switch (iocp->ioc_cmd) {
176 		case TCP_IOC_ABORT_CONN:
177 			tcp_ioctl_abort_conn(q, mp);
178 			return;
179 		case TI_GETPEERNAME:
180 		case TI_GETMYNAME:
181 			mi_copyin(q, mp, NULL,
182 			    SIZEOF_STRUCT(strbuf, iocp->ioc_flag));
183 			return;
184 
185 		default:
186 			output_proc = tcp_wput_ioctl;
187 			break;
188 		}
189 		break;
190 	default:
191 		output_proc = tcp_wput_nondata;
192 		break;
193 	}
194 
195 	CONN_INC_REF(connp);
196 	SQUEUE_ENTER_ONE(connp->conn_sqp, mp, output_proc, connp,
197 	    NULL, tcp_squeue_flag, SQTAG_TCP_WPUT_OTHER);
198 }
199 
200 /*
201  * The TCP normal data output path.
202  * NOTE: the logic of the fast path is duplicated from this function.
203  */
204 void
tcp_wput_data(tcp_t * tcp,mblk_t * mp,boolean_t urgent)205 tcp_wput_data(tcp_t *tcp, mblk_t *mp, boolean_t urgent)
206 {
207 	int		len;
208 	mblk_t		*local_time;
209 	mblk_t		*mp1;
210 	uint32_t	snxt;
211 	int		tail_unsent;
212 	int		tcpstate;
213 	int		usable = 0;
214 	mblk_t		*xmit_tail;
215 	int32_t		mss;
216 	int32_t		num_sack_blk = 0;
217 	int32_t		total_hdr_len;
218 	int32_t		tcp_hdr_len;
219 	int		rc;
220 	tcp_stack_t	*tcps = tcp->tcp_tcps;
221 	conn_t		*connp = tcp->tcp_connp;
222 	clock_t		now = LBOLT_FASTPATH;
223 
224 	tcpstate = tcp->tcp_state;
225 	if (mp == NULL) {
226 		/*
227 		 * tcp_wput_data() with NULL mp should only be called when
228 		 * there is unsent data.
229 		 */
230 		ASSERT(tcp->tcp_unsent > 0);
231 		/* Really tacky... but we need this for detached closes. */
232 		len = tcp->tcp_unsent;
233 		goto data_null;
234 	}
235 
236 	ASSERT(mp->b_datap->db_type == M_DATA);
237 	/*
238 	 * Don't allow data after T_ORDREL_REQ or T_DISCON_REQ,
239 	 * or before a connection attempt has begun.
240 	 */
241 	if (tcpstate < TCPS_SYN_SENT || tcpstate > TCPS_CLOSE_WAIT ||
242 	    (tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) {
243 		if ((tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) {
244 #ifdef DEBUG
245 			cmn_err(CE_WARN,
246 			    "tcp_wput_data: data after ordrel, %s",
247 			    tcp_display(tcp, NULL,
248 			    DISP_ADDR_AND_PORT));
249 #else
250 			if (connp->conn_debug) {
251 				(void) strlog(TCP_MOD_ID, 0, 1,
252 				    SL_TRACE|SL_ERROR,
253 				    "tcp_wput_data: data after ordrel, %s\n",
254 				    tcp_display(tcp, NULL,
255 				    DISP_ADDR_AND_PORT));
256 			}
257 #endif /* DEBUG */
258 		}
259 		if (tcp->tcp_snd_zcopy_aware &&
260 		    (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY))
261 			tcp_zcopy_notify(tcp);
262 		freemsg(mp);
263 		mutex_enter(&tcp->tcp_non_sq_lock);
264 		if (tcp->tcp_flow_stopped &&
265 		    TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
266 			tcp_clrqfull(tcp);
267 		}
268 		mutex_exit(&tcp->tcp_non_sq_lock);
269 		return;
270 	}
271 
272 	/* Strip empties */
273 	for (;;) {
274 		ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <=
275 		    (uintptr_t)INT_MAX);
276 		len = (int)(mp->b_wptr - mp->b_rptr);
277 		if (len > 0)
278 			break;
279 		mp1 = mp;
280 		mp = mp->b_cont;
281 		freeb(mp1);
282 		if (mp == NULL) {
283 			return;
284 		}
285 	}
286 
287 	/* If we are the first on the list ... */
288 	if (tcp->tcp_xmit_head == NULL) {
289 		tcp->tcp_xmit_head = mp;
290 		tcp->tcp_xmit_tail = mp;
291 		tcp->tcp_xmit_tail_unsent = len;
292 	} else {
293 		/* If tiny tx and room in txq tail, pullup to save mblks. */
294 		struct datab *dp;
295 
296 		mp1 = tcp->tcp_xmit_last;
297 		if (len < tcp_tx_pull_len &&
298 		    (dp = mp1->b_datap)->db_ref == 1 &&
299 		    dp->db_lim - mp1->b_wptr >= len) {
300 			ASSERT(len > 0);
301 			ASSERT(!mp1->b_cont);
302 			if (len == 1) {
303 				*mp1->b_wptr++ = *mp->b_rptr;
304 			} else {
305 				bcopy(mp->b_rptr, mp1->b_wptr, len);
306 				mp1->b_wptr += len;
307 			}
308 			if (mp1 == tcp->tcp_xmit_tail)
309 				tcp->tcp_xmit_tail_unsent += len;
310 			mp1->b_cont = mp->b_cont;
311 			if (tcp->tcp_snd_zcopy_aware &&
312 			    (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY))
313 				mp1->b_datap->db_struioflag |= STRUIO_ZCNOTIFY;
314 			freeb(mp);
315 			mp = mp1;
316 		} else {
317 			tcp->tcp_xmit_last->b_cont = mp;
318 		}
319 		len += tcp->tcp_unsent;
320 	}
321 
322 	/* Tack on however many more positive length mblks we have */
323 	if ((mp1 = mp->b_cont) != NULL) {
324 		do {
325 			int tlen;
326 			ASSERT((uintptr_t)(mp1->b_wptr - mp1->b_rptr) <=
327 			    (uintptr_t)INT_MAX);
328 			tlen = (int)(mp1->b_wptr - mp1->b_rptr);
329 			if (tlen <= 0) {
330 				mp->b_cont = mp1->b_cont;
331 				freeb(mp1);
332 			} else {
333 				len += tlen;
334 				mp = mp1;
335 			}
336 		} while ((mp1 = mp->b_cont) != NULL);
337 	}
338 	tcp->tcp_xmit_last = mp;
339 	tcp->tcp_unsent = len;
340 
341 	if (urgent)
342 		usable = 1;
343 
344 data_null:
345 	snxt = tcp->tcp_snxt;
346 	xmit_tail = tcp->tcp_xmit_tail;
347 	tail_unsent = tcp->tcp_xmit_tail_unsent;
348 
349 	/*
350 	 * Note that tcp_mss has been adjusted to take into account the
351 	 * timestamp option if applicable.  Because SACK options do not
352 	 * appear in every TCP segments and they are of variable lengths,
353 	 * they cannot be included in tcp_mss.  Thus we need to calculate
354 	 * the actual segment length when we need to send a segment which
355 	 * includes SACK options.
356 	 */
357 	if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) {
358 		int32_t	opt_len;
359 
360 		num_sack_blk = MIN(tcp->tcp_max_sack_blk,
361 		    tcp->tcp_num_sack_blk);
362 		opt_len = num_sack_blk * sizeof (sack_blk_t) + TCPOPT_NOP_LEN *
363 		    2 + TCPOPT_HEADER_LEN;
364 		mss = tcp->tcp_mss - opt_len;
365 		total_hdr_len = connp->conn_ht_iphc_len + opt_len;
366 		tcp_hdr_len = connp->conn_ht_ulp_len + opt_len;
367 	} else {
368 		mss = tcp->tcp_mss;
369 		total_hdr_len = connp->conn_ht_iphc_len;
370 		tcp_hdr_len = connp->conn_ht_ulp_len;
371 	}
372 
373 	if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet &&
374 	    (TICK_TO_MSEC(now - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) {
375 		TCP_SET_INIT_CWND(tcp, mss, tcps->tcps_slow_start_after_idle);
376 	}
377 	if (tcpstate == TCPS_SYN_RCVD) {
378 		/*
379 		 * The three-way connection establishment handshake is not
380 		 * complete yet. We want to queue the data for transmission
381 		 * after entering ESTABLISHED state (RFC793). A jump to
382 		 * "done" label effectively leaves data on the queue.
383 		 */
384 		goto done;
385 	} else {
386 		int usable_r;
387 
388 		/*
389 		 * In the special case when cwnd is zero, which can only
390 		 * happen if the connection is ECN capable, return now.
391 		 * New segments is sent using tcp_timer().  The timer
392 		 * is set in tcp_input_data().
393 		 */
394 		if (tcp->tcp_cwnd == 0) {
395 			/*
396 			 * Note that tcp_cwnd is 0 before 3-way handshake is
397 			 * finished.
398 			 */
399 			ASSERT(tcp->tcp_ecn_ok ||
400 			    tcp->tcp_state < TCPS_ESTABLISHED);
401 			return;
402 		}
403 
404 		/* NOTE: trouble if xmitting while SYN not acked? */
405 		usable_r = snxt - tcp->tcp_suna;
406 		usable_r = tcp->tcp_swnd - usable_r;
407 
408 		/*
409 		 * Check if the receiver has shrunk the window.  If
410 		 * tcp_wput_data() with NULL mp is called, tcp_fin_sent
411 		 * cannot be set as there is unsent data, so FIN cannot
412 		 * be sent out.  Otherwise, we need to take into account
413 		 * of FIN as it consumes an "invisible" sequence number.
414 		 */
415 		ASSERT(tcp->tcp_fin_sent == 0);
416 		if (usable_r < 0) {
417 			/*
418 			 * The receiver has shrunk the window and we have sent
419 			 * -usable_r date beyond the window, re-adjust.
420 			 *
421 			 * If TCP window scaling is enabled, there can be
422 			 * round down error as the advertised receive window
423 			 * is actually right shifted n bits.  This means that
424 			 * the lower n bits info is wiped out.  It will look
425 			 * like the window is shrunk.  Do a check here to
426 			 * see if the shrunk amount is actually within the
427 			 * error in window calculation.  If it is, just
428 			 * return.  Note that this check is inside the
429 			 * shrunk window check.  This makes sure that even
430 			 * though tcp_process_shrunk_swnd() is not called,
431 			 * we will stop further processing.
432 			 */
433 			if ((-usable_r >> tcp->tcp_snd_ws) > 0) {
434 				tcp_process_shrunk_swnd(tcp, -usable_r);
435 			}
436 			return;
437 		}
438 
439 		/* usable = MIN(swnd, cwnd) - unacked_bytes */
440 		if (tcp->tcp_swnd > tcp->tcp_cwnd)
441 			usable_r -= tcp->tcp_swnd - tcp->tcp_cwnd;
442 
443 		/* usable = MIN(usable, unsent) */
444 		if (usable_r > len)
445 			usable_r = len;
446 
447 		/* usable = MAX(usable, {1 for urgent, 0 for data}) */
448 		if (usable_r > 0) {
449 			usable = usable_r;
450 		} else {
451 			/* Bypass all other unnecessary processing. */
452 			goto done;
453 		}
454 	}
455 
456 	local_time = (mblk_t *)now;
457 
458 	/*
459 	 * "Our" Nagle Algorithm.  This is not the same as in the old
460 	 * BSD.  This is more in line with the true intent of Nagle.
461 	 *
462 	 * The conditions are:
463 	 * 1. The amount of unsent data (or amount of data which can be
464 	 *    sent, whichever is smaller) is less than Nagle limit.
465 	 * 2. The last sent size is also less than Nagle limit.
466 	 * 3. There is unack'ed data.
467 	 * 4. Urgent pointer is not set.  Send urgent data ignoring the
468 	 *    Nagle algorithm.  This reduces the probability that urgent
469 	 *    bytes get "merged" together.
470 	 * 5. The app has not closed the connection.  This eliminates the
471 	 *    wait time of the receiving side waiting for the last piece of
472 	 *    (small) data.
473 	 *
474 	 * If all are satisified, exit without sending anything.  Note
475 	 * that Nagle limit can be smaller than 1 MSS.  Nagle limit is
476 	 * the smaller of 1 MSS and global tcp_naglim_def (default to be
477 	 * 4095).
478 	 */
479 	if (usable < (int)tcp->tcp_naglim &&
480 	    tcp->tcp_naglim > tcp->tcp_last_sent_len &&
481 	    snxt != tcp->tcp_suna &&
482 	    !(tcp->tcp_valid_bits & TCP_URG_VALID) &&
483 	    !(tcp->tcp_valid_bits & TCP_FSS_VALID)) {
484 		goto done;
485 	}
486 
487 	/*
488 	 * If tcp_zero_win_probe is not set and the tcp->tcp_cork option
489 	 * is set, then we have to force TCP not to send partial segment
490 	 * (smaller than MSS bytes). We are calculating the usable now
491 	 * based on full mss and will save the rest of remaining data for
492 	 * later. When tcp_zero_win_probe is set, TCP needs to send out
493 	 * something to do zero window probe.
494 	 */
495 	if (tcp->tcp_cork && !tcp->tcp_zero_win_probe) {
496 		if (usable < mss)
497 			goto done;
498 		usable = (usable / mss) * mss;
499 	}
500 
501 	/* Update the latest receive window size in TCP header. */
502 	tcp->tcp_tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
503 
504 	/* Send the packet. */
505 	rc = tcp_send(tcp, mss, total_hdr_len, tcp_hdr_len,
506 	    num_sack_blk, &usable, &snxt, &tail_unsent, &xmit_tail,
507 	    local_time);
508 
509 	/* Pretend that all we were trying to send really got sent */
510 	if (rc < 0 && tail_unsent < 0) {
511 		do {
512 			xmit_tail = xmit_tail->b_cont;
513 			xmit_tail->b_prev = local_time;
514 			ASSERT((uintptr_t)(xmit_tail->b_wptr -
515 			    xmit_tail->b_rptr) <= (uintptr_t)INT_MAX);
516 			tail_unsent += (int)(xmit_tail->b_wptr -
517 			    xmit_tail->b_rptr);
518 		} while (tail_unsent < 0);
519 	}
520 done:;
521 	tcp->tcp_xmit_tail = xmit_tail;
522 	tcp->tcp_xmit_tail_unsent = tail_unsent;
523 	len = tcp->tcp_snxt - snxt;
524 	if (len) {
525 		/*
526 		 * If new data was sent, need to update the notsack
527 		 * list, which is, afterall, data blocks that have
528 		 * not been sack'ed by the receiver.  New data is
529 		 * not sack'ed.
530 		 */
531 		if (tcp->tcp_snd_sack_ok && tcp->tcp_notsack_list != NULL) {
532 			/* len is a negative value. */
533 			tcp->tcp_pipe -= len;
534 			tcp_notsack_update(&(tcp->tcp_notsack_list),
535 			    tcp->tcp_snxt, snxt,
536 			    &(tcp->tcp_num_notsack_blk),
537 			    &(tcp->tcp_cnt_notsack_list));
538 		}
539 		tcp->tcp_snxt = snxt + tcp->tcp_fin_sent;
540 		tcp->tcp_rack = tcp->tcp_rnxt;
541 		tcp->tcp_rack_cnt = 0;
542 		if ((snxt + len) == tcp->tcp_suna) {
543 			TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
544 		}
545 	} else if (snxt == tcp->tcp_suna && tcp->tcp_swnd == 0) {
546 		/*
547 		 * Didn't send anything. Make sure the timer is running
548 		 * so that we will probe a zero window.
549 		 */
550 		TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
551 	}
552 	/* Note that len is the amount we just sent but with a negative sign */
553 	tcp->tcp_unsent += len;
554 	mutex_enter(&tcp->tcp_non_sq_lock);
555 	if (tcp->tcp_flow_stopped) {
556 		if (TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
557 			tcp_clrqfull(tcp);
558 		}
559 	} else if (TCP_UNSENT_BYTES(tcp) >= connp->conn_sndbuf) {
560 		if (!(tcp->tcp_detached))
561 			tcp_setqfull(tcp);
562 	}
563 	mutex_exit(&tcp->tcp_non_sq_lock);
564 }
565 
566 /*
567  * Initial STREAMS write side put() procedure for sockets. It tries to
568  * handle the T_CAPABILITY_REQ which sockfs sends down while setting
569  * up the socket without using the squeue. Non T_CAPABILITY_REQ messages
570  * are handled by tcp_wput() as usual.
571  *
572  * All further messages will also be handled by tcp_wput() because we cannot
573  * be sure that the above short cut is safe later.
574  */
575 void
tcp_wput_sock(queue_t * wq,mblk_t * mp)576 tcp_wput_sock(queue_t *wq, mblk_t *mp)
577 {
578 	conn_t			*connp = Q_TO_CONN(wq);
579 	tcp_t			*tcp = connp->conn_tcp;
580 	struct T_capability_req	*car = (struct T_capability_req *)mp->b_rptr;
581 
582 	ASSERT(wq->q_qinfo == &tcp_sock_winit);
583 	wq->q_qinfo = &tcp_winit;
584 
585 	ASSERT(IPCL_IS_TCP(connp));
586 	ASSERT(TCP_IS_SOCKET(tcp));
587 
588 	if (DB_TYPE(mp) == M_PCPROTO &&
589 	    MBLKL(mp) == sizeof (struct T_capability_req) &&
590 	    car->PRIM_type == T_CAPABILITY_REQ) {
591 		tcp_capability_req(tcp, mp);
592 		return;
593 	}
594 
595 	tcp_wput(wq, mp);
596 }
597 
598 /* ARGSUSED */
599 void
tcp_wput_fallback(queue_t * wq,mblk_t * mp)600 tcp_wput_fallback(queue_t *wq, mblk_t *mp)
601 {
602 #ifdef DEBUG
603 	cmn_err(CE_CONT, "tcp_wput_fallback: Message during fallback \n");
604 #endif
605 	freemsg(mp);
606 }
607 
608 /*
609  * Call by tcp_wput() to handle misc non M_DATA messages.
610  */
611 /* ARGSUSED */
612 static void
tcp_wput_nondata(void * arg,mblk_t * mp,void * arg2,ip_recv_attr_t * dummy)613 tcp_wput_nondata(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
614 {
615 	conn_t	*connp = (conn_t *)arg;
616 	tcp_t	*tcp = connp->conn_tcp;
617 
618 	ASSERT(DB_TYPE(mp) != M_IOCTL);
619 	/*
620 	 * TCP is D_MP and qprocsoff() is done towards the end of the tcp_close.
621 	 * Once the close starts, streamhead and sockfs will not let any data
622 	 * packets come down (close ensures that there are no threads using the
623 	 * queue and no new threads will come down) but since qprocsoff()
624 	 * hasn't happened yet, a M_FLUSH or some non data message might
625 	 * get reflected back (in response to our own FLUSHRW) and get
626 	 * processed after tcp_close() is done. The conn would still be valid
627 	 * because a ref would have added but we need to check the state
628 	 * before actually processing the packet.
629 	 */
630 	if (TCP_IS_DETACHED(tcp) || (tcp->tcp_state == TCPS_CLOSED)) {
631 		freemsg(mp);
632 		return;
633 	}
634 
635 	switch (DB_TYPE(mp)) {
636 	case M_IOCDATA:
637 		tcp_wput_iocdata(tcp, mp);
638 		break;
639 	case M_FLUSH:
640 		tcp_wput_flush(tcp, mp);
641 		break;
642 	default:
643 		ip_wput_nondata(connp->conn_wq, mp);
644 		break;
645 	}
646 }
647 
648 /* tcp_wput_flush is called by tcp_wput_nondata to handle M_FLUSH messages. */
649 static void
tcp_wput_flush(tcp_t * tcp,mblk_t * mp)650 tcp_wput_flush(tcp_t *tcp, mblk_t *mp)
651 {
652 	uchar_t	fval = *mp->b_rptr;
653 	mblk_t	*tail;
654 	conn_t	*connp = tcp->tcp_connp;
655 	queue_t	*q = connp->conn_wq;
656 
657 	/* TODO: How should flush interact with urgent data? */
658 	if ((fval & FLUSHW) && tcp->tcp_xmit_head != NULL &&
659 	    !(tcp->tcp_valid_bits & TCP_URG_VALID)) {
660 		/*
661 		 * Flush only data that has not yet been put on the wire.  If
662 		 * we flush data that we have already transmitted, life, as we
663 		 * know it, may come to an end.
664 		 */
665 		tail = tcp->tcp_xmit_tail;
666 		tail->b_wptr -= tcp->tcp_xmit_tail_unsent;
667 		tcp->tcp_xmit_tail_unsent = 0;
668 		tcp->tcp_unsent = 0;
669 		if (tail->b_wptr != tail->b_rptr)
670 			tail = tail->b_cont;
671 		if (tail) {
672 			mblk_t **excess = &tcp->tcp_xmit_head;
673 			for (;;) {
674 				mblk_t *mp1 = *excess;
675 				if (mp1 == tail)
676 					break;
677 				tcp->tcp_xmit_tail = mp1;
678 				tcp->tcp_xmit_last = mp1;
679 				excess = &mp1->b_cont;
680 			}
681 			*excess = NULL;
682 			tcp_close_mpp(&tail);
683 			if (tcp->tcp_snd_zcopy_aware)
684 				tcp_zcopy_notify(tcp);
685 		}
686 		/*
687 		 * We have no unsent data, so unsent must be less than
688 		 * conn_sndlowat, so re-enable flow.
689 		 */
690 		mutex_enter(&tcp->tcp_non_sq_lock);
691 		if (tcp->tcp_flow_stopped) {
692 			tcp_clrqfull(tcp);
693 		}
694 		mutex_exit(&tcp->tcp_non_sq_lock);
695 	}
696 	/*
697 	 * TODO: you can't just flush these, you have to increase rwnd for one
698 	 * thing.  For another, how should urgent data interact?
699 	 */
700 	if (fval & FLUSHR) {
701 		*mp->b_rptr = fval & ~FLUSHW;
702 		/* XXX */
703 		qreply(q, mp);
704 		return;
705 	}
706 	freemsg(mp);
707 }
708 
709 /*
710  * tcp_wput_iocdata is called by tcp_wput_nondata to handle all M_IOCDATA
711  * messages.
712  */
713 static void
tcp_wput_iocdata(tcp_t * tcp,mblk_t * mp)714 tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp)
715 {
716 	mblk_t		*mp1;
717 	struct iocblk	*iocp = (struct iocblk *)mp->b_rptr;
718 	STRUCT_HANDLE(strbuf, sb);
719 	uint_t		addrlen;
720 	conn_t		*connp = tcp->tcp_connp;
721 	queue_t 	*q = connp->conn_wq;
722 
723 	/* Make sure it is one of ours. */
724 	switch (iocp->ioc_cmd) {
725 	case TI_GETMYNAME:
726 	case TI_GETPEERNAME:
727 		break;
728 	default:
729 		/*
730 		 * If the conn is closing, then error the ioctl here. Otherwise
731 		 * use the CONN_IOCTLREF_* macros to hold off tcp_close until
732 		 * we're done here.
733 		 */
734 		mutex_enter(&connp->conn_lock);
735 		if (connp->conn_state_flags & CONN_CLOSING) {
736 			mutex_exit(&connp->conn_lock);
737 			iocp->ioc_error = EINVAL;
738 			mp->b_datap->db_type = M_IOCNAK;
739 			iocp->ioc_count = 0;
740 			qreply(q, mp);
741 			return;
742 		}
743 
744 		CONN_INC_IOCTLREF_LOCKED(connp);
745 		ip_wput_nondata(q, mp);
746 		CONN_DEC_IOCTLREF(connp);
747 		return;
748 	}
749 	switch (mi_copy_state(q, mp, &mp1)) {
750 	case -1:
751 		return;
752 	case MI_COPY_CASE(MI_COPY_IN, 1):
753 		break;
754 	case MI_COPY_CASE(MI_COPY_OUT, 1):
755 		/* Copy out the strbuf. */
756 		mi_copyout(q, mp);
757 		return;
758 	case MI_COPY_CASE(MI_COPY_OUT, 2):
759 		/* All done. */
760 		mi_copy_done(q, mp, 0);
761 		return;
762 	default:
763 		mi_copy_done(q, mp, EPROTO);
764 		return;
765 	}
766 	/* Check alignment of the strbuf */
767 	if (!OK_32PTR(mp1->b_rptr)) {
768 		mi_copy_done(q, mp, EINVAL);
769 		return;
770 	}
771 
772 	STRUCT_SET_HANDLE(sb, iocp->ioc_flag, (void *)mp1->b_rptr);
773 
774 	if (connp->conn_family == AF_INET)
775 		addrlen = sizeof (sin_t);
776 	else
777 		addrlen = sizeof (sin6_t);
778 
779 	if (STRUCT_FGET(sb, maxlen) < addrlen) {
780 		mi_copy_done(q, mp, EINVAL);
781 		return;
782 	}
783 
784 	switch (iocp->ioc_cmd) {
785 	case TI_GETMYNAME:
786 		break;
787 	case TI_GETPEERNAME:
788 		if (tcp->tcp_state < TCPS_SYN_RCVD) {
789 			mi_copy_done(q, mp, ENOTCONN);
790 			return;
791 		}
792 		break;
793 	}
794 	mp1 = mi_copyout_alloc(q, mp, STRUCT_FGETP(sb, buf), addrlen, B_TRUE);
795 	if (!mp1)
796 		return;
797 
798 	STRUCT_FSET(sb, len, addrlen);
799 	switch (((struct iocblk *)mp->b_rptr)->ioc_cmd) {
800 	case TI_GETMYNAME:
801 		(void) conn_getsockname(connp, (struct sockaddr *)mp1->b_wptr,
802 		    &addrlen);
803 		break;
804 	case TI_GETPEERNAME:
805 		(void) conn_getpeername(connp, (struct sockaddr *)mp1->b_wptr,
806 		    &addrlen);
807 		break;
808 	}
809 	mp1->b_wptr += addrlen;
810 	/* Copy out the address */
811 	mi_copyout(q, mp);
812 }
813 
814 /*
815  * tcp_wput_ioctl is called by tcp_wput_nondata() to handle all M_IOCTL
816  * messages.
817  */
818 /* ARGSUSED */
819 static void
tcp_wput_ioctl(void * arg,mblk_t * mp,void * arg2,ip_recv_attr_t * dummy)820 tcp_wput_ioctl(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
821 {
822 	conn_t 		*connp = (conn_t *)arg;
823 	tcp_t		*tcp = connp->conn_tcp;
824 	queue_t		*q = connp->conn_wq;
825 	struct iocblk	*iocp;
826 
827 	ASSERT(DB_TYPE(mp) == M_IOCTL);
828 	/*
829 	 * Try and ASSERT the minimum possible references on the
830 	 * conn early enough. Since we are executing on write side,
831 	 * the connection is obviously not detached and that means
832 	 * there is a ref each for TCP and IP. Since we are behind
833 	 * the squeue, the minimum references needed are 3. If the
834 	 * conn is in classifier hash list, there should be an
835 	 * extra ref for that (we check both the possibilities).
836 	 */
837 	ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
838 	    (connp->conn_fanout == NULL && connp->conn_ref >= 3));
839 
840 	iocp = (struct iocblk *)mp->b_rptr;
841 	switch (iocp->ioc_cmd) {
842 	case _SIOCSOCKFALLBACK:
843 		/*
844 		 * Either sockmod is about to be popped and the socket
845 		 * would now be treated as a plain stream, or a module
846 		 * is about to be pushed so we could no longer use read-
847 		 * side synchronous streams for fused loopback tcp.
848 		 * Drain any queued data and disable direct sockfs
849 		 * interface from now on.
850 		 */
851 		if (!tcp->tcp_issocket) {
852 			DB_TYPE(mp) = M_IOCNAK;
853 			iocp->ioc_error = EINVAL;
854 		} else {
855 			tcp_use_pure_tpi(tcp);
856 			DB_TYPE(mp) = M_IOCACK;
857 			iocp->ioc_error = 0;
858 		}
859 		iocp->ioc_count = 0;
860 		iocp->ioc_rval = 0;
861 		qreply(q, mp);
862 		return;
863 	}
864 
865 	/*
866 	 * If the conn is closing, then error the ioctl here. Otherwise bump the
867 	 * conn_ioctlref to hold off tcp_close until we're done here.
868 	 */
869 	mutex_enter(&(connp)->conn_lock);
870 	if ((connp)->conn_state_flags & CONN_CLOSING) {
871 		mutex_exit(&(connp)->conn_lock);
872 		iocp->ioc_error = EINVAL;
873 		mp->b_datap->db_type = M_IOCNAK;
874 		iocp->ioc_count = 0;
875 		qreply(q, mp);
876 		return;
877 	}
878 
879 	CONN_INC_IOCTLREF_LOCKED(connp);
880 	ip_wput_nondata(q, mp);
881 	CONN_DEC_IOCTLREF(connp);
882 }
883 
884 /*
885  * This routine is called by tcp_wput() to handle all TPI requests.
886  */
887 /* ARGSUSED */
888 static void
tcp_wput_proto(void * arg,mblk_t * mp,void * arg2,ip_recv_attr_t * dummy)889 tcp_wput_proto(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
890 {
891 	conn_t		*connp = (conn_t *)arg;
892 	tcp_t		*tcp = connp->conn_tcp;
893 	union T_primitives *tprim = (union T_primitives *)mp->b_rptr;
894 	uchar_t		*rptr;
895 	t_scalar_t	type;
896 	cred_t		*cr;
897 
898 	/*
899 	 * Try and ASSERT the minimum possible references on the
900 	 * conn early enough. Since we are executing on write side,
901 	 * the connection is obviously not detached and that means
902 	 * there is a ref each for TCP and IP. Since we are behind
903 	 * the squeue, the minimum references needed are 3. If the
904 	 * conn is in classifier hash list, there should be an
905 	 * extra ref for that (we check both the possibilities).
906 	 */
907 	ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
908 	    (connp->conn_fanout == NULL && connp->conn_ref >= 3));
909 
910 	rptr = mp->b_rptr;
911 	ASSERT((uintptr_t)(mp->b_wptr - rptr) <= (uintptr_t)INT_MAX);
912 	if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) {
913 		type = ((union T_primitives *)rptr)->type;
914 		if (type == T_EXDATA_REQ) {
915 			tcp_output_urgent(connp, mp, arg2, NULL);
916 		} else if (type != T_DATA_REQ) {
917 			goto non_urgent_data;
918 		} else {
919 			/* TODO: options, flags, ... from user */
920 			/* Set length to zero for reclamation below */
921 			tcp_wput_data(tcp, mp->b_cont, B_TRUE);
922 			freeb(mp);
923 		}
924 		return;
925 	} else {
926 		if (connp->conn_debug) {
927 			(void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE,
928 			    "tcp_wput_proto, dropping one...");
929 		}
930 		freemsg(mp);
931 		return;
932 	}
933 
934 non_urgent_data:
935 
936 	switch ((int)tprim->type) {
937 	case O_T_BIND_REQ:	/* bind request */
938 	case T_BIND_REQ:	/* new semantics bind request */
939 		tcp_tpi_bind(tcp, mp);
940 		break;
941 	case T_UNBIND_REQ:	/* unbind request */
942 		tcp_tpi_unbind(tcp, mp);
943 		break;
944 	case O_T_CONN_RES:	/* old connection response XXX */
945 	case T_CONN_RES:	/* connection response */
946 		tcp_tli_accept(tcp, mp);
947 		break;
948 	case T_CONN_REQ:	/* connection request */
949 		tcp_tpi_connect(tcp, mp);
950 		break;
951 	case T_DISCON_REQ:	/* disconnect request */
952 		tcp_disconnect(tcp, mp);
953 		break;
954 	case T_CAPABILITY_REQ:
955 		tcp_capability_req(tcp, mp);	/* capability request */
956 		break;
957 	case T_INFO_REQ:	/* information request */
958 		tcp_info_req(tcp, mp);
959 		break;
960 	case T_SVR4_OPTMGMT_REQ:	/* manage options req */
961 	case T_OPTMGMT_REQ:
962 		/*
963 		 * Note:  no support for snmpcom_req() through new
964 		 * T_OPTMGMT_REQ. See comments in ip.c
965 		 */
966 
967 		/*
968 		 * All Solaris components should pass a db_credp
969 		 * for this TPI message, hence we ASSERT.
970 		 * But in case there is some other M_PROTO that looks
971 		 * like a TPI message sent by some other kernel
972 		 * component, we check and return an error.
973 		 */
974 		cr = msg_getcred(mp, NULL);
975 		ASSERT(cr != NULL);
976 		if (cr == NULL) {
977 			tcp_err_ack(tcp, mp, TSYSERR, EINVAL);
978 			return;
979 		}
980 		/*
981 		 * If EINPROGRESS is returned, the request has been queued
982 		 * for subsequent processing by ip_restart_optmgmt(), which
983 		 * will do the CONN_DEC_REF().
984 		 */
985 		if ((int)tprim->type == T_SVR4_OPTMGMT_REQ) {
986 			svr4_optcom_req(connp->conn_wq, mp, cr, &tcp_opt_obj);
987 		} else {
988 			tpi_optcom_req(connp->conn_wq, mp, cr, &tcp_opt_obj);
989 		}
990 		break;
991 
992 	case T_UNITDATA_REQ:	/* unitdata request */
993 		tcp_err_ack(tcp, mp, TNOTSUPPORT, 0);
994 		break;
995 	case T_ORDREL_REQ:	/* orderly release req */
996 		freemsg(mp);
997 
998 		if (tcp->tcp_fused)
999 			tcp_unfuse(tcp);
1000 
1001 		if (tcp_xmit_end(tcp) != 0) {
1002 			/*
1003 			 * We were crossing FINs and got a reset from
1004 			 * the other side. Just ignore it.
1005 			 */
1006 			if (connp->conn_debug) {
1007 				(void) strlog(TCP_MOD_ID, 0, 1,
1008 				    SL_ERROR|SL_TRACE,
1009 				    "tcp_wput_proto, T_ORDREL_REQ out of "
1010 				    "state %s",
1011 				    tcp_display(tcp, NULL,
1012 				    DISP_ADDR_AND_PORT));
1013 			}
1014 		}
1015 		break;
1016 	case T_ADDR_REQ:
1017 		tcp_addr_req(tcp, mp);
1018 		break;
1019 	default:
1020 		if (connp->conn_debug) {
1021 			(void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE,
1022 			    "tcp_wput_proto, bogus TPI msg, type %d",
1023 			    tprim->type);
1024 		}
1025 		/*
1026 		 * We used to M_ERROR.  Sending TNOTSUPPORT gives the user
1027 		 * to recover.
1028 		 */
1029 		tcp_err_ack(tcp, mp, TNOTSUPPORT, 0);
1030 		break;
1031 	}
1032 }
1033 
1034 /*
1035  * Handle special out-of-band ioctl requests (see PSARC/2008/265).
1036  */
1037 static void
tcp_wput_cmdblk(queue_t * q,mblk_t * mp)1038 tcp_wput_cmdblk(queue_t *q, mblk_t *mp)
1039 {
1040 	void	*data;
1041 	mblk_t	*datamp = mp->b_cont;
1042 	conn_t	*connp = Q_TO_CONN(q);
1043 	tcp_t	*tcp = connp->conn_tcp;
1044 	cmdblk_t *cmdp = (cmdblk_t *)mp->b_rptr;
1045 
1046 	if (datamp == NULL || MBLKL(datamp) < cmdp->cb_len) {
1047 		cmdp->cb_error = EPROTO;
1048 		qreply(q, mp);
1049 		return;
1050 	}
1051 
1052 	data = datamp->b_rptr;
1053 
1054 	switch (cmdp->cb_cmd) {
1055 	case TI_GETPEERNAME:
1056 		if (tcp->tcp_state < TCPS_SYN_RCVD)
1057 			cmdp->cb_error = ENOTCONN;
1058 		else
1059 			cmdp->cb_error = conn_getpeername(connp, data,
1060 			    &cmdp->cb_len);
1061 		break;
1062 	case TI_GETMYNAME:
1063 		cmdp->cb_error = conn_getsockname(connp, data, &cmdp->cb_len);
1064 		break;
1065 	default:
1066 		cmdp->cb_error = EINVAL;
1067 		break;
1068 	}
1069 
1070 	qreply(q, mp);
1071 }
1072 
1073 /*
1074  * The TCP fast path write put procedure.
1075  * NOTE: the logic of the fast path is duplicated from tcp_wput_data()
1076  */
1077 /* ARGSUSED */
1078 void
tcp_output(void * arg,mblk_t * mp,void * arg2,ip_recv_attr_t * dummy)1079 tcp_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1080 {
1081 	int		len;
1082 	int		hdrlen;
1083 	int		plen;
1084 	mblk_t		*mp1;
1085 	uchar_t		*rptr;
1086 	uint32_t	snxt;
1087 	tcpha_t		*tcpha;
1088 	struct datab	*db;
1089 	uint32_t	suna;
1090 	uint32_t	mss;
1091 	ipaddr_t	*dst;
1092 	ipaddr_t	*src;
1093 	uint32_t	sum;
1094 	int		usable;
1095 	conn_t		*connp = (conn_t *)arg;
1096 	tcp_t		*tcp = connp->conn_tcp;
1097 	uint32_t	msize;
1098 	tcp_stack_t	*tcps = tcp->tcp_tcps;
1099 	ip_xmit_attr_t	*ixa;
1100 	clock_t		now;
1101 
1102 	/*
1103 	 * Try and ASSERT the minimum possible references on the
1104 	 * conn early enough. Since we are executing on write side,
1105 	 * the connection is obviously not detached and that means
1106 	 * there is a ref each for TCP and IP. Since we are behind
1107 	 * the squeue, the minimum references needed are 3. If the
1108 	 * conn is in classifier hash list, there should be an
1109 	 * extra ref for that (we check both the possibilities).
1110 	 */
1111 	ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) ||
1112 	    (connp->conn_fanout == NULL && connp->conn_ref >= 3));
1113 
1114 	ASSERT(DB_TYPE(mp) == M_DATA);
1115 	msize = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp);
1116 
1117 	mutex_enter(&tcp->tcp_non_sq_lock);
1118 	tcp->tcp_squeue_bytes -= msize;
1119 	mutex_exit(&tcp->tcp_non_sq_lock);
1120 
1121 	/* Bypass tcp protocol for fused tcp loopback */
1122 	if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize))
1123 		return;
1124 
1125 	mss = tcp->tcp_mss;
1126 	/*
1127 	 * If ZEROCOPY has turned off, try not to send any zero-copy message
1128 	 * down. Do backoff, now.
1129 	 */
1130 	if (tcp->tcp_snd_zcopy_aware && !tcp->tcp_snd_zcopy_on)
1131 		mp = tcp_zcopy_backoff(tcp, mp, B_FALSE);
1132 
1133 
1134 	ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= (uintptr_t)INT_MAX);
1135 	len = (int)(mp->b_wptr - mp->b_rptr);
1136 
1137 	/*
1138 	 * Criteria for fast path:
1139 	 *
1140 	 *   1. no unsent data
1141 	 *   2. single mblk in request
1142 	 *   3. connection established
1143 	 *   4. data in mblk
1144 	 *   5. len <= mss
1145 	 *   6. no tcp_valid bits
1146 	 */
1147 	if ((tcp->tcp_unsent != 0) ||
1148 	    (tcp->tcp_cork) ||
1149 	    (mp->b_cont != NULL) ||
1150 	    (tcp->tcp_state != TCPS_ESTABLISHED) ||
1151 	    (len == 0) ||
1152 	    (len > mss) ||
1153 	    (tcp->tcp_valid_bits != 0)) {
1154 		tcp_wput_data(tcp, mp, B_FALSE);
1155 		return;
1156 	}
1157 
1158 	ASSERT(tcp->tcp_xmit_tail_unsent == 0);
1159 	ASSERT(tcp->tcp_fin_sent == 0);
1160 
1161 	/* queue new packet onto retransmission queue */
1162 	if (tcp->tcp_xmit_head == NULL) {
1163 		tcp->tcp_xmit_head = mp;
1164 	} else {
1165 		tcp->tcp_xmit_last->b_cont = mp;
1166 	}
1167 	tcp->tcp_xmit_last = mp;
1168 	tcp->tcp_xmit_tail = mp;
1169 
1170 	/* find out how much we can send */
1171 	/* BEGIN CSTYLED */
1172 	/*
1173 	 *    un-acked	   usable
1174 	 *  |--------------|-----------------|
1175 	 *  tcp_suna       tcp_snxt	  tcp_suna+tcp_swnd
1176 	 */
1177 	/* END CSTYLED */
1178 
1179 	/* start sending from tcp_snxt */
1180 	snxt = tcp->tcp_snxt;
1181 
1182 	/*
1183 	 * Check to see if this connection has been idled for some
1184 	 * time and no ACK is expected.  If it is, we need to slow
1185 	 * start again to get back the connection's "self-clock" as
1186 	 * described in VJ's paper.
1187 	 *
1188 	 * Reinitialize tcp_cwnd after idle.
1189 	 */
1190 	now = LBOLT_FASTPATH;
1191 	if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet &&
1192 	    (TICK_TO_MSEC(now - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) {
1193 		TCP_SET_INIT_CWND(tcp, mss, tcps->tcps_slow_start_after_idle);
1194 	}
1195 
1196 	usable = tcp->tcp_swnd;		/* tcp window size */
1197 	if (usable > tcp->tcp_cwnd)
1198 		usable = tcp->tcp_cwnd;	/* congestion window smaller */
1199 	usable -= snxt;		/* subtract stuff already sent */
1200 	suna = tcp->tcp_suna;
1201 	usable += suna;
1202 	/* usable can be < 0 if the congestion window is smaller */
1203 	if (len > usable) {
1204 		/* Can't send complete M_DATA in one shot */
1205 		goto slow;
1206 	}
1207 
1208 	mutex_enter(&tcp->tcp_non_sq_lock);
1209 	if (tcp->tcp_flow_stopped &&
1210 	    TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat) {
1211 		tcp_clrqfull(tcp);
1212 	}
1213 	mutex_exit(&tcp->tcp_non_sq_lock);
1214 
1215 	/*
1216 	 * determine if anything to send (Nagle).
1217 	 *
1218 	 *   1. len < tcp_mss (i.e. small)
1219 	 *   2. unacknowledged data present
1220 	 *   3. len < nagle limit
1221 	 *   4. last packet sent < nagle limit (previous packet sent)
1222 	 */
1223 	if ((len < mss) && (snxt != suna) &&
1224 	    (len < (int)tcp->tcp_naglim) &&
1225 	    (tcp->tcp_last_sent_len < tcp->tcp_naglim)) {
1226 		/*
1227 		 * This was the first unsent packet and normally
1228 		 * mss < xmit_hiwater so there is no need to worry
1229 		 * about flow control. The next packet will go
1230 		 * through the flow control check in tcp_wput_data().
1231 		 */
1232 		/* leftover work from above */
1233 		tcp->tcp_unsent = len;
1234 		tcp->tcp_xmit_tail_unsent = len;
1235 
1236 		return;
1237 	}
1238 
1239 	/*
1240 	 * len <= tcp->tcp_mss && len == unsent so no sender silly window.  Can
1241 	 * send now.
1242 	 */
1243 
1244 	if (snxt == suna) {
1245 		TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
1246 	}
1247 
1248 	/* we have always sent something */
1249 	tcp->tcp_rack_cnt = 0;
1250 
1251 	tcp->tcp_snxt = snxt + len;
1252 	tcp->tcp_rack = tcp->tcp_rnxt;
1253 
1254 	if ((mp1 = dupb(mp)) == 0)
1255 		goto no_memory;
1256 	mp->b_prev = (mblk_t *)(uintptr_t)now;
1257 	mp->b_next = (mblk_t *)(uintptr_t)snxt;
1258 
1259 	/* adjust tcp header information */
1260 	tcpha = tcp->tcp_tcpha;
1261 	tcpha->tha_flags = (TH_ACK|TH_PUSH);
1262 
1263 	sum = len + connp->conn_ht_ulp_len + connp->conn_sum;
1264 	sum = (sum >> 16) + (sum & 0xFFFF);
1265 	tcpha->tha_sum = htons(sum);
1266 
1267 	tcpha->tha_seq = htonl(snxt);
1268 
1269 	TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1270 	TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1271 	BUMP_LOCAL(tcp->tcp_obsegs);
1272 
1273 	/* Update the latest receive window size in TCP header. */
1274 	tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
1275 
1276 	tcp->tcp_last_sent_len = (ushort_t)len;
1277 
1278 	plen = len + connp->conn_ht_iphc_len;
1279 
1280 	ixa = connp->conn_ixa;
1281 	ixa->ixa_pktlen = plen;
1282 
1283 	if (ixa->ixa_flags & IXAF_IS_IPV4) {
1284 		tcp->tcp_ipha->ipha_length = htons(plen);
1285 	} else {
1286 		tcp->tcp_ip6h->ip6_plen = htons(plen - IPV6_HDR_LEN);
1287 	}
1288 
1289 	/* see if we need to allocate a mblk for the headers */
1290 	hdrlen = connp->conn_ht_iphc_len;
1291 	rptr = mp1->b_rptr - hdrlen;
1292 	db = mp1->b_datap;
1293 	if ((db->db_ref != 2) || rptr < db->db_base ||
1294 	    (!OK_32PTR(rptr))) {
1295 		/* NOTE: we assume allocb returns an OK_32PTR */
1296 		mp = allocb(hdrlen + tcps->tcps_wroff_xtra, BPRI_MED);
1297 		if (!mp) {
1298 			freemsg(mp1);
1299 			goto no_memory;
1300 		}
1301 		mp->b_cont = mp1;
1302 		mp1 = mp;
1303 		/* Leave room for Link Level header */
1304 		rptr = &mp1->b_rptr[tcps->tcps_wroff_xtra];
1305 		mp1->b_wptr = &rptr[hdrlen];
1306 	}
1307 	mp1->b_rptr = rptr;
1308 
1309 	/* Fill in the timestamp option. */
1310 	if (tcp->tcp_snd_ts_ok) {
1311 		uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
1312 
1313 		U32_TO_BE32(llbolt,
1314 		    (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
1315 		U32_TO_BE32(tcp->tcp_ts_recent,
1316 		    (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
1317 	} else {
1318 		ASSERT(connp->conn_ht_ulp_len == TCP_MIN_HEADER_LENGTH);
1319 	}
1320 
1321 	/* copy header into outgoing packet */
1322 	dst = (ipaddr_t *)rptr;
1323 	src = (ipaddr_t *)connp->conn_ht_iphc;
1324 	dst[0] = src[0];
1325 	dst[1] = src[1];
1326 	dst[2] = src[2];
1327 	dst[3] = src[3];
1328 	dst[4] = src[4];
1329 	dst[5] = src[5];
1330 	dst[6] = src[6];
1331 	dst[7] = src[7];
1332 	dst[8] = src[8];
1333 	dst[9] = src[9];
1334 	if (hdrlen -= 40) {
1335 		hdrlen >>= 2;
1336 		dst += 10;
1337 		src += 10;
1338 		do {
1339 			*dst++ = *src++;
1340 		} while (--hdrlen);
1341 	}
1342 
1343 	/*
1344 	 * Set the ECN info in the TCP header.  Note that this
1345 	 * is not the template header.
1346 	 */
1347 	if (tcp->tcp_ecn_ok) {
1348 		TCP_SET_ECT(tcp, rptr);
1349 
1350 		tcpha = (tcpha_t *)(rptr + ixa->ixa_ip_hdr_length);
1351 		if (tcp->tcp_ecn_echo_on)
1352 			tcpha->tha_flags |= TH_ECE;
1353 		if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
1354 			tcpha->tha_flags |= TH_CWR;
1355 			tcp->tcp_ecn_cwr_sent = B_TRUE;
1356 		}
1357 	}
1358 
1359 	if (tcp->tcp_ip_forward_progress) {
1360 		tcp->tcp_ip_forward_progress = B_FALSE;
1361 		connp->conn_ixa->ixa_flags |= IXAF_REACH_CONF;
1362 	} else {
1363 		connp->conn_ixa->ixa_flags &= ~IXAF_REACH_CONF;
1364 	}
1365 	tcp_send_data(tcp, mp1);
1366 	return;
1367 
1368 	/*
1369 	 * If we ran out of memory, we pretend to have sent the packet
1370 	 * and that it was lost on the wire.
1371 	 */
1372 no_memory:
1373 	return;
1374 
1375 slow:
1376 	/* leftover work from above */
1377 	tcp->tcp_unsent = len;
1378 	tcp->tcp_xmit_tail_unsent = len;
1379 	tcp_wput_data(tcp, NULL, B_FALSE);
1380 }
1381 
1382 /* ARGSUSED2 */
1383 void
tcp_output_urgent(void * arg,mblk_t * mp,void * arg2,ip_recv_attr_t * dummy)1384 tcp_output_urgent(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1385 {
1386 	int len;
1387 	uint32_t msize;
1388 	conn_t *connp = (conn_t *)arg;
1389 	tcp_t *tcp = connp->conn_tcp;
1390 
1391 	msize = msgdsize(mp);
1392 
1393 	len = msize - 1;
1394 	if (len < 0) {
1395 		freemsg(mp);
1396 		return;
1397 	}
1398 
1399 	/*
1400 	 * Try to force urgent data out on the wire. Even if we have unsent
1401 	 * data this will at least send the urgent flag.
1402 	 * XXX does not handle more flag correctly.
1403 	 */
1404 	len += tcp->tcp_unsent;
1405 	len += tcp->tcp_snxt;
1406 	tcp->tcp_urg = len;
1407 	tcp->tcp_valid_bits |= TCP_URG_VALID;
1408 
1409 	/* Bypass tcp protocol for fused tcp loopback */
1410 	if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize))
1411 		return;
1412 
1413 	/* Strip off the T_EXDATA_REQ if the data is from TPI */
1414 	if (DB_TYPE(mp) != M_DATA) {
1415 		mblk_t *mp1 = mp;
1416 		ASSERT(!IPCL_IS_NONSTR(connp));
1417 		mp = mp->b_cont;
1418 		freeb(mp1);
1419 	}
1420 	tcp_wput_data(tcp, mp, B_TRUE);
1421 }
1422 
1423 /*
1424  * Called by streams close routine via squeues when our client blows off her
1425  * descriptor, we take this to mean: "close the stream state NOW, close the tcp
1426  * connection politely" When SO_LINGER is set (with a non-zero linger time and
1427  * it is not a nonblocking socket) then this routine sleeps until the FIN is
1428  * acked.
1429  *
1430  * NOTE: tcp_close potentially returns error when lingering.
1431  * However, the stream head currently does not pass these errors
1432  * to the application. 4.4BSD only returns EINTR and EWOULDBLOCK
1433  * errors to the application (from tsleep()) and not errors
1434  * like ECONNRESET caused by receiving a reset packet.
1435  */
1436 
1437 /* ARGSUSED */
1438 void
tcp_close_output(void * arg,mblk_t * mp,void * arg2,ip_recv_attr_t * dummy)1439 tcp_close_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1440 {
1441 	char	*msg;
1442 	conn_t	*connp = (conn_t *)arg;
1443 	tcp_t	*tcp = connp->conn_tcp;
1444 	clock_t	delta = 0;
1445 	tcp_stack_t	*tcps = tcp->tcp_tcps;
1446 
1447 	/*
1448 	 * When a non-STREAMS socket is being closed, it does not always
1449 	 * stick around waiting for tcp_close_output to run and can therefore
1450 	 * have dropped a reference already. So adjust the asserts accordingly.
1451 	 */
1452 	ASSERT((connp->conn_fanout != NULL &&
1453 	    connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 3 : 4)) ||
1454 	    (connp->conn_fanout == NULL &&
1455 	    connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 2 : 3)));
1456 
1457 	mutex_enter(&tcp->tcp_eager_lock);
1458 	if (tcp->tcp_conn_req_cnt_q0 != 0 || tcp->tcp_conn_req_cnt_q != 0) {
1459 		/*
1460 		 * Cleanup for listener. For non-STREAM sockets sockfs will
1461 		 * close all the eagers on 'q', so in that case only deal
1462 		 * with 'q0'.
1463 		 */
1464 		tcp_eager_cleanup(tcp, IPCL_IS_NONSTR(connp) ? 1 : 0);
1465 		tcp->tcp_wait_for_eagers = 1;
1466 	}
1467 	mutex_exit(&tcp->tcp_eager_lock);
1468 
1469 	tcp->tcp_lso = B_FALSE;
1470 
1471 	msg = NULL;
1472 	switch (tcp->tcp_state) {
1473 	case TCPS_CLOSED:
1474 	case TCPS_IDLE:
1475 		break;
1476 	case TCPS_BOUND:
1477 		if (tcp->tcp_listener != NULL) {
1478 			ASSERT(IPCL_IS_NONSTR(connp));
1479 			/*
1480 			 * Unlink from the listener and drop the reference
1481 			 * put on it by the eager. tcp_closei_local will not
1482 			 * do it because tcp_tconnind_started is TRUE.
1483 			 */
1484 			mutex_enter(&tcp->tcp_saved_listener->tcp_eager_lock);
1485 			tcp_eager_unlink(tcp);
1486 			mutex_exit(&tcp->tcp_saved_listener->tcp_eager_lock);
1487 			CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp);
1488 		}
1489 		break;
1490 	case TCPS_LISTEN:
1491 		break;
1492 	case TCPS_SYN_SENT:
1493 		msg = "tcp_close, during connect";
1494 		break;
1495 	case TCPS_SYN_RCVD:
1496 		/*
1497 		 * Close during the connect 3-way handshake
1498 		 * but here there may or may not be pending data
1499 		 * already on queue. Process almost same as in
1500 		 * the ESTABLISHED state.
1501 		 */
1502 		/* FALLTHRU */
1503 	default:
1504 		if (tcp->tcp_fused)
1505 			tcp_unfuse(tcp);
1506 
1507 		/*
1508 		 * If SO_LINGER has set a zero linger time, abort the
1509 		 * connection with a reset.
1510 		 */
1511 		if (connp->conn_linger && connp->conn_lingertime == 0) {
1512 			msg = "tcp_close, zero lingertime";
1513 			break;
1514 		}
1515 
1516 		/*
1517 		 * Abort connection if there is unread data queued.
1518 		 */
1519 		if (tcp->tcp_rcv_list || tcp->tcp_reass_head) {
1520 			msg = "tcp_close, unread data";
1521 			break;
1522 		}
1523 
1524 		/*
1525 		 * Abort connection if it is being closed without first
1526 		 * being accepted. This can happen if a listening non-STREAM
1527 		 * socket wants to get rid of the socket, for example, if the
1528 		 * listener is closing.
1529 		 */
1530 		if (tcp->tcp_listener != NULL) {
1531 			ASSERT(IPCL_IS_NONSTR(connp));
1532 			msg = "tcp_close, close before accept";
1533 
1534 			/*
1535 			 * Unlink from the listener and drop the reference
1536 			 * put on it by the eager. tcp_closei_local will not
1537 			 * do it because tcp_tconnind_started is TRUE.
1538 			 */
1539 			mutex_enter(&tcp->tcp_saved_listener->tcp_eager_lock);
1540 			tcp_eager_unlink(tcp);
1541 			mutex_exit(&tcp->tcp_saved_listener->tcp_eager_lock);
1542 			CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp);
1543 			break;
1544 		}
1545 
1546 		/*
1547 		 * Transmit the FIN before detaching the tcp_t.
1548 		 * After tcp_detach returns this queue/perimeter
1549 		 * no longer owns the tcp_t thus others can modify it.
1550 		 */
1551 		(void) tcp_xmit_end(tcp);
1552 
1553 		/*
1554 		 * If lingering on close then wait until the fin is acked,
1555 		 * the SO_LINGER time passes, or a reset is sent/received.
1556 		 */
1557 		if (connp->conn_linger && connp->conn_lingertime > 0 &&
1558 		    !(tcp->tcp_fin_acked) &&
1559 		    tcp->tcp_state >= TCPS_ESTABLISHED) {
1560 			if (tcp->tcp_closeflags & (FNDELAY|FNONBLOCK)) {
1561 				tcp->tcp_client_errno = EWOULDBLOCK;
1562 			} else if (tcp->tcp_client_errno == 0) {
1563 
1564 				ASSERT(tcp->tcp_linger_tid == 0);
1565 
1566 				/* conn_lingertime is in sec. */
1567 				tcp->tcp_linger_tid = TCP_TIMER(tcp,
1568 				    tcp_close_linger_timeout,
1569 				    connp->conn_lingertime * MILLISEC);
1570 
1571 				/* tcp_close_linger_timeout will finish close */
1572 				if (tcp->tcp_linger_tid == 0)
1573 					tcp->tcp_client_errno = ENOSR;
1574 				else
1575 					return;
1576 			}
1577 
1578 			/*
1579 			 * Check if we need to detach or just close
1580 			 * the instance.
1581 			 */
1582 			if (tcp->tcp_state <= TCPS_LISTEN)
1583 				break;
1584 		}
1585 
1586 		/*
1587 		 * Make sure that no other thread will access the conn_rq of
1588 		 * this instance (through lookups etc.) as conn_rq will go
1589 		 * away shortly.
1590 		 */
1591 		tcp_acceptor_hash_remove(tcp);
1592 
1593 		mutex_enter(&tcp->tcp_non_sq_lock);
1594 		if (tcp->tcp_flow_stopped) {
1595 			tcp_clrqfull(tcp);
1596 		}
1597 		mutex_exit(&tcp->tcp_non_sq_lock);
1598 
1599 		if (tcp->tcp_timer_tid != 0) {
1600 			delta = TCP_TIMER_CANCEL(tcp, tcp->tcp_timer_tid);
1601 			tcp->tcp_timer_tid = 0;
1602 		}
1603 		/*
1604 		 * Need to cancel those timers which will not be used when
1605 		 * TCP is detached.  This has to be done before the conn_wq
1606 		 * is set to NULL.
1607 		 */
1608 		tcp_timers_stop(tcp);
1609 
1610 		tcp->tcp_detached = B_TRUE;
1611 		if (tcp->tcp_state == TCPS_TIME_WAIT) {
1612 			tcp_time_wait_append(tcp);
1613 			TCP_DBGSTAT(tcps, tcp_detach_time_wait);
1614 			ASSERT(connp->conn_ref >=
1615 			    (IPCL_IS_NONSTR(connp) ? 2 : 3));
1616 			goto finish;
1617 		}
1618 
1619 		/*
1620 		 * If delta is zero the timer event wasn't executed and was
1621 		 * successfully canceled. In this case we need to restart it
1622 		 * with the minimal delta possible.
1623 		 */
1624 		if (delta >= 0)
1625 			tcp->tcp_timer_tid = TCP_TIMER(tcp, tcp_timer,
1626 			    delta ? delta : 1);
1627 
1628 		ASSERT(connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 2 : 3));
1629 		goto finish;
1630 	}
1631 
1632 	/* Detach did not complete. Still need to remove q from stream. */
1633 	if (msg) {
1634 		if (tcp->tcp_state == TCPS_ESTABLISHED ||
1635 		    tcp->tcp_state == TCPS_CLOSE_WAIT)
1636 			TCPS_BUMP_MIB(tcps, tcpEstabResets);
1637 		if (tcp->tcp_state == TCPS_SYN_SENT ||
1638 		    tcp->tcp_state == TCPS_SYN_RCVD)
1639 			TCPS_BUMP_MIB(tcps, tcpAttemptFails);
1640 		tcp_xmit_ctl(msg, tcp,  tcp->tcp_snxt, 0, TH_RST);
1641 	}
1642 
1643 	tcp_closei_local(tcp);
1644 	CONN_DEC_REF(connp);
1645 	ASSERT(connp->conn_ref >= (IPCL_IS_NONSTR(connp) ? 1 : 2));
1646 
1647 finish:
1648 	/*
1649 	 * Don't change the queues in the case of a listener that has
1650 	 * eagers in its q or q0. It could surprise the eagers.
1651 	 * Instead wait for the eagers outside the squeue.
1652 	 *
1653 	 * For non-STREAMS sockets tcp_wait_for_eagers implies that
1654 	 * we should delay the su_closed upcall until all eagers have
1655 	 * dropped their references.
1656 	 */
1657 	if (!tcp->tcp_wait_for_eagers) {
1658 		tcp->tcp_detached = B_TRUE;
1659 		connp->conn_rq = NULL;
1660 		connp->conn_wq = NULL;
1661 
1662 		/* non-STREAM socket, release the upper handle */
1663 		if (IPCL_IS_NONSTR(connp)) {
1664 			ASSERT(connp->conn_upper_handle != NULL);
1665 			(*connp->conn_upcalls->su_closed)
1666 			    (connp->conn_upper_handle);
1667 			connp->conn_upper_handle = NULL;
1668 			connp->conn_upcalls = NULL;
1669 		}
1670 	}
1671 
1672 	/* Signal tcp_close() to finish closing. */
1673 	mutex_enter(&tcp->tcp_closelock);
1674 	tcp->tcp_closed = 1;
1675 	cv_signal(&tcp->tcp_closecv);
1676 	mutex_exit(&tcp->tcp_closelock);
1677 }
1678 
1679 /* ARGSUSED */
1680 void
tcp_shutdown_output(void * arg,mblk_t * mp,void * arg2,ip_recv_attr_t * dummy)1681 tcp_shutdown_output(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1682 {
1683 	conn_t 	*connp = (conn_t *)arg;
1684 	tcp_t	*tcp = connp->conn_tcp;
1685 
1686 	freemsg(mp);
1687 
1688 	if (tcp->tcp_fused)
1689 		tcp_unfuse(tcp);
1690 
1691 	if (tcp_xmit_end(tcp) != 0) {
1692 		/*
1693 		 * We were crossing FINs and got a reset from
1694 		 * the other side. Just ignore it.
1695 		 */
1696 		if (connp->conn_debug) {
1697 			(void) strlog(TCP_MOD_ID, 0, 1,
1698 			    SL_ERROR|SL_TRACE,
1699 			    "tcp_shutdown_output() out of state %s",
1700 			    tcp_display(tcp, NULL, DISP_ADDR_AND_PORT));
1701 		}
1702 	}
1703 }
1704 
1705 #pragma inline(tcp_send_data)
1706 
1707 void
tcp_send_data(tcp_t * tcp,mblk_t * mp)1708 tcp_send_data(tcp_t *tcp, mblk_t *mp)
1709 {
1710 	conn_t		*connp = tcp->tcp_connp;
1711 
1712 	/*
1713 	 * Check here to avoid sending zero-copy message down to IP when
1714 	 * ZEROCOPY capability has turned off. We only need to deal with
1715 	 * the race condition between sockfs and the notification here.
1716 	 * Since we have tried to backoff the tcp_xmit_head when turning
1717 	 * zero-copy off and new messages in tcp_output(), we simply drop
1718 	 * the dup'ed packet here and let tcp retransmit, if tcp_xmit_zc_clean
1719 	 * is not true.
1720 	 */
1721 	if (tcp->tcp_snd_zcopy_aware && !tcp->tcp_snd_zcopy_on &&
1722 	    !tcp->tcp_xmit_zc_clean) {
1723 		ip_drop_output("TCP ZC was disabled but not clean", mp, NULL);
1724 		freemsg(mp);
1725 		return;
1726 	}
1727 
1728 	DTRACE_TCP5(send, mblk_t *, NULL, ip_xmit_attr_t *, connp->conn_ixa,
1729 	    __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, tcp,
1730 	    __dtrace_tcp_tcph_t *,
1731 	    &mp->b_rptr[connp->conn_ixa->ixa_ip_hdr_length]);
1732 
1733 	ASSERT(connp->conn_ixa->ixa_notify_cookie == connp->conn_tcp);
1734 	(void) conn_ip_output(mp, connp->conn_ixa);
1735 }
1736 
1737 /* ARGSUSED2 */
1738 void
tcp_send_synack(void * arg,mblk_t * mp,void * arg2,ip_recv_attr_t * dummy)1739 tcp_send_synack(void *arg, mblk_t *mp, void *arg2, ip_recv_attr_t *dummy)
1740 {
1741 	conn_t	*econnp = (conn_t *)arg;
1742 	tcp_t	*tcp = econnp->conn_tcp;
1743 	ip_xmit_attr_t *ixa = econnp->conn_ixa;
1744 
1745 	/* Guard against a RST having blown it away while on the squeue */
1746 	if (tcp->tcp_state == TCPS_CLOSED) {
1747 		freemsg(mp);
1748 		return;
1749 	}
1750 
1751 	/*
1752 	 * In the off-chance that the eager received and responded to
1753 	 * some other packet while the SYN|ACK was queued, we recalculate
1754 	 * the ixa_pktlen. It would be better to fix the SYN/accept
1755 	 * multithreading scheme to avoid this complexity.
1756 	 */
1757 	ixa->ixa_pktlen = msgdsize(mp);
1758 	(void) conn_ip_output(mp, ixa);
1759 }
1760 
1761 /*
1762  * tcp_send() is called by tcp_wput_data() and returns one of the following:
1763  *
1764  * -1 = failed allocation.
1765  *  0 = We've either successfully sent data, or our usable send window is too
1766  *      small and we'd rather wait until later before sending again.
1767  */
1768 static int
tcp_send(tcp_t * tcp,const int mss,const int total_hdr_len,const int tcp_hdr_len,const int num_sack_blk,int * usable,uint_t * snxt,int * tail_unsent,mblk_t ** xmit_tail,mblk_t * local_time)1769 tcp_send(tcp_t *tcp, const int mss, const int total_hdr_len,
1770     const int tcp_hdr_len, const int num_sack_blk, int *usable,
1771     uint_t *snxt, int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time)
1772 {
1773 	int		num_lso_seg = 1;
1774 	uint_t		lso_usable;
1775 	boolean_t	do_lso_send = B_FALSE;
1776 	tcp_stack_t	*tcps = tcp->tcp_tcps;
1777 	conn_t		*connp = tcp->tcp_connp;
1778 	ip_xmit_attr_t	*ixa = connp->conn_ixa;
1779 
1780 	/*
1781 	 * Check LSO possibility. The value of tcp->tcp_lso indicates whether
1782 	 * the underlying connection is LSO capable. Will check whether having
1783 	 * enough available data to initiate LSO transmission in the for(){}
1784 	 * loops.
1785 	 */
1786 	if (tcp->tcp_lso && (tcp->tcp_valid_bits & ~TCP_FSS_VALID) == 0)
1787 		do_lso_send = B_TRUE;
1788 
1789 	for (;;) {
1790 		struct datab	*db;
1791 		tcpha_t		*tcpha;
1792 		uint32_t	sum;
1793 		mblk_t		*mp, *mp1;
1794 		uchar_t		*rptr;
1795 		int		len;
1796 
1797 		/*
1798 		 * Calculate the maximum payload length we can send at one
1799 		 * time.
1800 		 */
1801 		if (do_lso_send) {
1802 			/*
1803 			 * Determine whether or not it's possible to do LSO,
1804 			 * and if so, how much data we can send.
1805 			 */
1806 			if ((*usable - 1) / mss >= 1) {
1807 				lso_usable = MIN(tcp->tcp_lso_max, *usable);
1808 				num_lso_seg = lso_usable / mss;
1809 				if (lso_usable % mss) {
1810 					num_lso_seg++;
1811 					tcp->tcp_last_sent_len = (ushort_t)
1812 					    (lso_usable % mss);
1813 				} else {
1814 					tcp->tcp_last_sent_len = (ushort_t)mss;
1815 				}
1816 			} else {
1817 				do_lso_send = B_FALSE;
1818 				num_lso_seg = 1;
1819 				lso_usable = mss;
1820 			}
1821 		}
1822 
1823 		ASSERT(num_lso_seg <= IP_MAXPACKET / mss + 1);
1824 
1825 		len = mss;
1826 		if (len > *usable) {
1827 			ASSERT(do_lso_send == B_FALSE);
1828 
1829 			len = *usable;
1830 			if (len <= 0) {
1831 				/* Terminate the loop */
1832 				break;	/* success; too small */
1833 			}
1834 			/*
1835 			 * Sender silly-window avoidance.
1836 			 * Ignore this if we are going to send a
1837 			 * zero window probe out.
1838 			 *
1839 			 * TODO: force data into microscopic window?
1840 			 *	==> (!pushed || (unsent > usable))
1841 			 */
1842 			if (len < (tcp->tcp_max_swnd >> 1) &&
1843 			    (tcp->tcp_unsent - (*snxt - tcp->tcp_snxt)) > len &&
1844 			    !((tcp->tcp_valid_bits & TCP_URG_VALID) &&
1845 			    len == 1) && (! tcp->tcp_zero_win_probe)) {
1846 				/*
1847 				 * If the retransmit timer is not running
1848 				 * we start it so that we will retransmit
1849 				 * in the case when the receiver has
1850 				 * decremented the window.
1851 				 */
1852 				if (*snxt == tcp->tcp_snxt &&
1853 				    *snxt == tcp->tcp_suna) {
1854 					/*
1855 					 * We are not supposed to send
1856 					 * anything.  So let's wait a little
1857 					 * bit longer before breaking SWS
1858 					 * avoidance.
1859 					 *
1860 					 * What should the value be?
1861 					 * Suggestion: MAX(init rexmit time,
1862 					 * tcp->tcp_rto)
1863 					 */
1864 					TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
1865 				}
1866 				break;	/* success; too small */
1867 			}
1868 		}
1869 
1870 		tcpha = tcp->tcp_tcpha;
1871 
1872 		/*
1873 		 * The reason to adjust len here is that we need to set flags
1874 		 * and calculate checksum.
1875 		 */
1876 		if (do_lso_send)
1877 			len = lso_usable;
1878 
1879 		*usable -= len; /* Approximate - can be adjusted later */
1880 		if (*usable > 0)
1881 			tcpha->tha_flags = TH_ACK;
1882 		else
1883 			tcpha->tha_flags = (TH_ACK | TH_PUSH);
1884 
1885 		/*
1886 		 * Prime pump for IP's checksumming on our behalf.
1887 		 * Include the adjustment for a source route if any.
1888 		 * In case of LSO, the partial pseudo-header checksum should
1889 		 * exclusive TCP length, so zero tha_sum before IP calculate
1890 		 * pseudo-header checksum for partial checksum offload.
1891 		 */
1892 		if (do_lso_send) {
1893 			sum = 0;
1894 		} else {
1895 			sum = len + tcp_hdr_len + connp->conn_sum;
1896 			sum = (sum >> 16) + (sum & 0xFFFF);
1897 		}
1898 		tcpha->tha_sum = htons(sum);
1899 		tcpha->tha_seq = htonl(*snxt);
1900 
1901 		/*
1902 		 * Branch off to tcp_xmit_mp() if any of the VALID bits is
1903 		 * set.  For the case when TCP_FSS_VALID is the only valid
1904 		 * bit (normal active close), branch off only when we think
1905 		 * that the FIN flag needs to be set.  Note for this case,
1906 		 * that (snxt + len) may not reflect the actual seg_len,
1907 		 * as len may be further reduced in tcp_xmit_mp().  If len
1908 		 * gets modified, we will end up here again.
1909 		 */
1910 		if (tcp->tcp_valid_bits != 0 &&
1911 		    (tcp->tcp_valid_bits != TCP_FSS_VALID ||
1912 		    ((*snxt + len) == tcp->tcp_fss))) {
1913 			uchar_t		*prev_rptr;
1914 			uint32_t	prev_snxt = tcp->tcp_snxt;
1915 
1916 			if (*tail_unsent == 0) {
1917 				ASSERT((*xmit_tail)->b_cont != NULL);
1918 				*xmit_tail = (*xmit_tail)->b_cont;
1919 				prev_rptr = (*xmit_tail)->b_rptr;
1920 				*tail_unsent = (int)((*xmit_tail)->b_wptr -
1921 				    (*xmit_tail)->b_rptr);
1922 			} else {
1923 				prev_rptr = (*xmit_tail)->b_rptr;
1924 				(*xmit_tail)->b_rptr = (*xmit_tail)->b_wptr -
1925 				    *tail_unsent;
1926 			}
1927 			mp = tcp_xmit_mp(tcp, *xmit_tail, len, NULL, NULL,
1928 			    *snxt, B_FALSE, (uint32_t *)&len, B_FALSE);
1929 			/* Restore tcp_snxt so we get amount sent right. */
1930 			tcp->tcp_snxt = prev_snxt;
1931 			if (prev_rptr == (*xmit_tail)->b_rptr) {
1932 				/*
1933 				 * If the previous timestamp is still in use,
1934 				 * don't stomp on it.
1935 				 */
1936 				if ((*xmit_tail)->b_next == NULL) {
1937 					(*xmit_tail)->b_prev = local_time;
1938 					(*xmit_tail)->b_next =
1939 					    (mblk_t *)(uintptr_t)(*snxt);
1940 				}
1941 			} else
1942 				(*xmit_tail)->b_rptr = prev_rptr;
1943 
1944 			if (mp == NULL) {
1945 				return (-1);
1946 			}
1947 			mp1 = mp->b_cont;
1948 
1949 			if (len <= mss) /* LSO is unusable (!do_lso_send) */
1950 				tcp->tcp_last_sent_len = (ushort_t)len;
1951 			while (mp1->b_cont) {
1952 				*xmit_tail = (*xmit_tail)->b_cont;
1953 				(*xmit_tail)->b_prev = local_time;
1954 				(*xmit_tail)->b_next =
1955 				    (mblk_t *)(uintptr_t)(*snxt);
1956 				mp1 = mp1->b_cont;
1957 			}
1958 			*snxt += len;
1959 			*tail_unsent = (*xmit_tail)->b_wptr - mp1->b_wptr;
1960 			BUMP_LOCAL(tcp->tcp_obsegs);
1961 			TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1962 			TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1963 			tcp_send_data(tcp, mp);
1964 			continue;
1965 		}
1966 
1967 		*snxt += len;	/* Adjust later if we don't send all of len */
1968 		TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
1969 		TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, len);
1970 
1971 		if (*tail_unsent) {
1972 			/* Are the bytes above us in flight? */
1973 			rptr = (*xmit_tail)->b_wptr - *tail_unsent;
1974 			if (rptr != (*xmit_tail)->b_rptr) {
1975 				*tail_unsent -= len;
1976 				if (len <= mss) /* LSO is unusable */
1977 					tcp->tcp_last_sent_len = (ushort_t)len;
1978 				len += total_hdr_len;
1979 				ixa->ixa_pktlen = len;
1980 
1981 				if (ixa->ixa_flags & IXAF_IS_IPV4) {
1982 					tcp->tcp_ipha->ipha_length = htons(len);
1983 				} else {
1984 					tcp->tcp_ip6h->ip6_plen =
1985 					    htons(len - IPV6_HDR_LEN);
1986 				}
1987 
1988 				mp = dupb(*xmit_tail);
1989 				if (mp == NULL) {
1990 					return (-1);	/* out_of_mem */
1991 				}
1992 				mp->b_rptr = rptr;
1993 				/*
1994 				 * If the old timestamp is no longer in use,
1995 				 * sample a new timestamp now.
1996 				 */
1997 				if ((*xmit_tail)->b_next == NULL) {
1998 					(*xmit_tail)->b_prev = local_time;
1999 					(*xmit_tail)->b_next =
2000 					    (mblk_t *)(uintptr_t)(*snxt-len);
2001 				}
2002 				goto must_alloc;
2003 			}
2004 		} else {
2005 			*xmit_tail = (*xmit_tail)->b_cont;
2006 			ASSERT((uintptr_t)((*xmit_tail)->b_wptr -
2007 			    (*xmit_tail)->b_rptr) <= (uintptr_t)INT_MAX);
2008 			*tail_unsent = (int)((*xmit_tail)->b_wptr -
2009 			    (*xmit_tail)->b_rptr);
2010 		}
2011 
2012 		(*xmit_tail)->b_prev = local_time;
2013 		(*xmit_tail)->b_next = (mblk_t *)(uintptr_t)(*snxt - len);
2014 
2015 		*tail_unsent -= len;
2016 		if (len <= mss) /* LSO is unusable (!do_lso_send) */
2017 			tcp->tcp_last_sent_len = (ushort_t)len;
2018 
2019 		len += total_hdr_len;
2020 		ixa->ixa_pktlen = len;
2021 
2022 		if (ixa->ixa_flags & IXAF_IS_IPV4) {
2023 			tcp->tcp_ipha->ipha_length = htons(len);
2024 		} else {
2025 			tcp->tcp_ip6h->ip6_plen = htons(len - IPV6_HDR_LEN);
2026 		}
2027 
2028 		mp = dupb(*xmit_tail);
2029 		if (mp == NULL) {
2030 			return (-1);	/* out_of_mem */
2031 		}
2032 
2033 		len = total_hdr_len;
2034 		/*
2035 		 * There are four reasons to allocate a new hdr mblk:
2036 		 *  1) The bytes above us are in use by another packet
2037 		 *  2) We don't have good alignment
2038 		 *  3) The mblk is being shared
2039 		 *  4) We don't have enough room for a header
2040 		 */
2041 		rptr = mp->b_rptr - len;
2042 		if (!OK_32PTR(rptr) ||
2043 		    ((db = mp->b_datap), db->db_ref != 2) ||
2044 		    rptr < db->db_base) {
2045 			/* NOTE: we assume allocb returns an OK_32PTR */
2046 
2047 		must_alloc:;
2048 			mp1 = allocb(connp->conn_ht_iphc_allocated +
2049 			    tcps->tcps_wroff_xtra, BPRI_MED);
2050 			if (mp1 == NULL) {
2051 				freemsg(mp);
2052 				return (-1);	/* out_of_mem */
2053 			}
2054 			mp1->b_cont = mp;
2055 			mp = mp1;
2056 			/* Leave room for Link Level header */
2057 			len = total_hdr_len;
2058 			rptr = &mp->b_rptr[tcps->tcps_wroff_xtra];
2059 			mp->b_wptr = &rptr[len];
2060 		}
2061 
2062 		/*
2063 		 * Fill in the header using the template header, and add
2064 		 * options such as time-stamp, ECN and/or SACK, as needed.
2065 		 */
2066 		tcp_fill_header(tcp, rptr, (clock_t)local_time, num_sack_blk);
2067 
2068 		mp->b_rptr = rptr;
2069 
2070 		if (*tail_unsent) {
2071 			int spill = *tail_unsent;
2072 
2073 			mp1 = mp->b_cont;
2074 			if (mp1 == NULL)
2075 				mp1 = mp;
2076 
2077 			/*
2078 			 * If we're a little short, tack on more mblks until
2079 			 * there is no more spillover.
2080 			 */
2081 			while (spill < 0) {
2082 				mblk_t *nmp;
2083 				int nmpsz;
2084 
2085 				nmp = (*xmit_tail)->b_cont;
2086 				nmpsz = MBLKL(nmp);
2087 
2088 				/*
2089 				 * Excess data in mblk; can we split it?
2090 				 * If LSO is enabled for the connection,
2091 				 * keep on splitting as this is a transient
2092 				 * send path.
2093 				 */
2094 				if (!do_lso_send && (spill + nmpsz > 0)) {
2095 					/*
2096 					 * Don't split if stream head was
2097 					 * told to break up larger writes
2098 					 * into smaller ones.
2099 					 */
2100 					if (tcp->tcp_maxpsz_multiplier > 0)
2101 						break;
2102 
2103 					/*
2104 					 * Next mblk is less than SMSS/2
2105 					 * rounded up to nearest 64-byte;
2106 					 * let it get sent as part of the
2107 					 * next segment.
2108 					 */
2109 					if (tcp->tcp_localnet &&
2110 					    !tcp->tcp_cork &&
2111 					    (nmpsz < roundup((mss >> 1), 64)))
2112 						break;
2113 				}
2114 
2115 				*xmit_tail = nmp;
2116 				ASSERT((uintptr_t)nmpsz <= (uintptr_t)INT_MAX);
2117 				/* Stash for rtt use later */
2118 				(*xmit_tail)->b_prev = local_time;
2119 				(*xmit_tail)->b_next =
2120 				    (mblk_t *)(uintptr_t)(*snxt - len);
2121 				mp1->b_cont = dupb(*xmit_tail);
2122 				mp1 = mp1->b_cont;
2123 
2124 				spill += nmpsz;
2125 				if (mp1 == NULL) {
2126 					*tail_unsent = spill;
2127 					freemsg(mp);
2128 					return (-1);	/* out_of_mem */
2129 				}
2130 			}
2131 
2132 			/* Trim back any surplus on the last mblk */
2133 			if (spill >= 0) {
2134 				mp1->b_wptr -= spill;
2135 				*tail_unsent = spill;
2136 			} else {
2137 				/*
2138 				 * We did not send everything we could in
2139 				 * order to remain within the b_cont limit.
2140 				 */
2141 				*usable -= spill;
2142 				*snxt += spill;
2143 				tcp->tcp_last_sent_len += spill;
2144 				TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, spill);
2145 				/*
2146 				 * Adjust the checksum
2147 				 */
2148 				tcpha = (tcpha_t *)(rptr +
2149 				    ixa->ixa_ip_hdr_length);
2150 				sum += spill;
2151 				sum = (sum >> 16) + (sum & 0xFFFF);
2152 				tcpha->tha_sum = htons(sum);
2153 				if (connp->conn_ipversion == IPV4_VERSION) {
2154 					sum = ntohs(
2155 					    ((ipha_t *)rptr)->ipha_length) +
2156 					    spill;
2157 					((ipha_t *)rptr)->ipha_length =
2158 					    htons(sum);
2159 				} else {
2160 					sum = ntohs(
2161 					    ((ip6_t *)rptr)->ip6_plen) +
2162 					    spill;
2163 					((ip6_t *)rptr)->ip6_plen =
2164 					    htons(sum);
2165 				}
2166 				ixa->ixa_pktlen += spill;
2167 				*tail_unsent = 0;
2168 			}
2169 		}
2170 		if (tcp->tcp_ip_forward_progress) {
2171 			tcp->tcp_ip_forward_progress = B_FALSE;
2172 			ixa->ixa_flags |= IXAF_REACH_CONF;
2173 		} else {
2174 			ixa->ixa_flags &= ~IXAF_REACH_CONF;
2175 		}
2176 
2177 		if (do_lso_send) {
2178 			/* Append LSO information to the mp. */
2179 			lso_info_set(mp, mss, HW_LSO);
2180 			ixa->ixa_fragsize = IP_MAXPACKET;
2181 			ixa->ixa_extra_ident = num_lso_seg - 1;
2182 
2183 			DTRACE_PROBE2(tcp_send_lso, int, num_lso_seg,
2184 			    boolean_t, B_TRUE);
2185 
2186 			tcp_send_data(tcp, mp);
2187 
2188 			/*
2189 			 * Restore values of ixa_fragsize and ixa_extra_ident.
2190 			 */
2191 			ixa->ixa_fragsize = ixa->ixa_pmtu;
2192 			ixa->ixa_extra_ident = 0;
2193 			tcp->tcp_obsegs += num_lso_seg;
2194 			TCP_STAT(tcps, tcp_lso_times);
2195 			TCP_STAT_UPDATE(tcps, tcp_lso_pkt_out, num_lso_seg);
2196 		} else {
2197 			/*
2198 			 * Make sure to clean up LSO information. Wherever a
2199 			 * new mp uses the prepended header room after dupb(),
2200 			 * lso_info_cleanup() should be called.
2201 			 */
2202 			lso_info_cleanup(mp);
2203 			tcp_send_data(tcp, mp);
2204 			BUMP_LOCAL(tcp->tcp_obsegs);
2205 		}
2206 	}
2207 
2208 	return (0);
2209 }
2210 
2211 /*
2212  * Initiate closedown sequence on an active connection.  (May be called as
2213  * writer.)  Return value zero for OK return, non-zero for error return.
2214  */
2215 static int
tcp_xmit_end(tcp_t * tcp)2216 tcp_xmit_end(tcp_t *tcp)
2217 {
2218 	mblk_t		*mp;
2219 	tcp_stack_t	*tcps = tcp->tcp_tcps;
2220 	iulp_t		uinfo;
2221 	ip_stack_t	*ipst = tcps->tcps_netstack->netstack_ip;
2222 	conn_t		*connp = tcp->tcp_connp;
2223 
2224 	if (tcp->tcp_state < TCPS_SYN_RCVD ||
2225 	    tcp->tcp_state > TCPS_CLOSE_WAIT) {
2226 		/*
2227 		 * Invalid state, only states TCPS_SYN_RCVD,
2228 		 * TCPS_ESTABLISHED and TCPS_CLOSE_WAIT are valid
2229 		 */
2230 		return (-1);
2231 	}
2232 
2233 	tcp->tcp_fss = tcp->tcp_snxt + tcp->tcp_unsent;
2234 	tcp->tcp_valid_bits |= TCP_FSS_VALID;
2235 	/*
2236 	 * If there is nothing more unsent, send the FIN now.
2237 	 * Otherwise, it will go out with the last segment.
2238 	 */
2239 	if (tcp->tcp_unsent == 0) {
2240 		mp = tcp_xmit_mp(tcp, NULL, 0, NULL, NULL,
2241 		    tcp->tcp_fss, B_FALSE, NULL, B_FALSE);
2242 
2243 		if (mp) {
2244 			tcp_send_data(tcp, mp);
2245 		} else {
2246 			/*
2247 			 * Couldn't allocate msg.  Pretend we got it out.
2248 			 * Wait for rexmit timeout.
2249 			 */
2250 			tcp->tcp_snxt = tcp->tcp_fss + 1;
2251 			TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
2252 		}
2253 
2254 		/*
2255 		 * If needed, update tcp_rexmit_snxt as tcp_snxt is
2256 		 * changed.
2257 		 */
2258 		if (tcp->tcp_rexmit && tcp->tcp_rexmit_nxt == tcp->tcp_fss) {
2259 			tcp->tcp_rexmit_nxt = tcp->tcp_snxt;
2260 		}
2261 	} else {
2262 		/*
2263 		 * If tcp->tcp_cork is set, then the data will not get sent,
2264 		 * so we have to check that and unset it first.
2265 		 */
2266 		if (tcp->tcp_cork)
2267 			tcp->tcp_cork = B_FALSE;
2268 		tcp_wput_data(tcp, NULL, B_FALSE);
2269 	}
2270 
2271 	/*
2272 	 * If TCP does not get enough samples of RTT or tcp_rtt_updates
2273 	 * is 0, don't update the cache.
2274 	 */
2275 	if (tcps->tcps_rtt_updates == 0 ||
2276 	    tcp->tcp_rtt_update < tcps->tcps_rtt_updates)
2277 		return (0);
2278 
2279 	/*
2280 	 * We do not have a good algorithm to update ssthresh at this time.
2281 	 * So don't do any update.
2282 	 */
2283 	bzero(&uinfo, sizeof (uinfo));
2284 	uinfo.iulp_rtt = tcp->tcp_rtt_sa;
2285 	uinfo.iulp_rtt_sd = tcp->tcp_rtt_sd;
2286 
2287 	/*
2288 	 * Note that uinfo is kept for conn_faddr in the DCE. Could update even
2289 	 * if source routed but we don't.
2290 	 */
2291 	if (connp->conn_ipversion == IPV4_VERSION) {
2292 		if (connp->conn_faddr_v4 !=  tcp->tcp_ipha->ipha_dst) {
2293 			return (0);
2294 		}
2295 		(void) dce_update_uinfo_v4(connp->conn_faddr_v4, &uinfo, ipst);
2296 	} else {
2297 		uint_t ifindex;
2298 
2299 		if (!(IN6_ARE_ADDR_EQUAL(&connp->conn_faddr_v6,
2300 		    &tcp->tcp_ip6h->ip6_dst))) {
2301 			return (0);
2302 		}
2303 		ifindex = 0;
2304 		if (IN6_IS_ADDR_LINKSCOPE(&connp->conn_faddr_v6)) {
2305 			ip_xmit_attr_t *ixa = connp->conn_ixa;
2306 
2307 			/*
2308 			 * If we are going to create a DCE we'd better have
2309 			 * an ifindex
2310 			 */
2311 			if (ixa->ixa_nce != NULL) {
2312 				ifindex = ixa->ixa_nce->nce_common->ncec_ill->
2313 				    ill_phyint->phyint_ifindex;
2314 			} else {
2315 				return (0);
2316 			}
2317 		}
2318 
2319 		(void) dce_update_uinfo(&connp->conn_faddr_v6, ifindex, &uinfo,
2320 		    ipst);
2321 	}
2322 	return (0);
2323 }
2324 
2325 /*
2326  * Send out a control packet on the tcp connection specified.  This routine
2327  * is typically called where we need a simple ACK or RST generated.
2328  */
2329 void
tcp_xmit_ctl(char * str,tcp_t * tcp,uint32_t seq,uint32_t ack,int ctl)2330 tcp_xmit_ctl(char *str, tcp_t *tcp, uint32_t seq, uint32_t ack, int ctl)
2331 {
2332 	uchar_t		*rptr;
2333 	tcpha_t		*tcpha;
2334 	ipha_t		*ipha = NULL;
2335 	ip6_t		*ip6h = NULL;
2336 	uint32_t	sum;
2337 	int		total_hdr_len;
2338 	int		ip_hdr_len;
2339 	mblk_t		*mp;
2340 	tcp_stack_t	*tcps = tcp->tcp_tcps;
2341 	conn_t		*connp = tcp->tcp_connp;
2342 	ip_xmit_attr_t	*ixa = connp->conn_ixa;
2343 
2344 	/*
2345 	 * Save sum for use in source route later.
2346 	 */
2347 	sum = connp->conn_ht_ulp_len + connp->conn_sum;
2348 	total_hdr_len = connp->conn_ht_iphc_len;
2349 	ip_hdr_len = ixa->ixa_ip_hdr_length;
2350 
2351 	/* If a text string is passed in with the request, pass it to strlog. */
2352 	if (str != NULL && connp->conn_debug) {
2353 		(void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE,
2354 		    "tcp_xmit_ctl: '%s', seq 0x%x, ack 0x%x, ctl 0x%x",
2355 		    str, seq, ack, ctl);
2356 	}
2357 	mp = allocb(connp->conn_ht_iphc_allocated + tcps->tcps_wroff_xtra,
2358 	    BPRI_MED);
2359 	if (mp == NULL) {
2360 		return;
2361 	}
2362 	rptr = &mp->b_rptr[tcps->tcps_wroff_xtra];
2363 	mp->b_rptr = rptr;
2364 	mp->b_wptr = &rptr[total_hdr_len];
2365 	bcopy(connp->conn_ht_iphc, rptr, total_hdr_len);
2366 
2367 	ixa->ixa_pktlen = total_hdr_len;
2368 
2369 	if (ixa->ixa_flags & IXAF_IS_IPV4) {
2370 		ipha = (ipha_t *)rptr;
2371 		ipha->ipha_length = htons(total_hdr_len);
2372 	} else {
2373 		ip6h = (ip6_t *)rptr;
2374 		ip6h->ip6_plen = htons(total_hdr_len - IPV6_HDR_LEN);
2375 	}
2376 	tcpha = (tcpha_t *)&rptr[ip_hdr_len];
2377 	tcpha->tha_flags = (uint8_t)ctl;
2378 	if (ctl & TH_RST) {
2379 		TCPS_BUMP_MIB(tcps, tcpOutRsts);
2380 		TCPS_BUMP_MIB(tcps, tcpOutControl);
2381 		/*
2382 		 * Don't send TSopt w/ TH_RST packets per RFC 1323.
2383 		 */
2384 		if (tcp->tcp_snd_ts_ok &&
2385 		    tcp->tcp_state > TCPS_SYN_SENT) {
2386 			mp->b_wptr = &rptr[total_hdr_len - TCPOPT_REAL_TS_LEN];
2387 			*(mp->b_wptr) = TCPOPT_EOL;
2388 
2389 			ixa->ixa_pktlen = total_hdr_len - TCPOPT_REAL_TS_LEN;
2390 
2391 			if (connp->conn_ipversion == IPV4_VERSION) {
2392 				ipha->ipha_length = htons(total_hdr_len -
2393 				    TCPOPT_REAL_TS_LEN);
2394 			} else {
2395 				ip6h->ip6_plen = htons(total_hdr_len -
2396 				    IPV6_HDR_LEN - TCPOPT_REAL_TS_LEN);
2397 			}
2398 			tcpha->tha_offset_and_reserved -= (3 << 4);
2399 			sum -= TCPOPT_REAL_TS_LEN;
2400 		}
2401 	}
2402 	if (ctl & TH_ACK) {
2403 		if (tcp->tcp_snd_ts_ok) {
2404 			uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
2405 
2406 			U32_TO_BE32(llbolt,
2407 			    (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
2408 			U32_TO_BE32(tcp->tcp_ts_recent,
2409 			    (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
2410 		}
2411 
2412 		/* Update the latest receive window size in TCP header. */
2413 		tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
2414 		/* Track what we sent to the peer */
2415 		tcp->tcp_tcpha->tha_win = tcpha->tha_win;
2416 		tcp->tcp_rack = ack;
2417 		tcp->tcp_rack_cnt = 0;
2418 		TCPS_BUMP_MIB(tcps, tcpOutAck);
2419 	}
2420 	BUMP_LOCAL(tcp->tcp_obsegs);
2421 	tcpha->tha_seq = htonl(seq);
2422 	tcpha->tha_ack = htonl(ack);
2423 	/*
2424 	 * Include the adjustment for a source route if any.
2425 	 */
2426 	sum = (sum >> 16) + (sum & 0xFFFF);
2427 	tcpha->tha_sum = htons(sum);
2428 	tcp_send_data(tcp, mp);
2429 }
2430 
2431 /*
2432  * Generate a reset based on an inbound packet, connp is set by caller
2433  * when RST is in response to an unexpected inbound packet for which
2434  * there is active tcp state in the system.
2435  *
2436  * IPSEC NOTE : Try to send the reply with the same protection as it came
2437  * in.  We have the ip_recv_attr_t which is reversed to form the ip_xmit_attr_t.
2438  * That way the packet will go out at the same level of protection as it
2439  * came in with.
2440  */
2441 static void
tcp_xmit_early_reset(char * str,mblk_t * mp,uint32_t seq,uint32_t ack,int ctl,ip_recv_attr_t * ira,ip_stack_t * ipst,conn_t * connp)2442 tcp_xmit_early_reset(char *str, mblk_t *mp, uint32_t seq, uint32_t ack, int ctl,
2443     ip_recv_attr_t *ira, ip_stack_t *ipst, conn_t *connp)
2444 {
2445 	ipha_t		*ipha = NULL;
2446 	ip6_t		*ip6h = NULL;
2447 	ushort_t	len;
2448 	tcpha_t		*tcpha;
2449 	int		i;
2450 	ipaddr_t	v4addr;
2451 	in6_addr_t	v6addr;
2452 	netstack_t	*ns = ipst->ips_netstack;
2453 	tcp_stack_t	*tcps = ns->netstack_tcp;
2454 	ip_xmit_attr_t	ixas, *ixa;
2455 	uint_t		ip_hdr_len = ira->ira_ip_hdr_length;
2456 	boolean_t	need_refrele = B_FALSE;		/* ixa_refrele(ixa) */
2457 	ushort_t	port;
2458 
2459 	if (!tcp_send_rst_chk(tcps)) {
2460 		TCP_STAT(tcps, tcp_rst_unsent);
2461 		freemsg(mp);
2462 		return;
2463 	}
2464 
2465 	/*
2466 	 * If connp != NULL we use conn_ixa to keep IP_NEXTHOP and other
2467 	 * options from the listener. In that case the caller must ensure that
2468 	 * we are running on the listener = connp squeue.
2469 	 *
2470 	 * We get a safe copy of conn_ixa so we don't need to restore anything
2471 	 * we or ip_output_simple might change in the ixa.
2472 	 */
2473 	if (connp != NULL) {
2474 		ASSERT(connp->conn_on_sqp);
2475 
2476 		ixa = conn_get_ixa_exclusive(connp);
2477 		if (ixa == NULL) {
2478 			TCP_STAT(tcps, tcp_rst_unsent);
2479 			freemsg(mp);
2480 			return;
2481 		}
2482 		need_refrele = B_TRUE;
2483 	} else {
2484 		bzero(&ixas, sizeof (ixas));
2485 		ixa = &ixas;
2486 		/*
2487 		 * IXAF_VERIFY_SOURCE is overkill since we know the
2488 		 * packet was for us.
2489 		 */
2490 		ixa->ixa_flags |= IXAF_SET_ULP_CKSUM | IXAF_VERIFY_SOURCE;
2491 		ixa->ixa_protocol = IPPROTO_TCP;
2492 		ixa->ixa_zoneid = ira->ira_zoneid;
2493 		ixa->ixa_ifindex = 0;
2494 		ixa->ixa_ipst = ipst;
2495 		ixa->ixa_cred = kcred;
2496 		ixa->ixa_cpid = NOPID;
2497 	}
2498 
2499 	if (str && tcps->tcps_dbg) {
2500 		(void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE,
2501 		    "tcp_xmit_early_reset: '%s', seq 0x%x, ack 0x%x, "
2502 		    "flags 0x%x",
2503 		    str, seq, ack, ctl);
2504 	}
2505 	if (mp->b_datap->db_ref != 1) {
2506 		mblk_t *mp1 = copyb(mp);
2507 		freemsg(mp);
2508 		mp = mp1;
2509 		if (mp == NULL)
2510 			goto done;
2511 	} else if (mp->b_cont) {
2512 		freemsg(mp->b_cont);
2513 		mp->b_cont = NULL;
2514 		DB_CKSUMFLAGS(mp) = 0;
2515 	}
2516 	/*
2517 	 * We skip reversing source route here.
2518 	 * (for now we replace all IP options with EOL)
2519 	 */
2520 	if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2521 		ipha = (ipha_t *)mp->b_rptr;
2522 		for (i = IP_SIMPLE_HDR_LENGTH; i < (int)ip_hdr_len; i++)
2523 			mp->b_rptr[i] = IPOPT_EOL;
2524 		/*
2525 		 * Make sure that src address isn't flagrantly invalid.
2526 		 * Not all broadcast address checking for the src address
2527 		 * is possible, since we don't know the netmask of the src
2528 		 * addr.  No check for destination address is done, since
2529 		 * IP will not pass up a packet with a broadcast dest
2530 		 * address to TCP.  Similar checks are done below for IPv6.
2531 		 */
2532 		if (ipha->ipha_src == 0 || ipha->ipha_src == INADDR_BROADCAST ||
2533 		    CLASSD(ipha->ipha_src)) {
2534 			BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsInDiscards);
2535 			ip_drop_input("ipIfStatsInDiscards", mp, NULL);
2536 			freemsg(mp);
2537 			goto done;
2538 		}
2539 	} else {
2540 		ip6h = (ip6_t *)mp->b_rptr;
2541 
2542 		if (IN6_IS_ADDR_UNSPECIFIED(&ip6h->ip6_src) ||
2543 		    IN6_IS_ADDR_MULTICAST(&ip6h->ip6_src)) {
2544 			BUMP_MIB(&ipst->ips_ip6_mib, ipIfStatsInDiscards);
2545 			ip_drop_input("ipIfStatsInDiscards", mp, NULL);
2546 			freemsg(mp);
2547 			goto done;
2548 		}
2549 
2550 		/* Remove any extension headers assuming partial overlay */
2551 		if (ip_hdr_len > IPV6_HDR_LEN) {
2552 			uint8_t *to;
2553 
2554 			to = mp->b_rptr + ip_hdr_len - IPV6_HDR_LEN;
2555 			ovbcopy(ip6h, to, IPV6_HDR_LEN);
2556 			mp->b_rptr += ip_hdr_len - IPV6_HDR_LEN;
2557 			ip_hdr_len = IPV6_HDR_LEN;
2558 			ip6h = (ip6_t *)mp->b_rptr;
2559 			ip6h->ip6_nxt = IPPROTO_TCP;
2560 		}
2561 	}
2562 	tcpha = (tcpha_t *)&mp->b_rptr[ip_hdr_len];
2563 	if (tcpha->tha_flags & TH_RST) {
2564 		freemsg(mp);
2565 		goto done;
2566 	}
2567 	tcpha->tha_offset_and_reserved = (5 << 4);
2568 	len = ip_hdr_len + sizeof (tcpha_t);
2569 	mp->b_wptr = &mp->b_rptr[len];
2570 	if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2571 		ipha->ipha_length = htons(len);
2572 		/* Swap addresses */
2573 		v4addr = ipha->ipha_src;
2574 		ipha->ipha_src = ipha->ipha_dst;
2575 		ipha->ipha_dst = v4addr;
2576 		ipha->ipha_ident = 0;
2577 		ipha->ipha_ttl = (uchar_t)tcps->tcps_ipv4_ttl;
2578 		ixa->ixa_flags |= IXAF_IS_IPV4;
2579 		ixa->ixa_ip_hdr_length = ip_hdr_len;
2580 	} else {
2581 		ip6h->ip6_plen = htons(len - IPV6_HDR_LEN);
2582 		/* Swap addresses */
2583 		v6addr = ip6h->ip6_src;
2584 		ip6h->ip6_src = ip6h->ip6_dst;
2585 		ip6h->ip6_dst = v6addr;
2586 		ip6h->ip6_hops = (uchar_t)tcps->tcps_ipv6_hoplimit;
2587 		ixa->ixa_flags &= ~IXAF_IS_IPV4;
2588 
2589 		if (IN6_IS_ADDR_LINKSCOPE(&ip6h->ip6_dst)) {
2590 			ixa->ixa_flags |= IXAF_SCOPEID_SET;
2591 			ixa->ixa_scopeid = ira->ira_ruifindex;
2592 		}
2593 		ixa->ixa_ip_hdr_length = IPV6_HDR_LEN;
2594 	}
2595 	ixa->ixa_pktlen = len;
2596 
2597 	/* Swap the ports */
2598 	port = tcpha->tha_fport;
2599 	tcpha->tha_fport = tcpha->tha_lport;
2600 	tcpha->tha_lport = port;
2601 
2602 	tcpha->tha_ack = htonl(ack);
2603 	tcpha->tha_seq = htonl(seq);
2604 	tcpha->tha_win = 0;
2605 	tcpha->tha_sum = htons(sizeof (tcpha_t));
2606 	tcpha->tha_flags = (uint8_t)ctl;
2607 	if (ctl & TH_RST) {
2608 		if (ctl & TH_ACK) {
2609 			/*
2610 			 * Probe connection rejection here.
2611 			 * tcp_xmit_listeners_reset() drops non-SYN segments
2612 			 * that do not specify TH_ACK in their flags without
2613 			 * calling this function.  As a consequence, if this
2614 			 * function is called with a TH_RST|TH_ACK ctl argument,
2615 			 * it is being called in response to a SYN segment
2616 			 * and thus the tcp:::accept-refused probe point
2617 			 * is valid here.
2618 			 */
2619 			DTRACE_TCP5(accept__refused, mblk_t *, NULL,
2620 			    void, NULL, void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2621 			    tcph_t *, tcpha);
2622 		}
2623 		TCPS_BUMP_MIB(tcps, tcpOutRsts);
2624 		TCPS_BUMP_MIB(tcps, tcpOutControl);
2625 	}
2626 
2627 	/* Discard any old label */
2628 	if (ixa->ixa_free_flags & IXA_FREE_TSL) {
2629 		ASSERT(ixa->ixa_tsl != NULL);
2630 		label_rele(ixa->ixa_tsl);
2631 		ixa->ixa_free_flags &= ~IXA_FREE_TSL;
2632 	}
2633 	ixa->ixa_tsl = ira->ira_tsl;	/* Behave as a multi-level responder */
2634 
2635 	if (ira->ira_flags & IRAF_IPSEC_SECURE) {
2636 		/*
2637 		 * Apply IPsec based on how IPsec was applied to
2638 		 * the packet that caused the RST.
2639 		 */
2640 		if (!ipsec_in_to_out(ira, ixa, mp, ipha, ip6h)) {
2641 			BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsOutDiscards);
2642 			/* Note: mp already consumed and ip_drop_packet done */
2643 			goto done;
2644 		}
2645 	} else {
2646 		/*
2647 		 * This is in clear. The RST message we are building
2648 		 * here should go out in clear, independent of our policy.
2649 		 */
2650 		ixa->ixa_flags |= IXAF_NO_IPSEC;
2651 	}
2652 
2653 	DTRACE_TCP5(send, mblk_t *, NULL, ip_xmit_attr_t *, ixa,
2654 	    __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2655 	    __dtrace_tcp_tcph_t *, tcpha);
2656 
2657 	/*
2658 	 * NOTE:  one might consider tracing a TCP packet here, but
2659 	 * this function has no active TCP state and no tcp structure
2660 	 * that has a trace buffer.  If we traced here, we would have
2661 	 * to keep a local trace buffer in tcp_record_trace().
2662 	 */
2663 
2664 	(void) ip_output_simple(mp, ixa);
2665 done:
2666 	ixa_cleanup(ixa);
2667 	if (need_refrele) {
2668 		ASSERT(ixa != &ixas);
2669 		ixa_refrele(ixa);
2670 	}
2671 }
2672 
2673 /*
2674  * Generate a "no listener here" RST in response to an "unknown" segment.
2675  * connp is set by caller when RST is in response to an unexpected
2676  * inbound packet for which there is active tcp state in the system.
2677  * Note that we are reusing the incoming mp to construct the outgoing RST.
2678  */
2679 void
tcp_xmit_listeners_reset(mblk_t * mp,ip_recv_attr_t * ira,ip_stack_t * ipst,conn_t * connp)2680 tcp_xmit_listeners_reset(mblk_t *mp, ip_recv_attr_t *ira, ip_stack_t *ipst,
2681     conn_t *connp)
2682 {
2683 	uchar_t		*rptr;
2684 	uint32_t	seg_len;
2685 	tcpha_t		*tcpha;
2686 	uint32_t	seg_seq;
2687 	uint32_t	seg_ack;
2688 	uint_t		flags;
2689 	ipha_t 		*ipha;
2690 	ip6_t 		*ip6h;
2691 	boolean_t	policy_present;
2692 	netstack_t	*ns = ipst->ips_netstack;
2693 	tcp_stack_t	*tcps = ns->netstack_tcp;
2694 	ipsec_stack_t	*ipss = tcps->tcps_netstack->netstack_ipsec;
2695 	uint_t		ip_hdr_len = ira->ira_ip_hdr_length;
2696 
2697 	TCP_STAT(tcps, tcp_no_listener);
2698 
2699 	/*
2700 	 * DTrace this "unknown" segment as a tcp:::receive, as we did
2701 	 * just receive something that was TCP.
2702 	 */
2703 	DTRACE_TCP5(receive, mblk_t *, NULL, ip_xmit_attr_t *, NULL,
2704 	    __dtrace_tcp_void_ip_t *, mp->b_rptr, tcp_t *, NULL,
2705 	    __dtrace_tcp_tcph_t *, &mp->b_rptr[ip_hdr_len]);
2706 
2707 	if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) {
2708 		policy_present = ipss->ipsec_inbound_v4_policy_present;
2709 		ipha = (ipha_t *)mp->b_rptr;
2710 		ip6h = NULL;
2711 	} else {
2712 		policy_present = ipss->ipsec_inbound_v6_policy_present;
2713 		ipha = NULL;
2714 		ip6h = (ip6_t *)mp->b_rptr;
2715 	}
2716 
2717 	if (policy_present) {
2718 		/*
2719 		 * The conn_t parameter is NULL because we already know
2720 		 * nobody's home.
2721 		 */
2722 		mp = ipsec_check_global_policy(mp, (conn_t *)NULL, ipha, ip6h,
2723 		    ira, ns);
2724 		if (mp == NULL)
2725 			return;
2726 	}
2727 	if (is_system_labeled() && !tsol_can_reply_error(mp, ira)) {
2728 		DTRACE_PROBE2(
2729 		    tx__ip__log__error__nolistener__tcp,
2730 		    char *, "Could not reply with RST to mp(1)",
2731 		    mblk_t *, mp);
2732 		ip2dbg(("tcp_xmit_listeners_reset: not permitted to reply\n"));
2733 		freemsg(mp);
2734 		return;
2735 	}
2736 
2737 	rptr = mp->b_rptr;
2738 
2739 	tcpha = (tcpha_t *)&rptr[ip_hdr_len];
2740 	seg_seq = ntohl(tcpha->tha_seq);
2741 	seg_ack = ntohl(tcpha->tha_ack);
2742 	flags = tcpha->tha_flags;
2743 
2744 	seg_len = msgdsize(mp) - (TCP_HDR_LENGTH(tcpha) + ip_hdr_len);
2745 	if (flags & TH_RST) {
2746 		freemsg(mp);
2747 	} else if (flags & TH_ACK) {
2748 		tcp_xmit_early_reset("no tcp, reset", mp, seg_ack, 0, TH_RST,
2749 		    ira, ipst, connp);
2750 	} else {
2751 		if (flags & TH_SYN) {
2752 			seg_len++;
2753 		} else {
2754 			/*
2755 			 * Here we violate the RFC.  Note that a normal
2756 			 * TCP will never send a segment without the ACK
2757 			 * flag, except for RST or SYN segment.  This
2758 			 * segment is neither.  Just drop it on the
2759 			 * floor.
2760 			 */
2761 			freemsg(mp);
2762 			TCP_STAT(tcps, tcp_rst_unsent);
2763 			return;
2764 		}
2765 
2766 		tcp_xmit_early_reset("no tcp, reset/ack", mp, 0,
2767 		    seg_seq + seg_len, TH_RST | TH_ACK, ira, ipst, connp);
2768 	}
2769 }
2770 
2771 /*
2772  * Helper function for tcp_xmit_mp() in handling connection set up flag
2773  * options setting.
2774  */
2775 static void
tcp_xmit_mp_aux_iss(tcp_t * tcp,conn_t * connp,tcpha_t * tcpha,mblk_t * mp,uint_t * flags)2776 tcp_xmit_mp_aux_iss(tcp_t *tcp, conn_t *connp, tcpha_t *tcpha, mblk_t *mp,
2777     uint_t *flags)
2778 {
2779 	uint32_t u1;
2780 	uint8_t	*wptr = mp->b_wptr;
2781 	tcp_stack_t *tcps = tcp->tcp_tcps;
2782 	boolean_t add_sack = B_FALSE;
2783 
2784 	/*
2785 	 * If TCP_ISS_VALID and the seq number is tcp_iss,
2786 	 * TCP can only be in SYN-SENT, SYN-RCVD or
2787 	 * FIN-WAIT-1 state.  It can be FIN-WAIT-1 if
2788 	 * our SYN is not ack'ed but the app closes this
2789 	 * TCP connection.
2790 	 */
2791 	ASSERT(tcp->tcp_state == TCPS_SYN_SENT ||
2792 	    tcp->tcp_state == TCPS_SYN_RCVD ||
2793 	    tcp->tcp_state == TCPS_FIN_WAIT_1);
2794 
2795 	/*
2796 	 * Tack on the MSS option.  It is always needed
2797 	 * for both active and passive open.
2798 	 *
2799 	 * MSS option value should be interface MTU - MIN
2800 	 * TCP/IP header according to RFC 793 as it means
2801 	 * the maximum segment size TCP can receive.  But
2802 	 * to get around some broken middle boxes/end hosts
2803 	 * out there, we allow the option value to be the
2804 	 * same as the MSS option size on the peer side.
2805 	 * In this way, the other side will not send
2806 	 * anything larger than they can receive.
2807 	 *
2808 	 * Note that for SYN_SENT state, the ndd param
2809 	 * tcp_use_smss_as_mss_opt has no effect as we
2810 	 * don't know the peer's MSS option value. So
2811 	 * the only case we need to take care of is in
2812 	 * SYN_RCVD state, which is done later.
2813 	 */
2814 	wptr[0] = TCPOPT_MAXSEG;
2815 	wptr[1] = TCPOPT_MAXSEG_LEN;
2816 	wptr += 2;
2817 	u1 = tcp->tcp_initial_pmtu - (connp->conn_ipversion == IPV4_VERSION ?
2818 	    IP_SIMPLE_HDR_LENGTH : IPV6_HDR_LEN) - TCP_MIN_HEADER_LENGTH;
2819 	U16_TO_BE16(u1, wptr);
2820 	wptr += 2;
2821 
2822 	/* Update the offset to cover the additional word */
2823 	tcpha->tha_offset_and_reserved += (1 << 4);
2824 
2825 	switch (tcp->tcp_state) {
2826 	case TCPS_SYN_SENT:
2827 		*flags = TH_SYN;
2828 
2829 		if (tcp->tcp_snd_sack_ok)
2830 			add_sack = B_TRUE;
2831 
2832 		if (tcp->tcp_snd_ts_ok) {
2833 			uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
2834 
2835 			if (add_sack) {
2836 				wptr[0] = TCPOPT_SACK_PERMITTED;
2837 				wptr[1] = TCPOPT_SACK_OK_LEN;
2838 				add_sack = B_FALSE;
2839 			} else {
2840 				wptr[0] = TCPOPT_NOP;
2841 				wptr[1] = TCPOPT_NOP;
2842 			}
2843 			wptr[2] = TCPOPT_TSTAMP;
2844 			wptr[3] = TCPOPT_TSTAMP_LEN;
2845 			wptr += 4;
2846 			U32_TO_BE32(llbolt, wptr);
2847 			wptr += 4;
2848 			ASSERT(tcp->tcp_ts_recent == 0);
2849 			U32_TO_BE32(0L, wptr);
2850 			wptr += 4;
2851 			tcpha->tha_offset_and_reserved += (3 << 4);
2852 		}
2853 
2854 		/*
2855 		 * Set up all the bits to tell other side
2856 		 * we are ECN capable.
2857 		 */
2858 		if (tcp->tcp_ecn_ok)
2859 			*flags |= (TH_ECE | TH_CWR);
2860 
2861 		break;
2862 
2863 	case TCPS_SYN_RCVD:
2864 		*flags |= TH_SYN;
2865 
2866 		/*
2867 		 * Reset the MSS option value to be SMSS
2868 		 * We should probably add back the bytes
2869 		 * for timestamp option and IPsec.  We
2870 		 * don't do that as this is a workaround
2871 		 * for broken middle boxes/end hosts, it
2872 		 * is better for us to be more cautious.
2873 		 * They may not take these things into
2874 		 * account in their SMSS calculation.  Thus
2875 		 * the peer's calculated SMSS may be smaller
2876 		 * than what it can be.  This should be OK.
2877 		 */
2878 		if (tcps->tcps_use_smss_as_mss_opt) {
2879 			u1 = tcp->tcp_mss;
2880 			/*
2881 			 * Note that wptr points just past the MSS
2882 			 * option value.
2883 			 */
2884 			U16_TO_BE16(u1, wptr - 2);
2885 		}
2886 
2887 		/*
2888 		 * tcp_snd_ts_ok can only be set in TCPS_SYN_RCVD
2889 		 * when the peer also uses timestamps option.  And
2890 		 * the TCP header template must have already been
2891 		 * updated to include the timestamps option.
2892 		 */
2893 		if (tcp->tcp_snd_sack_ok) {
2894 			if (tcp->tcp_snd_ts_ok) {
2895 				uint8_t *tmp_wptr;
2896 
2897 				/*
2898 				 * Use the NOP in the header just
2899 				 * before timestamps opton.
2900 				 */
2901 				tmp_wptr = (uint8_t *)tcpha +
2902 				    TCP_MIN_HEADER_LENGTH;
2903 				ASSERT(tmp_wptr[0] == TCPOPT_NOP &&
2904 				    tmp_wptr[1] == TCPOPT_NOP);
2905 				tmp_wptr[0] = TCPOPT_SACK_PERMITTED;
2906 				tmp_wptr[1] = TCPOPT_SACK_OK_LEN;
2907 			} else {
2908 				add_sack = B_TRUE;
2909 			}
2910 		}
2911 
2912 
2913 		/*
2914 		 * If the other side is ECN capable, reply
2915 		 * that we are also ECN capable.
2916 		 */
2917 		if (tcp->tcp_ecn_ok)
2918 			*flags |= TH_ECE;
2919 		break;
2920 
2921 	default:
2922 		/*
2923 		 * The above ASSERT() makes sure that this
2924 		 * must be FIN-WAIT-1 state.  Our SYN has
2925 		 * not been ack'ed so retransmit it.
2926 		 */
2927 		*flags |= TH_SYN;
2928 		break;
2929 	}
2930 
2931 	if (add_sack) {
2932 		wptr[0] = TCPOPT_NOP;
2933 		wptr[1] = TCPOPT_NOP;
2934 		wptr[2] = TCPOPT_SACK_PERMITTED;
2935 		wptr[3] = TCPOPT_SACK_OK_LEN;
2936 		wptr += TCPOPT_REAL_SACK_OK_LEN;
2937 		tcpha->tha_offset_and_reserved += (1 << 4);
2938 	}
2939 
2940 	if (tcp->tcp_snd_ws_ok) {
2941 		wptr[0] =  TCPOPT_NOP;
2942 		wptr[1] =  TCPOPT_WSCALE;
2943 		wptr[2] =  TCPOPT_WS_LEN;
2944 		wptr[3] = (uchar_t)tcp->tcp_rcv_ws;
2945 		wptr += TCPOPT_REAL_WS_LEN;
2946 		tcpha->tha_offset_and_reserved += (1 << 4);
2947 	}
2948 
2949 	mp->b_wptr = wptr;
2950 	u1 = (int)(mp->b_wptr - mp->b_rptr);
2951 	/*
2952 	 * Get IP set to checksum on our behalf
2953 	 * Include the adjustment for a source route if any.
2954 	 */
2955 	u1 += connp->conn_sum;
2956 	u1 = (u1 >> 16) + (u1 & 0xFFFF);
2957 	tcpha->tha_sum = htons(u1);
2958 	TCPS_BUMP_MIB(tcps, tcpOutControl);
2959 }
2960 
2961 /*
2962  * Helper function for tcp_xmit_mp() in handling connection tear down
2963  * flag setting and state changes.
2964  */
2965 static void
tcp_xmit_mp_aux_fss(tcp_t * tcp,ip_xmit_attr_t * ixa,uint_t * flags)2966 tcp_xmit_mp_aux_fss(tcp_t *tcp, ip_xmit_attr_t *ixa, uint_t *flags)
2967 {
2968 	if (!tcp->tcp_fin_acked) {
2969 		*flags |= TH_FIN;
2970 		TCPS_BUMP_MIB(tcp->tcp_tcps, tcpOutControl);
2971 	}
2972 	if (!tcp->tcp_fin_sent) {
2973 		tcp->tcp_fin_sent = B_TRUE;
2974 		switch (tcp->tcp_state) {
2975 		case TCPS_SYN_RCVD:
2976 			tcp->tcp_state = TCPS_FIN_WAIT_1;
2977 			DTRACE_TCP6(state__change, void, NULL,
2978 			    ip_xmit_attr_t *, ixa, void, NULL,
2979 			    tcp_t *, tcp, void, NULL,
2980 			    int32_t, TCPS_SYN_RCVD);
2981 			break;
2982 		case TCPS_ESTABLISHED:
2983 			tcp->tcp_state = TCPS_FIN_WAIT_1;
2984 			DTRACE_TCP6(state__change, void, NULL,
2985 			    ip_xmit_attr_t *, ixa, void, NULL,
2986 			    tcp_t *, tcp, void, NULL,
2987 			    int32_t, TCPS_ESTABLISHED);
2988 			break;
2989 		case TCPS_CLOSE_WAIT:
2990 			tcp->tcp_state = TCPS_LAST_ACK;
2991 			DTRACE_TCP6(state__change, void, NULL,
2992 			    ip_xmit_attr_t *, ixa, void, NULL,
2993 			    tcp_t *, tcp, void, NULL,
2994 			    int32_t, TCPS_CLOSE_WAIT);
2995 			break;
2996 		}
2997 		if (tcp->tcp_suna == tcp->tcp_snxt)
2998 			TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
2999 		tcp->tcp_snxt = tcp->tcp_fss + 1;
3000 	}
3001 }
3002 
3003 /*
3004  * tcp_xmit_mp is called to return a pointer to an mblk chain complete with
3005  * ip and tcp header ready to pass down to IP.  If the mp passed in is
3006  * non-NULL, then up to max_to_send bytes of data will be dup'ed off that
3007  * mblk. (If sendall is not set the dup'ing will stop at an mblk boundary
3008  * otherwise it will dup partial mblks.)
3009  * Otherwise, an appropriate ACK packet will be generated.  This
3010  * routine is not usually called to send new data for the first time.  It
3011  * is mostly called out of the timer for retransmits, and to generate ACKs.
3012  *
3013  * If offset is not NULL, the returned mblk chain's first mblk's b_rptr will
3014  * be adjusted by *offset.  And after dupb(), the offset and the ending mblk
3015  * of the original mblk chain will be returned in *offset and *end_mp.
3016  */
3017 mblk_t *
tcp_xmit_mp(tcp_t * tcp,mblk_t * mp,int32_t max_to_send,int32_t * offset,mblk_t ** end_mp,uint32_t seq,boolean_t sendall,uint32_t * seg_len,boolean_t rexmit)3018 tcp_xmit_mp(tcp_t *tcp, mblk_t *mp, int32_t max_to_send, int32_t *offset,
3019     mblk_t **end_mp, uint32_t seq, boolean_t sendall, uint32_t *seg_len,
3020     boolean_t rexmit)
3021 {
3022 	int	data_length;
3023 	int32_t	off = 0;
3024 	uint_t	flags;
3025 	mblk_t	*mp1;
3026 	mblk_t	*mp2;
3027 	uchar_t	*rptr;
3028 	tcpha_t	*tcpha;
3029 	int32_t	num_sack_blk = 0;
3030 	int32_t	sack_opt_len = 0;
3031 	tcp_stack_t	*tcps = tcp->tcp_tcps;
3032 	conn_t		*connp = tcp->tcp_connp;
3033 	ip_xmit_attr_t	*ixa = connp->conn_ixa;
3034 
3035 	/* Allocate for our maximum TCP header + link-level */
3036 	mp1 = allocb(connp->conn_ht_iphc_allocated + tcps->tcps_wroff_xtra,
3037 	    BPRI_MED);
3038 	if (mp1 == NULL)
3039 		return (NULL);
3040 	data_length = 0;
3041 
3042 	/*
3043 	 * Note that tcp_mss has been adjusted to take into account the
3044 	 * timestamp option if applicable.  Because SACK options do not
3045 	 * appear in every TCP segments and they are of variable lengths,
3046 	 * they cannot be included in tcp_mss.  Thus we need to calculate
3047 	 * the actual segment length when we need to send a segment which
3048 	 * includes SACK options.
3049 	 */
3050 	if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) {
3051 		num_sack_blk = MIN(tcp->tcp_max_sack_blk,
3052 		    tcp->tcp_num_sack_blk);
3053 		sack_opt_len = num_sack_blk * sizeof (sack_blk_t) +
3054 		    TCPOPT_NOP_LEN * 2 + TCPOPT_HEADER_LEN;
3055 		if (max_to_send + sack_opt_len > tcp->tcp_mss)
3056 			max_to_send -= sack_opt_len;
3057 	}
3058 
3059 	if (offset != NULL) {
3060 		off = *offset;
3061 		/* We use offset as an indicator that end_mp is not NULL. */
3062 		*end_mp = NULL;
3063 	}
3064 	for (mp2 = mp1; mp && data_length != max_to_send; mp = mp->b_cont) {
3065 		/* This could be faster with cooperation from downstream */
3066 		if (mp2 != mp1 && !sendall &&
3067 		    data_length + (int)(mp->b_wptr - mp->b_rptr) >
3068 		    max_to_send)
3069 			/*
3070 			 * Don't send the next mblk since the whole mblk
3071 			 * does not fit.
3072 			 */
3073 			break;
3074 		mp2->b_cont = dupb(mp);
3075 		mp2 = mp2->b_cont;
3076 		if (!mp2) {
3077 			freemsg(mp1);
3078 			return (NULL);
3079 		}
3080 		mp2->b_rptr += off;
3081 		ASSERT((uintptr_t)(mp2->b_wptr - mp2->b_rptr) <=
3082 		    (uintptr_t)INT_MAX);
3083 
3084 		data_length += (int)(mp2->b_wptr - mp2->b_rptr);
3085 		if (data_length > max_to_send) {
3086 			mp2->b_wptr -= data_length - max_to_send;
3087 			data_length = max_to_send;
3088 			off = mp2->b_wptr - mp->b_rptr;
3089 			break;
3090 		} else {
3091 			off = 0;
3092 		}
3093 	}
3094 	if (offset != NULL) {
3095 		*offset = off;
3096 		*end_mp = mp;
3097 	}
3098 	if (seg_len != NULL) {
3099 		*seg_len = data_length;
3100 	}
3101 
3102 	/* Update the latest receive window size in TCP header. */
3103 	tcp->tcp_tcpha->tha_win = htons(tcp->tcp_rwnd >> tcp->tcp_rcv_ws);
3104 
3105 	rptr = mp1->b_rptr + tcps->tcps_wroff_xtra;
3106 	mp1->b_rptr = rptr;
3107 	mp1->b_wptr = rptr + connp->conn_ht_iphc_len + sack_opt_len;
3108 	bcopy(connp->conn_ht_iphc, rptr, connp->conn_ht_iphc_len);
3109 	tcpha = (tcpha_t *)&rptr[ixa->ixa_ip_hdr_length];
3110 	tcpha->tha_seq = htonl(seq);
3111 
3112 	/*
3113 	 * Use tcp_unsent to determine if the PUSH bit should be used assumes
3114 	 * that this function was called from tcp_wput_data. Thus, when called
3115 	 * to retransmit data the setting of the PUSH bit may appear some
3116 	 * what random in that it might get set when it should not. This
3117 	 * should not pose any performance issues.
3118 	 */
3119 	if (data_length != 0 && (tcp->tcp_unsent == 0 ||
3120 	    tcp->tcp_unsent == data_length)) {
3121 		flags = TH_ACK | TH_PUSH;
3122 	} else {
3123 		flags = TH_ACK;
3124 	}
3125 
3126 	if (tcp->tcp_ecn_ok) {
3127 		if (tcp->tcp_ecn_echo_on)
3128 			flags |= TH_ECE;
3129 
3130 		/*
3131 		 * Only set ECT bit and ECN_CWR if a segment contains new data.
3132 		 * There is no TCP flow control for non-data segments, and
3133 		 * only data segment is transmitted reliably.
3134 		 */
3135 		if (data_length > 0 && !rexmit) {
3136 			TCP_SET_ECT(tcp, rptr);
3137 			if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
3138 				flags |= TH_CWR;
3139 				tcp->tcp_ecn_cwr_sent = B_TRUE;
3140 			}
3141 		}
3142 	}
3143 
3144 	/* Check if there is any special processing needs to be done. */
3145 	if (tcp->tcp_valid_bits) {
3146 		uint32_t u1;
3147 
3148 		/* We don't allow having SYN and FIN in the same segment... */
3149 		if ((tcp->tcp_valid_bits & TCP_ISS_VALID) &&
3150 		    seq == tcp->tcp_iss) {
3151 			/* Need to do connection set up processing. */
3152 			tcp_xmit_mp_aux_iss(tcp, connp, tcpha, mp1, &flags);
3153 		} else if ((tcp->tcp_valid_bits & TCP_FSS_VALID) &&
3154 		    (seq + data_length) == tcp->tcp_fss) {
3155 			/* Need to do connection tear down processing. */
3156 			tcp_xmit_mp_aux_fss(tcp, ixa, &flags);
3157 		}
3158 
3159 		/*
3160 		 * Need to do urgent pointer processing.
3161 		 *
3162 		 * Note the trick here.  u1 is unsigned.  When tcp_urg
3163 		 * is smaller than seq, u1 will become a very huge value.
3164 		 * So the comparison will fail.  Also note that tcp_urp
3165 		 * should be positive, see RFC 793 page 17.
3166 		 */
3167 		u1 = tcp->tcp_urg - seq + TCP_OLD_URP_INTERPRETATION;
3168 		if ((tcp->tcp_valid_bits & TCP_URG_VALID) && u1 != 0 &&
3169 		    u1 < (uint32_t)(64 * 1024)) {
3170 			flags |= TH_URG;
3171 			TCPS_BUMP_MIB(tcps, tcpOutUrg);
3172 			tcpha->tha_urp = htons(u1);
3173 		}
3174 	}
3175 	tcpha->tha_flags = (uchar_t)flags;
3176 	tcp->tcp_rack = tcp->tcp_rnxt;
3177 	tcp->tcp_rack_cnt = 0;
3178 
3179 	/* Fill in the current value of timestamps option. */
3180 	if (tcp->tcp_snd_ts_ok) {
3181 		if (tcp->tcp_state != TCPS_SYN_SENT) {
3182 			uint32_t llbolt = (uint32_t)LBOLT_FASTPATH;
3183 
3184 			U32_TO_BE32(llbolt,
3185 			    (char *)tcpha + TCP_MIN_HEADER_LENGTH+4);
3186 			U32_TO_BE32(tcp->tcp_ts_recent,
3187 			    (char *)tcpha + TCP_MIN_HEADER_LENGTH+8);
3188 		}
3189 	}
3190 
3191 	/* Fill in the SACK blocks. */
3192 	if (num_sack_blk > 0) {
3193 		uchar_t *wptr = (uchar_t *)tcpha + connp->conn_ht_ulp_len;
3194 		sack_blk_t *tmp;
3195 		int32_t	i;
3196 
3197 		wptr[0] = TCPOPT_NOP;
3198 		wptr[1] = TCPOPT_NOP;
3199 		wptr[2] = TCPOPT_SACK;
3200 		wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk *
3201 		    sizeof (sack_blk_t);
3202 		wptr += TCPOPT_REAL_SACK_LEN;
3203 
3204 		tmp = tcp->tcp_sack_list;
3205 		for (i = 0; i < num_sack_blk; i++) {
3206 			U32_TO_BE32(tmp[i].begin, wptr);
3207 			wptr += sizeof (tcp_seq);
3208 			U32_TO_BE32(tmp[i].end, wptr);
3209 			wptr += sizeof (tcp_seq);
3210 		}
3211 		tcpha->tha_offset_and_reserved += ((num_sack_blk * 2 + 1) << 4);
3212 	}
3213 	ASSERT((uintptr_t)(mp1->b_wptr - rptr) <= (uintptr_t)INT_MAX);
3214 	data_length += (int)(mp1->b_wptr - rptr);
3215 
3216 	ixa->ixa_pktlen = data_length;
3217 
3218 	if (ixa->ixa_flags & IXAF_IS_IPV4) {
3219 		((ipha_t *)rptr)->ipha_length = htons(data_length);
3220 	} else {
3221 		ip6_t *ip6 = (ip6_t *)rptr;
3222 
3223 		ip6->ip6_plen = htons(data_length - IPV6_HDR_LEN);
3224 	}
3225 
3226 	/*
3227 	 * Prime pump for IP
3228 	 * Include the adjustment for a source route if any.
3229 	 */
3230 	data_length -= ixa->ixa_ip_hdr_length;
3231 	data_length += connp->conn_sum;
3232 	data_length = (data_length >> 16) + (data_length & 0xFFFF);
3233 	tcpha->tha_sum = htons(data_length);
3234 	if (tcp->tcp_ip_forward_progress) {
3235 		tcp->tcp_ip_forward_progress = B_FALSE;
3236 		connp->conn_ixa->ixa_flags |= IXAF_REACH_CONF;
3237 	} else {
3238 		connp->conn_ixa->ixa_flags &= ~IXAF_REACH_CONF;
3239 	}
3240 	return (mp1);
3241 }
3242 
3243 /*
3244  * If this routine returns B_TRUE, TCP can generate a RST in response
3245  * to a segment.  If it returns B_FALSE, TCP should not respond.
3246  */
3247 static boolean_t
tcp_send_rst_chk(tcp_stack_t * tcps)3248 tcp_send_rst_chk(tcp_stack_t *tcps)
3249 {
3250 	int64_t	now;
3251 
3252 	/*
3253 	 * TCP needs to protect itself from generating too many RSTs.
3254 	 * This can be a DoS attack by sending us random segments
3255 	 * soliciting RSTs.
3256 	 *
3257 	 * What we do here is to have a limit of tcp_rst_sent_rate RSTs
3258 	 * in each 1 second interval.  In this way, TCP still generate
3259 	 * RSTs in normal cases but when under attack, the impact is
3260 	 * limited.
3261 	 */
3262 	if (tcps->tcps_rst_sent_rate_enabled != 0) {
3263 		now = ddi_get_lbolt64();
3264 		if (TICK_TO_MSEC(now - tcps->tcps_last_rst_intrvl) >
3265 		    1*SECONDS) {
3266 			tcps->tcps_last_rst_intrvl = now;
3267 			tcps->tcps_rst_cnt = 1;
3268 		} else if (++tcps->tcps_rst_cnt > tcps->tcps_rst_sent_rate) {
3269 			return (B_FALSE);
3270 		}
3271 	}
3272 	return (B_TRUE);
3273 }
3274 
3275 /*
3276  * This function handles all retransmissions if SACK is enabled for this
3277  * connection.  First it calculates how many segments can be retransmitted
3278  * based on tcp_pipe.  Then it goes thru the notsack list to find eligible
3279  * segments.  A segment is eligible if sack_cnt for that segment is greater
3280  * than or equal tcp_dupack_fast_retransmit.  After it has retransmitted
3281  * all eligible segments, it checks to see if TCP can send some new segments
3282  * (fast recovery).  If it can, set the appropriate flag for tcp_input_data().
3283  *
3284  * Parameters:
3285  *	tcp_t *tcp: the tcp structure of the connection.
3286  *	uint_t *flags: in return, appropriate value will be set for
3287  *	tcp_input_data().
3288  */
3289 void
tcp_sack_rexmit(tcp_t * tcp,uint_t * flags)3290 tcp_sack_rexmit(tcp_t *tcp, uint_t *flags)
3291 {
3292 	notsack_blk_t	*notsack_blk;
3293 	int32_t		usable_swnd;
3294 	int32_t		mss;
3295 	uint32_t	seg_len;
3296 	mblk_t		*xmit_mp;
3297 	tcp_stack_t	*tcps = tcp->tcp_tcps;
3298 
3299 	ASSERT(tcp->tcp_notsack_list != NULL);
3300 	ASSERT(tcp->tcp_rexmit == B_FALSE);
3301 
3302 	/* Defensive coding in case there is a bug... */
3303 	if (tcp->tcp_notsack_list == NULL) {
3304 		return;
3305 	}
3306 	notsack_blk = tcp->tcp_notsack_list;
3307 	mss = tcp->tcp_mss;
3308 
3309 	/*
3310 	 * Limit the num of outstanding data in the network to be
3311 	 * tcp_cwnd_ssthresh, which is half of the original congestion wnd.
3312 	 */
3313 	usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe;
3314 
3315 	/* At least retransmit 1 MSS of data. */
3316 	if (usable_swnd <= 0) {
3317 		usable_swnd = mss;
3318 	}
3319 
3320 	/* Make sure no new RTT samples will be taken. */
3321 	tcp->tcp_csuna = tcp->tcp_snxt;
3322 
3323 	notsack_blk = tcp->tcp_notsack_list;
3324 	while (usable_swnd > 0) {
3325 		mblk_t		*snxt_mp, *tmp_mp;
3326 		tcp_seq		begin = tcp->tcp_sack_snxt;
3327 		tcp_seq		end;
3328 		int32_t		off;
3329 
3330 		for (; notsack_blk != NULL; notsack_blk = notsack_blk->next) {
3331 			if (SEQ_GT(notsack_blk->end, begin) &&
3332 			    (notsack_blk->sack_cnt >=
3333 			    tcps->tcps_dupack_fast_retransmit)) {
3334 				end = notsack_blk->end;
3335 				if (SEQ_LT(begin, notsack_blk->begin)) {
3336 					begin = notsack_blk->begin;
3337 				}
3338 				break;
3339 			}
3340 		}
3341 		/*
3342 		 * All holes are filled.  Manipulate tcp_cwnd to send more
3343 		 * if we can.  Note that after the SACK recovery, tcp_cwnd is
3344 		 * set to tcp_cwnd_ssthresh.
3345 		 */
3346 		if (notsack_blk == NULL) {
3347 			usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe;
3348 			if (usable_swnd <= 0 || tcp->tcp_unsent == 0) {
3349 				tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna;
3350 				ASSERT(tcp->tcp_cwnd > 0);
3351 				return;
3352 			} else {
3353 				usable_swnd = usable_swnd / mss;
3354 				tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna +
3355 				    MAX(usable_swnd * mss, mss);
3356 				*flags |= TH_XMIT_NEEDED;
3357 				return;
3358 			}
3359 		}
3360 
3361 		/*
3362 		 * Note that we may send more than usable_swnd allows here
3363 		 * because of round off, but no more than 1 MSS of data.
3364 		 */
3365 		seg_len = end - begin;
3366 		if (seg_len > mss)
3367 			seg_len = mss;
3368 		snxt_mp = tcp_get_seg_mp(tcp, begin, &off);
3369 		ASSERT(snxt_mp != NULL);
3370 		/* This should not happen.  Defensive coding again... */
3371 		if (snxt_mp == NULL) {
3372 			return;
3373 		}
3374 
3375 		xmit_mp = tcp_xmit_mp(tcp, snxt_mp, seg_len, &off,
3376 		    &tmp_mp, begin, B_TRUE, &seg_len, B_TRUE);
3377 		if (xmit_mp == NULL)
3378 			return;
3379 
3380 		usable_swnd -= seg_len;
3381 		tcp->tcp_pipe += seg_len;
3382 		tcp->tcp_sack_snxt = begin + seg_len;
3383 
3384 		tcp_send_data(tcp, xmit_mp);
3385 
3386 		/*
3387 		 * Update the send timestamp to avoid false retransmission.
3388 		 */
3389 		snxt_mp->b_prev = (mblk_t *)ddi_get_lbolt();
3390 
3391 		TCPS_BUMP_MIB(tcps, tcpRetransSegs);
3392 		TCPS_UPDATE_MIB(tcps, tcpRetransBytes, seg_len);
3393 		TCPS_BUMP_MIB(tcps, tcpOutSackRetransSegs);
3394 		/*
3395 		 * Update tcp_rexmit_max to extend this SACK recovery phase.
3396 		 * This happens when new data sent during fast recovery is
3397 		 * also lost.  If TCP retransmits those new data, it needs
3398 		 * to extend SACK recover phase to avoid starting another
3399 		 * fast retransmit/recovery unnecessarily.
3400 		 */
3401 		if (SEQ_GT(tcp->tcp_sack_snxt, tcp->tcp_rexmit_max)) {
3402 			tcp->tcp_rexmit_max = tcp->tcp_sack_snxt;
3403 		}
3404 	}
3405 }
3406 
3407 /*
3408  * tcp_ss_rexmit() is called to do slow start retransmission after a timeout
3409  * or ICMP errors.
3410  */
3411 void
tcp_ss_rexmit(tcp_t * tcp)3412 tcp_ss_rexmit(tcp_t *tcp)
3413 {
3414 	uint32_t	snxt;
3415 	uint32_t	smax;
3416 	int32_t		win;
3417 	int32_t		mss;
3418 	int32_t		off;
3419 	mblk_t		*snxt_mp;
3420 	tcp_stack_t	*tcps = tcp->tcp_tcps;
3421 
3422 	/*
3423 	 * Note that tcp_rexmit can be set even though TCP has retransmitted
3424 	 * all unack'ed segments.
3425 	 */
3426 	if (SEQ_LT(tcp->tcp_rexmit_nxt, tcp->tcp_rexmit_max)) {
3427 		smax = tcp->tcp_rexmit_max;
3428 		snxt = tcp->tcp_rexmit_nxt;
3429 		if (SEQ_LT(snxt, tcp->tcp_suna)) {
3430 			snxt = tcp->tcp_suna;
3431 		}
3432 		win = MIN(tcp->tcp_cwnd, tcp->tcp_swnd);
3433 		win -= snxt - tcp->tcp_suna;
3434 		mss = tcp->tcp_mss;
3435 		snxt_mp = tcp_get_seg_mp(tcp, snxt, &off);
3436 
3437 		while (SEQ_LT(snxt, smax) && (win > 0) && (snxt_mp != NULL)) {
3438 			mblk_t	*xmit_mp;
3439 			mblk_t	*old_snxt_mp = snxt_mp;
3440 			uint32_t cnt = mss;
3441 
3442 			if (win < cnt) {
3443 				cnt = win;
3444 			}
3445 			if (SEQ_GT(snxt + cnt, smax)) {
3446 				cnt = smax - snxt;
3447 			}
3448 			xmit_mp = tcp_xmit_mp(tcp, snxt_mp, cnt, &off,
3449 			    &snxt_mp, snxt, B_TRUE, &cnt, B_TRUE);
3450 			if (xmit_mp == NULL)
3451 				return;
3452 
3453 			tcp_send_data(tcp, xmit_mp);
3454 
3455 			snxt += cnt;
3456 			win -= cnt;
3457 			/*
3458 			 * Update the send timestamp to avoid false
3459 			 * retransmission.
3460 			 */
3461 			old_snxt_mp->b_prev = (mblk_t *)ddi_get_lbolt();
3462 			TCPS_BUMP_MIB(tcps, tcpRetransSegs);
3463 			TCPS_UPDATE_MIB(tcps, tcpRetransBytes, cnt);
3464 
3465 			tcp->tcp_rexmit_nxt = snxt;
3466 		}
3467 		/*
3468 		 * If we have transmitted all we have at the time
3469 		 * we started the retranmission, we can leave
3470 		 * the rest of the job to tcp_wput_data().  But we
3471 		 * need to check the send window first.  If the
3472 		 * win is not 0, go on with tcp_wput_data().
3473 		 */
3474 		if (SEQ_LT(snxt, smax) || win == 0) {
3475 			return;
3476 		}
3477 	}
3478 	/* Only call tcp_wput_data() if there is data to be sent. */
3479 	if (tcp->tcp_unsent) {
3480 		tcp_wput_data(tcp, NULL, B_FALSE);
3481 	}
3482 }
3483 
3484 /*
3485  * Do slow start retransmission after ICMP errors of PMTU changes.
3486  */
3487 void
tcp_rexmit_after_error(tcp_t * tcp)3488 tcp_rexmit_after_error(tcp_t *tcp)
3489 {
3490 	/*
3491 	 * All sent data has been acknowledged or no data left to send, just
3492 	 * to return.
3493 	 */
3494 	if (!SEQ_LT(tcp->tcp_suna, tcp->tcp_snxt) ||
3495 	    (tcp->tcp_xmit_head == NULL))
3496 		return;
3497 
3498 	if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && (tcp->tcp_unsent == 0))
3499 		tcp->tcp_rexmit_max = tcp->tcp_fss;
3500 	else
3501 		tcp->tcp_rexmit_max = tcp->tcp_snxt;
3502 
3503 	tcp->tcp_rexmit_nxt = tcp->tcp_suna;
3504 	tcp->tcp_rexmit = B_TRUE;
3505 	tcp->tcp_dupack_cnt = 0;
3506 	tcp_ss_rexmit(tcp);
3507 }
3508 
3509 /*
3510  * tcp_get_seg_mp() is called to get the pointer to a segment in the
3511  * send queue which starts at the given sequence number. If the given
3512  * sequence number is equal to last valid sequence number (tcp_snxt), the
3513  * returned mblk is the last valid mblk, and off is set to the length of
3514  * that mblk.
3515  *
3516  * send queue which starts at the given seq. no.
3517  *
3518  * Parameters:
3519  *	tcp_t *tcp: the tcp instance pointer.
3520  *	uint32_t seq: the starting seq. no of the requested segment.
3521  *	int32_t *off: after the execution, *off will be the offset to
3522  *		the returned mblk which points to the requested seq no.
3523  *		It is the caller's responsibility to send in a non-null off.
3524  *
3525  * Return:
3526  *	A mblk_t pointer pointing to the requested segment in send queue.
3527  */
3528 static mblk_t *
tcp_get_seg_mp(tcp_t * tcp,uint32_t seq,int32_t * off)3529 tcp_get_seg_mp(tcp_t *tcp, uint32_t seq, int32_t *off)
3530 {
3531 	int32_t	cnt;
3532 	mblk_t	*mp;
3533 
3534 	/* Defensive coding.  Make sure we don't send incorrect data. */
3535 	if (SEQ_LT(seq, tcp->tcp_suna) || SEQ_GT(seq, tcp->tcp_snxt))
3536 		return (NULL);
3537 
3538 	cnt = seq - tcp->tcp_suna;
3539 	mp = tcp->tcp_xmit_head;
3540 	while (cnt > 0 && mp != NULL) {
3541 		cnt -= mp->b_wptr - mp->b_rptr;
3542 		if (cnt <= 0) {
3543 			cnt += mp->b_wptr - mp->b_rptr;
3544 			break;
3545 		}
3546 		mp = mp->b_cont;
3547 	}
3548 	ASSERT(mp != NULL);
3549 	*off = cnt;
3550 	return (mp);
3551 }
3552 
3553 /*
3554  * This routine adjusts next-to-send sequence number variables, in the
3555  * case where the reciever has shrunk it's window.
3556  */
3557 void
tcp_update_xmit_tail(tcp_t * tcp,uint32_t snxt)3558 tcp_update_xmit_tail(tcp_t *tcp, uint32_t snxt)
3559 {
3560 	mblk_t *xmit_tail;
3561 	int32_t offset;
3562 
3563 	tcp->tcp_snxt = snxt;
3564 
3565 	/* Get the mblk, and the offset in it, as per the shrunk window */
3566 	xmit_tail = tcp_get_seg_mp(tcp, snxt, &offset);
3567 	ASSERT(xmit_tail != NULL);
3568 	tcp->tcp_xmit_tail = xmit_tail;
3569 	tcp->tcp_xmit_tail_unsent = xmit_tail->b_wptr -
3570 	    xmit_tail->b_rptr - offset;
3571 }
3572 
3573 /*
3574  * This handles the case when the receiver has shrunk its win. Per RFC 1122
3575  * if the receiver shrinks the window, i.e. moves the right window to the
3576  * left, the we should not send new data, but should retransmit normally the
3577  * old unacked data between suna and suna + swnd. We might has sent data
3578  * that is now outside the new window, pretend that we didn't send  it.
3579  */
3580 static void
tcp_process_shrunk_swnd(tcp_t * tcp,uint32_t shrunk_count)3581 tcp_process_shrunk_swnd(tcp_t *tcp, uint32_t shrunk_count)
3582 {
3583 	uint32_t	snxt = tcp->tcp_snxt;
3584 
3585 	ASSERT(shrunk_count > 0);
3586 
3587 	if (!tcp->tcp_is_wnd_shrnk) {
3588 		tcp->tcp_snxt_shrunk = snxt;
3589 		tcp->tcp_is_wnd_shrnk = B_TRUE;
3590 	} else if (SEQ_GT(snxt, tcp->tcp_snxt_shrunk)) {
3591 		tcp->tcp_snxt_shrunk = snxt;
3592 	}
3593 
3594 	/* Pretend we didn't send the data outside the window */
3595 	snxt -= shrunk_count;
3596 
3597 	/* Reset all the values per the now shrunk window */
3598 	tcp_update_xmit_tail(tcp, snxt);
3599 	tcp->tcp_unsent += shrunk_count;
3600 
3601 	/*
3602 	 * If the SACK option is set, delete the entire list of
3603 	 * notsack'ed blocks.
3604 	 */
3605 	TCP_NOTSACK_REMOVE_ALL(tcp->tcp_notsack_list, tcp);
3606 
3607 	if (tcp->tcp_suna == tcp->tcp_snxt && tcp->tcp_swnd == 0)
3608 		/*
3609 		 * Make sure the timer is running so that we will probe a zero
3610 		 * window.
3611 		 */
3612 		TCP_TIMER_RESTART(tcp, tcp->tcp_rto);
3613 }
3614 
3615 /*
3616  * tcp_fill_header is called by tcp_send() to fill the outgoing TCP header
3617  * with the template header, as well as other options such as time-stamp,
3618  * ECN and/or SACK.
3619  */
3620 static void
tcp_fill_header(tcp_t * tcp,uchar_t * rptr,clock_t now,int num_sack_blk)3621 tcp_fill_header(tcp_t *tcp, uchar_t *rptr, clock_t now, int num_sack_blk)
3622 {
3623 	tcpha_t *tcp_tmpl, *tcpha;
3624 	uint32_t *dst, *src;
3625 	int hdrlen;
3626 	conn_t *connp = tcp->tcp_connp;
3627 
3628 	ASSERT(OK_32PTR(rptr));
3629 
3630 	/* Template header */
3631 	tcp_tmpl = tcp->tcp_tcpha;
3632 
3633 	/* Header of outgoing packet */
3634 	tcpha = (tcpha_t *)(rptr + connp->conn_ixa->ixa_ip_hdr_length);
3635 
3636 	/* dst and src are opaque 32-bit fields, used for copying */
3637 	dst = (uint32_t *)rptr;
3638 	src = (uint32_t *)connp->conn_ht_iphc;
3639 	hdrlen = connp->conn_ht_iphc_len;
3640 
3641 	/* Fill time-stamp option if needed */
3642 	if (tcp->tcp_snd_ts_ok) {
3643 		U32_TO_BE32((uint32_t)now,
3644 		    (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 4);
3645 		U32_TO_BE32(tcp->tcp_ts_recent,
3646 		    (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 8);
3647 	} else {
3648 		ASSERT(connp->conn_ht_ulp_len == TCP_MIN_HEADER_LENGTH);
3649 	}
3650 
3651 	/*
3652 	 * Copy the template header; is this really more efficient than
3653 	 * calling bcopy()?  For simple IPv4/TCP, it may be the case,
3654 	 * but perhaps not for other scenarios.
3655 	 */
3656 	dst[0] = src[0];
3657 	dst[1] = src[1];
3658 	dst[2] = src[2];
3659 	dst[3] = src[3];
3660 	dst[4] = src[4];
3661 	dst[5] = src[5];
3662 	dst[6] = src[6];
3663 	dst[7] = src[7];
3664 	dst[8] = src[8];
3665 	dst[9] = src[9];
3666 	if (hdrlen -= 40) {
3667 		hdrlen >>= 2;
3668 		dst += 10;
3669 		src += 10;
3670 		do {
3671 			*dst++ = *src++;
3672 		} while (--hdrlen);
3673 	}
3674 
3675 	/*
3676 	 * Set the ECN info in the TCP header if it is not a zero
3677 	 * window probe.  Zero window probe is only sent in
3678 	 * tcp_wput_data() and tcp_timer().
3679 	 */
3680 	if (tcp->tcp_ecn_ok && !tcp->tcp_zero_win_probe) {
3681 		TCP_SET_ECT(tcp, rptr);
3682 
3683 		if (tcp->tcp_ecn_echo_on)
3684 			tcpha->tha_flags |= TH_ECE;
3685 		if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) {
3686 			tcpha->tha_flags |= TH_CWR;
3687 			tcp->tcp_ecn_cwr_sent = B_TRUE;
3688 		}
3689 	}
3690 
3691 	/* Fill in SACK options */
3692 	if (num_sack_blk > 0) {
3693 		uchar_t *wptr = rptr + connp->conn_ht_iphc_len;
3694 		sack_blk_t *tmp;
3695 		int32_t	i;
3696 
3697 		wptr[0] = TCPOPT_NOP;
3698 		wptr[1] = TCPOPT_NOP;
3699 		wptr[2] = TCPOPT_SACK;
3700 		wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk *
3701 		    sizeof (sack_blk_t);
3702 		wptr += TCPOPT_REAL_SACK_LEN;
3703 
3704 		tmp = tcp->tcp_sack_list;
3705 		for (i = 0; i < num_sack_blk; i++) {
3706 			U32_TO_BE32(tmp[i].begin, wptr);
3707 			wptr += sizeof (tcp_seq);
3708 			U32_TO_BE32(tmp[i].end, wptr);
3709 			wptr += sizeof (tcp_seq);
3710 		}
3711 		tcpha->tha_offset_and_reserved +=
3712 		    ((num_sack_blk * 2 + 1) << 4);
3713 	}
3714 }
3715