xref: /freebsd/sys/netinet/tcp_output.c (revision af700f430fd86ba3eae63e587985a12436db8f69)
1c398230bSWarner Losh /*-
251369649SPedro F. Giffuni  * SPDX-License-Identifier: BSD-3-Clause
351369649SPedro F. Giffuni  *
4d7f570e6SGarrett Wollman  * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
5df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
6df8bae1dSRodney W. Grimes  *
7df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
8df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
9df8bae1dSRodney W. Grimes  * are met:
10df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
11df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
12df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
13df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
14df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
15fbbd9655SWarner Losh  * 3. Neither the name of the University nor the names of its contributors
16df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
17df8bae1dSRodney W. Grimes  *    without specific prior written permission.
18df8bae1dSRodney W. Grimes  *
19df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
30df8bae1dSRodney W. Grimes  */
31df8bae1dSRodney W. Grimes 
324b421e2dSMike Silbersack #include <sys/cdefs.h>
331cfd4b53SBruce M Simpson #include "opt_inet.h"
34fb59c426SYoshinobu Inoue #include "opt_inet6.h"
353a2a9f79SYoshinobu Inoue #include "opt_ipsec.h"
36b2e60773SJohn Baldwin #include "opt_kern_tls.h"
370cc12cc5SJoerg Wunsch 
38df8bae1dSRodney W. Grimes #include <sys/param.h>
39df8bae1dSRodney W. Grimes #include <sys/systm.h>
40adc56f5aSEdward Tomasz Napierala #include <sys/arb.h>
41686cdd19SJun-ichiro itojun Hagino #include <sys/domain.h>
42bd79708dSJonathan T. Looney #ifdef TCP_HHOOK
4339bc9de5SLawrence Stewart #include <sys/hhook.h>
44bd79708dSJonathan T. Looney #endif
45fb919e4dSMark Murray #include <sys/kernel.h>
46b2e60773SJohn Baldwin #ifdef KERN_TLS
47b2e60773SJohn Baldwin #include <sys/ktls.h>
48b2e60773SJohn Baldwin #endif
49fb919e4dSMark Murray #include <sys/lock.h>
50fb919e4dSMark Murray #include <sys/mbuf.h>
51fb919e4dSMark Murray #include <sys/mutex.h>
52df8bae1dSRodney W. Grimes #include <sys/protosw.h>
53adc56f5aSEdward Tomasz Napierala #include <sys/qmath.h>
5457f60867SMark Johnston #include <sys/sdt.h>
55df8bae1dSRodney W. Grimes #include <sys/socket.h>
56df8bae1dSRodney W. Grimes #include <sys/socketvar.h>
57fb919e4dSMark Murray #include <sys/sysctl.h>
58adc56f5aSEdward Tomasz Napierala #include <sys/stats.h>
59df8bae1dSRodney W. Grimes 
604b79449eSBjoern A. Zeeb #include <net/if.h>
61df8bae1dSRodney W. Grimes #include <net/route.h>
62983066f0SAlexander V. Chernikov #include <net/route/nhop.h>
63530c0060SRobert Watson #include <net/vnet.h>
64df8bae1dSRodney W. Grimes 
65df8bae1dSRodney W. Grimes #include <netinet/in.h>
6657f60867SMark Johnston #include <netinet/in_kdtrace.h>
67df8bae1dSRodney W. Grimes #include <netinet/in_systm.h>
68df8bae1dSRodney W. Grimes #include <netinet/ip.h>
69df8bae1dSRodney W. Grimes #include <netinet/in_pcb.h>
70df8bae1dSRodney W. Grimes #include <netinet/ip_var.h>
71ef39adf0SAndre Oppermann #include <netinet/ip_options.h>
72fb59c426SYoshinobu Inoue #ifdef INET6
73686cdd19SJun-ichiro itojun Hagino #include <netinet6/in6_pcb.h>
74686cdd19SJun-ichiro itojun Hagino #include <netinet/ip6.h>
75fb59c426SYoshinobu Inoue #include <netinet6/ip6_var.h>
76fb59c426SYoshinobu Inoue #endif
772de3e790SGleb Smirnoff #include <netinet/tcp.h>
78a00f4ac2SGleb Smirnoff #define	TCPOUTFLAGS
79df8bae1dSRodney W. Grimes #include <netinet/tcp_fsm.h>
80df8bae1dSRodney W. Grimes #include <netinet/tcp_seq.h>
81df8bae1dSRodney W. Grimes #include <netinet/tcp_var.h>
8269c7c811SRandall Stewart #include <netinet/tcp_log_buf.h>
83f7220c48SRichard Scheffenegger #include <netinet/tcp_syncache.h>
84f7220c48SRichard Scheffenegger #include <netinet/tcp_timer.h>
85df8bae1dSRodney W. Grimes #include <netinet/tcpip.h>
864644fda3SGleb Smirnoff #include <netinet/cc/cc.h>
87c560df6fSPatrick Kelsey #include <netinet/tcp_fastopen.h>
8886a996e6SHiren Panchasara #ifdef TCPPCAP
8986a996e6SHiren Panchasara #include <netinet/tcp_pcap.h>
9086a996e6SHiren Panchasara #endif
9109fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD
9209fe6320SNavdeep Parhar #include <netinet/tcp_offload.h>
9309fe6320SNavdeep Parhar #endif
94f7220c48SRichard Scheffenegger #include <netinet/tcp_ecn.h>
95df8bae1dSRodney W. Grimes 
96fcf59617SAndrey V. Elsukov #include <netipsec/ipsec_support.h>
97b9234fafSSam Leffler 
989e644c23SMichael Tuexen #include <netinet/udp.h>
999e644c23SMichael Tuexen #include <netinet/udp_var.h>
100db4f9cc7SJonathan Lemon #include <machine/in_cksum.h>
101db4f9cc7SJonathan Lemon 
102aed55708SRobert Watson #include <security/mac/mac_framework.h>
103aed55708SRobert Watson 
10482cea7e6SBjoern A. Zeeb VNET_DEFINE(int, path_mtu_discovery) = 1;
1056df8a710SGleb Smirnoff SYSCTL_INT(_net_inet_tcp, OID_AUTO, path_mtu_discovery, CTLFLAG_VNET | CTLFLAG_RW,
106eddfbb76SRobert Watson 	&VNET_NAME(path_mtu_discovery), 1,
107eddfbb76SRobert Watson 	"Enable Path MTU Discovery");
1089a039a5fSDavid Greenman 
10982cea7e6SBjoern A. Zeeb VNET_DEFINE(int, tcp_do_tso) = 1;
1106df8a710SGleb Smirnoff SYSCTL_INT(_net_inet_tcp, OID_AUTO, tso, CTLFLAG_VNET | CTLFLAG_RW,
111eddfbb76SRobert Watson 	&VNET_NAME(tcp_do_tso), 0,
112eddfbb76SRobert Watson 	"Enable TCP Segmentation Offload");
113b3c0f300SAndre Oppermann 
114873789cbSAndre Oppermann VNET_DEFINE(int, tcp_sendspace) = 1024*32;
115873789cbSAndre Oppermann #define	V_tcp_sendspace	VNET(tcp_sendspace)
1166df8a710SGleb Smirnoff SYSCTL_INT(_net_inet_tcp, TCPCTL_SENDSPACE, sendspace, CTLFLAG_VNET | CTLFLAG_RW,
117873789cbSAndre Oppermann 	&VNET_NAME(tcp_sendspace), 0, "Initial send socket buffer size");
118873789cbSAndre Oppermann 
11982cea7e6SBjoern A. Zeeb VNET_DEFINE(int, tcp_do_autosndbuf) = 1;
1206df8a710SGleb Smirnoff SYSCTL_INT(_net_inet_tcp, OID_AUTO, sendbuf_auto, CTLFLAG_VNET | CTLFLAG_RW,
121eddfbb76SRobert Watson 	&VNET_NAME(tcp_do_autosndbuf), 0,
122eddfbb76SRobert Watson 	"Enable automatic send buffer sizing");
1236741ecf5SAndre Oppermann 
12482cea7e6SBjoern A. Zeeb VNET_DEFINE(int, tcp_autosndbuf_inc) = 8*1024;
1256df8a710SGleb Smirnoff SYSCTL_INT(_net_inet_tcp, OID_AUTO, sendbuf_inc, CTLFLAG_VNET | CTLFLAG_RW,
126eddfbb76SRobert Watson 	&VNET_NAME(tcp_autosndbuf_inc), 0,
1278b615593SMarko Zec 	"Incrementor step size of automatic send buffer");
1286741ecf5SAndre Oppermann 
129b233773bSBjoern A. Zeeb VNET_DEFINE(int, tcp_autosndbuf_max) = 2*1024*1024;
1306df8a710SGleb Smirnoff SYSCTL_INT(_net_inet_tcp, OID_AUTO, sendbuf_max, CTLFLAG_VNET | CTLFLAG_RW,
131eddfbb76SRobert Watson 	&VNET_NAME(tcp_autosndbuf_max), 0,
1328b615593SMarko Zec 	"Max size of automatic send buffer");
1336741ecf5SAndre Oppermann 
134ac952dd2SSean Bruno VNET_DEFINE(int, tcp_sendbuf_auto_lowat) = 0;
135ac952dd2SSean Bruno #define	V_tcp_sendbuf_auto_lowat	VNET(tcp_sendbuf_auto_lowat)
136ac952dd2SSean Bruno SYSCTL_INT(_net_inet_tcp, OID_AUTO, sendbuf_auto_lowat, CTLFLAG_VNET | CTLFLAG_RW,
137ac952dd2SSean Bruno 	&VNET_NAME(tcp_sendbuf_auto_lowat), 0,
138ac952dd2SSean Bruno 	"Modify threshold for auto send buffer growth to account for SO_SNDLOWAT");
139ac952dd2SSean Bruno 
140425b7639SSepherosa Ziehau /*
141425b7639SSepherosa Ziehau  * Make sure that either retransmit or persist timer is set for SYN, FIN and
142425b7639SSepherosa Ziehau  * non-ACK.
143425b7639SSepherosa Ziehau  */
144425b7639SSepherosa Ziehau #define TCP_XMIT_TIMER_ASSERT(tp, len, th_flags)			\
145fd29ff5dSRandall Stewart 	KASSERT(((len) == 0 && ((th_flags) & (TH_SYN | TH_FIN)) == 0) ||\
146425b7639SSepherosa Ziehau 	    tcp_timer_active((tp), TT_REXMT) ||				\
147425b7639SSepherosa Ziehau 	    tcp_timer_active((tp), TT_PERSIST),				\
148425b7639SSepherosa Ziehau 	    ("neither rexmt nor persist timer is set"))
149425b7639SSepherosa Ziehau 
150bd79708dSJonathan T. Looney #ifdef TCP_HHOOK
151dbc42409SLawrence Stewart /*
1520f5e7edcSKevin Lo  * Wrapper for the TCP established output helper hook.
15339bc9de5SLawrence Stewart  */
15489e560f4SRandall Stewart void
15539bc9de5SLawrence Stewart hhook_run_tcp_est_out(struct tcpcb *tp, struct tcphdr *th,
1563ac12506SJonathan T. Looney     struct tcpopt *to, uint32_t len, int tso)
15739bc9de5SLawrence Stewart {
15839bc9de5SLawrence Stewart 	struct tcp_hhook_data hhook_data;
15939bc9de5SLawrence Stewart 
16039bc9de5SLawrence Stewart 	if (V_tcp_hhh[HHOOK_TCP_EST_OUT]->hhh_nhooks > 0) {
16139bc9de5SLawrence Stewart 		hhook_data.tp = tp;
16239bc9de5SLawrence Stewart 		hhook_data.th = th;
16339bc9de5SLawrence Stewart 		hhook_data.to = to;
16439bc9de5SLawrence Stewart 		hhook_data.len = len;
16539bc9de5SLawrence Stewart 		hhook_data.tso = tso;
16639bc9de5SLawrence Stewart 
16739bc9de5SLawrence Stewart 		hhook_run_hooks(V_tcp_hhh[HHOOK_TCP_EST_OUT], &hhook_data,
168e68b3792SGleb Smirnoff 		    &tp->t_osd);
16939bc9de5SLawrence Stewart 	}
17039bc9de5SLawrence Stewart }
171bd79708dSJonathan T. Looney #endif
17239bc9de5SLawrence Stewart 
17339bc9de5SLawrence Stewart /*
174dbc42409SLawrence Stewart  * CC wrapper hook functions
175dbc42409SLawrence Stewart  */
176cd84e78fSRandall Stewart void
177dbc42409SLawrence Stewart cc_after_idle(struct tcpcb *tp)
178dbc42409SLawrence Stewart {
1799eb0e832SGleb Smirnoff 	INP_WLOCK_ASSERT(tptoinpcb(tp));
180dbc42409SLawrence Stewart 
181dbc42409SLawrence Stewart 	if (CC_ALGO(tp)->after_idle != NULL)
182e68b3792SGleb Smirnoff 		CC_ALGO(tp)->after_idle(&tp->t_ccv);
183dbc42409SLawrence Stewart }
1846741ecf5SAndre Oppermann 
185df8bae1dSRodney W. Grimes /*
186df8bae1dSRodney W. Grimes  * Tcp output routine: figure out what should be sent and send it.
187df8bae1dSRodney W. Grimes  */
188df8bae1dSRodney W. Grimes int
1895b08b46aSGleb Smirnoff tcp_default_output(struct tcpcb *tp)
190df8bae1dSRodney W. Grimes {
1919eb0e832SGleb Smirnoff 	struct socket *so = tptosocket(tp);
1929eb0e832SGleb Smirnoff 	struct inpcb *inp = tptoinpcb(tp);
1933ac12506SJonathan T. Looney 	int32_t len;
1943ac12506SJonathan T. Looney 	uint32_t recwin, sendwin;
195f7220c48SRichard Scheffenegger 	uint16_t flags;
196f7220c48SRichard Scheffenegger 	int off, error = 0;	/* Keep compiler happy */
197581a046aSRandall Stewart 	u_int if_hw_tsomaxsegcount = 0;
19882e837f8SWarner Losh 	u_int if_hw_tsomaxsegsize = 0;
199f10e85d7SLuigi Rizzo 	struct mbuf *m;
200fb59c426SYoshinobu Inoue 	struct ip *ip = NULL;
201f10e85d7SLuigi Rizzo 	struct tcphdr *th;
202a0292f23SGarrett Wollman 	u_char opt[TCP_MAXOLEN];
2039e644c23SMichael Tuexen 	unsigned ipoptlen, optlen, hdrlen, ulen;
204fcf59617SAndrey V. Elsukov #if defined(IPSEC) || defined(IPSEC_SUPPORT)
2059ad0173dSBjoern A. Zeeb 	unsigned ipsec_optlen = 0;
2069ad0173dSBjoern A. Zeeb #endif
20710d20c84SMatt Macy 	int idle, sendalot, curticks;
20802a1a643SAndre Oppermann 	int sack_rxmit, sack_bytes_rxmt;
2096d90faf3SPaul Saab 	struct sackhole *p;
210df0633a1SGleb Smirnoff 	int tso, mtu;
21102a1a643SAndre Oppermann 	struct tcpopt to;
2129e644c23SMichael Tuexen 	struct udphdr *udp = NULL;
213bd30a121SMichael Tuexen 	struct tcp_log_buffer *lgb;
214c560df6fSPatrick Kelsey 	unsigned int wanted_cookie = 0;
215c560df6fSPatrick Kelsey 	unsigned int dont_sendalot = 0;
216262c1c1aSMatthew Dillon #if 0
21746f58482SJonathan Lemon 	int maxburst = TCP_MAXBURST;
218262c1c1aSMatthew Dillon #endif
219fb59c426SYoshinobu Inoue #ifdef INET6
220f10e85d7SLuigi Rizzo 	struct ip6_hdr *ip6 = NULL;
2212f201df1SAlfonso 	const bool isipv6 = (inp->inp_vflag & INP_IPV6) != 0;
222fb59c426SYoshinobu Inoue #endif
223b2e60773SJohn Baldwin #ifdef KERN_TLS
224c0e4090eSAndrew Gallatin 	const bool hw_tls = tp->t_nic_ktls_xmit != 0;
225b2e60773SJohn Baldwin #else
226b2e60773SJohn Baldwin 	const bool hw_tls = false;
227b2e60773SJohn Baldwin #endif
228df8bae1dSRodney W. Grimes 
229109eb549SGleb Smirnoff 	NET_EPOCH_ASSERT();
2309eb0e832SGleb Smirnoff 	INP_WLOCK_ASSERT(inp);
2313d6ade3aSJennifer Yang 
23209fe6320SNavdeep Parhar #ifdef TCP_OFFLOAD
23309fe6320SNavdeep Parhar 	if (tp->t_flags & TF_TOE)
23409fe6320SNavdeep Parhar 		return (tcp_offload_output(tp));
23509fe6320SNavdeep Parhar #endif
23609fe6320SNavdeep Parhar 
237281a0fd4SPatrick Kelsey 	/*
2388db239dcSMichael Tuexen 	 * For TFO connections in SYN_SENT or SYN_RECEIVED,
2398db239dcSMichael Tuexen 	 * only allow the initial SYN or SYN|ACK and those sent
2408db239dcSMichael Tuexen 	 * by the retransmit timer.
241281a0fd4SPatrick Kelsey 	 */
242dd7b86e2SGleb Smirnoff 	if ((tp->t_flags & TF_FASTOPEN) &&
2438db239dcSMichael Tuexen 	    ((tp->t_state == TCPS_SYN_SENT) ||
2448db239dcSMichael Tuexen 	    (tp->t_state == TCPS_SYN_RECEIVED)) &&
245dd7b86e2SGleb Smirnoff 	    SEQ_GT(tp->snd_max, tp->snd_una) && /* SYN or SYN|ACK sent */
246281a0fd4SPatrick Kelsey 	    (tp->snd_nxt != tp->snd_una))       /* not a retransmit */
247281a0fd4SPatrick Kelsey 		return (0);
24818a75309SPatrick Kelsey 
249df8bae1dSRodney W. Grimes 	/*
250df8bae1dSRodney W. Grimes 	 * Determine length of data that should be transmitted,
251df8bae1dSRodney W. Grimes 	 * and flags that will be used.
252df8bae1dSRodney W. Grimes 	 * If there is some data or critical controls (SYN, RST)
253df8bae1dSRodney W. Grimes 	 * to send, then transmit; otherwise, investigate further.
254df8bae1dSRodney W. Grimes 	 */
255c24d5daeSJayanth Vijayaraghavan 	idle = (tp->t_flags & TF_LASTIDLE) || (tp->snd_max == tp->snd_una);
2569dc7d8a2SRichard Scheffenegger 	if (idle && (((ticks - tp->t_rcvtime) >= tp->t_rxtcur) ||
2579dc7d8a2SRichard Scheffenegger 	    (tp->t_sndtime && ((ticks - tp->t_sndtime) >= tp->t_rxtcur))))
2582ea8da28SLawrence Stewart 		cc_after_idle(tp);
259c24d5daeSJayanth Vijayaraghavan 	tp->t_flags &= ~TF_LASTIDLE;
260c24d5daeSJayanth Vijayaraghavan 	if (idle) {
261c24d5daeSJayanth Vijayaraghavan 		if (tp->t_flags & TF_MORETOCOME) {
262c24d5daeSJayanth Vijayaraghavan 			tp->t_flags |= TF_LASTIDLE;
263c24d5daeSJayanth Vijayaraghavan 			idle = 0;
264c24d5daeSJayanth Vijayaraghavan 		}
265c24d5daeSJayanth Vijayaraghavan 	}
266df8bae1dSRodney W. Grimes again:
2676d90faf3SPaul Saab 	/*
2686d90faf3SPaul Saab 	 * If we've recently taken a timeout, snd_max will be greater than
2696d90faf3SPaul Saab 	 * snd_nxt.  There may be SACK information that allows us to avoid
2706d90faf3SPaul Saab 	 * resending already delivered data.  Adjust snd_nxt accordingly.
2716d90faf3SPaul Saab 	 */
2723529149eSAndre Oppermann 	if ((tp->t_flags & TF_SACK_PERMIT) &&
2733529149eSAndre Oppermann 	    SEQ_LT(tp->snd_nxt, tp->snd_max))
2746d90faf3SPaul Saab 		tcp_sack_adjust(tp);
275df8bae1dSRodney W. Grimes 	sendalot = 0;
276153e5b57SAndre Oppermann 	tso = 0;
277df0633a1SGleb Smirnoff 	mtu = 0;
278df8bae1dSRodney W. Grimes 	off = tp->snd_nxt - tp->snd_una;
279201d185bSAndre Oppermann 	sendwin = min(tp->snd_wnd, tp->snd_cwnd);
280df8bae1dSRodney W. Grimes 
281df8bae1dSRodney W. Grimes 	flags = tcp_outflags[tp->t_state];
282a0292f23SGarrett Wollman 	/*
2836d90faf3SPaul Saab 	 * Send any SACK-generated retransmissions.  If we're explicitly trying
2846d90faf3SPaul Saab 	 * to send out new data (when sendalot is 1), bypass this function.
2856d90faf3SPaul Saab 	 * If we retransmit in fast recovery mode, decrement snd_cwnd, since
2866d90faf3SPaul Saab 	 * we're replacing a (future) new transmission with a retransmission
2876d90faf3SPaul Saab 	 * now, and we previously incremented snd_cwnd in tcp_input().
2886d90faf3SPaul Saab 	 */
2896d90faf3SPaul Saab 	/*
2906d90faf3SPaul Saab 	 * Still in sack recovery , reset rxmit flag to zero.
2916d90faf3SPaul Saab 	 */
2926d90faf3SPaul Saab 	sack_rxmit = 0;
293a55db2b6SPaul Saab 	sack_bytes_rxmt = 0;
2946d90faf3SPaul Saab 	len = 0;
2956d90faf3SPaul Saab 	p = NULL;
296dbc42409SLawrence Stewart 	if ((tp->t_flags & TF_SACK_PERMIT) && IN_FASTRECOVERY(tp->t_flags) &&
297a55db2b6SPaul Saab 	    (p = tcp_sack_output(tp, &sack_bytes_rxmt))) {
2983ac12506SJonathan T. Looney 		uint32_t cwin;
299a55db2b6SPaul Saab 
3003ac12506SJonathan T. Looney 		cwin =
3013ac12506SJonathan T. Looney 		    imax(min(tp->snd_wnd, tp->snd_cwnd) - sack_bytes_rxmt, 0);
302f787edd8SJayanth Vijayaraghavan 		/* Do not retransmit SACK segments beyond snd_recover */
303f787edd8SJayanth Vijayaraghavan 		if (SEQ_GT(p->end, tp->snd_recover)) {
304f787edd8SJayanth Vijayaraghavan 			/*
305f787edd8SJayanth Vijayaraghavan 			 * (At least) part of sack hole extends beyond
306f787edd8SJayanth Vijayaraghavan 			 * snd_recover. Check to see if we can rexmit data
307f787edd8SJayanth Vijayaraghavan 			 * for this hole.
308f787edd8SJayanth Vijayaraghavan 			 */
309f787edd8SJayanth Vijayaraghavan 			if (SEQ_GEQ(p->rxmit, tp->snd_recover)) {
310f787edd8SJayanth Vijayaraghavan 				/*
311f787edd8SJayanth Vijayaraghavan 				 * Can't rexmit any more data for this hole.
312f787edd8SJayanth Vijayaraghavan 				 * That data will be rexmitted in the next
313f787edd8SJayanth Vijayaraghavan 				 * sack recovery episode, when snd_recover
314f787edd8SJayanth Vijayaraghavan 				 * moves past p->rxmit.
315f787edd8SJayanth Vijayaraghavan 				 */
316f787edd8SJayanth Vijayaraghavan 				p = NULL;
317f787edd8SJayanth Vijayaraghavan 				goto after_sack_rexmit;
31828173d49SHans Petter Selasky 			} else {
319f787edd8SJayanth Vijayaraghavan 				/* Can rexmit part of the current hole */
3203ac12506SJonathan T. Looney 				len = ((int32_t)ulmin(cwin,
32128173d49SHans Petter Selasky 				    SEQ_SUB(tp->snd_recover, p->rxmit)));
32228173d49SHans Petter Selasky 			}
32328173d49SHans Petter Selasky 		} else {
32428173d49SHans Petter Selasky 			len = ((int32_t)ulmin(cwin,
32528173d49SHans Petter Selasky 			    SEQ_SUB(p->end, p->rxmit)));
32628173d49SHans Petter Selasky 		}
3276d9e911fSMichael Tuexen 		if (len > 0) {
32828173d49SHans Petter Selasky 			off = SEQ_SUB(p->rxmit, tp->snd_una);
329f787edd8SJayanth Vijayaraghavan 			KASSERT(off >= 0,("%s: sack block to the left of una : %d",
330f787edd8SJayanth Vijayaraghavan 			    __func__, off));
331ab5c14d8SRobert Watson 			sack_rxmit = 1;
332ab5c14d8SRobert Watson 			sendalot = 1;
3336d90faf3SPaul Saab 		}
3346d90faf3SPaul Saab 	}
335f787edd8SJayanth Vijayaraghavan after_sack_rexmit:
3366d90faf3SPaul Saab 	/*
337a0292f23SGarrett Wollman 	 * Get standard flags, and add SYN or FIN if requested by 'hidden'
338a0292f23SGarrett Wollman 	 * state flags.
339a0292f23SGarrett Wollman 	 */
340a0292f23SGarrett Wollman 	if (tp->t_flags & TF_NEEDFIN)
341a0292f23SGarrett Wollman 		flags |= TH_FIN;
342a0292f23SGarrett Wollman 	if (tp->t_flags & TF_NEEDSYN)
343a0292f23SGarrett Wollman 		flags |= TH_SYN;
344a0292f23SGarrett Wollman 
345cf2942b6SRobert Watson 	SOCKBUF_LOCK(&so->so_snd);
346df8bae1dSRodney W. Grimes 	/*
347df8bae1dSRodney W. Grimes 	 * If in persist timeout with window of 0, send 1 byte.
348df8bae1dSRodney W. Grimes 	 * Otherwise, if window is small but nonzero
349df8bae1dSRodney W. Grimes 	 * and timer expired, we will send what we can
350df8bae1dSRodney W. Grimes 	 * and go to transmit state.
351df8bae1dSRodney W. Grimes 	 */
3522cdbfa66SPaul Saab 	if (tp->t_flags & TF_FORCEDATA) {
353201d185bSAndre Oppermann 		if (sendwin == 0) {
354df8bae1dSRodney W. Grimes 			/*
355df8bae1dSRodney W. Grimes 			 * If we still have some data to send, then
356df8bae1dSRodney W. Grimes 			 * clear the FIN bit.  Usually this would
357df8bae1dSRodney W. Grimes 			 * happen below when it realizes that we
358df8bae1dSRodney W. Grimes 			 * aren't sending all the data.  However,
35929089b51SJulian Elischer 			 * if we have exactly 1 byte of unsent data,
360df8bae1dSRodney W. Grimes 			 * then it won't clear the FIN bit below,
361df8bae1dSRodney W. Grimes 			 * and if we are in persist state, we wind
362df8bae1dSRodney W. Grimes 			 * up sending the packet without recording
363df8bae1dSRodney W. Grimes 			 * that we sent the FIN bit.
364df8bae1dSRodney W. Grimes 			 *
365df8bae1dSRodney W. Grimes 			 * We can't just blindly clear the FIN bit,
366df8bae1dSRodney W. Grimes 			 * because if we don't have any more data
367df8bae1dSRodney W. Grimes 			 * to send then the probe will be the FIN
368df8bae1dSRodney W. Grimes 			 * itself.
369df8bae1dSRodney W. Grimes 			 */
370cfa6009eSGleb Smirnoff 			if (off < sbused(&so->so_snd))
371df8bae1dSRodney W. Grimes 				flags &= ~TH_FIN;
372201d185bSAndre Oppermann 			sendwin = 1;
373df8bae1dSRodney W. Grimes 		} else {
374b8152ba7SAndre Oppermann 			tcp_timer_activate(tp, TT_PERSIST, 0);
375df8bae1dSRodney W. Grimes 			tp->t_rxtshift = 0;
376df8bae1dSRodney W. Grimes 		}
377df8bae1dSRodney W. Grimes 	}
378df8bae1dSRodney W. Grimes 
37928257b5cSMatthew Dillon 	/*
38028257b5cSMatthew Dillon 	 * If snd_nxt == snd_max and we have transmitted a FIN, the
38128257b5cSMatthew Dillon 	 * offset will be > 0 even if so_snd.sb_cc is 0, resulting in
382201d185bSAndre Oppermann 	 * a negative length.  This can also occur when TCP opens up
38328257b5cSMatthew Dillon 	 * its congestion window while receiving additional duplicate
38428257b5cSMatthew Dillon 	 * acks after fast-retransmit because TCP will reset snd_nxt
38528257b5cSMatthew Dillon 	 * to snd_max after the fast-retransmit.
38628257b5cSMatthew Dillon 	 *
38728257b5cSMatthew Dillon 	 * In the normal retransmit-FIN-only case, however, snd_nxt will
38828257b5cSMatthew Dillon 	 * be set to snd_una, the offset will be 0, and the length may
38928257b5cSMatthew Dillon 	 * wind up 0.
3906d90faf3SPaul Saab 	 *
3916d90faf3SPaul Saab 	 * If sack_rxmit is true we are retransmitting from the scoreboard
3926d90faf3SPaul Saab 	 * in which case len is already set.
39328257b5cSMatthew Dillon 	 */
394a55db2b6SPaul Saab 	if (sack_rxmit == 0) {
3952d05a1c8SRichard Scheffenegger 		if (sack_bytes_rxmt == 0) {
396cb503ae2SJonathan T. Looney 			len = ((int32_t)min(sbavail(&so->so_snd), sendwin) -
397cfa6009eSGleb Smirnoff 			    off);
3982d05a1c8SRichard Scheffenegger 		} else {
3993ac12506SJonathan T. Looney 			int32_t cwin;
400a55db2b6SPaul Saab 
401a55db2b6SPaul Saab 			/*
402a55db2b6SPaul Saab 			 * We are inside of a SACK recovery episode and are
403a55db2b6SPaul Saab 			 * sending new data, having retransmitted all the
404a55db2b6SPaul Saab 			 * data possible in the scoreboard.
405a55db2b6SPaul Saab 			 */
4063ac12506SJonathan T. Looney 			len = ((int32_t)min(sbavail(&so->so_snd), tp->snd_wnd) -
407cfa6009eSGleb Smirnoff 			    off);
4088d03f2b5SPaul Saab 			/*
4098d03f2b5SPaul Saab 			 * Don't remove this (len > 0) check !
4108d03f2b5SPaul Saab 			 * We explicitly check for len > 0 here (although it
4118d03f2b5SPaul Saab 			 * isn't really necessary), to work around a gcc
4128d03f2b5SPaul Saab 			 * optimization issue - to force gcc to compute
4138d03f2b5SPaul Saab 			 * len above. Without this check, the computation
4148d03f2b5SPaul Saab 			 * of len is bungled by the optimizer.
4158d03f2b5SPaul Saab 			 */
4168d03f2b5SPaul Saab 			if (len > 0) {
417a743fc88SRichard Scheffenegger 				cwin = tp->snd_cwnd - imax(0, (int32_t)
418a743fc88SRichard Scheffenegger 					(tp->snd_nxt - tp->snd_recover)) -
419a55db2b6SPaul Saab 					sack_bytes_rxmt;
420a55db2b6SPaul Saab 				if (cwin < 0)
421a55db2b6SPaul Saab 					cwin = 0;
4223ac12506SJonathan T. Looney 				len = imin(len, cwin);
423a55db2b6SPaul Saab 			}
424a55db2b6SPaul Saab 		}
4258d03f2b5SPaul Saab 	}
426a0292f23SGarrett Wollman 
427a0292f23SGarrett Wollman 	/*
428a0292f23SGarrett Wollman 	 * Lop off SYN bit if it has already been sent.  However, if this
429a0292f23SGarrett Wollman 	 * is SYN-SENT state and if segment contains data and if we don't
430a0292f23SGarrett Wollman 	 * know that foreign host supports TAO, suppress sending segment.
431a0292f23SGarrett Wollman 	 */
432a0292f23SGarrett Wollman 	if ((flags & TH_SYN) && SEQ_GT(tp->snd_nxt, tp->snd_una)) {
4334b8e98d6SQing Li 		if (tp->t_state != TCPS_SYN_RECEIVED)
434a0292f23SGarrett Wollman 			flags &= ~TH_SYN;
435281a0fd4SPatrick Kelsey 		/*
436281a0fd4SPatrick Kelsey 		 * When sending additional segments following a TFO SYN|ACK,
437281a0fd4SPatrick Kelsey 		 * do not include the SYN bit.
438281a0fd4SPatrick Kelsey 		 */
439dd7b86e2SGleb Smirnoff 		if ((tp->t_flags & TF_FASTOPEN) &&
440281a0fd4SPatrick Kelsey 		    (tp->t_state == TCPS_SYN_RECEIVED))
441281a0fd4SPatrick Kelsey 			flags &= ~TH_SYN;
442a0292f23SGarrett Wollman 		off--, len++;
443a0292f23SGarrett Wollman 	}
444a0292f23SGarrett Wollman 
44581165e48SAndras Olah 	/*
446c94c54e4SAndre Oppermann 	 * Be careful not to send data and/or FIN on SYN segments.
44781165e48SAndras Olah 	 * This measure is needed to prevent interoperability problems
44881165e48SAndras Olah 	 * with not fully conformant TCP implementations.
44981165e48SAndras Olah 	 */
450c94c54e4SAndre Oppermann 	if ((flags & TH_SYN) && (tp->t_flags & TF_NOOPT)) {
45181165e48SAndras Olah 		len = 0;
45281165e48SAndras Olah 		flags &= ~TH_FIN;
45381165e48SAndras Olah 	}
45481165e48SAndras Olah 
455281a0fd4SPatrick Kelsey 	/*
456c560df6fSPatrick Kelsey 	 * On TFO sockets, ensure no data is sent in the following cases:
457c560df6fSPatrick Kelsey 	 *
458c560df6fSPatrick Kelsey 	 *  - When retransmitting SYN|ACK on a passively-created socket
459c560df6fSPatrick Kelsey 	 *
460c560df6fSPatrick Kelsey 	 *  - When retransmitting SYN on an actively created socket
461c560df6fSPatrick Kelsey 	 *
462c560df6fSPatrick Kelsey 	 *  - When sending a zero-length cookie (cookie request) on an
463c560df6fSPatrick Kelsey 	 *    actively created socket
464c560df6fSPatrick Kelsey 	 *
465c560df6fSPatrick Kelsey 	 *  - When the socket is in the CLOSED state (RST is being sent)
466281a0fd4SPatrick Kelsey 	 */
467dd7b86e2SGleb Smirnoff 	if ((tp->t_flags & TF_FASTOPEN) &&
468c560df6fSPatrick Kelsey 	    (((flags & TH_SYN) && (tp->t_rxtshift > 0)) ||
469c560df6fSPatrick Kelsey 	     ((tp->t_state == TCPS_SYN_SENT) &&
470c560df6fSPatrick Kelsey 	      (tp->t_tfo_client_cookie_len == 0)) ||
471281a0fd4SPatrick Kelsey 	     (flags & TH_RST)))
472281a0fd4SPatrick Kelsey 		len = 0;
473*af700f43SMichael Tuexen 
474*af700f43SMichael Tuexen 	/* Without fast-open there should never be data sent on a SYN. */
475*af700f43SMichael Tuexen 	if ((flags & TH_SYN) && !(tp->t_flags & TF_FASTOPEN)) {
476*af700f43SMichael Tuexen 		len = 0;
477*af700f43SMichael Tuexen 	}
478*af700f43SMichael Tuexen 
47947a8e865SXin LI 	if (len <= 0) {
480df8bae1dSRodney W. Grimes 		/*
481df8bae1dSRodney W. Grimes 		 * If FIN has been sent but not acked,
482df8bae1dSRodney W. Grimes 		 * but we haven't been called to retransmit,
48328257b5cSMatthew Dillon 		 * len will be < 0.  Otherwise, window shrank
484df8bae1dSRodney W. Grimes 		 * after we sent into it.  If window shrank to 0,
4852d8266afSDavid Greenman 		 * cancel pending retransmit, pull snd_nxt back
4862d8266afSDavid Greenman 		 * to (closed) window, and set the persist timer
4872d8266afSDavid Greenman 		 * if it isn't already going.  If the window didn't
4882d8266afSDavid Greenman 		 * close completely, just wait for an ACK.
48947a8e865SXin LI 		 *
49047a8e865SXin LI 		 * We also do a general check here to ensure that
49147a8e865SXin LI 		 * we will set the persist timer when we have data
49247a8e865SXin LI 		 * to send, but a 0-byte window. This makes sure
49347a8e865SXin LI 		 * the persist timer is set even if the packet
49447a8e865SXin LI 		 * hits one of the "goto send" lines below.
495df8bae1dSRodney W. Grimes 		 */
496df8bae1dSRodney W. Grimes 		len = 0;
49747a8e865SXin LI 		if ((sendwin == 0) && (TCPS_HAVEESTABLISHED(tp->t_state)) &&
49808af8aacSRandall Stewart 		    (off < (int) sbavail(&so->so_snd)) &&
49908af8aacSRandall Stewart 		    !tcp_timer_active(tp, TT_PERSIST)) {
500b8152ba7SAndre Oppermann 			tcp_timer_activate(tp, TT_REXMT, 0);
5012d8266afSDavid Greenman 			tp->t_rxtshift = 0;
502df8bae1dSRodney W. Grimes 			tp->snd_nxt = tp->snd_una;
503b8152ba7SAndre Oppermann 			if (!tcp_timer_active(tp, TT_PERSIST))
5042d8266afSDavid Greenman 				tcp_setpersist(tp);
505df8bae1dSRodney W. Grimes 		}
506df8bae1dSRodney W. Grimes 	}
50728257b5cSMatthew Dillon 
5086741ecf5SAndre Oppermann 	/* len will be >= 0 after this point. */
509c4982faeSBjoern A. Zeeb 	KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
5106741ecf5SAndre Oppermann 
51166492feaSGleb Smirnoff 	tcp_sndbuf_autoscale(tp, so, sendwin);
5126741ecf5SAndre Oppermann 
5136741ecf5SAndre Oppermann 	/*
514ed420311SAndre Oppermann 	 * Decide if we can use TCP Segmentation Offloading (if supported by
515ed420311SAndre Oppermann 	 * hardware).
516b3c0f300SAndre Oppermann 	 *
517b3c0f300SAndre Oppermann 	 * TSO may only be used if we are in a pure bulk sending state.  The
518b3c0f300SAndre Oppermann 	 * presence of TCP-MD5, SACK retransmits, SACK advertizements and
519b3c0f300SAndre Oppermann 	 * IP options prevent using TSO.  With TSO the TCP header is the same
520b3c0f300SAndre Oppermann 	 * (except for the sequence number) for all generated packets.  This
521b3c0f300SAndre Oppermann 	 * makes it impossible to transmit any options which vary per generated
522b3c0f300SAndre Oppermann 	 * segment or packet.
523b6ff6724SHiren Panchasara 	 *
524b6ff6724SHiren Panchasara 	 * IPv4 handling has a clear separation of ip options and ip header
525b6ff6724SHiren Panchasara 	 * flags while IPv6 combines both in in6p_outputopts. ip6_optlen() does
526b6ff6724SHiren Panchasara 	 * the right thing below to provide length of just ip options and thus
527b6ff6724SHiren Panchasara 	 * checking for ipoptlen is enough to decide if ip options are present.
52828257b5cSMatthew Dillon 	 */
529fcf59617SAndrey V. Elsukov #if defined(IPSEC) || defined(IPSEC_SUPPORT)
5309ad0173dSBjoern A. Zeeb 	/*
5319ad0173dSBjoern A. Zeeb 	 * Pre-calculate here as we save another lookup into the darknesses
5329ad0173dSBjoern A. Zeeb 	 * of IPsec that way and can actually decide if TSO is ok.
5339ad0173dSBjoern A. Zeeb 	 */
534fcf59617SAndrey V. Elsukov #ifdef INET6
535fcf59617SAndrey V. Elsukov 	if (isipv6 && IPSEC_ENABLED(ipv6))
5369eb0e832SGleb Smirnoff 		ipsec_optlen = IPSEC_HDRSIZE(ipv6, inp);
537fcf59617SAndrey V. Elsukov #ifdef INET
538fcf59617SAndrey V. Elsukov 	else
5399ad0173dSBjoern A. Zeeb #endif
540fcf59617SAndrey V. Elsukov #endif /* INET6 */
541fcf59617SAndrey V. Elsukov #ifdef INET
542fcf59617SAndrey V. Elsukov 	if (IPSEC_ENABLED(ipv4))
5439eb0e832SGleb Smirnoff 		ipsec_optlen = IPSEC_HDRSIZE(ipv4, inp);
544fcf59617SAndrey V. Elsukov #endif /* INET */
545fcf59617SAndrey V. Elsukov #endif /* IPSEC */
546b6ff6724SHiren Panchasara #ifdef INET6
547b6ff6724SHiren Panchasara 	if (isipv6)
5489eb0e832SGleb Smirnoff 		ipoptlen = ip6_optlen(inp);
549b6ff6724SHiren Panchasara 	else
550b6ff6724SHiren Panchasara #endif
5519eb0e832SGleb Smirnoff 	if (inp->inp_options)
5529eb0e832SGleb Smirnoff 		ipoptlen = inp->inp_options->m_len -
553b6ff6724SHiren Panchasara 				offsetof(struct ipoption, ipopt_list);
554b6ff6724SHiren Panchasara 	else
555b6ff6724SHiren Panchasara 		ipoptlen = 0;
556fcf59617SAndrey V. Elsukov #if defined(IPSEC) || defined(IPSEC_SUPPORT)
557b6ff6724SHiren Panchasara 	ipoptlen += ipsec_optlen;
558b6ff6724SHiren Panchasara #endif
559b6ff6724SHiren Panchasara 
560ed420311SAndre Oppermann 	if ((tp->t_flags & TF_TSO) && V_tcp_do_tso && len > tp->t_maxseg &&
5619e644c23SMichael Tuexen 	    (tp->t_port == 0) &&
562b3c0f300SAndre Oppermann 	    ((tp->t_flags & TF_SIGNATURE) == 0) &&
563b3c0f300SAndre Oppermann 	    tp->rcv_numsacks == 0 && sack_rxmit == 0 &&
564c560df6fSPatrick Kelsey 	    ipoptlen == 0 && !(flags & TH_SYN))
565b3c0f300SAndre Oppermann 		tso = 1;
566153e5b57SAndre Oppermann 
5672d05a1c8SRichard Scheffenegger 	if (SEQ_LT((sack_rxmit ? p->rxmit : tp->snd_nxt) + len,
5682d05a1c8SRichard Scheffenegger 		    tp->snd_una + sbused(&so->so_snd))) {
5695d3b1b75SJayanth Vijayaraghavan 			flags &= ~TH_FIN;
5705d3b1b75SJayanth Vijayaraghavan 	}
571df8bae1dSRodney W. Grimes 
5723ac12506SJonathan T. Looney 	recwin = lmin(lmax(sbspace(&so->so_rcv), 0),
5733ac12506SJonathan T. Looney 	    (long)TCP_MAXWIN << tp->rcv_scale);
574df8bae1dSRodney W. Grimes 
575df8bae1dSRodney W. Grimes 	/*
576262c1c1aSMatthew Dillon 	 * Sender silly window avoidance.   We transmit under the following
577262c1c1aSMatthew Dillon 	 * conditions when len is non-zero:
578262c1c1aSMatthew Dillon 	 *
579b3c0f300SAndre Oppermann 	 *	- We have a full segment (or more with TSO)
580262c1c1aSMatthew Dillon 	 *	- This is the last buffer in a write()/send() and we are
581262c1c1aSMatthew Dillon 	 *	  either idle or running NODELAY
582262c1c1aSMatthew Dillon 	 *	- we've timed out (e.g. persist timer)
583262c1c1aSMatthew Dillon 	 *	- we have more then 1/2 the maximum send window's worth of
584262c1c1aSMatthew Dillon 	 *	  data (receiver may be limited the window size)
585262c1c1aSMatthew Dillon 	 *	- we need to retransmit
586df8bae1dSRodney W. Grimes 	 */
587df8bae1dSRodney W. Grimes 	if (len) {
588b3c0f300SAndre Oppermann 		if (len >= tp->t_maxseg)
589df8bae1dSRodney W. Grimes 			goto send;
590262c1c1aSMatthew Dillon 		/*
591e3995661SRichard Scheffenegger 		 * As the TCP header options are now
592e3995661SRichard Scheffenegger 		 * considered when setting up the initial
593e3995661SRichard Scheffenegger 		 * window, we would not send the last segment
594e3995661SRichard Scheffenegger 		 * if we skip considering the option length here.
595e3995661SRichard Scheffenegger 		 * Note: this may not work when tcp headers change
596e3995661SRichard Scheffenegger 		 * very dynamically in the future.
597e3995661SRichard Scheffenegger 		 */
598e3995661SRichard Scheffenegger 		if ((((tp->t_flags & TF_SIGNATURE) ?
599e3995661SRichard Scheffenegger 			PADTCPOLEN(TCPOLEN_SIGNATURE) : 0) +
600e3995661SRichard Scheffenegger 		    ((tp->t_flags & TF_RCVD_TSTMP) ?
601e3995661SRichard Scheffenegger 			PADTCPOLEN(TCPOLEN_TIMESTAMP) : 0) +
602e3995661SRichard Scheffenegger 		    len) >= tp->t_maxseg)
603e3995661SRichard Scheffenegger 			goto send;
604e3995661SRichard Scheffenegger 		/*
605262c1c1aSMatthew Dillon 		 * NOTE! on localhost connections an 'ack' from the remote
606262c1c1aSMatthew Dillon 		 * end may occur synchronously with the output and cause
607262c1c1aSMatthew Dillon 		 * us to flush a buffer queued with moretocome.  XXX
608262c1c1aSMatthew Dillon 		 *
609262c1c1aSMatthew Dillon 		 * note: the len + off check is almost certainly unnecessary.
610262c1c1aSMatthew Dillon 		 */
611262c1c1aSMatthew Dillon 		if (!(tp->t_flags & TF_MORETOCOME) &&	/* normal case */
612262c1c1aSMatthew Dillon 		    (idle || (tp->t_flags & TF_NODELAY)) &&
6133ac12506SJonathan T. Looney 		    (uint32_t)len + (uint32_t)off >= sbavail(&so->so_snd) &&
614262c1c1aSMatthew Dillon 		    (tp->t_flags & TF_NOPUSH) == 0) {
615df8bae1dSRodney W. Grimes 			goto send;
616262c1c1aSMatthew Dillon 		}
6172cdbfa66SPaul Saab 		if (tp->t_flags & TF_FORCEDATA)		/* typ. timeout case */
618df8bae1dSRodney W. Grimes 			goto send;
619a0292f23SGarrett Wollman 		if (len >= tp->max_sndwnd / 2 && tp->max_sndwnd > 0)
620df8bae1dSRodney W. Grimes 			goto send;
621262c1c1aSMatthew Dillon 		if (SEQ_LT(tp->snd_nxt, tp->snd_max))	/* retransmit case */
622df8bae1dSRodney W. Grimes 			goto send;
6236d90faf3SPaul Saab 		if (sack_rxmit)
6246d90faf3SPaul Saab 			goto send;
625df8bae1dSRodney W. Grimes 	}
626df8bae1dSRodney W. Grimes 
627df8bae1dSRodney W. Grimes 	/*
628f62563d3SAndre Oppermann 	 * Sending of standalone window updates.
629f62563d3SAndre Oppermann 	 *
63078f59b4bSAndre Oppermann 	 * Window updates are important when we close our window due to a
63178f59b4bSAndre Oppermann 	 * full socket buffer and are opening it again after the application
632f62563d3SAndre Oppermann 	 * reads data from it.  Once the window has opened again and the
633f62563d3SAndre Oppermann 	 * remote end starts to send again the ACK clock takes over and
634f62563d3SAndre Oppermann 	 * provides the most current window information.
635f62563d3SAndre Oppermann 	 *
63678f59b4bSAndre Oppermann 	 * We must avoid the silly window syndrome whereas every read
637f62563d3SAndre Oppermann 	 * from the receive buffer, no matter how small, causes a window
638f62563d3SAndre Oppermann 	 * update to be sent.  We also should avoid sending a flurry of
639f62563d3SAndre Oppermann 	 * window updates when the socket buffer had queued a lot of data
640f62563d3SAndre Oppermann 	 * and the application is doing small reads.
641f62563d3SAndre Oppermann 	 *
642f62563d3SAndre Oppermann 	 * Prevent a flurry of pointless window updates by only sending
643f62563d3SAndre Oppermann 	 * an update when we can increase the advertized window by more
644f62563d3SAndre Oppermann 	 * than 1/4th of the socket buffer capacity.  When the buffer is
645f62563d3SAndre Oppermann 	 * getting full or is very small be more aggressive and send an
646f62563d3SAndre Oppermann 	 * update whenever we can increase by two mss sized segments.
647f62563d3SAndre Oppermann 	 * In all other situations the ACK's to new incoming data will
648f62563d3SAndre Oppermann 	 * carry further window increases.
6494249614cSAndre Oppermann 	 *
6504249614cSAndre Oppermann 	 * Don't send an independent window update if a delayed
6514249614cSAndre Oppermann 	 * ACK is pending (it will get piggy-backed on it) or the
6524249614cSAndre Oppermann 	 * remote side already has done a half-close and won't send
653c809435bSGleb Smirnoff 	 * more data.
654df8bae1dSRodney W. Grimes 	 */
655b7de7d87SAndre Oppermann 	if (recwin > 0 && !(tp->t_flags & TF_NEEDSYN) &&
6564249614cSAndre Oppermann 	    !(tp->t_flags & TF_DELACK) &&
657b7de7d87SAndre Oppermann 	    !TCPS_HAVERCVDFIN(tp->t_state)) {
658df8bae1dSRodney W. Grimes 		/*
659f62563d3SAndre Oppermann 		 * "adv" is the amount we could increase the window,
660df8bae1dSRodney W. Grimes 		 * taking into account that we are limited by
661df8bae1dSRodney W. Grimes 		 * TCP_MAXWIN << tp->rcv_scale.
662df8bae1dSRodney W. Grimes 		 */
6633ac12506SJonathan T. Looney 		int32_t adv;
664f701e30dSJohn Baldwin 		int oldwin;
665f701e30dSJohn Baldwin 
6663ac12506SJonathan T. Looney 		adv = recwin;
667f701e30dSJohn Baldwin 		if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt)) {
668f701e30dSJohn Baldwin 			oldwin = (tp->rcv_adv - tp->rcv_nxt);
669f092a3c7SRandall Stewart 			if (adv > oldwin)
670f701e30dSJohn Baldwin 				adv -= oldwin;
671f092a3c7SRandall Stewart 			else
672f092a3c7SRandall Stewart 				adv = 0;
673f701e30dSJohn Baldwin 		} else
674f701e30dSJohn Baldwin 			oldwin = 0;
675df8bae1dSRodney W. Grimes 
676da84b2e6SJohn Baldwin 		/*
6770dda76b8SJonathan T. Looney 		 * If the new window size ends up being the same as or less
6780dda76b8SJonathan T. Looney 		 * than the old size when it is scaled, then don't force
6790dda76b8SJonathan T. Looney 		 * a window update.
680da84b2e6SJohn Baldwin 		 */
6810dda76b8SJonathan T. Looney 		if (oldwin >> tp->rcv_scale >= (adv + oldwin) >> tp->rcv_scale)
682da84b2e6SJohn Baldwin 			goto dontupdate;
683f62563d3SAndre Oppermann 
6843ac12506SJonathan T. Looney 		if (adv >= (int32_t)(2 * tp->t_maxseg) &&
6853ac12506SJonathan T. Looney 		    (adv >= (int32_t)(so->so_rcv.sb_hiwat / 4) ||
6863ac12506SJonathan T. Looney 		     recwin <= (so->so_rcv.sb_hiwat / 8) ||
68750075939SStephen Hurd 		     so->so_rcv.sb_hiwat <= 8 * tp->t_maxseg ||
68850075939SStephen Hurd 		     adv >= TCP_MAXWIN << tp->rcv_scale))
689df8bae1dSRodney W. Grimes 			goto send;
6908d62aae8SMichael Tuexen 		if (2 * adv >= (int32_t)so->so_rcv.sb_hiwat)
6918d62aae8SMichael Tuexen 			goto send;
692df8bae1dSRodney W. Grimes 	}
693da84b2e6SJohn Baldwin dontupdate:
694df8bae1dSRodney W. Grimes 
695df8bae1dSRodney W. Grimes 	/*
69628257b5cSMatthew Dillon 	 * Send if we owe the peer an ACK, RST, SYN, or urgent data.  ACKNOW
69728257b5cSMatthew Dillon 	 * is also a catch-all for the retransmit timer timeout case.
698df8bae1dSRodney W. Grimes 	 */
699df8bae1dSRodney W. Grimes 	if (tp->t_flags & TF_ACKNOW)
700df8bae1dSRodney W. Grimes 		goto send;
701a0292f23SGarrett Wollman 	if ((flags & TH_RST) ||
702a0292f23SGarrett Wollman 	    ((flags & TH_SYN) && (tp->t_flags & TF_NEEDSYN) == 0))
703df8bae1dSRodney W. Grimes 		goto send;
704df8bae1dSRodney W. Grimes 	if (SEQ_GT(tp->snd_up, tp->snd_una))
705df8bae1dSRodney W. Grimes 		goto send;
706df8bae1dSRodney W. Grimes 	/*
707df8bae1dSRodney W. Grimes 	 * If our state indicates that FIN should be sent
70828257b5cSMatthew Dillon 	 * and we have not yet done so, then we need to send.
709df8bae1dSRodney W. Grimes 	 */
710df8bae1dSRodney W. Grimes 	if (flags & TH_FIN &&
711df8bae1dSRodney W. Grimes 	    ((tp->t_flags & TF_SENTFIN) == 0 || tp->snd_nxt == tp->snd_una))
712df8bae1dSRodney W. Grimes 		goto send;
7136d90faf3SPaul Saab 	/*
7146d90faf3SPaul Saab 	 * In SACK, it is possible for tcp_output to fail to send a segment
7156d90faf3SPaul Saab 	 * after the retransmission timer has been turned off.  Make sure
7166d90faf3SPaul Saab 	 * that the retransmission timer is set.
7176d90faf3SPaul Saab 	 */
7183529149eSAndre Oppermann 	if ((tp->t_flags & TF_SACK_PERMIT) &&
7193529149eSAndre Oppermann 	    SEQ_GT(tp->snd_max, tp->snd_una) &&
720b8152ba7SAndre Oppermann 	    !tcp_timer_active(tp, TT_REXMT) &&
721b8152ba7SAndre Oppermann 	    !tcp_timer_active(tp, TT_PERSIST)) {
72208af8aacSRandall Stewart 		tcp_timer_activate(tp, TT_REXMT, TP_RXTCUR(tp));
723cf2942b6SRobert Watson 		goto just_return;
7246d90faf3SPaul Saab 	}
725df8bae1dSRodney W. Grimes 	/*
726df8bae1dSRodney W. Grimes 	 * TCP window updates are not reliable, rather a polling protocol
727df8bae1dSRodney W. Grimes 	 * using ``persist'' packets is used to insure receipt of window
728df8bae1dSRodney W. Grimes 	 * updates.  The three ``states'' for the output side are:
729df8bae1dSRodney W. Grimes 	 *	idle			not doing retransmits or persists
730df8bae1dSRodney W. Grimes 	 *	persisting		to move a small or zero window
731df8bae1dSRodney W. Grimes 	 *	(re)transmitting	and thereby not persisting
732df8bae1dSRodney W. Grimes 	 *
733b8152ba7SAndre Oppermann 	 * tcp_timer_active(tp, TT_PERSIST)
7349b8b58e0SJonathan Lemon 	 *	is true when we are in persist state.
7352cdbfa66SPaul Saab 	 * (tp->t_flags & TF_FORCEDATA)
736df8bae1dSRodney W. Grimes 	 *	is set when we are called to send a persist packet.
737b8152ba7SAndre Oppermann 	 * tcp_timer_active(tp, TT_REXMT)
738df8bae1dSRodney W. Grimes 	 *	is set when we are retransmitting
739df8bae1dSRodney W. Grimes 	 * The output side is idle when both timers are zero.
740df8bae1dSRodney W. Grimes 	 *
741df8bae1dSRodney W. Grimes 	 * If send window is too small, there is data to transmit, and no
742df8bae1dSRodney W. Grimes 	 * retransmit or persist is pending, then go to persist state.
743df8bae1dSRodney W. Grimes 	 * If nothing happens soon, send when timer expires:
744df8bae1dSRodney W. Grimes 	 * if window is nonzero, transmit what we can,
745df8bae1dSRodney W. Grimes 	 * otherwise force out a byte.
746df8bae1dSRodney W. Grimes 	 */
747cfa6009eSGleb Smirnoff 	if (sbavail(&so->so_snd) && !tcp_timer_active(tp, TT_REXMT) &&
748b8152ba7SAndre Oppermann 	    !tcp_timer_active(tp, TT_PERSIST)) {
749df8bae1dSRodney W. Grimes 		tp->t_rxtshift = 0;
750df8bae1dSRodney W. Grimes 		tcp_setpersist(tp);
751df8bae1dSRodney W. Grimes 	}
752df8bae1dSRodney W. Grimes 
753df8bae1dSRodney W. Grimes 	/*
754df8bae1dSRodney W. Grimes 	 * No reason to send a segment, just return.
755df8bae1dSRodney W. Grimes 	 */
756cf2942b6SRobert Watson just_return:
757cf2942b6SRobert Watson 	SOCKBUF_UNLOCK(&so->so_snd);
758df8bae1dSRodney W. Grimes 	return (0);
759df8bae1dSRodney W. Grimes 
760df8bae1dSRodney W. Grimes send:
761cf2942b6SRobert Watson 	SOCKBUF_LOCK_ASSERT(&so->so_snd);
762f6f6703fSSean Bruno 	if (len > 0) {
763f6f6703fSSean Bruno 		if (len >= tp->t_maxseg)
764f6f6703fSSean Bruno 			tp->t_flags2 |= TF2_PLPMTU_MAXSEGSNT;
765f6f6703fSSean Bruno 		else
766f6f6703fSSean Bruno 			tp->t_flags2 &= ~TF2_PLPMTU_MAXSEGSNT;
767f6f6703fSSean Bruno 	}
768df8bae1dSRodney W. Grimes 	/*
769df8bae1dSRodney W. Grimes 	 * Before ESTABLISHED, force sending of initial options
770df8bae1dSRodney W. Grimes 	 * unless TCP set not to do any options.
771df8bae1dSRodney W. Grimes 	 * NOTE: we assume that the IP/TCP header plus TCP options
772df8bae1dSRodney W. Grimes 	 * always fit in a single mbuf, leaving room for a maximum
773df8bae1dSRodney W. Grimes 	 * link header, i.e.
77433841545SHajimu UMEMOTO 	 *	max_linkhdr + sizeof (struct tcpiphdr) + optlen <= MCLBYTES
775df8bae1dSRodney W. Grimes 	 */
776df8bae1dSRodney W. Grimes 	optlen = 0;
777fb59c426SYoshinobu Inoue #ifdef INET6
778fb59c426SYoshinobu Inoue 	if (isipv6)
779fb59c426SYoshinobu Inoue 		hdrlen = sizeof (struct ip6_hdr) + sizeof (struct tcphdr);
780fb59c426SYoshinobu Inoue 	else
781fb59c426SYoshinobu Inoue #endif
782df8bae1dSRodney W. Grimes 		hdrlen = sizeof (struct tcpiphdr);
78302a1a643SAndre Oppermann 
784ed782b9fSMichael Tuexen 	if (flags & TH_SYN) {
785ed782b9fSMichael Tuexen 		tp->snd_nxt = tp->iss;
786ed782b9fSMichael Tuexen 	}
787ed782b9fSMichael Tuexen 
78802a1a643SAndre Oppermann 	/*
78902a1a643SAndre Oppermann 	 * Compute options for segment.
79002a1a643SAndre Oppermann 	 * We only have to care about SYN and established connection
79102a1a643SAndre Oppermann 	 * segments.  Options for SYN-ACK segments are handled in TCP
79202a1a643SAndre Oppermann 	 * syncache.
79302a1a643SAndre Oppermann 	 */
79402a1a643SAndre Oppermann 	to.to_flags = 0;
795f73d9fd2SGleb Smirnoff 	if ((tp->t_flags & TF_NOOPT) == 0) {
79602a1a643SAndre Oppermann 		/* Maximum segment size. */
797df8bae1dSRodney W. Grimes 		if (flags & TH_SYN) {
7989eb0e832SGleb Smirnoff 			to.to_mss = tcp_mssopt(&inp->inp_inc);
7999e644c23SMichael Tuexen 			if (tp->t_port)
8009e644c23SMichael Tuexen 				to.to_mss -= V_tcp_udp_tunneling_overhead;
80102a1a643SAndre Oppermann 			to.to_flags |= TOF_MSS;
80218a75309SPatrick Kelsey 
803281a0fd4SPatrick Kelsey 			/*
804c560df6fSPatrick Kelsey 			 * On SYN or SYN|ACK transmits on TFO connections,
805c560df6fSPatrick Kelsey 			 * only include the TFO option if it is not a
806c560df6fSPatrick Kelsey 			 * retransmit, as the presence of the TFO option may
807c560df6fSPatrick Kelsey 			 * have caused the original SYN or SYN|ACK to have
808c560df6fSPatrick Kelsey 			 * been dropped by a middlebox.
809281a0fd4SPatrick Kelsey 			 */
810dd7b86e2SGleb Smirnoff 			if ((tp->t_flags & TF_FASTOPEN) &&
811281a0fd4SPatrick Kelsey 			    (tp->t_rxtshift == 0)) {
812c560df6fSPatrick Kelsey 				if (tp->t_state == TCPS_SYN_RECEIVED) {
813281a0fd4SPatrick Kelsey 					to.to_tfo_len = TCP_FASTOPEN_COOKIE_LEN;
814c560df6fSPatrick Kelsey 					to.to_tfo_cookie =
815c560df6fSPatrick Kelsey 					    (u_int8_t *)&tp->t_tfo_cookie.server;
816281a0fd4SPatrick Kelsey 					to.to_flags |= TOF_FASTOPEN;
817c560df6fSPatrick Kelsey 					wanted_cookie = 1;
818c560df6fSPatrick Kelsey 				} else if (tp->t_state == TCPS_SYN_SENT) {
819c560df6fSPatrick Kelsey 					to.to_tfo_len =
820c560df6fSPatrick Kelsey 					    tp->t_tfo_client_cookie_len;
821c560df6fSPatrick Kelsey 					to.to_tfo_cookie =
822c560df6fSPatrick Kelsey 					    tp->t_tfo_cookie.client;
823c560df6fSPatrick Kelsey 					to.to_flags |= TOF_FASTOPEN;
824c560df6fSPatrick Kelsey 					wanted_cookie = 1;
825c560df6fSPatrick Kelsey 					/*
826c560df6fSPatrick Kelsey 					 * If we wind up having more data to
827c560df6fSPatrick Kelsey 					 * send with the SYN than can fit in
828c560df6fSPatrick Kelsey 					 * one segment, don't send any more
829c560df6fSPatrick Kelsey 					 * until the SYN|ACK comes back from
830c560df6fSPatrick Kelsey 					 * the other end.
831c560df6fSPatrick Kelsey 					 */
832c560df6fSPatrick Kelsey 					dont_sendalot = 1;
833c560df6fSPatrick Kelsey 				}
834281a0fd4SPatrick Kelsey 			}
835df8bae1dSRodney W. Grimes 		}
83602a1a643SAndre Oppermann 		/* Window scaling. */
83702a1a643SAndre Oppermann 		if ((flags & TH_SYN) && (tp->t_flags & TF_REQ_SCALE)) {
83802a1a643SAndre Oppermann 			to.to_wscale = tp->request_r_scale;
83902a1a643SAndre Oppermann 			to.to_flags |= TOF_SCALE;
840df8bae1dSRodney W. Grimes 		}
84102a1a643SAndre Oppermann 		/* Timestamps. */
84202a1a643SAndre Oppermann 		if ((tp->t_flags & TF_RCVD_TSTMP) ||
84302a1a643SAndre Oppermann 		    ((flags & TH_SYN) && (tp->t_flags & TF_REQ_TSTMP))) {
84410d20c84SMatt Macy 			curticks = tcp_ts_getticks();
84510d20c84SMatt Macy 			to.to_tsval = curticks + tp->ts_offset;
84602a1a643SAndre Oppermann 			to.to_tsecr = tp->ts_recent;
84702a1a643SAndre Oppermann 			to.to_flags |= TOF_TS;
84810d20c84SMatt Macy 			if (tp->t_rxtshift == 1)
84910d20c84SMatt Macy 				tp->t_badrxtwin = curticks;
850e44c1887SSteven Hartland 		}
851e44c1887SSteven Hartland 
8526741ecf5SAndre Oppermann 		/* Set receive buffer autosizing timestamp. */
85302a1a643SAndre Oppermann 		if (tp->rfbuf_ts == 0 &&
85402a1a643SAndre Oppermann 		    (so->so_rcv.sb_flags & SB_AUTOSIZE))
855d8951c8aSBjoern A. Zeeb 			tp->rfbuf_ts = tcp_ts_getticks();
856e44c1887SSteven Hartland 
85702a1a643SAndre Oppermann 		/* Selective ACK's. */
8583529149eSAndre Oppermann 		if (tp->t_flags & TF_SACK_PERMIT) {
85902a1a643SAndre Oppermann 			if (flags & TH_SYN)
86002a1a643SAndre Oppermann 				to.to_flags |= TOF_SACKPERM;
86102a1a643SAndre Oppermann 			else if (TCPS_HAVEESTABLISHED(tp->t_state) &&
86202a1a643SAndre Oppermann 			    tp->rcv_numsacks > 0) {
86302a1a643SAndre Oppermann 				to.to_flags |= TOF_SACK;
86402a1a643SAndre Oppermann 				to.to_nsacks = tp->rcv_numsacks;
86502a1a643SAndre Oppermann 				to.to_sacks = (u_char *)tp->sackblks;
86602a1a643SAndre Oppermann 			}
86702a1a643SAndre Oppermann 		}
868fcf59617SAndrey V. Elsukov #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
86902a1a643SAndre Oppermann 		/* TCP-MD5 (RFC2385). */
870fcf59617SAndrey V. Elsukov 		/*
871fcf59617SAndrey V. Elsukov 		 * Check that TCP_MD5SIG is enabled in tcpcb to
872fcf59617SAndrey V. Elsukov 		 * account the size needed to set this TCP option.
873fcf59617SAndrey V. Elsukov 		 */
87402a1a643SAndre Oppermann 		if (tp->t_flags & TF_SIGNATURE)
87502a1a643SAndre Oppermann 			to.to_flags |= TOF_SIGNATURE;
8761cfd4b53SBruce M Simpson #endif /* TCP_SIGNATURE */
8771cfd4b53SBruce M Simpson 
87802a1a643SAndre Oppermann 		/* Processing the options. */
8794a411b9fSBjoern A. Zeeb 		hdrlen += optlen = tcp_addoptions(&to, opt);
880c560df6fSPatrick Kelsey 		/*
881c560df6fSPatrick Kelsey 		 * If we wanted a TFO option to be added, but it was unable
882c560df6fSPatrick Kelsey 		 * to fit, ensure no data is sent.
883c560df6fSPatrick Kelsey 		 */
884dd7b86e2SGleb Smirnoff 		if ((tp->t_flags & TF_FASTOPEN) && wanted_cookie &&
885c560df6fSPatrick Kelsey 		    !(to.to_flags & TOF_FASTOPEN))
886c560df6fSPatrick Kelsey 			len = 0;
887be3f3b5eSPaul Saab 	}
8889e644c23SMichael Tuexen 	if (tp->t_port) {
8899e644c23SMichael Tuexen 		if (V_tcp_udp_tunneling_port == 0) {
8909e644c23SMichael Tuexen 			/* The port was removed?? */
8919e644c23SMichael Tuexen 			SOCKBUF_UNLOCK(&so->so_snd);
8929e644c23SMichael Tuexen 			return (EHOSTUNREACH);
8939e644c23SMichael Tuexen 		}
8949e644c23SMichael Tuexen 		hdrlen += sizeof(struct udphdr);
8959e644c23SMichael Tuexen 	}
896df8bae1dSRodney W. Grimes 	/*
897df8bae1dSRodney W. Grimes 	 * Adjust data length if insertion of options will
8980c39d38dSGleb Smirnoff 	 * bump the packet length beyond the t_maxseg length.
899a0292f23SGarrett Wollman 	 * Clear the FIN bit because we cut off the tail of
900a0292f23SGarrett Wollman 	 * the segment.
901df8bae1dSRodney W. Grimes 	 */
9020c39d38dSGleb Smirnoff 	if (len + optlen + ipoptlen > tp->t_maxseg) {
903297a37f3SDavid Greenman 		flags &= ~TH_FIN;
904ed420311SAndre Oppermann 
905b3c0f300SAndre Oppermann 		if (tso) {
9069fd573c3SHans Petter Selasky 			u_int if_hw_tsomax;
9079fd573c3SHans Petter Selasky 			u_int moff;
9089fd573c3SHans Petter Selasky 			int max_len;
9099fd573c3SHans Petter Selasky 
9109fd573c3SHans Petter Selasky 			/* extract TSO information */
9119fd573c3SHans Petter Selasky 			if_hw_tsomax = tp->t_tsomax;
9129fd573c3SHans Petter Selasky 			if_hw_tsomaxsegcount = tp->t_tsomaxsegcount;
9139fd573c3SHans Petter Selasky 			if_hw_tsomaxsegsize = tp->t_tsomaxsegsize;
9149fd573c3SHans Petter Selasky 
9159fd573c3SHans Petter Selasky 			/*
9169fd573c3SHans Petter Selasky 			 * Limit a TSO burst to prevent it from
9179fd573c3SHans Petter Selasky 			 * overflowing or exceeding the maximum length
9189fd573c3SHans Petter Selasky 			 * allowed by the network interface:
9199fd573c3SHans Petter Selasky 			 */
920ed420311SAndre Oppermann 			KASSERT(ipoptlen == 0,
921ed420311SAndre Oppermann 			    ("%s: TSO can't do IP options", __func__));
922ed420311SAndre Oppermann 
923ed420311SAndre Oppermann 			/*
9249fd573c3SHans Petter Selasky 			 * Check if we should limit by maximum payload
9259fd573c3SHans Petter Selasky 			 * length:
926ed420311SAndre Oppermann 			 */
9279fd573c3SHans Petter Selasky 			if (if_hw_tsomax != 0) {
9289fd573c3SHans Petter Selasky 				/* compute maximum TSO length */
929d76d4012SHans Petter Selasky 				max_len = (if_hw_tsomax - hdrlen -
930d76d4012SHans Petter Selasky 				    max_linkhdr);
9319fd573c3SHans Petter Selasky 				if (max_len <= 0) {
9329fd573c3SHans Petter Selasky 					len = 0;
9333c7c188cSHans Petter Selasky 				} else if (len > max_len) {
934b3c0f300SAndre Oppermann 					sendalot = 1;
9353c7c188cSHans Petter Selasky 					len = max_len;
9369fd573c3SHans Petter Selasky 				}
9379fd573c3SHans Petter Selasky 			}
93874e10fb6SJohn Baldwin 
939ed420311SAndre Oppermann 			/*
940ed420311SAndre Oppermann 			 * Prevent the last segment from being
9419fd573c3SHans Petter Selasky 			 * fractional unless the send sockbuf can be
9429fd573c3SHans Petter Selasky 			 * emptied:
943ed420311SAndre Oppermann 			 */
9440c39d38dSGleb Smirnoff 			max_len = (tp->t_maxseg - optlen);
9453ac12506SJonathan T. Looney 			if (((uint32_t)off + (uint32_t)len) <
9463ac12506SJonathan T. Looney 			    sbavail(&so->so_snd)) {
9473c7c188cSHans Petter Selasky 				moff = len % max_len;
9489fd573c3SHans Petter Selasky 				if (moff != 0) {
9499fd573c3SHans Petter Selasky 					len -= moff;
950b3c0f300SAndre Oppermann 					sendalot = 1;
951ed420311SAndre Oppermann 				}
9529fd573c3SHans Petter Selasky 			}
9539fd573c3SHans Petter Selasky 
9549fd573c3SHans Petter Selasky 			/*
9559fd573c3SHans Petter Selasky 			 * In case there are too many small fragments
9569fd573c3SHans Petter Selasky 			 * don't use TSO:
9579fd573c3SHans Petter Selasky 			 */
9583c7c188cSHans Petter Selasky 			if (len <= max_len) {
9593c7c188cSHans Petter Selasky 				len = max_len;
9609fd573c3SHans Petter Selasky 				sendalot = 1;
9619fd573c3SHans Petter Selasky 				tso = 0;
9629fd573c3SHans Petter Selasky 			}
963ed420311SAndre Oppermann 
964ed420311SAndre Oppermann 			/*
965ed420311SAndre Oppermann 			 * Send the FIN in a separate segment
966ed420311SAndre Oppermann 			 * after the bulk sending is done.
967ed420311SAndre Oppermann 			 * We don't trust the TSO implementations
968ed420311SAndre Oppermann 			 * to clear the FIN flag on all but the
969ed420311SAndre Oppermann 			 * last segment.
970ed420311SAndre Oppermann 			 */
971ed420311SAndre Oppermann 			if (tp->t_flags & TF_NEEDFIN)
972ed420311SAndre Oppermann 				sendalot = 1;
973b3c0f300SAndre Oppermann 		} else {
97412a43d0dSMichael Tuexen 			if (optlen + ipoptlen >= tp->t_maxseg) {
97512a43d0dSMichael Tuexen 				/*
97612a43d0dSMichael Tuexen 				 * Since we don't have enough space to put
97712a43d0dSMichael Tuexen 				 * the IP header chain and the TCP header in
97812a43d0dSMichael Tuexen 				 * one packet as required by RFC 7112, don't
97912a43d0dSMichael Tuexen 				 * send it. Also ensure that at least one
98012a43d0dSMichael Tuexen 				 * byte of the payload can be put into the
98112a43d0dSMichael Tuexen 				 * TCP segment.
98212a43d0dSMichael Tuexen 				 */
98312a43d0dSMichael Tuexen 				SOCKBUF_UNLOCK(&so->so_snd);
98412a43d0dSMichael Tuexen 				error = EMSGSIZE;
98512a43d0dSMichael Tuexen 				sack_rxmit = 0;
98612a43d0dSMichael Tuexen 				goto out;
98712a43d0dSMichael Tuexen 			}
9880c39d38dSGleb Smirnoff 			len = tp->t_maxseg - optlen - ipoptlen;
989df8bae1dSRodney W. Grimes 			sendalot = 1;
990c560df6fSPatrick Kelsey 			if (dont_sendalot)
991c560df6fSPatrick Kelsey 				sendalot = 0;
992df8bae1dSRodney W. Grimes 		}
993ed420311SAndre Oppermann 	} else
994ed420311SAndre Oppermann 		tso = 0;
995ed420311SAndre Oppermann 
996ed420311SAndre Oppermann 	KASSERT(len + hdrlen + ipoptlen <= IP_MAXPACKET,
997ed420311SAndre Oppermann 	    ("%s: len > IP_MAXPACKET", __func__));
998df8bae1dSRodney W. Grimes 
999a0292f23SGarrett Wollman /*#ifdef DIAGNOSTIC*/
1000a683a7ddSYoshinobu Inoue #ifdef INET6
1001a683a7ddSYoshinobu Inoue 	if (max_linkhdr + hdrlen > MCLBYTES)
1002a683a7ddSYoshinobu Inoue #else
1003df8bae1dSRodney W. Grimes 	if (max_linkhdr + hdrlen > MHLEN)
1004a683a7ddSYoshinobu Inoue #endif
1005f10e85d7SLuigi Rizzo 		panic("tcphdr too big");
1006a0292f23SGarrett Wollman /*#endif*/
1007df8bae1dSRodney W. Grimes 
1008df8bae1dSRodney W. Grimes 	/*
1009c4982faeSBjoern A. Zeeb 	 * This KASSERT is here to catch edge cases at a well defined place.
1010c4982faeSBjoern A. Zeeb 	 * Before, those had triggered (random) panic conditions further down.
1011c4982faeSBjoern A. Zeeb 	 */
1012c4982faeSBjoern A. Zeeb 	KASSERT(len >= 0, ("[%s:%d]: len < 0", __func__, __LINE__));
1013c4982faeSBjoern A. Zeeb 
1014c4982faeSBjoern A. Zeeb 	/*
1015df8bae1dSRodney W. Grimes 	 * Grab a header mbuf, attaching a copy of data to
1016df8bae1dSRodney W. Grimes 	 * be transmitted, and initialize the header from
1017df8bae1dSRodney W. Grimes 	 * the template for sends on this connection.
1018df8bae1dSRodney W. Grimes 	 */
1019df8bae1dSRodney W. Grimes 	if (len) {
10204e023759SAndre Oppermann 		struct mbuf *mb;
1021581a046aSRandall Stewart 		struct sockbuf *msb;
10224e023759SAndre Oppermann 		u_int moff;
10234e023759SAndre Oppermann 
1024adc56f5aSEdward Tomasz Napierala 		if ((tp->t_flags & TF_FORCEDATA) && len == 1) {
102578b50714SRobert Watson 			TCPSTAT_INC(tcps_sndprobe);
1026adc56f5aSEdward Tomasz Napierala #ifdef STATS
1027adc56f5aSEdward Tomasz Napierala 			if (SEQ_LT(tp->snd_nxt, tp->snd_max))
1028adc56f5aSEdward Tomasz Napierala 				stats_voi_update_abs_u32(tp->t_stats,
1029adc56f5aSEdward Tomasz Napierala 				VOI_TCP_RETXPB, len);
1030adc56f5aSEdward Tomasz Napierala 			else
1031adc56f5aSEdward Tomasz Napierala 				stats_voi_update_abs_u64(tp->t_stats,
1032adc56f5aSEdward Tomasz Napierala 				    VOI_TCP_TXPB, len);
1033adc56f5aSEdward Tomasz Napierala #endif /* STATS */
1034adc56f5aSEdward Tomasz Napierala 		} else if (SEQ_LT(tp->snd_nxt, tp->snd_max) || sack_rxmit) {
1035f5d34df5SGeorge V. Neville-Neil 			tp->t_sndrexmitpack++;
103678b50714SRobert Watson 			TCPSTAT_INC(tcps_sndrexmitpack);
103778b50714SRobert Watson 			TCPSTAT_ADD(tcps_sndrexmitbyte, len);
10380932fb56SRichard Scheffenegger 			if (sack_rxmit) {
10390932fb56SRichard Scheffenegger 				TCPSTAT_INC(tcps_sack_rexmits);
10400932fb56SRichard Scheffenegger 				TCPSTAT_ADD(tcps_sack_rexmit_bytes, len);
10410932fb56SRichard Scheffenegger 			}
1042adc56f5aSEdward Tomasz Napierala #ifdef STATS
1043adc56f5aSEdward Tomasz Napierala 			stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_RETXPB,
1044adc56f5aSEdward Tomasz Napierala 			    len);
1045adc56f5aSEdward Tomasz Napierala #endif /* STATS */
1046df8bae1dSRodney W. Grimes 		} else {
104778b50714SRobert Watson 			TCPSTAT_INC(tcps_sndpack);
104878b50714SRobert Watson 			TCPSTAT_ADD(tcps_sndbyte, len);
1049adc56f5aSEdward Tomasz Napierala #ifdef STATS
1050adc56f5aSEdward Tomasz Napierala 			stats_voi_update_abs_u64(tp->t_stats, VOI_TCP_TXPB,
1051adc56f5aSEdward Tomasz Napierala 			    len);
1052adc56f5aSEdward Tomasz Napierala #endif /* STATS */
1053df8bae1dSRodney W. Grimes 		}
105439f6074eSGleb Smirnoff #ifdef INET6
105539f6074eSGleb Smirnoff 		if (MHLEN < hdrlen + max_linkhdr)
105639f6074eSGleb Smirnoff 			m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
105739f6074eSGleb Smirnoff 		else
105839f6074eSGleb Smirnoff #endif
105939f6074eSGleb Smirnoff 			m = m_gethdr(M_NOWAIT, MT_DATA);
106039f6074eSGleb Smirnoff 
1061df8bae1dSRodney W. Grimes 		if (m == NULL) {
1062cf2942b6SRobert Watson 			SOCKBUF_UNLOCK(&so->so_snd);
1063df8bae1dSRodney W. Grimes 			error = ENOBUFS;
10640e2bc05cSGleb Smirnoff 			sack_rxmit = 0;
1065df8bae1dSRodney W. Grimes 			goto out;
1066df8bae1dSRodney W. Grimes 		}
106739f6074eSGleb Smirnoff 
1068df8bae1dSRodney W. Grimes 		m->m_data += max_linkhdr;
1069df8bae1dSRodney W. Grimes 		m->m_len = hdrlen;
10704e023759SAndre Oppermann 
10714e023759SAndre Oppermann 		/*
10724e023759SAndre Oppermann 		 * Start the m_copy functions from the closest mbuf
10734e023759SAndre Oppermann 		 * to the offset in the socket buffer chain.
10744e023759SAndre Oppermann 		 */
1075581a046aSRandall Stewart 		mb = sbsndptr_noadv(&so->so_snd, off, &moff);
1076b2e60773SJohn Baldwin 		if (len <= MHLEN - hdrlen - max_linkhdr && !hw_tls) {
10773ac12506SJonathan T. Looney 			m_copydata(mb, moff, len,
1078df8bae1dSRodney W. Grimes 			    mtod(m, caddr_t) + hdrlen);
1079581a046aSRandall Stewart 			if (SEQ_LT(tp->snd_nxt, tp->snd_max))
1080581a046aSRandall Stewart 				sbsndptr_adv(&so->so_snd, mb, len);
1081df8bae1dSRodney W. Grimes 			m->m_len += len;
1082df8bae1dSRodney W. Grimes 		} else {
1083581a046aSRandall Stewart 			if (SEQ_LT(tp->snd_nxt, tp->snd_max))
1084581a046aSRandall Stewart 				msb = NULL;
1085581a046aSRandall Stewart 			else
1086581a046aSRandall Stewart 				msb = &so->so_snd;
1087581a046aSRandall Stewart 			m->m_next = tcp_m_copym(mb, moff,
1088581a046aSRandall Stewart 			    &len, if_hw_tsomaxsegcount,
1089b2e60773SJohn Baldwin 			    if_hw_tsomaxsegsize, msb, hw_tls);
1090581a046aSRandall Stewart 			if (len <= (tp->t_maxseg - optlen)) {
1091581a046aSRandall Stewart 				/*
1092581a046aSRandall Stewart 				 * Must have ran out of mbufs for the copy
1093581a046aSRandall Stewart 				 * shorten it to no longer need tso. Lets
1094581a046aSRandall Stewart 				 * not put on sendalot since we are low on
1095581a046aSRandall Stewart 				 * mbufs.
1096581a046aSRandall Stewart 				 */
1097581a046aSRandall Stewart 				tso = 0;
1098581a046aSRandall Stewart 			}
10994e023759SAndre Oppermann 			if (m->m_next == NULL) {
1100cf2942b6SRobert Watson 				SOCKBUF_UNLOCK(&so->so_snd);
1101d7f570e6SGarrett Wollman 				(void) m_free(m);
110251823c3aSGarrett Wollman 				error = ENOBUFS;
11030e2bc05cSGleb Smirnoff 				sack_rxmit = 0;
110451823c3aSGarrett Wollman 				goto out;
110551823c3aSGarrett Wollman 			}
1106df8bae1dSRodney W. Grimes 		}
1107472ea5beSColin Percival 
1108df8bae1dSRodney W. Grimes 		/*
1109df8bae1dSRodney W. Grimes 		 * If we're sending everything we've got, set PUSH.
1110df8bae1dSRodney W. Grimes 		 * (This will keep happy those implementations which only
1111df8bae1dSRodney W. Grimes 		 * give data to the user when a buffer fills or
1112df8bae1dSRodney W. Grimes 		 * a PUSH comes in.)
1113df8bae1dSRodney W. Grimes 		 */
11143ac12506SJonathan T. Looney 		if (((uint32_t)off + (uint32_t)len == sbused(&so->so_snd)) &&
11153ac12506SJonathan T. Looney 		    !(flags & TH_SYN))
1116df8bae1dSRodney W. Grimes 			flags |= TH_PUSH;
1117cf2942b6SRobert Watson 		SOCKBUF_UNLOCK(&so->so_snd);
1118df8bae1dSRodney W. Grimes 	} else {
1119cf2942b6SRobert Watson 		SOCKBUF_UNLOCK(&so->so_snd);
1120df8bae1dSRodney W. Grimes 		if (tp->t_flags & TF_ACKNOW)
112178b50714SRobert Watson 			TCPSTAT_INC(tcps_sndacks);
1122df8bae1dSRodney W. Grimes 		else if (flags & (TH_SYN|TH_FIN|TH_RST))
112378b50714SRobert Watson 			TCPSTAT_INC(tcps_sndctrl);
1124df8bae1dSRodney W. Grimes 		else if (SEQ_GT(tp->snd_up, tp->snd_una))
112578b50714SRobert Watson 			TCPSTAT_INC(tcps_sndurg);
1126df8bae1dSRodney W. Grimes 		else
112778b50714SRobert Watson 			TCPSTAT_INC(tcps_sndwinup);
1128df8bae1dSRodney W. Grimes 
1129aa8bd99dSGleb Smirnoff 		m = m_gethdr(M_NOWAIT, MT_DATA);
1130df8bae1dSRodney W. Grimes 		if (m == NULL) {
1131df8bae1dSRodney W. Grimes 			error = ENOBUFS;
11320e2bc05cSGleb Smirnoff 			sack_rxmit = 0;
1133df8bae1dSRodney W. Grimes 			goto out;
1134df8bae1dSRodney W. Grimes 		}
1135fb59c426SYoshinobu Inoue #ifdef INET6
1136fb59c426SYoshinobu Inoue 		if (isipv6 && (MHLEN < hdrlen + max_linkhdr) &&
1137fb59c426SYoshinobu Inoue 		    MHLEN >= hdrlen) {
1138ed6a66caSRobert Watson 			M_ALIGN(m, hdrlen);
1139fb59c426SYoshinobu Inoue 		} else
1140fb59c426SYoshinobu Inoue #endif
1141df8bae1dSRodney W. Grimes 		m->m_data += max_linkhdr;
1142df8bae1dSRodney W. Grimes 		m->m_len = hdrlen;
1143df8bae1dSRodney W. Grimes 	}
1144cf2942b6SRobert Watson 	SOCKBUF_UNLOCK_ASSERT(&so->so_snd);
1145df8bae1dSRodney W. Grimes 	m->m_pkthdr.rcvif = (struct ifnet *)0;
1146c488362eSRobert Watson #ifdef MAC
11479eb0e832SGleb Smirnoff 	mac_inpcb_create_mbuf(inp, m);
1148c488362eSRobert Watson #endif
1149fb59c426SYoshinobu Inoue #ifdef INET6
1150fb59c426SYoshinobu Inoue 	if (isipv6) {
1151fb59c426SYoshinobu Inoue 		ip6 = mtod(m, struct ip6_hdr *);
11529e644c23SMichael Tuexen 		if (tp->t_port) {
1153500eb6ddSMichael Tuexen 			udp = (struct udphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
11549e644c23SMichael Tuexen 			udp->uh_sport = htons(V_tcp_udp_tunneling_port);
11559e644c23SMichael Tuexen 			udp->uh_dport = tp->t_port;
11569e644c23SMichael Tuexen 			ulen = hdrlen + len - sizeof(struct ip6_hdr);
11579e644c23SMichael Tuexen 			udp->uh_ulen = htons(ulen);
11589e644c23SMichael Tuexen 			th = (struct tcphdr *)(udp + 1);
11599e644c23SMichael Tuexen 		} else {
1160fb59c426SYoshinobu Inoue 			th = (struct tcphdr *)(ip6 + 1);
11619e644c23SMichael Tuexen 		}
11629eb0e832SGleb Smirnoff 		tcpip_fillheaders(inp, tp->t_port, ip6, th);
1163fb59c426SYoshinobu Inoue 	} else
1164fb59c426SYoshinobu Inoue #endif /* INET6 */
1165fb59c426SYoshinobu Inoue 	{
1166fb59c426SYoshinobu Inoue 		ip = mtod(m, struct ip *);
11679e644c23SMichael Tuexen 		if (tp->t_port) {
1168500eb6ddSMichael Tuexen 			udp = (struct udphdr *)((caddr_t)ip + sizeof(struct ip));
11699e644c23SMichael Tuexen 			udp->uh_sport = htons(V_tcp_udp_tunneling_port);
11709e644c23SMichael Tuexen 			udp->uh_dport = tp->t_port;
11719e644c23SMichael Tuexen 			ulen = hdrlen + len - sizeof(struct ip);
11729e644c23SMichael Tuexen 			udp->uh_ulen = htons(ulen);
11739e644c23SMichael Tuexen 			th = (struct tcphdr *)(udp + 1);
11749e644c23SMichael Tuexen 		} else
1175fb59c426SYoshinobu Inoue 			th = (struct tcphdr *)(ip + 1);
11769eb0e832SGleb Smirnoff 		tcpip_fillheaders(inp, tp->t_port, ip, th);
1177fb59c426SYoshinobu Inoue 	}
1178df8bae1dSRodney W. Grimes 
1179df8bae1dSRodney W. Grimes 	/*
1180df8bae1dSRodney W. Grimes 	 * Fill in fields, remembering maximum advertised
1181df8bae1dSRodney W. Grimes 	 * window for use in delaying messages about window sizes.
1182df8bae1dSRodney W. Grimes 	 * If resending a FIN, be sure not to use a new sequence number.
1183df8bae1dSRodney W. Grimes 	 */
1184df8bae1dSRodney W. Grimes 	if (flags & TH_FIN && tp->t_flags & TF_SENTFIN &&
1185df8bae1dSRodney W. Grimes 	    tp->snd_nxt == tp->snd_max)
1186df8bae1dSRodney W. Grimes 		tp->snd_nxt--;
1187df8bae1dSRodney W. Grimes 	/*
1188f2512ba1SRui Paulo 	 * If we are starting a connection, send ECN setup
1189f2512ba1SRui Paulo 	 * SYN packet. If we are on a retransmit, we may
1190f2512ba1SRui Paulo 	 * resend those bits a number of times as per
1191f2512ba1SRui Paulo 	 * RFC 3168.
1192f2512ba1SRui Paulo 	 */
1193f7220c48SRichard Scheffenegger 	if (tp->t_state == TCPS_SYN_SENT && V_tcp_do_ecn) {
1194f7220c48SRichard Scheffenegger 		flags |= tcp_ecn_output_syn_sent(tp);
1195f2512ba1SRui Paulo 	}
1196f7220c48SRichard Scheffenegger 	/* Also handle parallel SYN for ECN */
1197f7220c48SRichard Scheffenegger 	if ((TCPS_HAVERCVDSYN(tp->t_state)) &&
11984012ef77SRichard Scheffenegger 	    (tp->t_flags2 & (TF2_ECN_PERMIT | TF2_ACE_PERMIT))) {
11992ff07d92SRichard Scheffenegger 		int ect = tcp_ecn_output_established(tp, &flags, len, sack_rxmit);
1200f7220c48SRichard Scheffenegger 		if ((tp->t_state == TCPS_SYN_RECEIVED) &&
1201f7220c48SRichard Scheffenegger 		    (tp->t_flags2 & TF2_ECN_SND_ECE))
1202f7220c48SRichard Scheffenegger 			tp->t_flags2 &= ~TF2_ECN_SND_ECE;
1203f2512ba1SRui Paulo #ifdef INET6
1204f026275eSRichard Scheffenegger 		if (isipv6) {
12052169f712SRichard Scheffenegger 			ip6->ip6_flow &= ~htonl(IPTOS_ECN_MASK << IPV6_FLOWLABEL_LEN);
12062169f712SRichard Scheffenegger 			ip6->ip6_flow |= htonl(ect << IPV6_FLOWLABEL_LEN);
1207f026275eSRichard Scheffenegger 		}
1208f2512ba1SRui Paulo 		else
1209f2512ba1SRui Paulo #endif
1210f026275eSRichard Scheffenegger 		{
1211f026275eSRichard Scheffenegger 			ip->ip_tos &= ~IPTOS_ECN_MASK;
1212f7220c48SRichard Scheffenegger 			ip->ip_tos |= ect;
1213f026275eSRichard Scheffenegger 		}
1214f2512ba1SRui Paulo 	}
1215f2512ba1SRui Paulo 
1216f2512ba1SRui Paulo 	/*
1217df8bae1dSRodney W. Grimes 	 * If we are doing retransmissions, then snd_nxt will
1218df8bae1dSRodney W. Grimes 	 * not reflect the first unsent octet.  For ACK only
1219df8bae1dSRodney W. Grimes 	 * packets, we do not want the sequence number of the
1220df8bae1dSRodney W. Grimes 	 * retransmitted packet, we want the sequence number
1221df8bae1dSRodney W. Grimes 	 * of the next unsent octet.  So, if there is no data
1222df8bae1dSRodney W. Grimes 	 * (and no SYN or FIN), use snd_max instead of snd_nxt
1223df8bae1dSRodney W. Grimes 	 * when filling in ti_seq.  But if we are in persist
1224df8bae1dSRodney W. Grimes 	 * state, snd_max might reflect one byte beyond the
1225df8bae1dSRodney W. Grimes 	 * right edge of the window, so use snd_nxt in that
1226df8bae1dSRodney W. Grimes 	 * case, since we know we aren't doing a retransmission.
1227df8bae1dSRodney W. Grimes 	 * (retransmit and persist are mutually exclusive...)
1228df8bae1dSRodney W. Grimes 	 */
1229a55db2b6SPaul Saab 	if (sack_rxmit == 0) {
1230b8152ba7SAndre Oppermann 		if (len || (flags & (TH_SYN|TH_FIN)) ||
1231b8152ba7SAndre Oppermann 		    tcp_timer_active(tp, TT_PERSIST))
1232fb59c426SYoshinobu Inoue 			th->th_seq = htonl(tp->snd_nxt);
1233df8bae1dSRodney W. Grimes 		else
1234fb59c426SYoshinobu Inoue 			th->th_seq = htonl(tp->snd_max);
1235a55db2b6SPaul Saab 	} else {
12366d90faf3SPaul Saab 		th->th_seq = htonl(p->rxmit);
12376d90faf3SPaul Saab 		p->rxmit += len;
12380471a8c7SRichard Scheffenegger 		/*
12390471a8c7SRichard Scheffenegger 		 * Lost Retransmission Detection
12400471a8c7SRichard Scheffenegger 		 * trigger resending of a (then
12410471a8c7SRichard Scheffenegger 		 * still existing) hole, when
12420471a8c7SRichard Scheffenegger 		 * fack acks recoverypoint.
12430471a8c7SRichard Scheffenegger 		 */
12440471a8c7SRichard Scheffenegger 		if ((tp->t_flags & TF_LRD) && SEQ_GEQ(p->rxmit, p->end))
12450471a8c7SRichard Scheffenegger 			p->rxmit = tp->snd_recover;
12460077b016SPaul Saab 		tp->sackhint.sack_bytes_rexmit += len;
12476d90faf3SPaul Saab 	}
1248e5313869SRichard Scheffenegger 	if (IN_RECOVERY(tp->t_flags)) {
1249e5313869SRichard Scheffenegger 		/*
1250e5313869SRichard Scheffenegger 		 * Account all bytes transmitted while
1251e5313869SRichard Scheffenegger 		 * IN_RECOVERY, simplifying PRR and
1252e5313869SRichard Scheffenegger 		 * Lost Retransmit Detection
1253e5313869SRichard Scheffenegger 		 */
1254e5313869SRichard Scheffenegger 		tp->sackhint.prr_out += len;
1255e5313869SRichard Scheffenegger 	}
1256fb59c426SYoshinobu Inoue 	th->th_ack = htonl(tp->rcv_nxt);
1257df8bae1dSRodney W. Grimes 	if (optlen) {
1258fb59c426SYoshinobu Inoue 		bcopy(opt, th + 1, optlen);
1259fb59c426SYoshinobu Inoue 		th->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
1260df8bae1dSRodney W. Grimes 	}
12611ebf4607SRichard Scheffenegger 	tcp_set_flags(th, flags);
1262df8bae1dSRodney W. Grimes 	/*
1263df8bae1dSRodney W. Grimes 	 * Calculate receive window.  Don't shrink window,
1264df8bae1dSRodney W. Grimes 	 * but avoid silly window syndrome.
126579410718SMichael Tuexen 	 * If a RST segment is sent, advertise a window of zero.
1266df8bae1dSRodney W. Grimes 	 */
126779410718SMichael Tuexen 	if (flags & TH_RST) {
126879410718SMichael Tuexen 		recwin = 0;
126979410718SMichael Tuexen 	} else {
12703ac12506SJonathan T. Looney 		if (recwin < (so->so_rcv.sb_hiwat / 4) &&
12713ac12506SJonathan T. Looney 		    recwin < tp->t_maxseg)
1272201d185bSAndre Oppermann 			recwin = 0;
1273f701e30dSJohn Baldwin 		if (SEQ_GT(tp->rcv_adv, tp->rcv_nxt) &&
12743ac12506SJonathan T. Looney 		    recwin < (tp->rcv_adv - tp->rcv_nxt))
12753ac12506SJonathan T. Looney 			recwin = (tp->rcv_adv - tp->rcv_nxt);
127679410718SMichael Tuexen 	}
1277104ebb2aSAndre Oppermann 	/*
1278104ebb2aSAndre Oppermann 	 * According to RFC1323 the window field in a SYN (i.e., a <SYN>
1279104ebb2aSAndre Oppermann 	 * or <SYN,ACK>) segment itself is never scaled.  The <SYN,ACK>
1280104ebb2aSAndre Oppermann 	 * case is handled in syncache.
1281104ebb2aSAndre Oppermann 	 */
1282104ebb2aSAndre Oppermann 	if (flags & TH_SYN)
1283104ebb2aSAndre Oppermann 		th->th_win = htons((u_short)
1284104ebb2aSAndre Oppermann 				(min(sbspace(&so->so_rcv), TCP_MAXWIN)));
12859028b6e0SRichard Scheffenegger 	else {
12869028b6e0SRichard Scheffenegger 		/* Avoid shrinking window with window scaling. */
12879028b6e0SRichard Scheffenegger 		recwin = roundup2(recwin, 1 << tp->rcv_scale);
1288104ebb2aSAndre Oppermann 		th->th_win = htons((u_short)(recwin >> tp->rcv_scale));
12899028b6e0SRichard Scheffenegger 	}
1290262c1c1aSMatthew Dillon 
1291262c1c1aSMatthew Dillon 	/*
1292262c1c1aSMatthew Dillon 	 * Adjust the RXWIN0SENT flag - indicate that we have advertised
1293262c1c1aSMatthew Dillon 	 * a 0 window.  This may cause the remote transmitter to stall.  This
1294262c1c1aSMatthew Dillon 	 * flag tells soreceive() to disable delayed acknowledgements when
1295262c1c1aSMatthew Dillon 	 * draining the buffer.  This can occur if the receiver is attempting
1296b2722702SRui Paulo 	 * to read more data than can be buffered prior to transmitting on
1297262c1c1aSMatthew Dillon 	 * the connection.
1298262c1c1aSMatthew Dillon 	 */
1299f5d34df5SGeorge V. Neville-Neil 	if (th->th_win == 0) {
1300f5d34df5SGeorge V. Neville-Neil 		tp->t_sndzerowin++;
1301262c1c1aSMatthew Dillon 		tp->t_flags |= TF_RXWIN0SENT;
1302f5d34df5SGeorge V. Neville-Neil 	} else
1303262c1c1aSMatthew Dillon 		tp->t_flags &= ~TF_RXWIN0SENT;
1304df8bae1dSRodney W. Grimes 	if (SEQ_GT(tp->snd_up, tp->snd_nxt)) {
1305fb59c426SYoshinobu Inoue 		th->th_urp = htons((u_short)(tp->snd_up - tp->snd_nxt));
1306fb59c426SYoshinobu Inoue 		th->th_flags |= TH_URG;
1307df8bae1dSRodney W. Grimes 	} else
1308df8bae1dSRodney W. Grimes 		/*
1309df8bae1dSRodney W. Grimes 		 * If no urgent pointer to send, then we pull
1310df8bae1dSRodney W. Grimes 		 * the urgent pointer to the left edge of the send window
1311df8bae1dSRodney W. Grimes 		 * so that it doesn't drift into the send window on sequence
1312df8bae1dSRodney W. Grimes 		 * number wraparound.
1313df8bae1dSRodney W. Grimes 		 */
1314df8bae1dSRodney W. Grimes 		tp->snd_up = tp->snd_una;		/* drag it along */
1315df8bae1dSRodney W. Grimes 
1316df8bae1dSRodney W. Grimes 	/*
1317df8bae1dSRodney W. Grimes 	 * Put TCP length in extended header, and then
1318df8bae1dSRodney W. Grimes 	 * checksum extended header and data.
1319df8bae1dSRodney W. Grimes 	 */
1320fb59c426SYoshinobu Inoue 	m->m_pkthdr.len = hdrlen + len; /* in6_cksum() need this */
1321fcf59617SAndrey V. Elsukov 
1322fcf59617SAndrey V. Elsukov #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
1323fcf59617SAndrey V. Elsukov 	if (to.to_flags & TOF_SIGNATURE) {
1324fcf59617SAndrey V. Elsukov 		/*
1325fcf59617SAndrey V. Elsukov 		 * Calculate MD5 signature and put it into the place
1326fcf59617SAndrey V. Elsukov 		 * determined before.
1327fcf59617SAndrey V. Elsukov 		 * NOTE: since TCP options buffer doesn't point into
1328fcf59617SAndrey V. Elsukov 		 * mbuf's data, calculate offset and use it.
1329fcf59617SAndrey V. Elsukov 		 */
13302aad6240SAndrey V. Elsukov 		if (!TCPMD5_ENABLED() || (error = TCPMD5_OUTPUT(m, th,
13312aad6240SAndrey V. Elsukov 		    (u_char *)(th + 1) + (to.to_signature - opt))) != 0) {
1332fcf59617SAndrey V. Elsukov 			/*
1333fcf59617SAndrey V. Elsukov 			 * Do not send segment if the calculation of MD5
1334fcf59617SAndrey V. Elsukov 			 * digest has failed.
1335fcf59617SAndrey V. Elsukov 			 */
13362aad6240SAndrey V. Elsukov 			m_freem(m);
1337fcf59617SAndrey V. Elsukov 			goto out;
1338fcf59617SAndrey V. Elsukov 		}
1339fcf59617SAndrey V. Elsukov 	}
1340fcf59617SAndrey V. Elsukov #endif
1341fb59c426SYoshinobu Inoue #ifdef INET6
134245747ba5SBjoern A. Zeeb 	if (isipv6) {
1343fb59c426SYoshinobu Inoue 		/*
13443df96ee6SCy Schubert 		 * There is no need to fill in ip6_plen right now.
13453df96ee6SCy Schubert 		 * It will be filled later by ip6_output.
1346fb59c426SYoshinobu Inoue 		 */
13479e644c23SMichael Tuexen 		if (tp->t_port) {
13489e644c23SMichael Tuexen 			m->m_pkthdr.csum_flags = CSUM_UDP_IPV6;
13499e644c23SMichael Tuexen 			m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
13509e644c23SMichael Tuexen 			udp->uh_sum = in6_cksum_pseudo(ip6, ulen, IPPROTO_UDP, 0);
13519e644c23SMichael Tuexen 			th->th_sum = htons(0);
13529e644c23SMichael Tuexen 			UDPSTAT_INC(udps_opackets);
13539e644c23SMichael Tuexen 		} else {
1354356ab07eSBjoern A. Zeeb 			m->m_pkthdr.csum_flags = CSUM_TCP_IPV6;
13559e644c23SMichael Tuexen 			m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
13569e644c23SMichael Tuexen 			th->th_sum = in6_cksum_pseudo(ip6,
13579e644c23SMichael Tuexen 			    sizeof(struct tcphdr) + optlen + len, IPPROTO_TCP,
13589e644c23SMichael Tuexen 			    0);
13599e644c23SMichael Tuexen 		}
136045747ba5SBjoern A. Zeeb 	}
136145747ba5SBjoern A. Zeeb #endif
136245747ba5SBjoern A. Zeeb #if defined(INET6) && defined(INET)
1363fb59c426SYoshinobu Inoue 	else
136445747ba5SBjoern A. Zeeb #endif
136545747ba5SBjoern A. Zeeb #ifdef INET
1366fb59c426SYoshinobu Inoue 	{
13679e644c23SMichael Tuexen 		if (tp->t_port) {
13689e644c23SMichael Tuexen 			m->m_pkthdr.csum_flags = CSUM_UDP;
13699e644c23SMichael Tuexen 			m->m_pkthdr.csum_data = offsetof(struct udphdr, uh_sum);
13709e644c23SMichael Tuexen 			udp->uh_sum = in_pseudo(ip->ip_src.s_addr,
13719e644c23SMichael Tuexen 			   ip->ip_dst.s_addr, htons(ulen + IPPROTO_UDP));
13729e644c23SMichael Tuexen 			th->th_sum = htons(0);
13739e644c23SMichael Tuexen 			UDPSTAT_INC(udps_opackets);
13749e644c23SMichael Tuexen 		} else {
1375356ab07eSBjoern A. Zeeb 			m->m_pkthdr.csum_flags = CSUM_TCP;
13769e644c23SMichael Tuexen 			m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
13779e644c23SMichael Tuexen 			th->th_sum = in_pseudo(ip->ip_src.s_addr,
13789e644c23SMichael Tuexen 			    ip->ip_dst.s_addr, htons(sizeof(struct tcphdr) +
13799e644c23SMichael Tuexen 			    IPPROTO_TCP + len + optlen));
13809e644c23SMichael Tuexen 		}
1381fb59c426SYoshinobu Inoue 
1382db4f9cc7SJonathan Lemon 		/* IP version must be set here for ipv4/ipv6 checking later */
1383db4f9cc7SJonathan Lemon 		KASSERT(ip->ip_v == IPVERSION,
13846e551fb6SDavid E. O'Brien 		    ("%s: IP version incorrect: %d", __func__, ip->ip_v));
1385fb59c426SYoshinobu Inoue 	}
138645747ba5SBjoern A. Zeeb #endif
1387df8bae1dSRodney W. Grimes 
1388df8bae1dSRodney W. Grimes 	/*
1389b3c0f300SAndre Oppermann 	 * Enable TSO and specify the size of the segments.
1390b3c0f300SAndre Oppermann 	 * The TCP pseudo header checksum is always provided.
1391b3c0f300SAndre Oppermann 	 */
1392b3c0f300SAndre Oppermann 	if (tso) {
13930c39d38dSGleb Smirnoff 		KASSERT(len > tp->t_maxseg - optlen,
1394153e5b57SAndre Oppermann 		    ("%s: len <= tso_segsz", __func__));
13953579cf4cSKenneth D. Merry 		m->m_pkthdr.csum_flags |= CSUM_TSO;
13960c39d38dSGleb Smirnoff 		m->m_pkthdr.tso_segsz = tp->t_maxseg - optlen;
1397b3c0f300SAndre Oppermann 	}
1398b3c0f300SAndre Oppermann 
139905fb056cSMichael Tuexen 	KASSERT(len + hdrlen == m_length(m, NULL),
140005fb056cSMichael Tuexen 	    ("%s: mbuf chain shorter than expected: %d + %u != %u",
140105fb056cSMichael Tuexen 	    __func__, len, hdrlen, m_length(m, NULL)));
1402ed420311SAndre Oppermann 
1403bd79708dSJonathan T. Looney #ifdef TCP_HHOOK
140439bc9de5SLawrence Stewart 	/* Run HHOOK_TCP_ESTABLISHED_OUT helper hooks. */
140539bc9de5SLawrence Stewart 	hhook_run_tcp_est_out(tp, th, &to, len, tso);
1406bd79708dSJonathan T. Looney #endif
140739bc9de5SLawrence Stewart 
14082b9c9984SGeorge V. Neville-Neil 	TCP_PROBE3(debug__output, tp, th, m);
1409df8bae1dSRodney W. Grimes 
1410dcaffbd6SJonathan T. Looney 	/* We're getting ready to send; log now. */
1411bd30a121SMichael Tuexen 	/* XXXMT: We are not honoring verbose logging. */
141269c7c811SRandall Stewart 
141369c7c811SRandall Stewart 	if (tcp_bblogging_on(tp))
141469c7c811SRandall Stewart 		lgb = tcp_log_event(tp, th, &so->so_rcv, &so->so_snd,
1415bd30a121SMichael Tuexen 		    TCP_LOG_OUT, ERRNO_UNK, len, NULL, false, NULL, NULL, 0,
1416bd30a121SMichael Tuexen 		    NULL);
1417bd30a121SMichael Tuexen 	else
1418bd30a121SMichael Tuexen 		lgb = NULL;
1419dcaffbd6SJonathan T. Looney 
1420df8bae1dSRodney W. Grimes 	/*
1421df8bae1dSRodney W. Grimes 	 * Fill in IP length and desired time to live and
1422df8bae1dSRodney W. Grimes 	 * send to IP level.  There should be a better way
1423df8bae1dSRodney W. Grimes 	 * to handle ttl and tos; we could keep them in
1424df8bae1dSRodney W. Grimes 	 * the template, but need a way to checksum without them.
1425df8bae1dSRodney W. Grimes 	 */
1426fb59c426SYoshinobu Inoue 	/*
142743630e62SHiren Panchasara 	 * m->m_pkthdr.len should have been set before checksum calculation,
1428fb59c426SYoshinobu Inoue 	 * because in6_cksum() need it.
1429fb59c426SYoshinobu Inoue 	 */
1430fb59c426SYoshinobu Inoue #ifdef INET6
1431fb59c426SYoshinobu Inoue 	if (isipv6) {
1432fb59c426SYoshinobu Inoue 		/*
1433fb59c426SYoshinobu Inoue 		 * we separately set hoplimit for every segment, since the
1434fb59c426SYoshinobu Inoue 		 * user might want to change the value via setsockopt.
1435fb59c426SYoshinobu Inoue 		 * Also, desired default hop limit might be changed via
1436fb59c426SYoshinobu Inoue 		 * Neighbor Discovery.
1437fb59c426SYoshinobu Inoue 		 */
14389eb0e832SGleb Smirnoff 		ip6->ip6_hlim = in6_selecthlim(inp, NULL);
1439fb59c426SYoshinobu Inoue 
144057f60867SMark Johnston 		/*
144157f60867SMark Johnston 		 * Set the packet size here for the benefit of DTrace probes.
144257f60867SMark Johnston 		 * ip6_output() will set it properly; it's supposed to include
144357f60867SMark Johnston 		 * the option header lengths as well.
144457f60867SMark Johnston 		 */
144557f60867SMark Johnston 		ip6->ip6_plen = htons(m->m_pkthdr.len - sizeof(*ip6));
144657f60867SMark Johnston 
14470c39d38dSGleb Smirnoff 		if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss)
14480f3e3bc5SSean Bruno 			tp->t_flags2 |= TF2_PLPMTU_PMTUD;
14490f3e3bc5SSean Bruno 		else
14500f3e3bc5SSean Bruno 			tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
14510f3e3bc5SSean Bruno 
145257f60867SMark Johnston 		if (tp->t_state == TCPS_SYN_SENT)
1453d9fae5abSAndriy Gapon 			TCP_PROBE5(connect__request, NULL, tp, ip6, tp, th);
145457f60867SMark Johnston 
145557f60867SMark Johnston 		TCP_PROBE5(send, NULL, tp, ip6, tp, th);
145657f60867SMark Johnston 
145786a996e6SHiren Panchasara #ifdef TCPPCAP
145886a996e6SHiren Panchasara 		/* Save packet, if requested. */
145986a996e6SHiren Panchasara 		tcp_pcap_add(th, m, &(tp->t_outpkts));
146086a996e6SHiren Panchasara #endif
146186a996e6SHiren Panchasara 
1462fb59c426SYoshinobu Inoue 		/* TODO: IPv6 IP6TOS_ECT bit on */
14639eb0e832SGleb Smirnoff 		error = ip6_output(m, inp->in6p_outputopts, &inp->inp_route6,
1464df0633a1SGleb Smirnoff 		    ((so->so_options & SO_DONTROUTE) ?  IP_ROUTETOIF : 0),
14659eb0e832SGleb Smirnoff 		    NULL, NULL, inp);
1466df0633a1SGleb Smirnoff 
14679eb0e832SGleb Smirnoff 		if (error == EMSGSIZE && inp->inp_route6.ro_nh != NULL)
14689eb0e832SGleb Smirnoff 			mtu = inp->inp_route6.ro_nh->nh_mtu;
1469b287c6c7SBjoern A. Zeeb 	}
1470fb59c426SYoshinobu Inoue #endif /* INET6 */
1471b287c6c7SBjoern A. Zeeb #if defined(INET) && defined(INET6)
1472b287c6c7SBjoern A. Zeeb 	else
1473b287c6c7SBjoern A. Zeeb #endif
1474b287c6c7SBjoern A. Zeeb #ifdef INET
1475df8bae1dSRodney W. Grimes     {
14768f134647SGleb Smirnoff 	ip->ip_len = htons(m->m_pkthdr.len);
1477686cdd19SJun-ichiro itojun Hagino #ifdef INET6
14789eb0e832SGleb Smirnoff 	if (inp->inp_vflag & INP_IPV6PROTO)
14799eb0e832SGleb Smirnoff 		ip->ip_ttl = in6_selecthlim(inp, NULL);
1480686cdd19SJun-ichiro itojun Hagino #endif /* INET6 */
1481f138387aSGarrett Wollman 	/*
148297d8d152SAndre Oppermann 	 * If we do path MTU discovery, then we set DF on every packet.
148397d8d152SAndre Oppermann 	 * This might not be the best thing to do according to RFC3390
148497d8d152SAndre Oppermann 	 * Section 2. However the tcp hostcache migitates the problem
148597d8d152SAndre Oppermann 	 * so it affects only the first tcp connection with a host.
1486e4e92660SAndre Oppermann 	 *
1487e4e92660SAndre Oppermann 	 * NB: Don't set DF on small MTU/MSS to have a safe fallback.
1488f138387aSGarrett Wollman 	 */
14890c39d38dSGleb Smirnoff 	if (V_path_mtu_discovery && tp->t_maxseg > V_tcp_minmss) {
1490f6f6703fSSean Bruno 		tp->t_flags2 |= TF2_PLPMTU_PMTUD;
14919e644c23SMichael Tuexen 		if (tp->t_port == 0 || len < V_tcp_minmss) {
14929e644c23SMichael Tuexen 			ip->ip_off |= htons(IP_DF);
14939e644c23SMichael Tuexen 		}
1494f6f6703fSSean Bruno 	} else {
1495f6f6703fSSean Bruno 		tp->t_flags2 &= ~TF2_PLPMTU_PMTUD;
1496f6f6703fSSean Bruno 	}
149797d8d152SAndre Oppermann 
149857f60867SMark Johnston 	if (tp->t_state == TCPS_SYN_SENT)
1499d9fae5abSAndriy Gapon 		TCP_PROBE5(connect__request, NULL, tp, ip, tp, th);
150057f60867SMark Johnston 
150157f60867SMark Johnston 	TCP_PROBE5(send, NULL, tp, ip, tp, th);
150257f60867SMark Johnston 
150386a996e6SHiren Panchasara #ifdef TCPPCAP
150486a996e6SHiren Panchasara 	/* Save packet, if requested. */
150586a996e6SHiren Panchasara 	tcp_pcap_add(th, m, &(tp->t_outpkts));
150686a996e6SHiren Panchasara #endif
150786a996e6SHiren Panchasara 
15089eb0e832SGleb Smirnoff 	error = ip_output(m, inp->inp_options, &inp->inp_route,
15099eb0e832SGleb Smirnoff 	    ((so->so_options & SO_DONTROUTE) ? IP_ROUTETOIF : 0), 0, inp);
1510df0633a1SGleb Smirnoff 
15119eb0e832SGleb Smirnoff 	if (error == EMSGSIZE && inp->inp_route.ro_nh != NULL)
15129eb0e832SGleb Smirnoff 		mtu = inp->inp_route.ro_nh->nh_mtu;
1513df8bae1dSRodney W. Grimes     }
1514b287c6c7SBjoern A. Zeeb #endif /* INET */
15150e2bc05cSGleb Smirnoff 
1516bd30a121SMichael Tuexen 	if (lgb != NULL) {
1517bd30a121SMichael Tuexen 		lgb->tlb_errno = error;
1518bd30a121SMichael Tuexen 		lgb = NULL;
1519bd30a121SMichael Tuexen 	}
15200e2bc05cSGleb Smirnoff out:
152167e89281SRandall Stewart 	if (error == 0)
15229e4d9e4cSRandall Stewart 		tcp_account_for_send(tp, len, (tp->snd_nxt != tp->snd_max), 0, hw_tls);
15230e2bc05cSGleb Smirnoff 	/*
15240e2bc05cSGleb Smirnoff 	 * In transmit state, time the transmission and arrange for
1525a8e817cfSRichard Scheffenegger 	 * the retransmit.  In persist state, just set snd_max.  In a closed
1526a8e817cfSRichard Scheffenegger 	 * state just return.
15270e2bc05cSGleb Smirnoff 	 */
1528a8e817cfSRichard Scheffenegger 	if (flags & TH_RST) {
1529a8e817cfSRichard Scheffenegger 		TCPSTAT_INC(tcps_sndtotal);
1530a8e817cfSRichard Scheffenegger 		return (0);
1531a8e817cfSRichard Scheffenegger 	} else if ((tp->t_flags & TF_FORCEDATA) == 0 ||
15320e2bc05cSGleb Smirnoff 	    !tcp_timer_active(tp, TT_PERSIST)) {
15330e2bc05cSGleb Smirnoff 		tcp_seq startseq = tp->snd_nxt;
15340e2bc05cSGleb Smirnoff 
15350e2bc05cSGleb Smirnoff 		/*
15360e2bc05cSGleb Smirnoff 		 * Advance snd_nxt over sequence space of this segment.
15370e2bc05cSGleb Smirnoff 		 */
15380e2bc05cSGleb Smirnoff 		if (flags & (TH_SYN|TH_FIN)) {
15390e2bc05cSGleb Smirnoff 			if (flags & TH_SYN)
15400e2bc05cSGleb Smirnoff 				tp->snd_nxt++;
15410e2bc05cSGleb Smirnoff 			if (flags & TH_FIN) {
15420e2bc05cSGleb Smirnoff 				tp->snd_nxt++;
15430e2bc05cSGleb Smirnoff 				tp->t_flags |= TF_SENTFIN;
15440e2bc05cSGleb Smirnoff 			}
15450e2bc05cSGleb Smirnoff 		}
15460e2bc05cSGleb Smirnoff 		if (sack_rxmit)
15470e2bc05cSGleb Smirnoff 			goto timer;
15480e2bc05cSGleb Smirnoff 		tp->snd_nxt += len;
15490e2bc05cSGleb Smirnoff 		if (SEQ_GT(tp->snd_nxt, tp->snd_max)) {
155008af8aacSRandall Stewart 			/*
155108af8aacSRandall Stewart 			 * Update "made progress" indication if we just
155208af8aacSRandall Stewart 			 * added new data to an empty socket buffer.
155308af8aacSRandall Stewart 			 */
155408af8aacSRandall Stewart 			if (tp->snd_una == tp->snd_max)
155508af8aacSRandall Stewart 				tp->t_acktime = ticks;
15560e2bc05cSGleb Smirnoff 			tp->snd_max = tp->snd_nxt;
15570e2bc05cSGleb Smirnoff 			/*
15580e2bc05cSGleb Smirnoff 			 * Time this transmission if not a retransmission and
15590e2bc05cSGleb Smirnoff 			 * not currently timing anything.
15600e2bc05cSGleb Smirnoff 			 */
15619dc7d8a2SRichard Scheffenegger 			tp->t_sndtime = ticks;
15620e2bc05cSGleb Smirnoff 			if (tp->t_rtttime == 0) {
15630e2bc05cSGleb Smirnoff 				tp->t_rtttime = ticks;
15640e2bc05cSGleb Smirnoff 				tp->t_rtseq = startseq;
15650e2bc05cSGleb Smirnoff 				TCPSTAT_INC(tcps_segstimed);
15660e2bc05cSGleb Smirnoff 			}
1567adc56f5aSEdward Tomasz Napierala #ifdef STATS
1568adc56f5aSEdward Tomasz Napierala 			if (!(tp->t_flags & TF_GPUTINPROG) && len) {
1569adc56f5aSEdward Tomasz Napierala 				tp->t_flags |= TF_GPUTINPROG;
1570adc56f5aSEdward Tomasz Napierala 				tp->gput_seq = startseq;
1571adc56f5aSEdward Tomasz Napierala 				tp->gput_ack = startseq +
1572adc56f5aSEdward Tomasz Napierala 				    ulmin(sbavail(&so->so_snd) - off, sendwin);
1573adc56f5aSEdward Tomasz Napierala 				tp->gput_ts = tcp_ts_getticks();
1574adc56f5aSEdward Tomasz Napierala 			}
1575adc56f5aSEdward Tomasz Napierala #endif /* STATS */
15760e2bc05cSGleb Smirnoff 		}
15770e2bc05cSGleb Smirnoff 
15780e2bc05cSGleb Smirnoff 		/*
15790e2bc05cSGleb Smirnoff 		 * Set retransmit timer if not currently set,
15800e2bc05cSGleb Smirnoff 		 * and not doing a pure ack or a keep-alive probe.
15810e2bc05cSGleb Smirnoff 		 * Initial value for retransmit timer is smoothed
15820e2bc05cSGleb Smirnoff 		 * round-trip time + 2 * round-trip time variance.
15830e2bc05cSGleb Smirnoff 		 * Initialize shift counter which is used for backoff
15840e2bc05cSGleb Smirnoff 		 * of retransmit time.
15850e2bc05cSGleb Smirnoff 		 */
15860e2bc05cSGleb Smirnoff timer:
15870e2bc05cSGleb Smirnoff 		if (!tcp_timer_active(tp, TT_REXMT) &&
15880e2bc05cSGleb Smirnoff 		    ((sack_rxmit && tp->snd_nxt != tp->snd_max) ||
15890e2bc05cSGleb Smirnoff 		     (tp->snd_nxt != tp->snd_una))) {
15900e2bc05cSGleb Smirnoff 			if (tcp_timer_active(tp, TT_PERSIST)) {
15910e2bc05cSGleb Smirnoff 				tcp_timer_activate(tp, TT_PERSIST, 0);
15920e2bc05cSGleb Smirnoff 				tp->t_rxtshift = 0;
15930e2bc05cSGleb Smirnoff 			}
159408af8aacSRandall Stewart 			tcp_timer_activate(tp, TT_REXMT, TP_RXTCUR(tp));
1595f8568079SHiren Panchasara 		} else if (len == 0 && sbavail(&so->so_snd) &&
1596f8568079SHiren Panchasara 		    !tcp_timer_active(tp, TT_REXMT) &&
1597f8568079SHiren Panchasara 		    !tcp_timer_active(tp, TT_PERSIST)) {
1598f8568079SHiren Panchasara 			/*
1599f8568079SHiren Panchasara 			 * Avoid a situation where we do not set persist timer
1600f8568079SHiren Panchasara 			 * after a zero window condition. For example:
1601f8568079SHiren Panchasara 			 * 1) A -> B: packet with enough data to fill the window
1602f8568079SHiren Panchasara 			 * 2) B -> A: ACK for #1 + new data (0 window
1603f8568079SHiren Panchasara 			 *    advertisement)
1604f8568079SHiren Panchasara 			 * 3) A -> B: ACK for #2, 0 len packet
1605f8568079SHiren Panchasara 			 *
1606f8568079SHiren Panchasara 			 * In this case, A will not activate the persist timer,
1607f8568079SHiren Panchasara 			 * because it chose to send a packet. Unless tcp_output
1608f8568079SHiren Panchasara 			 * is called for some other reason (delayed ack timer,
1609f8568079SHiren Panchasara 			 * another input packet from B, socket syscall), A will
1610f8568079SHiren Panchasara 			 * not send zero window probes.
1611f8568079SHiren Panchasara 			 *
1612f8568079SHiren Panchasara 			 * So, if you send a 0-length packet, but there is data
1613f8568079SHiren Panchasara 			 * in the socket buffer, and neither the rexmt or
1614f8568079SHiren Panchasara 			 * persist timer is already set, then activate the
1615f8568079SHiren Panchasara 			 * persist timer.
1616f8568079SHiren Panchasara 			 */
1617f8568079SHiren Panchasara 			tp->t_rxtshift = 0;
1618f8568079SHiren Panchasara 			tcp_setpersist(tp);
16190e2bc05cSGleb Smirnoff 		}
16200e2bc05cSGleb Smirnoff 	} else {
16210e2bc05cSGleb Smirnoff 		/*
16220e2bc05cSGleb Smirnoff 		 * Persist case, update snd_max but since we are in
16230e2bc05cSGleb Smirnoff 		 * persist mode (no window) we do not update snd_nxt.
16240e2bc05cSGleb Smirnoff 		 */
16250e2bc05cSGleb Smirnoff 		int xlen = len;
16260e2bc05cSGleb Smirnoff 		if (flags & TH_SYN)
16270e2bc05cSGleb Smirnoff 			++xlen;
16280e2bc05cSGleb Smirnoff 		if (flags & TH_FIN) {
16290e2bc05cSGleb Smirnoff 			++xlen;
16300e2bc05cSGleb Smirnoff 			tp->t_flags |= TF_SENTFIN;
16310e2bc05cSGleb Smirnoff 		}
16320e2bc05cSGleb Smirnoff 		if (SEQ_GT(tp->snd_nxt + xlen, tp->snd_max))
163315c82571SJonathan T. Looney 			tp->snd_max = tp->snd_nxt + xlen;
16340e2bc05cSGleb Smirnoff 	}
1635e5926fd3SRandall Stewart 	if ((error == 0) &&
1636e5926fd3SRandall Stewart 	    (TCPS_HAVEESTABLISHED(tp->t_state) &&
1637e5926fd3SRandall Stewart 	     (tp->t_flags & TF_SACK_PERMIT) &&
1638e5926fd3SRandall Stewart 	     tp->rcv_numsacks > 0)) {
1639e5926fd3SRandall Stewart 		    /* Clean up any DSACK's sent */
1640e5926fd3SRandall Stewart 		    tcp_clean_dsack_blocks(tp);
1641e5926fd3SRandall Stewart 	}
1642df8bae1dSRodney W. Grimes 	if (error) {
16437734ea06SArchie Cobbs 		/*
16447734ea06SArchie Cobbs 		 * We know that the packet was lost, so back out the
16457734ea06SArchie Cobbs 		 * sequence number advance, if any.
16462c30ec0aSAndre Oppermann 		 *
16472c30ec0aSAndre Oppermann 		 * If the error is EPERM the packet got blocked by the
16482c30ec0aSAndre Oppermann 		 * local firewall.  Normally we should terminate the
16492c30ec0aSAndre Oppermann 		 * connection but the blocking may have been spurious
16502c30ec0aSAndre Oppermann 		 * due to a firewall reconfiguration cycle.  So we treat
16512c30ec0aSAndre Oppermann 		 * it like a packet loss and let the retransmit timer and
16522c30ec0aSAndre Oppermann 		 * timeouts do their work over time.
16532c30ec0aSAndre Oppermann 		 * XXX: It is a POLA question whether calling tcp_drop right
16542c30ec0aSAndre Oppermann 		 * away would be the really correct behavior instead.
16557734ea06SArchie Cobbs 		 */
165672757d9aSGleb Smirnoff 		if (((tp->t_flags & TF_FORCEDATA) == 0 ||
1657b8152ba7SAndre Oppermann 		    !tcp_timer_active(tp, TT_PERSIST)) &&
165872757d9aSGleb Smirnoff 		    ((flags & TH_SYN) == 0) &&
165972757d9aSGleb Smirnoff 		    (error != EPERM)) {
16600077b016SPaul Saab 			if (sack_rxmit) {
1661e3b9058eSRichard Scheffenegger 				p->rxmit = SEQ_MIN(p->end, p->rxmit) - len;
16620077b016SPaul Saab 				tp->sackhint.sack_bytes_rexmit -= len;
166372757d9aSGleb Smirnoff 				KASSERT(tp->sackhint.sack_bytes_rexmit >= 0,
16640077b016SPaul Saab 				    ("sackhint bytes rtx >= 0"));
166566605ff7SRichard Scheffenegger 				KASSERT((flags & TH_FIN) == 0,
166666605ff7SRichard Scheffenegger 				    ("error while FIN with SACK rxmit"));
166766605ff7SRichard Scheffenegger 			} else {
16687734ea06SArchie Cobbs 				tp->snd_nxt -= len;
166966605ff7SRichard Scheffenegger 				if (flags & TH_FIN)
167066605ff7SRichard Scheffenegger 					tp->snd_nxt--;
167166605ff7SRichard Scheffenegger 			}
1672429f14f8SRichard Scheffenegger 			if (IN_RECOVERY(tp->t_flags))
1673429f14f8SRichard Scheffenegger 				tp->sackhint.prr_out -= len;
16747734ea06SArchie Cobbs 		}
1675cf2942b6SRobert Watson 		SOCKBUF_UNLOCK_ASSERT(&so->so_snd);	/* Check gotos. */
167672757d9aSGleb Smirnoff 		switch (error) {
1677fcf59617SAndrey V. Elsukov 		case EACCES:
167872757d9aSGleb Smirnoff 		case EPERM:
167972757d9aSGleb Smirnoff 			tp->t_softerror = error;
168072757d9aSGleb Smirnoff 			return (error);
168172757d9aSGleb Smirnoff 		case ENOBUFS:
1682425b7639SSepherosa Ziehau 			TCP_XMIT_TIMER_ASSERT(tp, len, flags);
16831600372bSAndre Oppermann 			tp->snd_cwnd = tp->t_maxseg;
1684df8bae1dSRodney W. Grimes 			return (0);
168572757d9aSGleb Smirnoff 		case EMSGSIZE:
16863d1f141bSGarrett Wollman 			/*
1687b3c0f300SAndre Oppermann 			 * For some reason the interface we used initially
1688b3c0f300SAndre Oppermann 			 * to send segments changed to another or lowered
1689b3c0f300SAndre Oppermann 			 * its MTU.
1690b3c0f300SAndre Oppermann 			 * If TSO was active we either got an interface
1691b3c0f300SAndre Oppermann 			 * without TSO capabilits or TSO was turned off.
1692df0633a1SGleb Smirnoff 			 * If we obtained mtu from ip_output() then update
1693df0633a1SGleb Smirnoff 			 * it and try again.
16943d1f141bSGarrett Wollman 			 */
1695b3c0f300SAndre Oppermann 			if (tso)
1696b3c0f300SAndre Oppermann 				tp->t_flags &= ~TF_TSO;
1697df0633a1SGleb Smirnoff 			if (mtu != 0) {
1698df0633a1SGleb Smirnoff 				tcp_mss_update(tp, -1, mtu, NULL, NULL);
1699df0633a1SGleb Smirnoff 				goto again;
1700df0633a1SGleb Smirnoff 			}
1701df0633a1SGleb Smirnoff 			return (error);
17028bec3467SGleb Smirnoff 		case EHOSTDOWN:
170372757d9aSGleb Smirnoff 		case EHOSTUNREACH:
170472757d9aSGleb Smirnoff 		case ENETDOWN:
17058bec3467SGleb Smirnoff 		case ENETUNREACH:
170672757d9aSGleb Smirnoff 			if (TCPS_HAVERCVDSYN(tp->t_state)) {
1707df8bae1dSRodney W. Grimes 				tp->t_softerror = error;
1708df8bae1dSRodney W. Grimes 				return (0);
1709df8bae1dSRodney W. Grimes 			}
171072757d9aSGleb Smirnoff 			/* FALLTHROUGH */
171172757d9aSGleb Smirnoff 		default:
1712df8bae1dSRodney W. Grimes 			return (error);
1713df8bae1dSRodney W. Grimes 		}
171472757d9aSGleb Smirnoff 	}
171578b50714SRobert Watson 	TCPSTAT_INC(tcps_sndtotal);
1716df8bae1dSRodney W. Grimes 
1717df8bae1dSRodney W. Grimes 	/*
1718df8bae1dSRodney W. Grimes 	 * Data sent (as far as we can tell).
1719df8bae1dSRodney W. Grimes 	 * If this advertises a larger window than any other segment,
1720df8bae1dSRodney W. Grimes 	 * then remember the size of the advertised window.
1721df8bae1dSRodney W. Grimes 	 * Any pending ACK has now been sent.
1722df8bae1dSRodney W. Grimes 	 */
17233ac12506SJonathan T. Looney 	if (SEQ_GT(tp->rcv_nxt + recwin, tp->rcv_adv))
1724201d185bSAndre Oppermann 		tp->rcv_adv = tp->rcv_nxt + recwin;
1725df8bae1dSRodney W. Grimes 	tp->last_ack_sent = tp->rcv_nxt;
17263bfd6421SJonathan Lemon 	tp->t_flags &= ~(TF_ACKNOW | TF_DELACK);
1727b8152ba7SAndre Oppermann 	if (tcp_timer_active(tp, TT_DELACK))
1728b8152ba7SAndre Oppermann 		tcp_timer_activate(tp, TT_DELACK, 0);
1729d912c694SMatthew Dillon 	if (sendalot)
1730d912c694SMatthew Dillon 		goto again;
1731df8bae1dSRodney W. Grimes 	return (0);
1732df8bae1dSRodney W. Grimes }
1733df8bae1dSRodney W. Grimes 
1734df8bae1dSRodney W. Grimes void
1735ad3f9ab3SAndre Oppermann tcp_setpersist(struct tcpcb *tp)
1736df8bae1dSRodney W. Grimes {
17379b8b58e0SJonathan Lemon 	int t = ((tp->t_srtt >> 2) + tp->t_rttvar) >> 1;
17389b8b58e0SJonathan Lemon 	int tt;
173908af8aacSRandall Stewart 	int maxunacktime;
1740df8bae1dSRodney W. Grimes 
1741672dc4aeSJohn Baldwin 	tp->t_flags &= ~TF_PREVVALID;
1742b8152ba7SAndre Oppermann 	if (tcp_timer_active(tp, TT_REXMT))
17439b8b58e0SJonathan Lemon 		panic("tcp_setpersist: retransmit pending");
1744df8bae1dSRodney W. Grimes 	/*
174508af8aacSRandall Stewart 	 * If the state is already closed, don't bother.
174608af8aacSRandall Stewart 	 */
174708af8aacSRandall Stewart 	if (tp->t_state == TCPS_CLOSED)
174808af8aacSRandall Stewart 		return;
174908af8aacSRandall Stewart 
175008af8aacSRandall Stewart 	/*
1751a4641f4eSPedro F. Giffuni 	 * Start/restart persistence timer.
1752df8bae1dSRodney W. Grimes 	 */
17539b8b58e0SJonathan Lemon 	TCPT_RANGESET(tt, t * tcp_backoff[tp->t_rxtshift],
17540645c604SHiren Panchasara 		      tcp_persmin, tcp_persmax);
175508af8aacSRandall Stewart 	if (TP_MAXUNACKTIME(tp) && tp->t_acktime) {
175608af8aacSRandall Stewart 		maxunacktime = tp->t_acktime + TP_MAXUNACKTIME(tp) - ticks;
175708af8aacSRandall Stewart 		if (maxunacktime < 1)
175808af8aacSRandall Stewart 			maxunacktime = 1;
175908af8aacSRandall Stewart 		if (maxunacktime < tt)
176008af8aacSRandall Stewart 			tt = maxunacktime;
176108af8aacSRandall Stewart 	}
1762b8152ba7SAndre Oppermann 	tcp_timer_activate(tp, TT_PERSIST, tt);
176343b117f8SRichard Scheffenegger 	if (tp->t_rxtshift < V_tcp_retries)
1764df8bae1dSRodney W. Grimes 		tp->t_rxtshift++;
1765df8bae1dSRodney W. Grimes }
176602a1a643SAndre Oppermann 
176702a1a643SAndre Oppermann /*
176802a1a643SAndre Oppermann  * Insert TCP options according to the supplied parameters to the place
176902a1a643SAndre Oppermann  * optp in a consistent way.  Can handle unaligned destinations.
177002a1a643SAndre Oppermann  *
177102a1a643SAndre Oppermann  * The order of the option processing is crucial for optimal packing and
177202a1a643SAndre Oppermann  * alignment for the scarce option space.
177302a1a643SAndre Oppermann  *
177402a1a643SAndre Oppermann  * The optimal order for a SYN/SYN-ACK segment is:
177502a1a643SAndre Oppermann  *   MSS (4) + NOP (1) + Window scale (3) + SACK permitted (2) +
177602a1a643SAndre Oppermann  *   Timestamp (10) + Signature (18) = 38 bytes out of a maximum of 40.
177702a1a643SAndre Oppermann  *
177802a1a643SAndre Oppermann  * The SACK options should be last.  SACK blocks consume 8*n+2 bytes.
177902a1a643SAndre Oppermann  * So a full size SACK blocks option is 34 bytes (with 4 SACK blocks).
178002a1a643SAndre Oppermann  * At minimum we need 10 bytes (to generate 1 SACK block).  If both
178102a1a643SAndre Oppermann  * TCP Timestamps (12 bytes) and TCP Signatures (18 bytes) are present,
178202a1a643SAndre Oppermann  * we only have 10 bytes for SACK options (40 - (12 + 18)).
178302a1a643SAndre Oppermann  */
178402a1a643SAndre Oppermann int
178502a1a643SAndre Oppermann tcp_addoptions(struct tcpopt *to, u_char *optp)
178602a1a643SAndre Oppermann {
17875d20f974SJonathan T. Looney 	u_int32_t mask, optlen = 0;
178802a1a643SAndre Oppermann 
178902a1a643SAndre Oppermann 	for (mask = 1; mask < TOF_MAXOPT; mask <<= 1) {
179002a1a643SAndre Oppermann 		if ((to->to_flags & mask) != mask)
179102a1a643SAndre Oppermann 			continue;
17923a4018c4SAndre Oppermann 		if (optlen == TCP_MAXOLEN)
17933a4018c4SAndre Oppermann 			break;
179402a1a643SAndre Oppermann 		switch (to->to_flags & mask) {
179502a1a643SAndre Oppermann 		case TOF_MSS:
179602a1a643SAndre Oppermann 			while (optlen % 4) {
179702a1a643SAndre Oppermann 				optlen += TCPOLEN_NOP;
179802a1a643SAndre Oppermann 				*optp++ = TCPOPT_NOP;
179902a1a643SAndre Oppermann 			}
18003a4018c4SAndre Oppermann 			if (TCP_MAXOLEN - optlen < TCPOLEN_MAXSEG)
18013a4018c4SAndre Oppermann 				continue;
180202a1a643SAndre Oppermann 			optlen += TCPOLEN_MAXSEG;
180302a1a643SAndre Oppermann 			*optp++ = TCPOPT_MAXSEG;
180402a1a643SAndre Oppermann 			*optp++ = TCPOLEN_MAXSEG;
180502a1a643SAndre Oppermann 			to->to_mss = htons(to->to_mss);
180602a1a643SAndre Oppermann 			bcopy((u_char *)&to->to_mss, optp, sizeof(to->to_mss));
180702a1a643SAndre Oppermann 			optp += sizeof(to->to_mss);
180802a1a643SAndre Oppermann 			break;
180902a1a643SAndre Oppermann 		case TOF_SCALE:
181002a1a643SAndre Oppermann 			while (!optlen || optlen % 2 != 1) {
181102a1a643SAndre Oppermann 				optlen += TCPOLEN_NOP;
181202a1a643SAndre Oppermann 				*optp++ = TCPOPT_NOP;
181302a1a643SAndre Oppermann 			}
18143a4018c4SAndre Oppermann 			if (TCP_MAXOLEN - optlen < TCPOLEN_WINDOW)
18153a4018c4SAndre Oppermann 				continue;
181602a1a643SAndre Oppermann 			optlen += TCPOLEN_WINDOW;
181702a1a643SAndre Oppermann 			*optp++ = TCPOPT_WINDOW;
181802a1a643SAndre Oppermann 			*optp++ = TCPOLEN_WINDOW;
181902a1a643SAndre Oppermann 			*optp++ = to->to_wscale;
182002a1a643SAndre Oppermann 			break;
182102a1a643SAndre Oppermann 		case TOF_SACKPERM:
182202a1a643SAndre Oppermann 			while (optlen % 2) {
182302a1a643SAndre Oppermann 				optlen += TCPOLEN_NOP;
182402a1a643SAndre Oppermann 				*optp++ = TCPOPT_NOP;
182502a1a643SAndre Oppermann 			}
18263a4018c4SAndre Oppermann 			if (TCP_MAXOLEN - optlen < TCPOLEN_SACK_PERMITTED)
18273a4018c4SAndre Oppermann 				continue;
182802a1a643SAndre Oppermann 			optlen += TCPOLEN_SACK_PERMITTED;
182902a1a643SAndre Oppermann 			*optp++ = TCPOPT_SACK_PERMITTED;
183002a1a643SAndre Oppermann 			*optp++ = TCPOLEN_SACK_PERMITTED;
183102a1a643SAndre Oppermann 			break;
183202a1a643SAndre Oppermann 		case TOF_TS:
183302a1a643SAndre Oppermann 			while (!optlen || optlen % 4 != 2) {
183402a1a643SAndre Oppermann 				optlen += TCPOLEN_NOP;
183502a1a643SAndre Oppermann 				*optp++ = TCPOPT_NOP;
183602a1a643SAndre Oppermann 			}
18373a4018c4SAndre Oppermann 			if (TCP_MAXOLEN - optlen < TCPOLEN_TIMESTAMP)
18383a4018c4SAndre Oppermann 				continue;
183902a1a643SAndre Oppermann 			optlen += TCPOLEN_TIMESTAMP;
184002a1a643SAndre Oppermann 			*optp++ = TCPOPT_TIMESTAMP;
184102a1a643SAndre Oppermann 			*optp++ = TCPOLEN_TIMESTAMP;
184202a1a643SAndre Oppermann 			to->to_tsval = htonl(to->to_tsval);
184302a1a643SAndre Oppermann 			to->to_tsecr = htonl(to->to_tsecr);
184402a1a643SAndre Oppermann 			bcopy((u_char *)&to->to_tsval, optp, sizeof(to->to_tsval));
184502a1a643SAndre Oppermann 			optp += sizeof(to->to_tsval);
184602a1a643SAndre Oppermann 			bcopy((u_char *)&to->to_tsecr, optp, sizeof(to->to_tsecr));
184702a1a643SAndre Oppermann 			optp += sizeof(to->to_tsecr);
184802a1a643SAndre Oppermann 			break;
184902a1a643SAndre Oppermann 		case TOF_SIGNATURE:
185002a1a643SAndre Oppermann 			{
185102a1a643SAndre Oppermann 			int siglen = TCPOLEN_SIGNATURE - 2;
185202a1a643SAndre Oppermann 
185302a1a643SAndre Oppermann 			while (!optlen || optlen % 4 != 2) {
185402a1a643SAndre Oppermann 				optlen += TCPOLEN_NOP;
185502a1a643SAndre Oppermann 				*optp++ = TCPOPT_NOP;
185602a1a643SAndre Oppermann 			}
1857fcf59617SAndrey V. Elsukov 			if (TCP_MAXOLEN - optlen < TCPOLEN_SIGNATURE) {
1858fcf59617SAndrey V. Elsukov 				to->to_flags &= ~TOF_SIGNATURE;
185902a1a643SAndre Oppermann 				continue;
1860fcf59617SAndrey V. Elsukov 			}
186102a1a643SAndre Oppermann 			optlen += TCPOLEN_SIGNATURE;
186202a1a643SAndre Oppermann 			*optp++ = TCPOPT_SIGNATURE;
186302a1a643SAndre Oppermann 			*optp++ = TCPOLEN_SIGNATURE;
186402a1a643SAndre Oppermann 			to->to_signature = optp;
186502a1a643SAndre Oppermann 			while (siglen--)
186602a1a643SAndre Oppermann 				 *optp++ = 0;
186702a1a643SAndre Oppermann 			break;
186802a1a643SAndre Oppermann 			}
186902a1a643SAndre Oppermann 		case TOF_SACK:
187002a1a643SAndre Oppermann 			{
187102a1a643SAndre Oppermann 			int sackblks = 0;
187202a1a643SAndre Oppermann 			struct sackblk *sack = (struct sackblk *)to->to_sacks;
187302a1a643SAndre Oppermann 			tcp_seq sack_seq;
187402a1a643SAndre Oppermann 
187502a1a643SAndre Oppermann 			while (!optlen || optlen % 4 != 2) {
187602a1a643SAndre Oppermann 				optlen += TCPOLEN_NOP;
187702a1a643SAndre Oppermann 				*optp++ = TCPOPT_NOP;
187802a1a643SAndre Oppermann 			}
18793a4018c4SAndre Oppermann 			if (TCP_MAXOLEN - optlen < TCPOLEN_SACKHDR + TCPOLEN_SACK)
188002a1a643SAndre Oppermann 				continue;
188102a1a643SAndre Oppermann 			optlen += TCPOLEN_SACKHDR;
188202a1a643SAndre Oppermann 			*optp++ = TCPOPT_SACK;
188302a1a643SAndre Oppermann 			sackblks = min(to->to_nsacks,
18840d957bbaSAndre Oppermann 					(TCP_MAXOLEN - optlen) / TCPOLEN_SACK);
188502a1a643SAndre Oppermann 			*optp++ = TCPOLEN_SACKHDR + sackblks * TCPOLEN_SACK;
188602a1a643SAndre Oppermann 			while (sackblks--) {
188702a1a643SAndre Oppermann 				sack_seq = htonl(sack->start);
188802a1a643SAndre Oppermann 				bcopy((u_char *)&sack_seq, optp, sizeof(sack_seq));
188902a1a643SAndre Oppermann 				optp += sizeof(sack_seq);
189002a1a643SAndre Oppermann 				sack_seq = htonl(sack->end);
189102a1a643SAndre Oppermann 				bcopy((u_char *)&sack_seq, optp, sizeof(sack_seq));
189202a1a643SAndre Oppermann 				optp += sizeof(sack_seq);
189302a1a643SAndre Oppermann 				optlen += TCPOLEN_SACK;
189402a1a643SAndre Oppermann 				sack++;
189502a1a643SAndre Oppermann 			}
189678b50714SRobert Watson 			TCPSTAT_INC(tcps_sack_send_blocks);
189702a1a643SAndre Oppermann 			break;
189802a1a643SAndre Oppermann 			}
1899281a0fd4SPatrick Kelsey 		case TOF_FASTOPEN:
1900281a0fd4SPatrick Kelsey 			{
1901281a0fd4SPatrick Kelsey 			int total_len;
1902281a0fd4SPatrick Kelsey 
1903281a0fd4SPatrick Kelsey 			/* XXX is there any point to aligning this option? */
1904281a0fd4SPatrick Kelsey 			total_len = TCPOLEN_FAST_OPEN_EMPTY + to->to_tfo_len;
1905c560df6fSPatrick Kelsey 			if (TCP_MAXOLEN - optlen < total_len) {
1906c560df6fSPatrick Kelsey 				to->to_flags &= ~TOF_FASTOPEN;
1907281a0fd4SPatrick Kelsey 				continue;
1908c560df6fSPatrick Kelsey 			}
1909281a0fd4SPatrick Kelsey 			*optp++ = TCPOPT_FAST_OPEN;
1910281a0fd4SPatrick Kelsey 			*optp++ = total_len;
1911281a0fd4SPatrick Kelsey 			if (to->to_tfo_len > 0) {
1912281a0fd4SPatrick Kelsey 				bcopy(to->to_tfo_cookie, optp, to->to_tfo_len);
1913281a0fd4SPatrick Kelsey 				optp += to->to_tfo_len;
1914281a0fd4SPatrick Kelsey 			}
1915281a0fd4SPatrick Kelsey 			optlen += total_len;
1916281a0fd4SPatrick Kelsey 			break;
1917281a0fd4SPatrick Kelsey 			}
191802a1a643SAndre Oppermann 		default:
191902a1a643SAndre Oppermann 			panic("%s: unknown TCP option type", __func__);
192002a1a643SAndre Oppermann 			break;
192102a1a643SAndre Oppermann 		}
192202a1a643SAndre Oppermann 	}
192302a1a643SAndre Oppermann 
192402a1a643SAndre Oppermann 	/* Terminate and pad TCP options to a 4 byte boundary. */
192502a1a643SAndre Oppermann 	if (optlen % 4) {
192602a1a643SAndre Oppermann 		optlen += TCPOLEN_EOL;
192702a1a643SAndre Oppermann 		*optp++ = TCPOPT_EOL;
192802a1a643SAndre Oppermann 	}
1929413deb12SBjoern A. Zeeb 	/*
1930413deb12SBjoern A. Zeeb 	 * According to RFC 793 (STD0007):
1931413deb12SBjoern A. Zeeb 	 *   "The content of the header beyond the End-of-Option option
1932413deb12SBjoern A. Zeeb 	 *    must be header padding (i.e., zero)."
1933413deb12SBjoern A. Zeeb 	 *   and later: "The padding is composed of zeros."
1934413deb12SBjoern A. Zeeb 	 */
193502a1a643SAndre Oppermann 	while (optlen % 4) {
1936c343c524SAndre Oppermann 		optlen += TCPOLEN_PAD;
1937c343c524SAndre Oppermann 		*optp++ = TCPOPT_PAD;
193802a1a643SAndre Oppermann 	}
193902a1a643SAndre Oppermann 
19400d957bbaSAndre Oppermann 	KASSERT(optlen <= TCP_MAXOLEN, ("%s: TCP options too long", __func__));
194102a1a643SAndre Oppermann 	return (optlen);
194202a1a643SAndre Oppermann }
194366492feaSGleb Smirnoff 
194489e560f4SRandall Stewart /*
194589e560f4SRandall Stewart  * This is a copy of m_copym(), taking the TSO segment size/limit
194689e560f4SRandall Stewart  * constraints into account, and advancing the sndptr as it goes.
194789e560f4SRandall Stewart  */
194889e560f4SRandall Stewart struct mbuf *
194989e560f4SRandall Stewart tcp_m_copym(struct mbuf *m, int32_t off0, int32_t *plen,
1950b2e60773SJohn Baldwin     int32_t seglimit, int32_t segsize, struct sockbuf *sb, bool hw_tls)
195189e560f4SRandall Stewart {
1952b2e60773SJohn Baldwin #ifdef KERN_TLS
1953b2e60773SJohn Baldwin 	struct ktls_session *tls, *ntls;
1954732b6d4dSJohn Baldwin 	struct mbuf *start __diagused;
1955b2e60773SJohn Baldwin #endif
195689e560f4SRandall Stewart 	struct mbuf *n, **np;
195789e560f4SRandall Stewart 	struct mbuf *top;
195889e560f4SRandall Stewart 	int32_t off = off0;
195989e560f4SRandall Stewart 	int32_t len = *plen;
196089e560f4SRandall Stewart 	int32_t fragsize;
196189e560f4SRandall Stewart 	int32_t len_cp = 0;
196289e560f4SRandall Stewart 	int32_t *pkthdrlen;
196389e560f4SRandall Stewart 	uint32_t mlen, frags;
196489e560f4SRandall Stewart 	bool copyhdr;
196589e560f4SRandall Stewart 
196689e560f4SRandall Stewart 	KASSERT(off >= 0, ("tcp_m_copym, negative off %d", off));
196789e560f4SRandall Stewart 	KASSERT(len >= 0, ("tcp_m_copym, negative len %d", len));
196889e560f4SRandall Stewart 	if (off == 0 && m->m_flags & M_PKTHDR)
196989e560f4SRandall Stewart 		copyhdr = true;
197089e560f4SRandall Stewart 	else
197189e560f4SRandall Stewart 		copyhdr = false;
197289e560f4SRandall Stewart 	while (off > 0) {
197389e560f4SRandall Stewart 		KASSERT(m != NULL, ("tcp_m_copym, offset > size of mbuf chain"));
197489e560f4SRandall Stewart 		if (off < m->m_len)
197589e560f4SRandall Stewart 			break;
197689e560f4SRandall Stewart 		off -= m->m_len;
197789e560f4SRandall Stewart 		if ((sb) && (m == sb->sb_sndptr)) {
197889e560f4SRandall Stewart 			sb->sb_sndptroff += m->m_len;
197989e560f4SRandall Stewart 			sb->sb_sndptr = m->m_next;
198089e560f4SRandall Stewart 		}
198189e560f4SRandall Stewart 		m = m->m_next;
198289e560f4SRandall Stewart 	}
198389e560f4SRandall Stewart 	np = &top;
198489e560f4SRandall Stewart 	top = NULL;
198589e560f4SRandall Stewart 	pkthdrlen = NULL;
1986b2e60773SJohn Baldwin #ifdef KERN_TLS
19876edfd179SGleb Smirnoff 	if (hw_tls && (m->m_flags & M_EXTPG))
19887b6c99d0SGleb Smirnoff 		tls = m->m_epg_tls;
1989b2e60773SJohn Baldwin 	else
1990b2e60773SJohn Baldwin 		tls = NULL;
1991b2e60773SJohn Baldwin 	start = m;
1992b2e60773SJohn Baldwin #endif
199389e560f4SRandall Stewart 	while (len > 0) {
199489e560f4SRandall Stewart 		if (m == NULL) {
199589e560f4SRandall Stewart 			KASSERT(len == M_COPYALL,
199689e560f4SRandall Stewart 			    ("tcp_m_copym, length > size of mbuf chain"));
199789e560f4SRandall Stewart 			*plen = len_cp;
199889e560f4SRandall Stewart 			if (pkthdrlen != NULL)
199989e560f4SRandall Stewart 				*pkthdrlen = len_cp;
200089e560f4SRandall Stewart 			break;
200189e560f4SRandall Stewart 		}
2002b2e60773SJohn Baldwin #ifdef KERN_TLS
2003b2e60773SJohn Baldwin 		if (hw_tls) {
20046edfd179SGleb Smirnoff 			if (m->m_flags & M_EXTPG)
20057b6c99d0SGleb Smirnoff 				ntls = m->m_epg_tls;
2006b2e60773SJohn Baldwin 			else
2007b2e60773SJohn Baldwin 				ntls = NULL;
2008b2e60773SJohn Baldwin 
2009b2e60773SJohn Baldwin 			/*
2010b2e60773SJohn Baldwin 			 * Avoid mixing TLS records with handshake
2011b2e60773SJohn Baldwin 			 * data or TLS records from different
2012b2e60773SJohn Baldwin 			 * sessions.
2013b2e60773SJohn Baldwin 			 */
2014b2e60773SJohn Baldwin 			if (tls != ntls) {
2015b2e60773SJohn Baldwin 				MPASS(m != start);
2016b2e60773SJohn Baldwin 				*plen = len_cp;
2017b2e60773SJohn Baldwin 				if (pkthdrlen != NULL)
2018b2e60773SJohn Baldwin 					*pkthdrlen = len_cp;
2019b2e60773SJohn Baldwin 				break;
2020b2e60773SJohn Baldwin 			}
2021b2e60773SJohn Baldwin 		}
2022b2e60773SJohn Baldwin #endif
202389e560f4SRandall Stewart 		mlen = min(len, m->m_len - off);
202489e560f4SRandall Stewart 		if (seglimit) {
202589e560f4SRandall Stewart 			/*
20266edfd179SGleb Smirnoff 			 * For M_EXTPG mbufs, add 3 segments
202789e560f4SRandall Stewart 			 * + 1 in case we are crossing page boundaries
202889e560f4SRandall Stewart 			 * + 2 in case the TLS hdr/trailer are used
202989e560f4SRandall Stewart 			 * It is cheaper to just add the segments
203089e560f4SRandall Stewart 			 * than it is to take the cache miss to look
203189e560f4SRandall Stewart 			 * at the mbuf ext_pgs state in detail.
203289e560f4SRandall Stewart 			 */
20336edfd179SGleb Smirnoff 			if (m->m_flags & M_EXTPG) {
203489e560f4SRandall Stewart 				fragsize = min(segsize, PAGE_SIZE);
203589e560f4SRandall Stewart 				frags = 3;
203689e560f4SRandall Stewart 			} else {
203789e560f4SRandall Stewart 				fragsize = segsize;
203889e560f4SRandall Stewart 				frags = 0;
203989e560f4SRandall Stewart 			}
204089e560f4SRandall Stewart 
204189e560f4SRandall Stewart 			/* Break if we really can't fit anymore. */
204289e560f4SRandall Stewart 			if ((frags + 1) >= seglimit) {
204389e560f4SRandall Stewart 				*plen =	len_cp;
204489e560f4SRandall Stewart 				if (pkthdrlen != NULL)
204589e560f4SRandall Stewart 					*pkthdrlen = len_cp;
204689e560f4SRandall Stewart 				break;
204789e560f4SRandall Stewart 			}
204889e560f4SRandall Stewart 
204989e560f4SRandall Stewart 			/*
205089e560f4SRandall Stewart 			 * Reduce size if you can't copy the whole
205189e560f4SRandall Stewart 			 * mbuf. If we can't copy the whole mbuf, also
205289e560f4SRandall Stewart 			 * adjust len so the loop will end after this
205389e560f4SRandall Stewart 			 * mbuf.
205489e560f4SRandall Stewart 			 */
205589e560f4SRandall Stewart 			if ((frags + howmany(mlen, fragsize)) >= seglimit) {
205689e560f4SRandall Stewart 				mlen = (seglimit - frags - 1) * fragsize;
205789e560f4SRandall Stewart 				len = mlen;
205889e560f4SRandall Stewart 				*plen = len_cp + len;
205989e560f4SRandall Stewart 				if (pkthdrlen != NULL)
206089e560f4SRandall Stewart 					*pkthdrlen = *plen;
206189e560f4SRandall Stewart 			}
206289e560f4SRandall Stewart 			frags += howmany(mlen, fragsize);
206389e560f4SRandall Stewart 			if (frags == 0)
206489e560f4SRandall Stewart 				frags++;
206589e560f4SRandall Stewart 			seglimit -= frags;
206689e560f4SRandall Stewart 			KASSERT(seglimit > 0,
206789e560f4SRandall Stewart 			    ("%s: seglimit went too low", __func__));
206889e560f4SRandall Stewart 		}
206989e560f4SRandall Stewart 		if (copyhdr)
207089e560f4SRandall Stewart 			n = m_gethdr(M_NOWAIT, m->m_type);
207189e560f4SRandall Stewart 		else
207289e560f4SRandall Stewart 			n = m_get(M_NOWAIT, m->m_type);
207389e560f4SRandall Stewart 		*np = n;
207489e560f4SRandall Stewart 		if (n == NULL)
207589e560f4SRandall Stewart 			goto nospace;
207689e560f4SRandall Stewart 		if (copyhdr) {
207789e560f4SRandall Stewart 			if (!m_dup_pkthdr(n, m, M_NOWAIT))
207889e560f4SRandall Stewart 				goto nospace;
207989e560f4SRandall Stewart 			if (len == M_COPYALL)
208089e560f4SRandall Stewart 				n->m_pkthdr.len -= off0;
208189e560f4SRandall Stewart 			else
208289e560f4SRandall Stewart 				n->m_pkthdr.len = len;
208389e560f4SRandall Stewart 			pkthdrlen = &n->m_pkthdr.len;
208489e560f4SRandall Stewart 			copyhdr = false;
208589e560f4SRandall Stewart 		}
208689e560f4SRandall Stewart 		n->m_len = mlen;
208789e560f4SRandall Stewart 		len_cp += n->m_len;
208861664ee7SGleb Smirnoff 		if (m->m_flags & (M_EXT | M_EXTPG)) {
208989e560f4SRandall Stewart 			n->m_data = m->m_data + off;
209089e560f4SRandall Stewart 			mb_dupcl(n, m);
209189e560f4SRandall Stewart 		} else
209289e560f4SRandall Stewart 			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
209389e560f4SRandall Stewart 			    (u_int)n->m_len);
209489e560f4SRandall Stewart 
209589e560f4SRandall Stewart 		if (sb && (sb->sb_sndptr == m) &&
209689e560f4SRandall Stewart 		    ((n->m_len + off) >= m->m_len) && m->m_next) {
209789e560f4SRandall Stewart 			sb->sb_sndptroff += m->m_len;
209889e560f4SRandall Stewart 			sb->sb_sndptr = m->m_next;
209989e560f4SRandall Stewart 		}
210089e560f4SRandall Stewart 		off = 0;
210189e560f4SRandall Stewart 		if (len != M_COPYALL) {
210289e560f4SRandall Stewart 			len -= n->m_len;
210389e560f4SRandall Stewart 		}
210489e560f4SRandall Stewart 		m = m->m_next;
210589e560f4SRandall Stewart 		np = &n->m_next;
210689e560f4SRandall Stewart 	}
210789e560f4SRandall Stewart 	return (top);
210889e560f4SRandall Stewart nospace:
210989e560f4SRandall Stewart 	m_freem(top);
211089e560f4SRandall Stewart 	return (NULL);
211189e560f4SRandall Stewart }
211289e560f4SRandall Stewart 
211366492feaSGleb Smirnoff void
211466492feaSGleb Smirnoff tcp_sndbuf_autoscale(struct tcpcb *tp, struct socket *so, uint32_t sendwin)
211566492feaSGleb Smirnoff {
211666492feaSGleb Smirnoff 
211766492feaSGleb Smirnoff 	/*
211866492feaSGleb Smirnoff 	 * Automatic sizing of send socket buffer.  Often the send buffer
211966492feaSGleb Smirnoff 	 * size is not optimally adjusted to the actual network conditions
212066492feaSGleb Smirnoff 	 * at hand (delay bandwidth product).  Setting the buffer size too
212166492feaSGleb Smirnoff 	 * small limits throughput on links with high bandwidth and high
212266492feaSGleb Smirnoff 	 * delay (eg. trans-continental/oceanic links).  Setting the
212366492feaSGleb Smirnoff 	 * buffer size too big consumes too much real kernel memory,
212466492feaSGleb Smirnoff 	 * especially with many connections on busy servers.
212566492feaSGleb Smirnoff 	 *
212666492feaSGleb Smirnoff 	 * The criteria to step up the send buffer one notch are:
212766492feaSGleb Smirnoff 	 *  1. receive window of remote host is larger than send buffer
212866492feaSGleb Smirnoff 	 *     (with a fudge factor of 5/4th);
212966492feaSGleb Smirnoff 	 *  2. send buffer is filled to 7/8th with data (so we actually
213066492feaSGleb Smirnoff 	 *     have data to make use of it);
213166492feaSGleb Smirnoff 	 *  3. send buffer fill has not hit maximal automatic size;
213266492feaSGleb Smirnoff 	 *  4. our send window (slow start and cogestion controlled) is
213366492feaSGleb Smirnoff 	 *     larger than sent but unacknowledged data in send buffer.
213466492feaSGleb Smirnoff 	 *
213566492feaSGleb Smirnoff 	 * The remote host receive window scaling factor may limit the
213666492feaSGleb Smirnoff 	 * growing of the send buffer before it reaches its allowed
213766492feaSGleb Smirnoff 	 * maximum.
213866492feaSGleb Smirnoff 	 *
213966492feaSGleb Smirnoff 	 * It scales directly with slow start or congestion window
214066492feaSGleb Smirnoff 	 * and does at most one step per received ACK.  This fast
214166492feaSGleb Smirnoff 	 * scaling has the drawback of growing the send buffer beyond
214266492feaSGleb Smirnoff 	 * what is strictly necessary to make full use of a given
214366492feaSGleb Smirnoff 	 * delay*bandwidth product.  However testing has shown this not
214466492feaSGleb Smirnoff 	 * to be much of an problem.  At worst we are trading wasting
214566492feaSGleb Smirnoff 	 * of available bandwidth (the non-use of it) for wasting some
214666492feaSGleb Smirnoff 	 * socket buffer memory.
214766492feaSGleb Smirnoff 	 *
214866492feaSGleb Smirnoff 	 * TODO: Shrink send buffer during idle periods together
214966492feaSGleb Smirnoff 	 * with congestion window.  Requires another timer.  Has to
215066492feaSGleb Smirnoff 	 * wait for upcoming tcp timer rewrite.
215166492feaSGleb Smirnoff 	 *
215266492feaSGleb Smirnoff 	 * XXXGL: should there be used sbused() or sbavail()?
215366492feaSGleb Smirnoff 	 */
215466492feaSGleb Smirnoff 	if (V_tcp_do_autosndbuf && so->so_snd.sb_flags & SB_AUTOSIZE) {
215566492feaSGleb Smirnoff 		int lowat;
215666492feaSGleb Smirnoff 
215766492feaSGleb Smirnoff 		lowat = V_tcp_sendbuf_auto_lowat ? so->so_snd.sb_lowat : 0;
215866492feaSGleb Smirnoff 		if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat - lowat &&
215966492feaSGleb Smirnoff 		    sbused(&so->so_snd) >=
216066492feaSGleb Smirnoff 		    (so->so_snd.sb_hiwat / 8 * 7) - lowat &&
216166492feaSGleb Smirnoff 		    sbused(&so->so_snd) < V_tcp_autosndbuf_max &&
216266492feaSGleb Smirnoff 		    sendwin >= (sbused(&so->so_snd) -
216366492feaSGleb Smirnoff 		    (tp->snd_nxt - tp->snd_una))) {
216443283184SGleb Smirnoff 			if (!sbreserve_locked(so, SO_SND,
216566492feaSGleb Smirnoff 			    min(so->so_snd.sb_hiwat + V_tcp_autosndbuf_inc,
216643283184SGleb Smirnoff 			     V_tcp_autosndbuf_max), curthread))
216766492feaSGleb Smirnoff 				so->so_snd.sb_flags &= ~SB_AUTOSIZE;
216866492feaSGleb Smirnoff 		}
216966492feaSGleb Smirnoff 	}
217066492feaSGleb Smirnoff }
2171