1ff550d0eSmasputra /*
2ff550d0eSmasputra * CDDL HEADER START
3ff550d0eSmasputra *
4ff550d0eSmasputra * The contents of this file are subject to the terms of the
5c203fc81Skrishna * Common Development and Distribution License (the "License").
6c203fc81Skrishna * You may not use this file except in compliance with the License.
7ff550d0eSmasputra *
8ff550d0eSmasputra * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9ff550d0eSmasputra * or http://www.opensolaris.org/os/licensing.
10ff550d0eSmasputra * See the License for the specific language governing permissions
11ff550d0eSmasputra * and limitations under the License.
12ff550d0eSmasputra *
13ff550d0eSmasputra * When distributing Covered Code, include this CDDL HEADER in each
14ff550d0eSmasputra * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15ff550d0eSmasputra * If applicable, add the following below this CDDL HEADER, with the
16ff550d0eSmasputra * fields enclosed by brackets "[]" replaced with your own identifying
17ff550d0eSmasputra * information: Portions Copyright [yyyy] [name of copyright owner]
18ff550d0eSmasputra *
19ff550d0eSmasputra * CDDL HEADER END
20ff550d0eSmasputra */
21ff550d0eSmasputra /*
2266cd0f60SKacheong Poon * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23a2f04351SSebastien Roy * Copyright (c) 2015 by Delphix. All rights reserved.
24e8249070SRobert Mustacchi * Copyright 2024 Oxide Computer Company
25ff550d0eSmasputra */
26ff550d0eSmasputra
27ff550d0eSmasputra #include <sys/types.h>
28ff550d0eSmasputra #include <sys/stream.h>
29ff550d0eSmasputra #include <sys/strsun.h>
30ff550d0eSmasputra #include <sys/strsubr.h>
31ff550d0eSmasputra #include <sys/debug.h>
32381a2a9aSdr146992 #include <sys/sdt.h>
33ff550d0eSmasputra #include <sys/cmn_err.h>
34ff550d0eSmasputra #include <sys/tihdr.h>
35ff550d0eSmasputra
36ff550d0eSmasputra #include <inet/common.h>
37fc80c0dfSnordmark #include <inet/optcom.h>
38ff550d0eSmasputra #include <inet/ip.h>
39e11c3f44Smeem #include <inet/ip_if.h>
40ff550d0eSmasputra #include <inet/ip_impl.h>
41ff550d0eSmasputra #include <inet/tcp.h>
42ff550d0eSmasputra #include <inet/tcp_impl.h>
43ff550d0eSmasputra #include <inet/ipsec_impl.h>
44ff550d0eSmasputra #include <inet/ipclassifier.h>
45ff550d0eSmasputra #include <inet/ipp_common.h>
4691762968SBrian Ruthven #include <inet/ip_if.h>
47ff550d0eSmasputra
48ff550d0eSmasputra /*
49ff550d0eSmasputra * This file implements TCP fusion - a protocol-less data path for TCP
50ff550d0eSmasputra * loopback connections. The fusion of two local TCP endpoints occurs
51ff550d0eSmasputra * at connection establishment time. Various conditions (see details
52ff550d0eSmasputra * in tcp_fuse()) need to be met for fusion to be successful. If it
53ff550d0eSmasputra * fails, we fall back to the regular TCP data path; if it succeeds,
54ff550d0eSmasputra * both endpoints proceed to use tcp_fuse_output() as the transmit path.
55ff550d0eSmasputra * tcp_fuse_output() enqueues application data directly onto the peer's
567b8f5432SAnders Persson * receive queue; no protocol processing is involved.
57ff550d0eSmasputra *
58e0968231Svi117747 * Sychronization is handled by squeue and the mutex tcp_non_sq_lock.
59ff550d0eSmasputra * One of the requirements for fusion to succeed is that both endpoints
60ff550d0eSmasputra * need to be using the same squeue. This ensures that neither side
617b8f5432SAnders Persson * can disappear while the other side is still sending data. Flow
627b8f5432SAnders Persson * control information is manipulated outside the squeue, so the
637b8f5432SAnders Persson * tcp_non_sq_lock must be held when touching tcp_flow_stopped.
64ff550d0eSmasputra */
65ff550d0eSmasputra
66ff550d0eSmasputra /*
67ff550d0eSmasputra * Setting this to false means we disable fusion altogether and
68ff550d0eSmasputra * loopback connections would go through the protocol paths.
69ff550d0eSmasputra */
70ff550d0eSmasputra boolean_t do_tcp_fusion = B_TRUE;
71ff550d0eSmasputra
72ff550d0eSmasputra /*
73ff550d0eSmasputra * This routine gets called by the eager tcp upon changing state from
74ff550d0eSmasputra * SYN_RCVD to ESTABLISHED. It fuses a direct path between itself
75ff550d0eSmasputra * and the active connect tcp such that the regular tcp processings
76ff550d0eSmasputra * may be bypassed under allowable circumstances. Because the fusion
77ff550d0eSmasputra * requires both endpoints to be in the same squeue, it does not work
78ff550d0eSmasputra * for simultaneous active connects because there is no easy way to
79ff550d0eSmasputra * switch from one squeue to another once the connection is created.
80ff550d0eSmasputra * This is different from the eager tcp case where we assign it the
81ff550d0eSmasputra * same squeue as the one given to the active connect tcp during open.
82ff550d0eSmasputra */
83ff550d0eSmasputra void
tcp_fuse(tcp_t * tcp,uchar_t * iphdr,tcpha_t * tcpha)84bd670b35SErik Nordmark tcp_fuse(tcp_t *tcp, uchar_t *iphdr, tcpha_t *tcpha)
85ff550d0eSmasputra {
86ff550d0eSmasputra conn_t *peer_connp, *connp = tcp->tcp_connp;
87ff550d0eSmasputra tcp_t *peer_tcp;
88f4b3ec61Sdh155122 tcp_stack_t *tcps = tcp->tcp_tcps;
89f4b3ec61Sdh155122 netstack_t *ns;
90f4b3ec61Sdh155122 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip;
91ff550d0eSmasputra
92ff550d0eSmasputra ASSERT(!tcp->tcp_fused);
93ff550d0eSmasputra ASSERT(tcp->tcp_loopback);
94ff550d0eSmasputra ASSERT(tcp->tcp_loopback_peer == NULL);
95ff550d0eSmasputra /*
96bd670b35SErik Nordmark * We need to inherit conn_rcvbuf of the listener tcp,
9779c0745dSRao Shoaib * but we can't really use tcp_listener since we get here after
98bd670b35SErik Nordmark * sending up T_CONN_IND and tcp_tli_accept() may be called
9979c0745dSRao Shoaib * independently, at which point tcp_listener is cleared;
10079c0745dSRao Shoaib * this is why we use tcp_saved_listener. The listener itself
10179c0745dSRao Shoaib * is guaranteed to be around until tcp_accept_finish() is called
10279c0745dSRao Shoaib * on this eager -- this won't happen until we're done since we're
10379c0745dSRao Shoaib * inside the eager's perimeter now.
104ff550d0eSmasputra */
105bd670b35SErik Nordmark ASSERT(tcp->tcp_saved_listener != NULL);
106ff550d0eSmasputra /*
107ff550d0eSmasputra * Lookup peer endpoint; search for the remote endpoint having
108ff550d0eSmasputra * the reversed address-port quadruplet in ESTABLISHED state,
109ff550d0eSmasputra * which is guaranteed to be unique in the system. Zone check
110ff550d0eSmasputra * is applied accordingly for loopback address, but not for
111ff550d0eSmasputra * local address since we want fusion to happen across Zones.
112ff550d0eSmasputra */
113bd670b35SErik Nordmark if (connp->conn_ipversion == IPV4_VERSION) {
114ff550d0eSmasputra peer_connp = ipcl_conn_tcp_lookup_reversed_ipv4(connp,
115bd670b35SErik Nordmark (ipha_t *)iphdr, tcpha, ipst);
116ff550d0eSmasputra } else {
117ff550d0eSmasputra peer_connp = ipcl_conn_tcp_lookup_reversed_ipv6(connp,
118bd670b35SErik Nordmark (ip6_t *)iphdr, tcpha, ipst);
119ff550d0eSmasputra }
120ff550d0eSmasputra
121ff550d0eSmasputra /*
122ff550d0eSmasputra * We can only proceed if peer exists, resides in the same squeue
1234da9f95bSAnders Persson * as our conn and is not raw-socket. We also restrict fusion to
1244da9f95bSAnders Persson * endpoints of the same type (STREAMS or non-STREAMS). The squeue
1254da9f95bSAnders Persson * assignment of this eager tcp was done earlier at the time of SYN
1264da9f95bSAnders Persson * processing in ip_fanout_tcp{_v6}. Note that similar squeues by
1274da9f95bSAnders Persson * itself doesn't guarantee a safe condition to fuse, hence we perform
128ff550d0eSmasputra * additional tests below.
129ff550d0eSmasputra */
130ff550d0eSmasputra ASSERT(peer_connp == NULL || peer_connp != connp);
131ff550d0eSmasputra if (peer_connp == NULL || peer_connp->conn_sqp != connp->conn_sqp ||
1324da9f95bSAnders Persson !IPCL_IS_TCP(peer_connp) ||
1334da9f95bSAnders Persson IPCL_IS_NONSTR(connp) != IPCL_IS_NONSTR(peer_connp)) {
134ff550d0eSmasputra if (peer_connp != NULL) {
135f4b3ec61Sdh155122 TCP_STAT(tcps, tcp_fusion_unqualified);
136ff550d0eSmasputra CONN_DEC_REF(peer_connp);
137ff550d0eSmasputra }
138ff550d0eSmasputra return;
139ff550d0eSmasputra }
140ff550d0eSmasputra peer_tcp = peer_connp->conn_tcp; /* active connect tcp */
141ff550d0eSmasputra
142ff550d0eSmasputra ASSERT(peer_tcp != NULL && peer_tcp != tcp && !peer_tcp->tcp_fused);
143ee6f0546SErik Nordmark ASSERT(peer_tcp->tcp_loopback_peer == NULL);
144ff550d0eSmasputra ASSERT(peer_connp->conn_sqp == connp->conn_sqp);
145ff550d0eSmasputra
146ff550d0eSmasputra /*
147ee6f0546SErik Nordmark * Due to IRE changes the peer and us might not agree on tcp_loopback.
148ee6f0546SErik Nordmark * We bail in that case.
149ee6f0546SErik Nordmark */
150ee6f0546SErik Nordmark if (!peer_tcp->tcp_loopback) {
151ee6f0546SErik Nordmark TCP_STAT(tcps, tcp_fusion_unqualified);
152ee6f0546SErik Nordmark CONN_DEC_REF(peer_connp);
153ee6f0546SErik Nordmark return;
154ee6f0546SErik Nordmark }
155*1edba515SAndy Fiddaman
156*1edba515SAndy Fiddaman /*
157*1edba515SAndy Fiddaman * If we need to add MD5 Signature options, don't allow fusion.
158*1edba515SAndy Fiddaman */
159*1edba515SAndy Fiddaman if (tcp->tcp_md5sig || peer_tcp->tcp_md5sig) {
160*1edba515SAndy Fiddaman TCP_STAT(tcps, tcp_fusion_unqualified);
161*1edba515SAndy Fiddaman CONN_DEC_REF(peer_connp);
162*1edba515SAndy Fiddaman return;
163*1edba515SAndy Fiddaman }
164*1edba515SAndy Fiddaman
165ee6f0546SErik Nordmark /*
166ff550d0eSmasputra * Fuse the endpoints; we perform further checks against both
167ff550d0eSmasputra * tcp endpoints to ensure that a fusion is allowed to happen.
168ff550d0eSmasputra */
169f4b3ec61Sdh155122 ns = tcps->tcps_netstack;
170f4b3ec61Sdh155122 ipst = ns->netstack_ip;
171f4b3ec61Sdh155122
172ff550d0eSmasputra if (!tcp->tcp_unfusable && !peer_tcp->tcp_unfusable &&
173dd49f125SAnders Persson tcp->tcp_xmit_head == NULL && peer_tcp->tcp_xmit_head == NULL) {
174ab82c29bSToomas Soome mblk_t *mp = NULL;
175bd670b35SErik Nordmark queue_t *peer_rq = peer_connp->conn_rq;
176ff550d0eSmasputra
1770f1702c5SYu Xiangning ASSERT(!TCP_IS_DETACHED(peer_tcp));
178bd670b35SErik Nordmark ASSERT(tcp->tcp_fused_sigurg_mp == NULL);
179bd670b35SErik Nordmark ASSERT(peer_tcp->tcp_fused_sigurg_mp == NULL);
180ff550d0eSmasputra
181ff550d0eSmasputra /*
182ff550d0eSmasputra * We need to drain data on both endpoints during unfuse.
183ff550d0eSmasputra * If we need to send up SIGURG at the time of draining,
184ff550d0eSmasputra * we want to be sure that an mblk is readily available.
185ff550d0eSmasputra * This is why we pre-allocate the M_PCSIG mblks for both
186ff550d0eSmasputra * endpoints which will only be used during/after unfuse.
18701765833SAnders Persson * The mblk might already exist if we are doing a re-fuse.
188ff550d0eSmasputra */
1890f1702c5SYu Xiangning if (!IPCL_IS_NONSTR(tcp->tcp_connp)) {
1904da9f95bSAnders Persson ASSERT(!IPCL_IS_NONSTR(peer_tcp->tcp_connp));
1914da9f95bSAnders Persson
19201765833SAnders Persson if (tcp->tcp_fused_sigurg_mp == NULL) {
193ff550d0eSmasputra if ((mp = allocb(1, BPRI_HI)) == NULL)
194ff550d0eSmasputra goto failed;
195ff550d0eSmasputra tcp->tcp_fused_sigurg_mp = mp;
19601765833SAnders Persson }
197ff550d0eSmasputra
19801765833SAnders Persson if (peer_tcp->tcp_fused_sigurg_mp == NULL) {
199ff550d0eSmasputra if ((mp = allocb(1, BPRI_HI)) == NULL)
200ff550d0eSmasputra goto failed;
201ff550d0eSmasputra peer_tcp->tcp_fused_sigurg_mp = mp;
20201765833SAnders Persson }
203ff550d0eSmasputra
2044da9f95bSAnders Persson if ((mp = allocb(sizeof (struct stroptions),
2054da9f95bSAnders Persson BPRI_HI)) == NULL)
206ff550d0eSmasputra goto failed;
2070f1702c5SYu Xiangning }
208ff550d0eSmasputra
209ff550d0eSmasputra /* Fuse both endpoints */
210ff550d0eSmasputra peer_tcp->tcp_loopback_peer = tcp;
211ff550d0eSmasputra tcp->tcp_loopback_peer = peer_tcp;
212ff550d0eSmasputra peer_tcp->tcp_fused = tcp->tcp_fused = B_TRUE;
213ff550d0eSmasputra
214ff550d0eSmasputra /*
215ff550d0eSmasputra * We never use regular tcp paths in fusion and should
216ff550d0eSmasputra * therefore clear tcp_unsent on both endpoints. Having
217ff550d0eSmasputra * them set to non-zero values means asking for trouble
218ff550d0eSmasputra * especially after unfuse, where we may end up sending
219ff550d0eSmasputra * through regular tcp paths which expect xmit_list and
220ff550d0eSmasputra * friends to be correctly setup.
221ff550d0eSmasputra */
222ff550d0eSmasputra peer_tcp->tcp_unsent = tcp->tcp_unsent = 0;
223ff550d0eSmasputra
224ff550d0eSmasputra tcp_timers_stop(tcp);
225ff550d0eSmasputra tcp_timers_stop(peer_tcp);
226ff550d0eSmasputra
22779c0745dSRao Shoaib /*
22879c0745dSRao Shoaib * Set receive buffer and max packet size for the
22979c0745dSRao Shoaib * active open tcp.
23079c0745dSRao Shoaib * eager's values will be set in tcp_accept_finish.
23179c0745dSRao Shoaib */
232bd670b35SErik Nordmark (void) tcp_rwnd_set(peer_tcp, peer_tcp->tcp_connp->conn_rcvbuf);
233ff550d0eSmasputra
234ff550d0eSmasputra /*
23579c0745dSRao Shoaib * Set the write offset value to zero since we won't
23679c0745dSRao Shoaib * be needing any room for TCP/IP headers.
237ff550d0eSmasputra */
2380f1702c5SYu Xiangning if (!IPCL_IS_NONSTR(peer_tcp->tcp_connp)) {
2390f1702c5SYu Xiangning struct stroptions *stropt;
2400f1702c5SYu Xiangning
241ff550d0eSmasputra DB_TYPE(mp) = M_SETOPTS;
242ff550d0eSmasputra mp->b_wptr += sizeof (*stropt);
243ff550d0eSmasputra
244ff550d0eSmasputra stropt = (struct stroptions *)mp->b_rptr;
2453e95bd4aSAnders Persson stropt->so_flags = SO_WROFF | SO_MAXBLK;
246ff550d0eSmasputra stropt->so_wroff = 0;
2473e95bd4aSAnders Persson stropt->so_maxblk = INFPSZ;
248ff550d0eSmasputra
249ff550d0eSmasputra /* Send the options up */
250ff550d0eSmasputra putnext(peer_rq, mp);
2510f1702c5SYu Xiangning } else {
2520f1702c5SYu Xiangning struct sock_proto_props sopp;
2530f1702c5SYu Xiangning
2540f1702c5SYu Xiangning /* The peer is a non-STREAMS end point */
2550f1702c5SYu Xiangning ASSERT(IPCL_IS_TCP(peer_connp));
2560f1702c5SYu Xiangning
2573e95bd4aSAnders Persson sopp.sopp_flags = SOCKOPT_WROFF | SOCKOPT_MAXBLK;
2580f1702c5SYu Xiangning sopp.sopp_wroff = 0;
2593e95bd4aSAnders Persson sopp.sopp_maxblk = INFPSZ;
2600f1702c5SYu Xiangning (*peer_connp->conn_upcalls->su_set_proto_props)
2610f1702c5SYu Xiangning (peer_connp->conn_upper_handle, &sopp);
2620f1702c5SYu Xiangning }
2632f9e7e9bSAnders Persson } else {
264f4b3ec61Sdh155122 TCP_STAT(tcps, tcp_fusion_unqualified);
265ff550d0eSmasputra }
266ff550d0eSmasputra CONN_DEC_REF(peer_connp);
267ff550d0eSmasputra return;
268ff550d0eSmasputra
269ff550d0eSmasputra failed:
270ff550d0eSmasputra if (tcp->tcp_fused_sigurg_mp != NULL) {
271ff550d0eSmasputra freeb(tcp->tcp_fused_sigurg_mp);
272ff550d0eSmasputra tcp->tcp_fused_sigurg_mp = NULL;
273ff550d0eSmasputra }
274ff550d0eSmasputra if (peer_tcp->tcp_fused_sigurg_mp != NULL) {
275ff550d0eSmasputra freeb(peer_tcp->tcp_fused_sigurg_mp);
276ff550d0eSmasputra peer_tcp->tcp_fused_sigurg_mp = NULL;
277ff550d0eSmasputra }
278ff550d0eSmasputra CONN_DEC_REF(peer_connp);
279ff550d0eSmasputra }
280ff550d0eSmasputra
281ff550d0eSmasputra /*
282ff550d0eSmasputra * Unfuse a previously-fused pair of tcp loopback endpoints.
283ff550d0eSmasputra */
284ff550d0eSmasputra void
tcp_unfuse(tcp_t * tcp)285ff550d0eSmasputra tcp_unfuse(tcp_t *tcp)
286ff550d0eSmasputra {
287ff550d0eSmasputra tcp_t *peer_tcp = tcp->tcp_loopback_peer;
2887b8f5432SAnders Persson tcp_stack_t *tcps = tcp->tcp_tcps;
289ff550d0eSmasputra
290ff550d0eSmasputra ASSERT(tcp->tcp_fused && peer_tcp != NULL);
291ff550d0eSmasputra ASSERT(peer_tcp->tcp_fused && peer_tcp->tcp_loopback_peer == tcp);
292ff550d0eSmasputra ASSERT(tcp->tcp_connp->conn_sqp == peer_tcp->tcp_connp->conn_sqp);
293ff550d0eSmasputra ASSERT(tcp->tcp_unsent == 0 && peer_tcp->tcp_unsent == 0);
294ff550d0eSmasputra
295ff550d0eSmasputra /*
2967b8f5432SAnders Persson * Cancel any pending push timers.
297ff550d0eSmasputra */
2987b8f5432SAnders Persson if (tcp->tcp_push_tid != 0) {
2997b8f5432SAnders Persson (void) TCP_TIMER_CANCEL(tcp, tcp->tcp_push_tid);
3007b8f5432SAnders Persson tcp->tcp_push_tid = 0;
3017b8f5432SAnders Persson }
3027b8f5432SAnders Persson if (peer_tcp->tcp_push_tid != 0) {
3037b8f5432SAnders Persson (void) TCP_TIMER_CANCEL(peer_tcp, peer_tcp->tcp_push_tid);
3047b8f5432SAnders Persson peer_tcp->tcp_push_tid = 0;
3057b8f5432SAnders Persson }
3067b8f5432SAnders Persson
3077b8f5432SAnders Persson /*
3087b8f5432SAnders Persson * Drain any pending data; Note that in case of a detached tcp, the
3097b8f5432SAnders Persson * draining will happen later after the tcp is unfused. For non-
3107b8f5432SAnders Persson * urgent data, this can be handled by the regular tcp_rcv_drain().
3117b8f5432SAnders Persson * If we have urgent data sitting in the receive list, we will
3127b8f5432SAnders Persson * need to send up a SIGURG signal first before draining the data.
3137b8f5432SAnders Persson * All of these will be handled by the code in tcp_fuse_rcv_drain()
3147b8f5432SAnders Persson * when called from tcp_rcv_drain().
3157b8f5432SAnders Persson */
3167b8f5432SAnders Persson if (!TCP_IS_DETACHED(tcp)) {
317bd670b35SErik Nordmark (void) tcp_fuse_rcv_drain(tcp->tcp_connp->conn_rq, tcp,
3187b8f5432SAnders Persson &tcp->tcp_fused_sigurg_mp);
3197b8f5432SAnders Persson }
3207b8f5432SAnders Persson if (!TCP_IS_DETACHED(peer_tcp)) {
321bd670b35SErik Nordmark (void) tcp_fuse_rcv_drain(peer_tcp->tcp_connp->conn_rq,
322bd670b35SErik Nordmark peer_tcp, &peer_tcp->tcp_fused_sigurg_mp);
3237b8f5432SAnders Persson }
3247b8f5432SAnders Persson
3257b8f5432SAnders Persson /* Lift up any flow-control conditions */
3267b8f5432SAnders Persson mutex_enter(&tcp->tcp_non_sq_lock);
3277b8f5432SAnders Persson if (tcp->tcp_flow_stopped) {
3287b8f5432SAnders Persson tcp_clrqfull(tcp);
3297b8f5432SAnders Persson TCP_STAT(tcps, tcp_fusion_backenabled);
3307b8f5432SAnders Persson }
3317b8f5432SAnders Persson mutex_exit(&tcp->tcp_non_sq_lock);
3327b8f5432SAnders Persson
3337b8f5432SAnders Persson mutex_enter(&peer_tcp->tcp_non_sq_lock);
3347b8f5432SAnders Persson if (peer_tcp->tcp_flow_stopped) {
3357b8f5432SAnders Persson tcp_clrqfull(peer_tcp);
3367b8f5432SAnders Persson TCP_STAT(tcps, tcp_fusion_backenabled);
3377b8f5432SAnders Persson }
3387b8f5432SAnders Persson mutex_exit(&peer_tcp->tcp_non_sq_lock);
339ff550d0eSmasputra
340ff550d0eSmasputra /*
341bd670b35SErik Nordmark * Update tha_seq and tha_ack in the header template
342ff550d0eSmasputra */
343bd670b35SErik Nordmark tcp->tcp_tcpha->tha_seq = htonl(tcp->tcp_snxt);
344bd670b35SErik Nordmark tcp->tcp_tcpha->tha_ack = htonl(tcp->tcp_rnxt);
345bd670b35SErik Nordmark peer_tcp->tcp_tcpha->tha_seq = htonl(peer_tcp->tcp_snxt);
346bd670b35SErik Nordmark peer_tcp->tcp_tcpha->tha_ack = htonl(peer_tcp->tcp_rnxt);
347ff550d0eSmasputra
348ff550d0eSmasputra /* Unfuse the endpoints */
349ff550d0eSmasputra peer_tcp->tcp_fused = tcp->tcp_fused = B_FALSE;
350ff550d0eSmasputra peer_tcp->tcp_loopback_peer = tcp->tcp_loopback_peer = NULL;
3510f1702c5SYu Xiangning }
352ff550d0eSmasputra
353ff550d0eSmasputra /*
3544da9f95bSAnders Persson * Fusion output routine used to handle urgent data sent by STREAMS based
3554da9f95bSAnders Persson * endpoints. This routine is called by tcp_fuse_output() for handling
3564da9f95bSAnders Persson * non-M_DATA mblks.
357ff550d0eSmasputra */
358ff550d0eSmasputra void
tcp_fuse_output_urg(tcp_t * tcp,mblk_t * mp)359ff550d0eSmasputra tcp_fuse_output_urg(tcp_t *tcp, mblk_t *mp)
360ff550d0eSmasputra {
361ff550d0eSmasputra mblk_t *mp1;
362ff550d0eSmasputra struct T_exdata_ind *tei;
363ff550d0eSmasputra tcp_t *peer_tcp = tcp->tcp_loopback_peer;
364ff550d0eSmasputra mblk_t *head, *prev_head = NULL;
365f4b3ec61Sdh155122 tcp_stack_t *tcps = tcp->tcp_tcps;
366ff550d0eSmasputra
367ff550d0eSmasputra ASSERT(tcp->tcp_fused);
368ff550d0eSmasputra ASSERT(peer_tcp != NULL && peer_tcp->tcp_loopback_peer == tcp);
3697b8f5432SAnders Persson ASSERT(!IPCL_IS_NONSTR(tcp->tcp_connp));
370ff550d0eSmasputra ASSERT(DB_TYPE(mp) == M_PROTO || DB_TYPE(mp) == M_PCPROTO);
371ff550d0eSmasputra ASSERT(mp->b_cont != NULL && DB_TYPE(mp->b_cont) == M_DATA);
372ff550d0eSmasputra ASSERT(MBLKL(mp) >= sizeof (*tei) && MBLKL(mp->b_cont) > 0);
373ff550d0eSmasputra
374ff550d0eSmasputra /*
375ff550d0eSmasputra * Urgent data arrives in the form of T_EXDATA_REQ from above.
376ff550d0eSmasputra * Each occurence denotes a new urgent pointer. For each new
377ff550d0eSmasputra * urgent pointer we signal (SIGURG) the receiving app to indicate
378ff550d0eSmasputra * that it needs to go into urgent mode. This is similar to the
379ff550d0eSmasputra * urgent data handling in the regular tcp. We don't need to keep
380ff550d0eSmasputra * track of where the urgent pointer is, because each T_EXDATA_REQ
381ff550d0eSmasputra * "advances" the urgent pointer for us.
382ff550d0eSmasputra *
383ff550d0eSmasputra * The actual urgent data carried by T_EXDATA_REQ is then prepended
384ff550d0eSmasputra * by a T_EXDATA_IND before being enqueued behind any existing data
385ff550d0eSmasputra * destined for the receiving app. There is only a single urgent
386ff550d0eSmasputra * pointer (out-of-band mark) for a given tcp. If the new urgent
387ff550d0eSmasputra * data arrives before the receiving app reads some existing urgent
388ff550d0eSmasputra * data, the previous marker is lost. This behavior is emulated
389ff550d0eSmasputra * accordingly below, by removing any existing T_EXDATA_IND messages
390ff550d0eSmasputra * and essentially converting old urgent data into non-urgent.
391ff550d0eSmasputra */
392ff550d0eSmasputra ASSERT(tcp->tcp_valid_bits & TCP_URG_VALID);
393ff550d0eSmasputra /* Let sender get out of urgent mode */
394ff550d0eSmasputra tcp->tcp_valid_bits &= ~TCP_URG_VALID;
395ff550d0eSmasputra
396ff550d0eSmasputra /*
397ff550d0eSmasputra * This flag indicates that a signal needs to be sent up.
398ff550d0eSmasputra * This flag will only get cleared once SIGURG is delivered and
399ff550d0eSmasputra * is not affected by the tcp_fused flag -- delivery will still
400ff550d0eSmasputra * happen even after an endpoint is unfused, to handle the case
401ff550d0eSmasputra * where the sending endpoint immediately closes/unfuses after
402ff550d0eSmasputra * sending urgent data and the accept is not yet finished.
403ff550d0eSmasputra */
404ff550d0eSmasputra peer_tcp->tcp_fused_sigurg = B_TRUE;
405ff550d0eSmasputra
406ff550d0eSmasputra /* Reuse T_EXDATA_REQ mblk for T_EXDATA_IND */
407ff550d0eSmasputra DB_TYPE(mp) = M_PROTO;
408ff550d0eSmasputra tei = (struct T_exdata_ind *)mp->b_rptr;
409ff550d0eSmasputra tei->PRIM_type = T_EXDATA_IND;
410ff550d0eSmasputra tei->MORE_flag = 0;
411ff550d0eSmasputra mp->b_wptr = (uchar_t *)&tei[1];
412ff550d0eSmasputra
413f4b3ec61Sdh155122 TCP_STAT(tcps, tcp_fusion_urg);
414721fffe3SKacheong Poon TCPS_BUMP_MIB(tcps, tcpOutUrg);
415ff550d0eSmasputra
416ff550d0eSmasputra head = peer_tcp->tcp_rcv_list;
417ff550d0eSmasputra while (head != NULL) {
418ff550d0eSmasputra /*
419ff550d0eSmasputra * Remove existing T_EXDATA_IND, keep the data which follows
420ff550d0eSmasputra * it and relink our list. Note that we don't modify the
421ff550d0eSmasputra * tcp_rcv_last_tail since it never points to T_EXDATA_IND.
422ff550d0eSmasputra */
423ff550d0eSmasputra if (DB_TYPE(head) != M_DATA) {
424ff550d0eSmasputra mp1 = head;
425ff550d0eSmasputra
426ff550d0eSmasputra ASSERT(DB_TYPE(mp1->b_cont) == M_DATA);
427ff550d0eSmasputra head = mp1->b_cont;
428ff550d0eSmasputra mp1->b_cont = NULL;
429ff550d0eSmasputra head->b_next = mp1->b_next;
430ff550d0eSmasputra mp1->b_next = NULL;
431ff550d0eSmasputra if (prev_head != NULL)
432ff550d0eSmasputra prev_head->b_next = head;
433ff550d0eSmasputra if (peer_tcp->tcp_rcv_list == mp1)
434ff550d0eSmasputra peer_tcp->tcp_rcv_list = head;
435ff550d0eSmasputra if (peer_tcp->tcp_rcv_last_head == mp1)
436ff550d0eSmasputra peer_tcp->tcp_rcv_last_head = head;
437ff550d0eSmasputra freeb(mp1);
438ff550d0eSmasputra }
439ff550d0eSmasputra prev_head = head;
440ff550d0eSmasputra head = head->b_next;
441ff550d0eSmasputra }
442ff550d0eSmasputra }
443ff550d0eSmasputra
444ff550d0eSmasputra /*
445ff550d0eSmasputra * Fusion output routine, called by tcp_output() and tcp_wput_proto().
446e0968231Svi117747 * If we are modifying any member that can be changed outside the squeue,
447e0968231Svi117747 * like tcp_flow_stopped, we need to take tcp_non_sq_lock.
448ff550d0eSmasputra */
449ff550d0eSmasputra boolean_t
tcp_fuse_output(tcp_t * tcp,mblk_t * mp,uint32_t send_size)450ff550d0eSmasputra tcp_fuse_output(tcp_t *tcp, mblk_t *mp, uint32_t send_size)
451ff550d0eSmasputra {
452bd670b35SErik Nordmark conn_t *connp = tcp->tcp_connp;
453ff550d0eSmasputra tcp_t *peer_tcp = tcp->tcp_loopback_peer;
454bd670b35SErik Nordmark conn_t *peer_connp = peer_tcp->tcp_connp;
4552c5134dbSudpa boolean_t flow_stopped, peer_data_queued = B_FALSE;
456ff550d0eSmasputra boolean_t urgent = (DB_TYPE(mp) != M_DATA);
4579910327fSanders boolean_t push = B_TRUE;
458381a2a9aSdr146992 mblk_t *mp1 = mp;
459381a2a9aSdr146992 uint_t ip_hdr_len;
460381a2a9aSdr146992 uint32_t recv_size = send_size;
461f4b3ec61Sdh155122 tcp_stack_t *tcps = tcp->tcp_tcps;
462f4b3ec61Sdh155122 netstack_t *ns = tcps->tcps_netstack;
463f4b3ec61Sdh155122 ip_stack_t *ipst = ns->netstack_ip;
464bd670b35SErik Nordmark ipsec_stack_t *ipss = ns->netstack_ipsec;
465bd670b35SErik Nordmark iaflags_t ixaflags = connp->conn_ixa->ixa_flags;
466bd670b35SErik Nordmark boolean_t do_ipsec, hooks_out, hooks_in, ipobs_enabled;
467ff550d0eSmasputra
468ff550d0eSmasputra ASSERT(tcp->tcp_fused);
469ff550d0eSmasputra ASSERT(peer_tcp != NULL && peer_tcp->tcp_loopback_peer == tcp);
470bd670b35SErik Nordmark ASSERT(connp->conn_sqp == peer_connp->conn_sqp);
471ff550d0eSmasputra ASSERT(DB_TYPE(mp) == M_DATA || DB_TYPE(mp) == M_PROTO ||
472ff550d0eSmasputra DB_TYPE(mp) == M_PCPROTO);
473ff550d0eSmasputra
474ff550d0eSmasputra if (send_size == 0) {
475ff550d0eSmasputra freemsg(mp);
476ff550d0eSmasputra return (B_TRUE);
477ff550d0eSmasputra }
478ff550d0eSmasputra
479ff550d0eSmasputra /*
480e8249070SRobert Mustacchi * Check enforcement of the minimum TTL policy differences in the
481e8249070SRobert Mustacchi * connection as this can change even after fusion. If we detect a
482e8249070SRobert Mustacchi * mismatch, unfuse and allow normal stack processing to handle this.
483e8249070SRobert Mustacchi */
484e8249070SRobert Mustacchi if (peer_connp->conn_min_ttl != 0 && peer_connp->conn_min_ttl >
485e8249070SRobert Mustacchi connp->conn_xmit_ipp.ipp_unicast_hops) {
486e8249070SRobert Mustacchi goto unfuse;
487e8249070SRobert Mustacchi }
488e8249070SRobert Mustacchi
489e8249070SRobert Mustacchi /*
490ff550d0eSmasputra * Handle urgent data; we either send up SIGURG to the peer now
491ff550d0eSmasputra * or do it later when we drain, in case the peer is detached
492ff550d0eSmasputra * or if we're short of memory for M_PCSIG mblk.
493ff550d0eSmasputra */
494ff550d0eSmasputra if (urgent) {
495ff550d0eSmasputra tcp_fuse_output_urg(tcp, mp);
496381a2a9aSdr146992
497381a2a9aSdr146992 mp1 = mp->b_cont;
498381a2a9aSdr146992 }
499381a2a9aSdr146992
500381a2a9aSdr146992 /*
501bd670b35SErik Nordmark * Check that we are still using an IRE_LOCAL or IRE_LOOPBACK before
502bd670b35SErik Nordmark * further processes.
503381a2a9aSdr146992 */
504bd670b35SErik Nordmark if (!ip_output_verify_local(connp->conn_ixa))
505bd670b35SErik Nordmark goto unfuse;
506bd670b35SErik Nordmark
507bd670b35SErik Nordmark /*
508bd670b35SErik Nordmark * Build IP and TCP header in case we have something that needs the
509bd670b35SErik Nordmark * headers. Those cases are:
510bd670b35SErik Nordmark * 1. IPsec
511bd670b35SErik Nordmark * 2. IPobs
512bd670b35SErik Nordmark * 3. FW_HOOKS
513bd670b35SErik Nordmark *
514bd670b35SErik Nordmark * If tcp_xmit_mp() fails to dupb() the message, unfuse the connection
515bd670b35SErik Nordmark * and back to regular path.
516bd670b35SErik Nordmark */
517bd670b35SErik Nordmark if (ixaflags & IXAF_IS_IPV4) {
518bd670b35SErik Nordmark do_ipsec = (ixaflags & IXAF_IPSEC_SECURE) ||
519bd670b35SErik Nordmark CONN_INBOUND_POLICY_PRESENT(peer_connp, ipss);
520bd670b35SErik Nordmark
521bd670b35SErik Nordmark hooks_out = HOOKS4_INTERESTED_LOOPBACK_OUT(ipst);
522bd670b35SErik Nordmark hooks_in = HOOKS4_INTERESTED_LOOPBACK_IN(ipst);
523bd670b35SErik Nordmark ipobs_enabled = (ipst->ips_ip4_observe.he_interested != 0);
524bd670b35SErik Nordmark } else {
525bd670b35SErik Nordmark do_ipsec = (ixaflags & IXAF_IPSEC_SECURE) ||
526bd670b35SErik Nordmark CONN_INBOUND_POLICY_PRESENT_V6(peer_connp, ipss);
527bd670b35SErik Nordmark
528bd670b35SErik Nordmark hooks_out = HOOKS6_INTERESTED_LOOPBACK_OUT(ipst);
529bd670b35SErik Nordmark hooks_in = HOOKS6_INTERESTED_LOOPBACK_IN(ipst);
530bd670b35SErik Nordmark ipobs_enabled = (ipst->ips_ip6_observe.he_interested != 0);
531bd670b35SErik Nordmark }
532bd670b35SErik Nordmark
533bd670b35SErik Nordmark /* We do logical 'or' for efficiency */
534bd670b35SErik Nordmark if (ipobs_enabled | do_ipsec | hooks_in | hooks_out) {
535381a2a9aSdr146992 if ((mp1 = tcp_xmit_mp(tcp, mp1, tcp->tcp_mss, NULL, NULL,
536381a2a9aSdr146992 tcp->tcp_snxt, B_TRUE, NULL, B_FALSE)) == NULL)
537381a2a9aSdr146992 /* If tcp_xmit_mp fails, use regular path */
538381a2a9aSdr146992 goto unfuse;
539381a2a9aSdr146992
54091762968SBrian Ruthven /*
541bd670b35SErik Nordmark * Leave all IP relevant processes to ip_output_process_local(),
542bd670b35SErik Nordmark * which handles IPsec, IPobs, and FW_HOOKS.
54391762968SBrian Ruthven */
544bd670b35SErik Nordmark mp1 = ip_output_process_local(mp1, connp->conn_ixa, hooks_out,
545bd670b35SErik Nordmark hooks_in, do_ipsec ? peer_connp : NULL);
54691762968SBrian Ruthven
547bd670b35SErik Nordmark /* If the message is dropped for any reason. */
548381a2a9aSdr146992 if (mp1 == NULL)
549381a2a9aSdr146992 goto unfuse;
550381a2a9aSdr146992
55191762968SBrian Ruthven /*
552bd670b35SErik Nordmark * Data length might have been changed by FW_HOOKS.
553bd670b35SErik Nordmark * We assume that the first mblk contains the TCP/IP headers.
55491762968SBrian Ruthven */
555bd670b35SErik Nordmark if (hooks_in || hooks_out) {
556bd670b35SErik Nordmark tcpha_t *tcpha;
557bd670b35SErik Nordmark
558bd670b35SErik Nordmark ip_hdr_len = (ixaflags & IXAF_IS_IPV4) ?
559bd670b35SErik Nordmark IPH_HDR_LENGTH((ipha_t *)mp1->b_rptr) :
560bd670b35SErik Nordmark ip_hdr_length_v6(mp1, (ip6_t *)mp1->b_rptr);
561bd670b35SErik Nordmark
562bd670b35SErik Nordmark tcpha = (tcpha_t *)&mp1->b_rptr[ip_hdr_len];
563bd670b35SErik Nordmark ASSERT((uchar_t *)tcpha + sizeof (tcpha_t) <=
564bd670b35SErik Nordmark mp1->b_wptr);
565bd670b35SErik Nordmark recv_size += htonl(tcpha->tha_seq) - tcp->tcp_snxt;
566bd670b35SErik Nordmark
56791762968SBrian Ruthven }
568381a2a9aSdr146992
569381a2a9aSdr146992 /*
570381a2a9aSdr146992 * The message duplicated by tcp_xmit_mp is freed.
571381a2a9aSdr146992 * Note: the original message passed in remains unchanged.
572381a2a9aSdr146992 */
573381a2a9aSdr146992 freemsg(mp1);
574ff550d0eSmasputra }
575ff550d0eSmasputra
576ff550d0eSmasputra /*
577ff550d0eSmasputra * Enqueue data into the peer's receive list; we may or may not
578ff550d0eSmasputra * drain the contents depending on the conditions below.
5797cae9885SAnders Persson *
5807b8f5432SAnders Persson * For non-STREAMS sockets we normally queue data directly in the
5817b8f5432SAnders Persson * socket by calling the su_recv upcall. However, if the peer is
5827b8f5432SAnders Persson * detached we use tcp_rcv_enqueue() instead. Queued data will be
5837b8f5432SAnders Persson * drained when the accept completes (in tcp_accept_finish()).
584ff550d0eSmasputra */
585bd670b35SErik Nordmark if (IPCL_IS_NONSTR(peer_connp) &&
5867b8f5432SAnders Persson !TCP_IS_DETACHED(peer_tcp)) {
5870f1702c5SYu Xiangning int error;
5880f1702c5SYu Xiangning int flags = 0;
5890f1702c5SYu Xiangning
5900f1702c5SYu Xiangning if ((tcp->tcp_valid_bits & TCP_URG_VALID) &&
5910f1702c5SYu Xiangning (tcp->tcp_urg == tcp->tcp_snxt)) {
5920f1702c5SYu Xiangning flags = MSG_OOB;
593bd670b35SErik Nordmark (*peer_connp->conn_upcalls->su_signal_oob)
594bd670b35SErik Nordmark (peer_connp->conn_upper_handle, 0);
5950f1702c5SYu Xiangning tcp->tcp_valid_bits &= ~TCP_URG_VALID;
5960f1702c5SYu Xiangning }
597bd670b35SErik Nordmark if ((*peer_connp->conn_upcalls->su_recv)(
598bd670b35SErik Nordmark peer_connp->conn_upper_handle, mp, recv_size,
599f3124163SAnders Persson flags, &error, &push) < 0) {
60041174437SAnders Persson ASSERT(error != EOPNOTSUPP);
601f3124163SAnders Persson peer_data_queued = B_TRUE;
602f3124163SAnders Persson }
6030f1702c5SYu Xiangning } else {
604bd670b35SErik Nordmark if (IPCL_IS_NONSTR(peer_connp) &&
6050f1702c5SYu Xiangning (tcp->tcp_valid_bits & TCP_URG_VALID) &&
6060f1702c5SYu Xiangning (tcp->tcp_urg == tcp->tcp_snxt)) {
6070f1702c5SYu Xiangning /*
6080f1702c5SYu Xiangning * Can not deal with urgent pointers
6090f1702c5SYu Xiangning * that arrive before the connection has been
6100f1702c5SYu Xiangning * accept()ed.
6110f1702c5SYu Xiangning */
6120f1702c5SYu Xiangning tcp->tcp_valid_bits &= ~TCP_URG_VALID;
6130f1702c5SYu Xiangning freemsg(mp);
6140f1702c5SYu Xiangning return (B_TRUE);
6150f1702c5SYu Xiangning }
6160f1702c5SYu Xiangning
617bd670b35SErik Nordmark tcp_rcv_enqueue(peer_tcp, mp, recv_size,
618bd670b35SErik Nordmark tcp->tcp_connp->conn_cred);
619ff550d0eSmasputra
620ff550d0eSmasputra /* In case it wrapped around and also to keep it constant */
621381a2a9aSdr146992 peer_tcp->tcp_rwnd += recv_size;
6227b8f5432SAnders Persson }
623ff550d0eSmasputra
624ff550d0eSmasputra /*
625ff550d0eSmasputra * Exercise flow-control when needed; we will get back-enabled
6267b8f5432SAnders Persson * in either tcp_accept_finish(), tcp_unfuse(), or when data is
6277b8f5432SAnders Persson * consumed. If peer endpoint is detached, we emulate streams flow
6287b8f5432SAnders Persson * control by checking the peer's queue size and high water mark;
6297b8f5432SAnders Persson * otherwise we simply use canputnext() to decide if we need to stop
6307b8f5432SAnders Persson * our flow.
631ff550d0eSmasputra *
632e0968231Svi117747 * Since we are accessing our tcp_flow_stopped and might modify it,
6337b8f5432SAnders Persson * we need to take tcp->tcp_non_sq_lock.
634e0968231Svi117747 */
635e0968231Svi117747 mutex_enter(&tcp->tcp_non_sq_lock);
636ff550d0eSmasputra flow_stopped = tcp->tcp_flow_stopped;
6377b8f5432SAnders Persson if ((TCP_IS_DETACHED(peer_tcp) &&
638bd670b35SErik Nordmark (peer_tcp->tcp_rcv_cnt >= peer_connp->conn_rcvbuf)) ||
6397b8f5432SAnders Persson (!TCP_IS_DETACHED(peer_tcp) &&
640bd670b35SErik Nordmark !IPCL_IS_NONSTR(peer_connp) && !canputnext(peer_connp->conn_rq))) {
6412c5134dbSudpa peer_data_queued = B_TRUE;
6422c5134dbSudpa }
6432c5134dbSudpa
6442c5134dbSudpa if (!flow_stopped && (peer_data_queued ||
645bd670b35SErik Nordmark (TCP_UNSENT_BYTES(tcp) >= connp->conn_sndbuf))) {
646ff550d0eSmasputra tcp_setqfull(tcp);
647ff550d0eSmasputra flow_stopped = B_TRUE;
648f4b3ec61Sdh155122 TCP_STAT(tcps, tcp_fusion_flowctl);
6497b8f5432SAnders Persson DTRACE_PROBE3(tcp__fuse__output__flowctl, tcp_t *, tcp,
6507b8f5432SAnders Persson uint_t, send_size, uint_t, peer_tcp->tcp_rcv_cnt);
6512c5134dbSudpa } else if (flow_stopped && !peer_data_queued &&
652bd670b35SErik Nordmark (TCP_UNSENT_BYTES(tcp) <= connp->conn_sndlowat)) {
653ff550d0eSmasputra tcp_clrqfull(tcp);
6548c0bf406Sja97890 TCP_STAT(tcps, tcp_fusion_backenabled);
655a2036d4dSmeem flow_stopped = B_FALSE;
656ff550d0eSmasputra }
657e0968231Svi117747 mutex_exit(&tcp->tcp_non_sq_lock);
6588c0bf406Sja97890
659f4b3ec61Sdh155122 ipst->ips_loopback_packets++;
660ff550d0eSmasputra tcp->tcp_last_sent_len = send_size;
661ff550d0eSmasputra
662ff550d0eSmasputra /* Need to adjust the following SNMP MIB-related variables */
663ff550d0eSmasputra tcp->tcp_snxt += send_size;
664ff550d0eSmasputra tcp->tcp_suna = tcp->tcp_snxt;
665381a2a9aSdr146992 peer_tcp->tcp_rnxt += recv_size;
6669cd928feSAlan Maguire peer_tcp->tcp_last_recv_len = recv_size;
667ff550d0eSmasputra peer_tcp->tcp_rack = peer_tcp->tcp_rnxt;
668ff550d0eSmasputra
669721fffe3SKacheong Poon TCPS_BUMP_MIB(tcps, tcpOutDataSegs);
670a2f04351SSebastien Roy TCPS_BUMP_MIB(tcps, tcpHCOutSegs);
671721fffe3SKacheong Poon TCPS_UPDATE_MIB(tcps, tcpOutDataBytes, send_size);
672a2f04351SSebastien Roy tcp->tcp_cs.tcp_out_data_bytes += send_size;
673a2f04351SSebastien Roy tcp->tcp_cs.tcp_out_data_segs++;
674ff550d0eSmasputra
675721fffe3SKacheong Poon TCPS_BUMP_MIB(tcps, tcpHCInSegs);
676721fffe3SKacheong Poon TCPS_BUMP_MIB(tcps, tcpInDataInorderSegs);
677721fffe3SKacheong Poon TCPS_UPDATE_MIB(tcps, tcpInDataInorderBytes, send_size);
678a2f04351SSebastien Roy peer_tcp->tcp_cs.tcp_in_data_inorder_bytes += send_size;
679a2f04351SSebastien Roy peer_tcp->tcp_cs.tcp_in_data_inorder_segs++;
680ff550d0eSmasputra
6819cd928feSAlan Maguire DTRACE_TCP5(send, void, NULL, ip_xmit_attr_t *, connp->conn_ixa,
6829cd928feSAlan Maguire __dtrace_tcp_void_ip_t *, NULL, tcp_t *, tcp,
6839cd928feSAlan Maguire __dtrace_tcp_tcph_t *, NULL);
6849cd928feSAlan Maguire DTRACE_TCP5(receive, void, NULL, ip_xmit_attr_t *,
6859cd928feSAlan Maguire peer_connp->conn_ixa, __dtrace_tcp_void_ip_t *, NULL,
6869cd928feSAlan Maguire tcp_t *, peer_tcp, __dtrace_tcp_tcph_t *, NULL);
687ff550d0eSmasputra
6884da9f95bSAnders Persson if (!IPCL_IS_NONSTR(peer_tcp->tcp_connp) &&
6894da9f95bSAnders Persson !TCP_IS_DETACHED(peer_tcp)) {
690ff550d0eSmasputra /*
691ff550d0eSmasputra * Drain the peer's receive queue it has urgent data or if
6927b8f5432SAnders Persson * we're not flow-controlled.
693ff550d0eSmasputra */
6947b8f5432SAnders Persson if (urgent || !flow_stopped) {
6954da9f95bSAnders Persson ASSERT(peer_tcp->tcp_rcv_list != NULL);
69681d28f7bSmeem /*
69781d28f7bSmeem * For TLI-based streams, a thread in tcp_accept_swap()
69881d28f7bSmeem * can race with us. That thread will ensure that the
699bd670b35SErik Nordmark * correct peer_connp->conn_rq is globally visible
700bd670b35SErik Nordmark * before peer_tcp->tcp_detached is visible as clear,
701bd670b35SErik Nordmark * but we must also ensure that the load of conn_rq
702bd670b35SErik Nordmark * cannot be reordered to be before the tcp_detached
703bd670b35SErik Nordmark * check.
70481d28f7bSmeem */
70581d28f7bSmeem membar_consumer();
706bd670b35SErik Nordmark (void) tcp_fuse_rcv_drain(peer_connp->conn_rq, peer_tcp,
70781d28f7bSmeem NULL);
708ff550d0eSmasputra }
709ff550d0eSmasputra }
710ff550d0eSmasputra return (B_TRUE);
711381a2a9aSdr146992 unfuse:
712381a2a9aSdr146992 tcp_unfuse(tcp);
713381a2a9aSdr146992 return (B_FALSE);
714ff550d0eSmasputra }
715ff550d0eSmasputra
716ff550d0eSmasputra /*
717ff550d0eSmasputra * This routine gets called to deliver data upstream on a fused or
718ff550d0eSmasputra * previously fused tcp loopback endpoint; the latter happens only
719ff550d0eSmasputra * when there is a pending SIGURG signal plus urgent data that can't
720ff550d0eSmasputra * be sent upstream in the past.
721ff550d0eSmasputra */
722ff550d0eSmasputra boolean_t
tcp_fuse_rcv_drain(queue_t * q,tcp_t * tcp,mblk_t ** sigurg_mpp)723ff550d0eSmasputra tcp_fuse_rcv_drain(queue_t *q, tcp_t *tcp, mblk_t **sigurg_mpp)
724ff550d0eSmasputra {
725ff550d0eSmasputra mblk_t *mp;
7260f1702c5SYu Xiangning conn_t *connp = tcp->tcp_connp;
7270f1702c5SYu Xiangning
728ff550d0eSmasputra #ifdef DEBUG
729ff550d0eSmasputra uint_t cnt = 0;
730ff550d0eSmasputra #endif
731f4b3ec61Sdh155122 tcp_stack_t *tcps = tcp->tcp_tcps;
7328c0bf406Sja97890 tcp_t *peer_tcp = tcp->tcp_loopback_peer;
733ff550d0eSmasputra
734ff550d0eSmasputra ASSERT(tcp->tcp_loopback);
735ff550d0eSmasputra ASSERT(tcp->tcp_fused || tcp->tcp_fused_sigurg);
736ff550d0eSmasputra ASSERT(!tcp->tcp_fused || tcp->tcp_loopback_peer != NULL);
7370f1702c5SYu Xiangning ASSERT(IPCL_IS_NONSTR(connp) || sigurg_mpp != NULL || tcp->tcp_fused);
738ff550d0eSmasputra
739ff550d0eSmasputra /* No need for the push timer now, in case it was scheduled */
740ff550d0eSmasputra if (tcp->tcp_push_tid != 0) {
741ff550d0eSmasputra (void) TCP_TIMER_CANCEL(tcp, tcp->tcp_push_tid);
742ff550d0eSmasputra tcp->tcp_push_tid = 0;
743ff550d0eSmasputra }
744ff550d0eSmasputra /*
745ff550d0eSmasputra * If there's urgent data sitting in receive list and we didn't
746ff550d0eSmasputra * get a chance to send up a SIGURG signal, make sure we send
747ff550d0eSmasputra * it first before draining in order to ensure that SIOCATMARK
748ff550d0eSmasputra * works properly.
749ff550d0eSmasputra */
750ff550d0eSmasputra if (tcp->tcp_fused_sigurg) {
7514da9f95bSAnders Persson ASSERT(!IPCL_IS_NONSTR(tcp->tcp_connp));
7524da9f95bSAnders Persson
7530f1702c5SYu Xiangning tcp->tcp_fused_sigurg = B_FALSE;
754ff550d0eSmasputra /*
755ff550d0eSmasputra * sigurg_mpp is normally NULL, i.e. when we're still
756ff550d0eSmasputra * fused and didn't get here because of tcp_unfuse().
757ff550d0eSmasputra * In this case try hard to allocate the M_PCSIG mblk.
758ff550d0eSmasputra */
759ff550d0eSmasputra if (sigurg_mpp == NULL &&
760ff550d0eSmasputra (mp = allocb(1, BPRI_HI)) == NULL &&
761ff550d0eSmasputra (mp = allocb_tryhard(1)) == NULL) {
762ff550d0eSmasputra /* Alloc failed; try again next time */
7630f1702c5SYu Xiangning tcp->tcp_push_tid = TCP_TIMER(tcp,
76466cd0f60SKacheong Poon tcp_push_timer, tcps->tcps_push_timer_interval);
765ff550d0eSmasputra return (B_TRUE);
766ff550d0eSmasputra } else if (sigurg_mpp != NULL) {
767ff550d0eSmasputra /*
768ff550d0eSmasputra * Use the supplied M_PCSIG mblk; it means we're
769ff550d0eSmasputra * either unfused or in the process of unfusing,
770ff550d0eSmasputra * and the drain must happen now.
771ff550d0eSmasputra */
772ff550d0eSmasputra mp = *sigurg_mpp;
773ff550d0eSmasputra *sigurg_mpp = NULL;
774ff550d0eSmasputra }
775ff550d0eSmasputra ASSERT(mp != NULL);
776ff550d0eSmasputra
777ff550d0eSmasputra /* Send up the signal */
778ff550d0eSmasputra DB_TYPE(mp) = M_PCSIG;
779ff550d0eSmasputra *mp->b_wptr++ = (uchar_t)SIGURG;
780ff550d0eSmasputra putnext(q, mp);
7814da9f95bSAnders Persson
782ff550d0eSmasputra /*
783ff550d0eSmasputra * Let the regular tcp_rcv_drain() path handle
784ff550d0eSmasputra * draining the data if we're no longer fused.
785ff550d0eSmasputra */
786ff550d0eSmasputra if (!tcp->tcp_fused)
787ff550d0eSmasputra return (B_FALSE);
788ff550d0eSmasputra }
789ff550d0eSmasputra
790ff550d0eSmasputra /* Drain the data */
791ff550d0eSmasputra while ((mp = tcp->tcp_rcv_list) != NULL) {
792ff550d0eSmasputra tcp->tcp_rcv_list = mp->b_next;
793ff550d0eSmasputra mp->b_next = NULL;
794ff550d0eSmasputra #ifdef DEBUG
795ff550d0eSmasputra cnt += msgdsize(mp);
796ff550d0eSmasputra #endif
7970f1702c5SYu Xiangning ASSERT(!IPCL_IS_NONSTR(connp));
798ff550d0eSmasputra putnext(q, mp);
799f4b3ec61Sdh155122 TCP_STAT(tcps, tcp_fusion_putnext);
800ff550d0eSmasputra }
801ff550d0eSmasputra
8020f1702c5SYu Xiangning #ifdef DEBUG
803ff550d0eSmasputra ASSERT(cnt == tcp->tcp_rcv_cnt);
8040f1702c5SYu Xiangning #endif
805ff550d0eSmasputra tcp->tcp_rcv_last_head = NULL;
806ff550d0eSmasputra tcp->tcp_rcv_last_tail = NULL;
807ff550d0eSmasputra tcp->tcp_rcv_cnt = 0;
808bd670b35SErik Nordmark tcp->tcp_rwnd = tcp->tcp_connp->conn_rcvbuf;
809ff550d0eSmasputra
8107b8f5432SAnders Persson mutex_enter(&peer_tcp->tcp_non_sq_lock);
8118c0bf406Sja97890 if (peer_tcp->tcp_flow_stopped && (TCP_UNSENT_BYTES(peer_tcp) <=
812bd670b35SErik Nordmark peer_tcp->tcp_connp->conn_sndlowat)) {
8138c0bf406Sja97890 tcp_clrqfull(peer_tcp);
8148c0bf406Sja97890 TCP_STAT(tcps, tcp_fusion_backenabled);
8158c0bf406Sja97890 }
8167b8f5432SAnders Persson mutex_exit(&peer_tcp->tcp_non_sq_lock);
8178c0bf406Sja97890
818ff550d0eSmasputra return (B_TRUE);
819ff550d0eSmasputra }
820ff550d0eSmasputra
821ff550d0eSmasputra /*
822ff550d0eSmasputra * Calculate the size of receive buffer for a fused tcp endpoint.
823ff550d0eSmasputra */
824ff550d0eSmasputra size_t
tcp_fuse_set_rcv_hiwat(tcp_t * tcp,size_t rwnd)825ff550d0eSmasputra tcp_fuse_set_rcv_hiwat(tcp_t *tcp, size_t rwnd)
826ff550d0eSmasputra {
827f4b3ec61Sdh155122 tcp_stack_t *tcps = tcp->tcp_tcps;
82893fcb0b9SKacheong Poon uint32_t max_win;
829f4b3ec61Sdh155122
830ff550d0eSmasputra ASSERT(tcp->tcp_fused);
831ff550d0eSmasputra
832ff550d0eSmasputra /* Ensure that value is within the maximum upper bound */
833f4b3ec61Sdh155122 if (rwnd > tcps->tcps_max_buf)
834f4b3ec61Sdh155122 rwnd = tcps->tcps_max_buf;
835ff550d0eSmasputra /*
836ff550d0eSmasputra * Round up to system page size in case SO_RCVBUF is modified
837ff550d0eSmasputra * after SO_SNDBUF; the latter is also similarly rounded up.
838ff550d0eSmasputra */
839ff550d0eSmasputra rwnd = P2ROUNDUP_TYPED(rwnd, PAGESIZE, size_t);
84093fcb0b9SKacheong Poon max_win = TCP_MAXWIN << tcp->tcp_rcv_ws;
84193fcb0b9SKacheong Poon if (rwnd > max_win) {
84293fcb0b9SKacheong Poon rwnd = max_win - (max_win % tcp->tcp_mss);
84393fcb0b9SKacheong Poon if (rwnd < tcp->tcp_mss)
84493fcb0b9SKacheong Poon rwnd = max_win;
84593fcb0b9SKacheong Poon }
84679c0745dSRao Shoaib
84779c0745dSRao Shoaib /*
84879c0745dSRao Shoaib * Record high water mark, this is used for flow-control
84979c0745dSRao Shoaib * purposes in tcp_fuse_output().
85079c0745dSRao Shoaib */
851bd670b35SErik Nordmark tcp->tcp_connp->conn_rcvbuf = rwnd;
852bd670b35SErik Nordmark tcp->tcp_rwnd = rwnd;
853ff550d0eSmasputra return (rwnd);
854ff550d0eSmasputra }
855ff550d0eSmasputra
856ff550d0eSmasputra /*
857ff550d0eSmasputra * Calculate the maximum outstanding unread data block for a fused tcp endpoint.
858ff550d0eSmasputra */
859ff550d0eSmasputra int
tcp_fuse_maxpsz(tcp_t * tcp)86079c0745dSRao Shoaib tcp_fuse_maxpsz(tcp_t *tcp)
861ff550d0eSmasputra {
862ff550d0eSmasputra tcp_t *peer_tcp = tcp->tcp_loopback_peer;
863bd670b35SErik Nordmark conn_t *connp = tcp->tcp_connp;
864bd670b35SErik Nordmark uint_t sndbuf = connp->conn_sndbuf;
865ff550d0eSmasputra uint_t maxpsz = sndbuf;
866ff550d0eSmasputra
867ff550d0eSmasputra ASSERT(tcp->tcp_fused);
868ff550d0eSmasputra ASSERT(peer_tcp != NULL);
869bd670b35SErik Nordmark ASSERT(peer_tcp->tcp_connp->conn_rcvbuf != 0);
870ff550d0eSmasputra /*
871ff550d0eSmasputra * In the fused loopback case, we want the stream head to split
872ff550d0eSmasputra * up larger writes into smaller chunks for a more accurate flow-
873ff550d0eSmasputra * control accounting. Our maxpsz is half of the sender's send
874ff550d0eSmasputra * buffer or the receiver's receive buffer, whichever is smaller.
875ff550d0eSmasputra * We round up the buffer to system page size due to the lack of
876ff550d0eSmasputra * TCP MSS concept in Fusion.
877ff550d0eSmasputra */
878bd670b35SErik Nordmark if (maxpsz > peer_tcp->tcp_connp->conn_rcvbuf)
879bd670b35SErik Nordmark maxpsz = peer_tcp->tcp_connp->conn_rcvbuf;
880ff550d0eSmasputra maxpsz = P2ROUNDUP_TYPED(maxpsz, PAGESIZE, uint_t) >> 1;
881ff550d0eSmasputra
882ff550d0eSmasputra return (maxpsz);
883ff550d0eSmasputra }
884f3124163SAnders Persson
885f3124163SAnders Persson /*
886f3124163SAnders Persson * Called to release flow control.
887f3124163SAnders Persson */
888f3124163SAnders Persson void
tcp_fuse_backenable(tcp_t * tcp)889f3124163SAnders Persson tcp_fuse_backenable(tcp_t *tcp)
890f3124163SAnders Persson {
891f3124163SAnders Persson tcp_t *peer_tcp = tcp->tcp_loopback_peer;
892f3124163SAnders Persson
893f3124163SAnders Persson ASSERT(tcp->tcp_fused);
894f3124163SAnders Persson ASSERT(peer_tcp != NULL && peer_tcp->tcp_fused);
895f3124163SAnders Persson ASSERT(peer_tcp->tcp_loopback_peer == tcp);
896f3124163SAnders Persson ASSERT(!TCP_IS_DETACHED(tcp));
897f3124163SAnders Persson ASSERT(tcp->tcp_connp->conn_sqp ==
898f3124163SAnders Persson peer_tcp->tcp_connp->conn_sqp);
899f3124163SAnders Persson
900f3124163SAnders Persson if (tcp->tcp_rcv_list != NULL)
901bd670b35SErik Nordmark (void) tcp_fuse_rcv_drain(tcp->tcp_connp->conn_rq, tcp, NULL);
902f3124163SAnders Persson
903f3124163SAnders Persson mutex_enter(&peer_tcp->tcp_non_sq_lock);
904f3124163SAnders Persson if (peer_tcp->tcp_flow_stopped &&
905f3124163SAnders Persson (TCP_UNSENT_BYTES(peer_tcp) <=
906bd670b35SErik Nordmark peer_tcp->tcp_connp->conn_sndlowat)) {
907f3124163SAnders Persson tcp_clrqfull(peer_tcp);
908f3124163SAnders Persson }
909f3124163SAnders Persson mutex_exit(&peer_tcp->tcp_non_sq_lock);
910f3124163SAnders Persson
911f3124163SAnders Persson TCP_STAT(tcp->tcp_tcps, tcp_fusion_backenabled);
912f3124163SAnders Persson }
913