1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 /* Copyright (c) 1990 Mentat Inc. */ 27 28 #include <sys/types.h> 29 #include <sys/stream.h> 30 #include <sys/strsun.h> 31 #include <sys/strsubr.h> 32 #include <sys/stropts.h> 33 #include <sys/strlog.h> 34 #include <sys/strsun.h> 35 #define _SUN_TPI_VERSION 2 36 #include <sys/tihdr.h> 37 #include <sys/timod.h> 38 #include <sys/ddi.h> 39 #include <sys/sunddi.h> 40 #include <sys/suntpi.h> 41 #include <sys/xti_inet.h> 42 #include <sys/cmn_err.h> 43 #include <sys/debug.h> 44 #include <sys/sdt.h> 45 #include <sys/vtrace.h> 46 #include <sys/kmem.h> 47 #include <sys/ethernet.h> 48 #include <sys/cpuvar.h> 49 #include <sys/dlpi.h> 50 #include <sys/multidata.h> 51 #include <sys/multidata_impl.h> 52 #include <sys/pattr.h> 53 #include <sys/policy.h> 54 #include <sys/priv.h> 55 #include <sys/zone.h> 56 #include <sys/sunldi.h> 57 58 #include <sys/errno.h> 59 #include <sys/signal.h> 60 #include <sys/socket.h> 61 #include <sys/sockio.h> 62 #include <sys/isa_defs.h> 63 #include <sys/md5.h> 64 #include <sys/random.h> 65 #include <sys/sodirect.h> 66 #include <sys/uio.h> 67 #include <netinet/in.h> 68 #include <netinet/tcp.h> 69 #include <netinet/ip6.h> 70 #include <netinet/icmp6.h> 71 #include <net/if.h> 72 #include <net/route.h> 73 #include <inet/ipsec_impl.h> 74 75 #include <inet/common.h> 76 #include <inet/ip.h> 77 #include <inet/ip_impl.h> 78 #include <inet/ip6.h> 79 #include <inet/ip_ndp.h> 80 #include <inet/mi.h> 81 #include <inet/mib2.h> 82 #include <inet/nd.h> 83 #include <inet/optcom.h> 84 #include <inet/snmpcom.h> 85 #include <inet/kstatcom.h> 86 #include <inet/tcp.h> 87 #include <inet/tcp_impl.h> 88 #include <net/pfkeyv2.h> 89 #include <inet/ipsec_info.h> 90 #include <inet/ipdrop.h> 91 92 #include <inet/ipclassifier.h> 93 #include <inet/ip_ire.h> 94 #include <inet/ip_ftable.h> 95 #include <inet/ip_if.h> 96 #include <inet/ipp_common.h> 97 #include <inet/ip_netinfo.h> 98 #include <sys/squeue.h> 99 #include <inet/kssl/ksslapi.h> 100 #include <sys/tsol/label.h> 101 #include <sys/tsol/tnet.h> 102 #include <rpc/pmap_prot.h> 103 104 /* 105 * TCP Notes: aka FireEngine Phase I (PSARC 2002/433) 106 * 107 * (Read the detailed design doc in PSARC case directory) 108 * 109 * The entire tcp state is contained in tcp_t and conn_t structure 110 * which are allocated in tandem using ipcl_conn_create() and passing 111 * IPCL_CONNTCP as a flag. We use 'conn_ref' and 'conn_lock' to protect 112 * the references on the tcp_t. The tcp_t structure is never compressed 113 * and packets always land on the correct TCP perimeter from the time 114 * eager is created till the time tcp_t dies (as such the old mentat 115 * TCP global queue is not used for detached state and no IPSEC checking 116 * is required). The global queue is still allocated to send out resets 117 * for connection which have no listeners and IP directly calls 118 * tcp_xmit_listeners_reset() which does any policy check. 119 * 120 * Protection and Synchronisation mechanism: 121 * 122 * The tcp data structure does not use any kind of lock for protecting 123 * its state but instead uses 'squeues' for mutual exclusion from various 124 * read and write side threads. To access a tcp member, the thread should 125 * always be behind squeue (via squeue_enter, squeue_enter_nodrain, or 126 * squeue_fill). Since the squeues allow a direct function call, caller 127 * can pass any tcp function having prototype of edesc_t as argument 128 * (different from traditional STREAMs model where packets come in only 129 * designated entry points). The list of functions that can be directly 130 * called via squeue are listed before the usual function prototype. 131 * 132 * Referencing: 133 * 134 * TCP is MT-Hot and we use a reference based scheme to make sure that the 135 * tcp structure doesn't disappear when its needed. When the application 136 * creates an outgoing connection or accepts an incoming connection, we 137 * start out with 2 references on 'conn_ref'. One for TCP and one for IP. 138 * The IP reference is just a symbolic reference since ip_tcpclose() 139 * looks at tcp structure after tcp_close_output() returns which could 140 * have dropped the last TCP reference. So as long as the connection is 141 * in attached state i.e. !TCP_IS_DETACHED, we have 2 references on the 142 * conn_t. The classifier puts its own reference when the connection is 143 * inserted in listen or connected hash. Anytime a thread needs to enter 144 * the tcp connection perimeter, it retrieves the conn/tcp from q->ptr 145 * on write side or by doing a classify on read side and then puts a 146 * reference on the conn before doing squeue_enter/tryenter/fill. For 147 * read side, the classifier itself puts the reference under fanout lock 148 * to make sure that tcp can't disappear before it gets processed. The 149 * squeue will drop this reference automatically so the called function 150 * doesn't have to do a DEC_REF. 151 * 152 * Opening a new connection: 153 * 154 * The outgoing connection open is pretty simple. tcp_open() does the 155 * work in creating the conn/tcp structure and initializing it. The 156 * squeue assignment is done based on the CPU the application 157 * is running on. So for outbound connections, processing is always done 158 * on application CPU which might be different from the incoming CPU 159 * being interrupted by the NIC. An optimal way would be to figure out 160 * the NIC <-> CPU binding at listen time, and assign the outgoing 161 * connection to the squeue attached to the CPU that will be interrupted 162 * for incoming packets (we know the NIC based on the bind IP address). 163 * This might seem like a problem if more data is going out but the 164 * fact is that in most cases the transmit is ACK driven transmit where 165 * the outgoing data normally sits on TCP's xmit queue waiting to be 166 * transmitted. 167 * 168 * Accepting a connection: 169 * 170 * This is a more interesting case because of various races involved in 171 * establishing a eager in its own perimeter. Read the meta comment on 172 * top of tcp_conn_request(). But briefly, the squeue is picked by 173 * ip_tcp_input()/ip_fanout_tcp_v6() based on the interrupted CPU. 174 * 175 * Closing a connection: 176 * 177 * The close is fairly straight forward. tcp_close() calls tcp_close_output() 178 * via squeue to do the close and mark the tcp as detached if the connection 179 * was in state TCPS_ESTABLISHED or greater. In the later case, TCP keep its 180 * reference but tcp_close() drop IP's reference always. So if tcp was 181 * not killed, it is sitting in time_wait list with 2 reference - 1 for TCP 182 * and 1 because it is in classifier's connected hash. This is the condition 183 * we use to determine that its OK to clean up the tcp outside of squeue 184 * when time wait expires (check the ref under fanout and conn_lock and 185 * if it is 2, remove it from fanout hash and kill it). 186 * 187 * Although close just drops the necessary references and marks the 188 * tcp_detached state, tcp_close needs to know the tcp_detached has been 189 * set (under squeue) before letting the STREAM go away (because a 190 * inbound packet might attempt to go up the STREAM while the close 191 * has happened and tcp_detached is not set). So a special lock and 192 * flag is used along with a condition variable (tcp_closelock, tcp_closed, 193 * and tcp_closecv) to signal tcp_close that tcp_close_out() has marked 194 * tcp_detached. 195 * 196 * Special provisions and fast paths: 197 * 198 * We make special provision for (AF_INET, SOCK_STREAM) sockets which 199 * can't have 'ipv6_recvpktinfo' set and for these type of sockets, IP 200 * will never send a M_CTL to TCP. As such, ip_tcp_input() which handles 201 * all TCP packets from the wire makes a IPCL_IS_TCP4_CONNECTED_NO_POLICY 202 * check to send packets directly to tcp_rput_data via squeue. Everyone 203 * else comes through tcp_input() on the read side. 204 * 205 * We also make special provisions for sockfs by marking tcp_issocket 206 * whenever we have only sockfs on top of TCP. This allows us to skip 207 * putting the tcp in acceptor hash since a sockfs listener can never 208 * become acceptor and also avoid allocating a tcp_t for acceptor STREAM 209 * since eager has already been allocated and the accept now happens 210 * on acceptor STREAM. There is a big blob of comment on top of 211 * tcp_conn_request explaining the new accept. When socket is POP'd, 212 * sockfs sends us an ioctl to mark the fact and we go back to old 213 * behaviour. Once tcp_issocket is unset, its never set for the 214 * life of that connection. 215 * 216 * In support of on-board asynchronous DMA hardware (e.g. Intel I/OAT) 217 * two consoldiation private KAPIs are used to enqueue M_DATA mblk_t's 218 * directly to the socket (sodirect) and start an asynchronous copyout 219 * to a user-land receive-side buffer (uioa) when a blocking socket read 220 * (e.g. read, recv, ...) is pending. 221 * 222 * This is accomplished when tcp_issocket is set and tcp_sodirect is not 223 * NULL so points to an sodirect_t and if marked enabled then we enqueue 224 * all mblk_t's directly to the socket. 225 * 226 * Further, if the sodirect_t sod_uioa and if marked enabled (due to a 227 * blocking socket read, e.g. user-land read, recv, ...) then an asynchronous 228 * copyout will be started directly to the user-land uio buffer. Also, as we 229 * have a pending read, TCP's push logic can take into account the number of 230 * bytes to be received and only awake the blocked read()er when the uioa_t 231 * byte count has been satisfied. 232 * 233 * IPsec notes : 234 * 235 * Since a packet is always executed on the correct TCP perimeter 236 * all IPsec processing is defered to IP including checking new 237 * connections and setting IPSEC policies for new connection. The 238 * only exception is tcp_xmit_listeners_reset() which is called 239 * directly from IP and needs to policy check to see if TH_RST 240 * can be sent out. 241 * 242 * PFHooks notes : 243 * 244 * For mdt case, one meta buffer contains multiple packets. Mblks for every 245 * packet are assembled and passed to the hooks. When packets are blocked, 246 * or boundary of any packet is changed, the mdt processing is stopped, and 247 * packets of the meta buffer are send to the IP path one by one. 248 */ 249 250 /* 251 * Values for squeue switch: 252 * 1: squeue_enter_nodrain 253 * 2: squeue_enter 254 * 3: squeue_fill 255 */ 256 int tcp_squeue_close = 2; /* Setable in /etc/system */ 257 int tcp_squeue_wput = 2; 258 259 squeue_func_t tcp_squeue_close_proc; 260 squeue_func_t tcp_squeue_wput_proc; 261 262 /* 263 * Macros for sodirect: 264 * 265 * SOD_PTR_ENTER(tcp, sodp) - for the tcp_t pointer "tcp" set the 266 * sodirect_t pointer "sodp" to the socket/tcp shared sodirect_t 267 * if it exists and is enabled, else to NULL. Note, in the current 268 * sodirect implementation the sod_lock must not be held across any 269 * STREAMS call (e.g. putnext) else a "recursive mutex_enter" PANIC 270 * will result as sod_lock is the streamhead stdata.sd_lock. 271 * 272 * SOD_NOT_ENABLED(tcp) - return true if not a sodirect tcp_t or the 273 * sodirect_t isn't enabled, usefull for ASSERT()ing that a recieve 274 * side tcp code path dealing with a tcp_rcv_list or putnext() isn't 275 * being used when sodirect code paths should be. 276 */ 277 278 #define SOD_PTR_ENTER(tcp, sodp) \ 279 (sodp) = (tcp)->tcp_sodirect; \ 280 \ 281 if ((sodp) != NULL) { \ 282 mutex_enter((sodp)->sod_lock); \ 283 if (!((sodp)->sod_state & SOD_ENABLED)) { \ 284 mutex_exit((sodp)->sod_lock); \ 285 (sodp) = NULL; \ 286 } \ 287 } 288 289 #define SOD_NOT_ENABLED(tcp) \ 290 ((tcp)->tcp_sodirect == NULL || \ 291 !((tcp)->tcp_sodirect->sod_state & SOD_ENABLED)) 292 293 /* 294 * This controls how tiny a write must be before we try to copy it 295 * into the the mblk on the tail of the transmit queue. Not much 296 * speedup is observed for values larger than sixteen. Zero will 297 * disable the optimisation. 298 */ 299 int tcp_tx_pull_len = 16; 300 301 /* 302 * TCP Statistics. 303 * 304 * How TCP statistics work. 305 * 306 * There are two types of statistics invoked by two macros. 307 * 308 * TCP_STAT(name) does non-atomic increment of a named stat counter. It is 309 * supposed to be used in non MT-hot paths of the code. 310 * 311 * TCP_DBGSTAT(name) does atomic increment of a named stat counter. It is 312 * supposed to be used for DEBUG purposes and may be used on a hot path. 313 * 314 * Both TCP_STAT and TCP_DBGSTAT counters are available using kstat 315 * (use "kstat tcp" to get them). 316 * 317 * There is also additional debugging facility that marks tcp_clean_death() 318 * instances and saves them in tcp_t structure. It is triggered by 319 * TCP_TAG_CLEAN_DEATH define. Also, there is a global array of counters for 320 * tcp_clean_death() calls that counts the number of times each tag was hit. It 321 * is triggered by TCP_CLD_COUNTERS define. 322 * 323 * How to add new counters. 324 * 325 * 1) Add a field in the tcp_stat structure describing your counter. 326 * 2) Add a line in the template in tcp_kstat2_init() with the name 327 * of the counter. 328 * 329 * IMPORTANT!! - make sure that both are in sync !! 330 * 3) Use either TCP_STAT or TCP_DBGSTAT with the name. 331 * 332 * Please avoid using private counters which are not kstat-exported. 333 * 334 * TCP_TAG_CLEAN_DEATH set to 1 enables tagging of tcp_clean_death() instances 335 * in tcp_t structure. 336 * 337 * TCP_MAX_CLEAN_DEATH_TAG is the maximum number of possible clean death tags. 338 */ 339 340 #ifndef TCP_DEBUG_COUNTER 341 #ifdef DEBUG 342 #define TCP_DEBUG_COUNTER 1 343 #else 344 #define TCP_DEBUG_COUNTER 0 345 #endif 346 #endif 347 348 #define TCP_CLD_COUNTERS 0 349 350 #define TCP_TAG_CLEAN_DEATH 1 351 #define TCP_MAX_CLEAN_DEATH_TAG 32 352 353 #ifdef lint 354 static int _lint_dummy_; 355 #endif 356 357 #if TCP_CLD_COUNTERS 358 static uint_t tcp_clean_death_stat[TCP_MAX_CLEAN_DEATH_TAG]; 359 #define TCP_CLD_STAT(x) tcp_clean_death_stat[x]++ 360 #elif defined(lint) 361 #define TCP_CLD_STAT(x) ASSERT(_lint_dummy_ == 0); 362 #else 363 #define TCP_CLD_STAT(x) 364 #endif 365 366 #if TCP_DEBUG_COUNTER 367 #define TCP_DBGSTAT(tcps, x) \ 368 atomic_add_64(&((tcps)->tcps_statistics.x.value.ui64), 1) 369 #define TCP_G_DBGSTAT(x) \ 370 atomic_add_64(&(tcp_g_statistics.x.value.ui64), 1) 371 #elif defined(lint) 372 #define TCP_DBGSTAT(tcps, x) ASSERT(_lint_dummy_ == 0); 373 #define TCP_G_DBGSTAT(x) ASSERT(_lint_dummy_ == 0); 374 #else 375 #define TCP_DBGSTAT(tcps, x) 376 #define TCP_G_DBGSTAT(x) 377 #endif 378 379 #define TCP_G_STAT(x) (tcp_g_statistics.x.value.ui64++) 380 381 tcp_g_stat_t tcp_g_statistics; 382 kstat_t *tcp_g_kstat; 383 384 /* 385 * Call either ip_output or ip_output_v6. This replaces putnext() calls on the 386 * tcp write side. 387 */ 388 #define CALL_IP_WPUT(connp, q, mp) { \ 389 tcp_stack_t *tcps; \ 390 \ 391 tcps = connp->conn_netstack->netstack_tcp; \ 392 ASSERT(((q)->q_flag & QREADR) == 0); \ 393 TCP_DBGSTAT(tcps, tcp_ip_output); \ 394 connp->conn_send(connp, (mp), (q), IP_WPUT); \ 395 } 396 397 /* Macros for timestamp comparisons */ 398 #define TSTMP_GEQ(a, b) ((int32_t)((a)-(b)) >= 0) 399 #define TSTMP_LT(a, b) ((int32_t)((a)-(b)) < 0) 400 401 /* 402 * Parameters for TCP Initial Send Sequence number (ISS) generation. When 403 * tcp_strong_iss is set to 1, which is the default, the ISS is calculated 404 * by adding three components: a time component which grows by 1 every 4096 405 * nanoseconds (versus every 4 microseconds suggested by RFC 793, page 27); 406 * a per-connection component which grows by 125000 for every new connection; 407 * and an "extra" component that grows by a random amount centered 408 * approximately on 64000. This causes the the ISS generator to cycle every 409 * 4.89 hours if no TCP connections are made, and faster if connections are 410 * made. 411 * 412 * When tcp_strong_iss is set to 0, ISS is calculated by adding two 413 * components: a time component which grows by 250000 every second; and 414 * a per-connection component which grows by 125000 for every new connections. 415 * 416 * A third method, when tcp_strong_iss is set to 2, for generating ISS is 417 * prescribed by Steve Bellovin. This involves adding time, the 125000 per 418 * connection, and a one-way hash (MD5) of the connection ID <sport, dport, 419 * src, dst>, a "truly" random (per RFC 1750) number, and a console-entered 420 * password. 421 */ 422 #define ISS_INCR 250000 423 #define ISS_NSEC_SHT 12 424 425 static sin_t sin_null; /* Zero address for quick clears */ 426 static sin6_t sin6_null; /* Zero address for quick clears */ 427 428 /* 429 * This implementation follows the 4.3BSD interpretation of the urgent 430 * pointer and not RFC 1122. Switching to RFC 1122 behavior would cause 431 * incompatible changes in protocols like telnet and rlogin. 432 */ 433 #define TCP_OLD_URP_INTERPRETATION 1 434 435 #define TCP_IS_DETACHED_NONEAGER(tcp) \ 436 (TCP_IS_DETACHED(tcp) && \ 437 (!(tcp)->tcp_hard_binding)) 438 439 /* 440 * TCP reassembly macros. We hide starting and ending sequence numbers in 441 * b_next and b_prev of messages on the reassembly queue. The messages are 442 * chained using b_cont. These macros are used in tcp_reass() so we don't 443 * have to see the ugly casts and assignments. 444 */ 445 #define TCP_REASS_SEQ(mp) ((uint32_t)(uintptr_t)((mp)->b_next)) 446 #define TCP_REASS_SET_SEQ(mp, u) ((mp)->b_next = \ 447 (mblk_t *)(uintptr_t)(u)) 448 #define TCP_REASS_END(mp) ((uint32_t)(uintptr_t)((mp)->b_prev)) 449 #define TCP_REASS_SET_END(mp, u) ((mp)->b_prev = \ 450 (mblk_t *)(uintptr_t)(u)) 451 452 /* 453 * Implementation of TCP Timers. 454 * ============================= 455 * 456 * INTERFACE: 457 * 458 * There are two basic functions dealing with tcp timers: 459 * 460 * timeout_id_t tcp_timeout(connp, func, time) 461 * clock_t tcp_timeout_cancel(connp, timeout_id) 462 * TCP_TIMER_RESTART(tcp, intvl) 463 * 464 * tcp_timeout() starts a timer for the 'tcp' instance arranging to call 'func' 465 * after 'time' ticks passed. The function called by timeout() must adhere to 466 * the same restrictions as a driver soft interrupt handler - it must not sleep 467 * or call other functions that might sleep. The value returned is the opaque 468 * non-zero timeout identifier that can be passed to tcp_timeout_cancel() to 469 * cancel the request. The call to tcp_timeout() may fail in which case it 470 * returns zero. This is different from the timeout(9F) function which never 471 * fails. 472 * 473 * The call-back function 'func' always receives 'connp' as its single 474 * argument. It is always executed in the squeue corresponding to the tcp 475 * structure. The tcp structure is guaranteed to be present at the time the 476 * call-back is called. 477 * 478 * NOTE: The call-back function 'func' is never called if tcp is in 479 * the TCPS_CLOSED state. 480 * 481 * tcp_timeout_cancel() attempts to cancel a pending tcp_timeout() 482 * request. locks acquired by the call-back routine should not be held across 483 * the call to tcp_timeout_cancel() or a deadlock may result. 484 * 485 * tcp_timeout_cancel() returns -1 if it can not cancel the timeout request. 486 * Otherwise, it returns an integer value greater than or equal to 0. In 487 * particular, if the call-back function is already placed on the squeue, it can 488 * not be canceled. 489 * 490 * NOTE: both tcp_timeout() and tcp_timeout_cancel() should always be called 491 * within squeue context corresponding to the tcp instance. Since the 492 * call-back is also called via the same squeue, there are no race 493 * conditions described in untimeout(9F) manual page since all calls are 494 * strictly serialized. 495 * 496 * TCP_TIMER_RESTART() is a macro that attempts to cancel a pending timeout 497 * stored in tcp_timer_tid and starts a new one using 498 * MSEC_TO_TICK(intvl). It always uses tcp_timer() function as a call-back 499 * and stores the return value of tcp_timeout() in the tcp->tcp_timer_tid 500 * field. 501 * 502 * NOTE: since the timeout cancellation is not guaranteed, the cancelled 503 * call-back may still be called, so it is possible tcp_timer() will be 504 * called several times. This should not be a problem since tcp_timer() 505 * should always check the tcp instance state. 506 * 507 * 508 * IMPLEMENTATION: 509 * 510 * TCP timers are implemented using three-stage process. The call to 511 * tcp_timeout() uses timeout(9F) function to call tcp_timer_callback() function 512 * when the timer expires. The tcp_timer_callback() arranges the call of the 513 * tcp_timer_handler() function via squeue corresponding to the tcp 514 * instance. The tcp_timer_handler() calls actual requested timeout call-back 515 * and passes tcp instance as an argument to it. Information is passed between 516 * stages using the tcp_timer_t structure which contains the connp pointer, the 517 * tcp call-back to call and the timeout id returned by the timeout(9F). 518 * 519 * The tcp_timer_t structure is not used directly, it is embedded in an mblk_t - 520 * like structure that is used to enter an squeue. The mp->b_rptr of this pseudo 521 * mblk points to the beginning of tcp_timer_t structure. The tcp_timeout() 522 * returns the pointer to this mblk. 523 * 524 * The pseudo mblk is allocated from a special tcp_timer_cache kmem cache. It 525 * looks like a normal mblk without actual dblk attached to it. 526 * 527 * To optimize performance each tcp instance holds a small cache of timer 528 * mblocks. In the current implementation it caches up to two timer mblocks per 529 * tcp instance. The cache is preserved over tcp frees and is only freed when 530 * the whole tcp structure is destroyed by its kmem destructor. Since all tcp 531 * timer processing happens on a corresponding squeue, the cache manipulation 532 * does not require any locks. Experiments show that majority of timer mblocks 533 * allocations are satisfied from the tcp cache and do not involve kmem calls. 534 * 535 * The tcp_timeout() places a refhold on the connp instance which guarantees 536 * that it will be present at the time the call-back function fires. The 537 * tcp_timer_handler() drops the reference after calling the call-back, so the 538 * call-back function does not need to manipulate the references explicitly. 539 */ 540 541 typedef struct tcp_timer_s { 542 conn_t *connp; 543 void (*tcpt_proc)(void *); 544 timeout_id_t tcpt_tid; 545 } tcp_timer_t; 546 547 static kmem_cache_t *tcp_timercache; 548 kmem_cache_t *tcp_sack_info_cache; 549 kmem_cache_t *tcp_iphc_cache; 550 551 /* 552 * For scalability, we must not run a timer for every TCP connection 553 * in TIME_WAIT state. To see why, consider (for time wait interval of 554 * 4 minutes): 555 * 1000 connections/sec * 240 seconds/time wait = 240,000 active conn's 556 * 557 * This list is ordered by time, so you need only delete from the head 558 * until you get to entries which aren't old enough to delete yet. 559 * The list consists of only the detached TIME_WAIT connections. 560 * 561 * Note that the timer (tcp_time_wait_expire) is started when the tcp_t 562 * becomes detached TIME_WAIT (either by changing the state and already 563 * being detached or the other way around). This means that the TIME_WAIT 564 * state can be extended (up to doubled) if the connection doesn't become 565 * detached for a long time. 566 * 567 * The list manipulations (including tcp_time_wait_next/prev) 568 * are protected by the tcp_time_wait_lock. The content of the 569 * detached TIME_WAIT connections is protected by the normal perimeters. 570 * 571 * This list is per squeue and squeues are shared across the tcp_stack_t's. 572 * Things on tcp_time_wait_head remain associated with the tcp_stack_t 573 * and conn_netstack. 574 * The tcp_t's that are added to tcp_free_list are disassociated and 575 * have NULL tcp_tcps and conn_netstack pointers. 576 */ 577 typedef struct tcp_squeue_priv_s { 578 kmutex_t tcp_time_wait_lock; 579 timeout_id_t tcp_time_wait_tid; 580 tcp_t *tcp_time_wait_head; 581 tcp_t *tcp_time_wait_tail; 582 tcp_t *tcp_free_list; 583 uint_t tcp_free_list_cnt; 584 } tcp_squeue_priv_t; 585 586 /* 587 * TCP_TIME_WAIT_DELAY governs how often the time_wait_collector runs. 588 * Running it every 5 seconds seems to give the best results. 589 */ 590 #define TCP_TIME_WAIT_DELAY drv_usectohz(5000000) 591 592 /* 593 * To prevent memory hog, limit the number of entries in tcp_free_list 594 * to 1% of available memory / number of cpus 595 */ 596 uint_t tcp_free_list_max_cnt = 0; 597 598 #define TCP_XMIT_LOWATER 4096 599 #define TCP_XMIT_HIWATER 49152 600 #define TCP_RECV_LOWATER 2048 601 #define TCP_RECV_HIWATER 49152 602 603 /* 604 * PAWS needs a timer for 24 days. This is the number of ticks in 24 days 605 */ 606 #define PAWS_TIMEOUT ((clock_t)(24*24*60*60*hz)) 607 608 #define TIDUSZ 4096 /* transport interface data unit size */ 609 610 /* 611 * Bind hash list size and has function. It has to be a power of 2 for 612 * hashing. 613 */ 614 #define TCP_BIND_FANOUT_SIZE 512 615 #define TCP_BIND_HASH(lport) (ntohs(lport) & (TCP_BIND_FANOUT_SIZE - 1)) 616 /* 617 * Size of listen and acceptor hash list. It has to be a power of 2 for 618 * hashing. 619 */ 620 #define TCP_FANOUT_SIZE 256 621 622 #ifdef _ILP32 623 #define TCP_ACCEPTOR_HASH(accid) \ 624 (((uint_t)(accid) >> 8) & (TCP_FANOUT_SIZE - 1)) 625 #else 626 #define TCP_ACCEPTOR_HASH(accid) \ 627 ((uint_t)(accid) & (TCP_FANOUT_SIZE - 1)) 628 #endif /* _ILP32 */ 629 630 #define IP_ADDR_CACHE_SIZE 2048 631 #define IP_ADDR_CACHE_HASH(faddr) \ 632 (ntohl(faddr) & (IP_ADDR_CACHE_SIZE -1)) 633 634 /* Hash for HSPs uses all 32 bits, since both networks and hosts are in table */ 635 #define TCP_HSP_HASH_SIZE 256 636 637 #define TCP_HSP_HASH(addr) \ 638 (((addr>>24) ^ (addr >>16) ^ \ 639 (addr>>8) ^ (addr)) % TCP_HSP_HASH_SIZE) 640 641 /* 642 * TCP options struct returned from tcp_parse_options. 643 */ 644 typedef struct tcp_opt_s { 645 uint32_t tcp_opt_mss; 646 uint32_t tcp_opt_wscale; 647 uint32_t tcp_opt_ts_val; 648 uint32_t tcp_opt_ts_ecr; 649 tcp_t *tcp; 650 } tcp_opt_t; 651 652 /* 653 * RFC1323-recommended phrasing of TSTAMP option, for easier parsing 654 */ 655 656 #ifdef _BIG_ENDIAN 657 #define TCPOPT_NOP_NOP_TSTAMP ((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | \ 658 (TCPOPT_TSTAMP << 8) | 10) 659 #else 660 #define TCPOPT_NOP_NOP_TSTAMP ((10 << 24) | (TCPOPT_TSTAMP << 16) | \ 661 (TCPOPT_NOP << 8) | TCPOPT_NOP) 662 #endif 663 664 /* 665 * Flags returned from tcp_parse_options. 666 */ 667 #define TCP_OPT_MSS_PRESENT 1 668 #define TCP_OPT_WSCALE_PRESENT 2 669 #define TCP_OPT_TSTAMP_PRESENT 4 670 #define TCP_OPT_SACK_OK_PRESENT 8 671 #define TCP_OPT_SACK_PRESENT 16 672 673 /* TCP option length */ 674 #define TCPOPT_NOP_LEN 1 675 #define TCPOPT_MAXSEG_LEN 4 676 #define TCPOPT_WS_LEN 3 677 #define TCPOPT_REAL_WS_LEN (TCPOPT_WS_LEN+1) 678 #define TCPOPT_TSTAMP_LEN 10 679 #define TCPOPT_REAL_TS_LEN (TCPOPT_TSTAMP_LEN+2) 680 #define TCPOPT_SACK_OK_LEN 2 681 #define TCPOPT_REAL_SACK_OK_LEN (TCPOPT_SACK_OK_LEN+2) 682 #define TCPOPT_REAL_SACK_LEN 4 683 #define TCPOPT_MAX_SACK_LEN 36 684 #define TCPOPT_HEADER_LEN 2 685 686 /* TCP cwnd burst factor. */ 687 #define TCP_CWND_INFINITE 65535 688 #define TCP_CWND_SS 3 689 #define TCP_CWND_NORMAL 5 690 691 /* Maximum TCP initial cwin (start/restart). */ 692 #define TCP_MAX_INIT_CWND 8 693 694 /* 695 * Initialize cwnd according to RFC 3390. def_max_init_cwnd is 696 * either tcp_slow_start_initial or tcp_slow_start_after idle 697 * depending on the caller. If the upper layer has not used the 698 * TCP_INIT_CWND option to change the initial cwnd, tcp_init_cwnd 699 * should be 0 and we use the formula in RFC 3390 to set tcp_cwnd. 700 * If the upper layer has changed set the tcp_init_cwnd, just use 701 * it to calculate the tcp_cwnd. 702 */ 703 #define SET_TCP_INIT_CWND(tcp, mss, def_max_init_cwnd) \ 704 { \ 705 if ((tcp)->tcp_init_cwnd == 0) { \ 706 (tcp)->tcp_cwnd = MIN(def_max_init_cwnd * (mss), \ 707 MIN(4 * (mss), MAX(2 * (mss), 4380 / (mss) * (mss)))); \ 708 } else { \ 709 (tcp)->tcp_cwnd = (tcp)->tcp_init_cwnd * (mss); \ 710 } \ 711 tcp->tcp_cwnd_cnt = 0; \ 712 } 713 714 /* TCP Timer control structure */ 715 typedef struct tcpt_s { 716 pfv_t tcpt_pfv; /* The routine we are to call */ 717 tcp_t *tcpt_tcp; /* The parameter we are to pass in */ 718 } tcpt_t; 719 720 /* Host Specific Parameter structure */ 721 typedef struct tcp_hsp { 722 struct tcp_hsp *tcp_hsp_next; 723 in6_addr_t tcp_hsp_addr_v6; 724 in6_addr_t tcp_hsp_subnet_v6; 725 uint_t tcp_hsp_vers; /* IPV4_VERSION | IPV6_VERSION */ 726 int32_t tcp_hsp_sendspace; 727 int32_t tcp_hsp_recvspace; 728 int32_t tcp_hsp_tstamp; 729 } tcp_hsp_t; 730 #define tcp_hsp_addr V4_PART_OF_V6(tcp_hsp_addr_v6) 731 #define tcp_hsp_subnet V4_PART_OF_V6(tcp_hsp_subnet_v6) 732 733 /* 734 * Functions called directly via squeue having a prototype of edesc_t. 735 */ 736 void tcp_conn_request(void *arg, mblk_t *mp, void *arg2); 737 static void tcp_wput_nondata(void *arg, mblk_t *mp, void *arg2); 738 void tcp_accept_finish(void *arg, mblk_t *mp, void *arg2); 739 static void tcp_wput_ioctl(void *arg, mblk_t *mp, void *arg2); 740 static void tcp_wput_proto(void *arg, mblk_t *mp, void *arg2); 741 void tcp_input(void *arg, mblk_t *mp, void *arg2); 742 void tcp_rput_data(void *arg, mblk_t *mp, void *arg2); 743 static void tcp_close_output(void *arg, mblk_t *mp, void *arg2); 744 void tcp_output(void *arg, mblk_t *mp, void *arg2); 745 static void tcp_rsrv_input(void *arg, mblk_t *mp, void *arg2); 746 static void tcp_timer_handler(void *arg, mblk_t *mp, void *arg2); 747 static void tcp_linger_interrupted(void *arg, mblk_t *mp, void *arg2); 748 749 750 /* Prototype for TCP functions */ 751 static void tcp_random_init(void); 752 int tcp_random(void); 753 static void tcp_accept(tcp_t *tcp, mblk_t *mp); 754 static void tcp_accept_swap(tcp_t *listener, tcp_t *acceptor, 755 tcp_t *eager); 756 static int tcp_adapt_ire(tcp_t *tcp, mblk_t *ire_mp); 757 static in_port_t tcp_bindi(tcp_t *tcp, in_port_t port, const in6_addr_t *laddr, 758 int reuseaddr, boolean_t quick_connect, boolean_t bind_to_req_port_only, 759 boolean_t user_specified); 760 static void tcp_closei_local(tcp_t *tcp); 761 static void tcp_close_detached(tcp_t *tcp); 762 static boolean_t tcp_conn_con(tcp_t *tcp, uchar_t *iphdr, tcph_t *tcph, 763 mblk_t *idmp, mblk_t **defermp); 764 static void tcp_connect(tcp_t *tcp, mblk_t *mp); 765 static void tcp_connect_ipv4(tcp_t *tcp, mblk_t *mp, ipaddr_t *dstaddrp, 766 in_port_t dstport, uint_t srcid); 767 static void tcp_connect_ipv6(tcp_t *tcp, mblk_t *mp, in6_addr_t *dstaddrp, 768 in_port_t dstport, uint32_t flowinfo, uint_t srcid, 769 uint32_t scope_id); 770 static int tcp_clean_death(tcp_t *tcp, int err, uint8_t tag); 771 static void tcp_def_q_set(tcp_t *tcp, mblk_t *mp); 772 static void tcp_disconnect(tcp_t *tcp, mblk_t *mp); 773 static char *tcp_display(tcp_t *tcp, char *, char); 774 static boolean_t tcp_eager_blowoff(tcp_t *listener, t_scalar_t seqnum); 775 static void tcp_eager_cleanup(tcp_t *listener, boolean_t q0_only); 776 static void tcp_eager_unlink(tcp_t *tcp); 777 static void tcp_err_ack(tcp_t *tcp, mblk_t *mp, int tlierr, 778 int unixerr); 779 static void tcp_err_ack_prim(tcp_t *tcp, mblk_t *mp, int primitive, 780 int tlierr, int unixerr); 781 static int tcp_extra_priv_ports_get(queue_t *q, mblk_t *mp, caddr_t cp, 782 cred_t *cr); 783 static int tcp_extra_priv_ports_add(queue_t *q, mblk_t *mp, 784 char *value, caddr_t cp, cred_t *cr); 785 static int tcp_extra_priv_ports_del(queue_t *q, mblk_t *mp, 786 char *value, caddr_t cp, cred_t *cr); 787 static int tcp_tpistate(tcp_t *tcp); 788 static void tcp_bind_hash_insert(tf_t *tf, tcp_t *tcp, 789 int caller_holds_lock); 790 static void tcp_bind_hash_remove(tcp_t *tcp); 791 static tcp_t *tcp_acceptor_hash_lookup(t_uscalar_t id, tcp_stack_t *); 792 void tcp_acceptor_hash_insert(t_uscalar_t id, tcp_t *tcp); 793 static void tcp_acceptor_hash_remove(tcp_t *tcp); 794 static void tcp_capability_req(tcp_t *tcp, mblk_t *mp); 795 static void tcp_info_req(tcp_t *tcp, mblk_t *mp); 796 static void tcp_addr_req(tcp_t *tcp, mblk_t *mp); 797 static void tcp_addr_req_ipv6(tcp_t *tcp, mblk_t *mp); 798 void tcp_g_q_setup(tcp_stack_t *); 799 void tcp_g_q_create(tcp_stack_t *); 800 void tcp_g_q_destroy(tcp_stack_t *); 801 static int tcp_header_init_ipv4(tcp_t *tcp); 802 static int tcp_header_init_ipv6(tcp_t *tcp); 803 int tcp_init(tcp_t *tcp, queue_t *q); 804 static int tcp_init_values(tcp_t *tcp); 805 static mblk_t *tcp_ip_advise_mblk(void *addr, int addr_len, ipic_t **ipic); 806 static mblk_t *tcp_ip_bind_mp(tcp_t *tcp, t_scalar_t bind_prim, 807 t_scalar_t addr_length); 808 static void tcp_ip_ire_mark_advice(tcp_t *tcp); 809 static void tcp_ip_notify(tcp_t *tcp); 810 static mblk_t *tcp_ire_mp(mblk_t *mp); 811 static void tcp_iss_init(tcp_t *tcp); 812 static void tcp_keepalive_killer(void *arg); 813 static int tcp_parse_options(tcph_t *tcph, tcp_opt_t *tcpopt); 814 static void tcp_mss_set(tcp_t *tcp, uint32_t size, boolean_t do_ss); 815 static int tcp_conprim_opt_process(tcp_t *tcp, mblk_t *mp, 816 int *do_disconnectp, int *t_errorp, int *sys_errorp); 817 static boolean_t tcp_allow_connopt_set(int level, int name); 818 int tcp_opt_default(queue_t *q, int level, int name, uchar_t *ptr); 819 int tcp_opt_get(queue_t *q, int level, int name, uchar_t *ptr); 820 int tcp_opt_set(queue_t *q, uint_t optset_context, int level, 821 int name, uint_t inlen, uchar_t *invalp, uint_t *outlenp, 822 uchar_t *outvalp, void *thisdg_attrs, cred_t *cr, 823 mblk_t *mblk); 824 static void tcp_opt_reverse(tcp_t *tcp, ipha_t *ipha); 825 static int tcp_opt_set_header(tcp_t *tcp, boolean_t checkonly, 826 uchar_t *ptr, uint_t len); 827 static int tcp_param_get(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr); 828 static boolean_t tcp_param_register(IDP *ndp, tcpparam_t *tcppa, int cnt, 829 tcp_stack_t *); 830 static int tcp_param_set(queue_t *q, mblk_t *mp, char *value, 831 caddr_t cp, cred_t *cr); 832 static int tcp_param_set_aligned(queue_t *q, mblk_t *mp, char *value, 833 caddr_t cp, cred_t *cr); 834 static void tcp_iss_key_init(uint8_t *phrase, int len, tcp_stack_t *); 835 static int tcp_1948_phrase_set(queue_t *q, mblk_t *mp, char *value, 836 caddr_t cp, cred_t *cr); 837 static void tcp_process_shrunk_swnd(tcp_t *tcp, uint32_t shrunk_cnt); 838 static mblk_t *tcp_reass(tcp_t *tcp, mblk_t *mp, uint32_t start); 839 static void tcp_reass_elim_overlap(tcp_t *tcp, mblk_t *mp); 840 static void tcp_reinit(tcp_t *tcp); 841 static void tcp_reinit_values(tcp_t *tcp); 842 static void tcp_report_item(mblk_t *mp, tcp_t *tcp, int hashval, 843 tcp_t *thisstream, cred_t *cr); 844 845 static uint_t tcp_rcv_drain(queue_t *q, tcp_t *tcp); 846 static void tcp_sack_rxmit(tcp_t *tcp, uint_t *flags); 847 static boolean_t tcp_send_rst_chk(tcp_stack_t *); 848 static void tcp_ss_rexmit(tcp_t *tcp); 849 static mblk_t *tcp_rput_add_ancillary(tcp_t *tcp, mblk_t *mp, ip6_pkt_t *ipp); 850 static void tcp_process_options(tcp_t *, tcph_t *); 851 static void tcp_rput_common(tcp_t *tcp, mblk_t *mp); 852 static void tcp_rsrv(queue_t *q); 853 static int tcp_rwnd_set(tcp_t *tcp, uint32_t rwnd); 854 static int tcp_snmp_state(tcp_t *tcp); 855 static int tcp_status_report(queue_t *q, mblk_t *mp, caddr_t cp, 856 cred_t *cr); 857 static int tcp_bind_hash_report(queue_t *q, mblk_t *mp, caddr_t cp, 858 cred_t *cr); 859 static int tcp_listen_hash_report(queue_t *q, mblk_t *mp, caddr_t cp, 860 cred_t *cr); 861 static int tcp_conn_hash_report(queue_t *q, mblk_t *mp, caddr_t cp, 862 cred_t *cr); 863 static int tcp_acceptor_hash_report(queue_t *q, mblk_t *mp, caddr_t cp, 864 cred_t *cr); 865 static void tcp_timer(void *arg); 866 static void tcp_timer_callback(void *); 867 static in_port_t tcp_update_next_port(in_port_t port, const tcp_t *tcp, 868 boolean_t random); 869 static in_port_t tcp_get_next_priv_port(const tcp_t *); 870 static void tcp_wput_sock(queue_t *q, mblk_t *mp); 871 void tcp_wput_accept(queue_t *q, mblk_t *mp); 872 static void tcp_wput_data(tcp_t *tcp, mblk_t *mp, boolean_t urgent); 873 static void tcp_wput_flush(tcp_t *tcp, mblk_t *mp); 874 static void tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp); 875 static int tcp_send(queue_t *q, tcp_t *tcp, const int mss, 876 const int tcp_hdr_len, const int tcp_tcp_hdr_len, 877 const int num_sack_blk, int *usable, uint_t *snxt, 878 int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time, 879 const int mdt_thres); 880 static int tcp_multisend(queue_t *q, tcp_t *tcp, const int mss, 881 const int tcp_hdr_len, const int tcp_tcp_hdr_len, 882 const int num_sack_blk, int *usable, uint_t *snxt, 883 int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time, 884 const int mdt_thres); 885 static void tcp_fill_header(tcp_t *tcp, uchar_t *rptr, clock_t now, 886 int num_sack_blk); 887 static void tcp_wsrv(queue_t *q); 888 static int tcp_xmit_end(tcp_t *tcp); 889 static void tcp_ack_timer(void *arg); 890 static mblk_t *tcp_ack_mp(tcp_t *tcp); 891 static void tcp_xmit_early_reset(char *str, mblk_t *mp, 892 uint32_t seq, uint32_t ack, int ctl, uint_t ip_hdr_len, 893 zoneid_t zoneid, tcp_stack_t *, conn_t *connp); 894 static void tcp_xmit_ctl(char *str, tcp_t *tcp, uint32_t seq, 895 uint32_t ack, int ctl); 896 static tcp_hsp_t *tcp_hsp_lookup(ipaddr_t addr, tcp_stack_t *); 897 static tcp_hsp_t *tcp_hsp_lookup_ipv6(in6_addr_t *addr, tcp_stack_t *); 898 static int setmaxps(queue_t *q, int maxpsz); 899 static void tcp_set_rto(tcp_t *, time_t); 900 static boolean_t tcp_check_policy(tcp_t *, mblk_t *, ipha_t *, ip6_t *, 901 boolean_t, boolean_t); 902 static void tcp_icmp_error_ipv6(tcp_t *tcp, mblk_t *mp, 903 boolean_t ipsec_mctl); 904 static mblk_t *tcp_setsockopt_mp(int level, int cmd, 905 char *opt, int optlen); 906 static int tcp_build_hdrs(queue_t *, tcp_t *); 907 static void tcp_time_wait_processing(tcp_t *tcp, mblk_t *mp, 908 uint32_t seg_seq, uint32_t seg_ack, int seg_len, 909 tcph_t *tcph); 910 boolean_t tcp_paws_check(tcp_t *tcp, tcph_t *tcph, tcp_opt_t *tcpoptp); 911 static mblk_t *tcp_mdt_info_mp(mblk_t *); 912 static void tcp_mdt_update(tcp_t *, ill_mdt_capab_t *, boolean_t); 913 static int tcp_mdt_add_attrs(multidata_t *, const mblk_t *, 914 const boolean_t, const uint32_t, const uint32_t, 915 const uint32_t, const uint32_t, tcp_stack_t *); 916 static void tcp_multisend_data(tcp_t *, ire_t *, const ill_t *, mblk_t *, 917 const uint_t, const uint_t, boolean_t *); 918 static mblk_t *tcp_lso_info_mp(mblk_t *); 919 static void tcp_lso_update(tcp_t *, ill_lso_capab_t *); 920 static void tcp_send_data(tcp_t *, queue_t *, mblk_t *); 921 extern mblk_t *tcp_timermp_alloc(int); 922 extern void tcp_timermp_free(tcp_t *); 923 static void tcp_timer_free(tcp_t *tcp, mblk_t *mp); 924 static void tcp_stop_lingering(tcp_t *tcp); 925 static void tcp_close_linger_timeout(void *arg); 926 static void *tcp_stack_init(netstackid_t stackid, netstack_t *ns); 927 static void tcp_stack_shutdown(netstackid_t stackid, void *arg); 928 static void tcp_stack_fini(netstackid_t stackid, void *arg); 929 static void *tcp_g_kstat_init(tcp_g_stat_t *); 930 static void tcp_g_kstat_fini(kstat_t *); 931 static void *tcp_kstat_init(netstackid_t, tcp_stack_t *); 932 static void tcp_kstat_fini(netstackid_t, kstat_t *); 933 static void *tcp_kstat2_init(netstackid_t, tcp_stat_t *); 934 static void tcp_kstat2_fini(netstackid_t, kstat_t *); 935 static int tcp_kstat_update(kstat_t *kp, int rw); 936 void tcp_reinput(conn_t *connp, mblk_t *mp, squeue_t *sqp); 937 static int tcp_conn_create_v6(conn_t *lconnp, conn_t *connp, mblk_t *mp, 938 tcph_t *tcph, uint_t ipvers, mblk_t *idmp); 939 static int tcp_conn_create_v4(conn_t *lconnp, conn_t *connp, ipha_t *ipha, 940 tcph_t *tcph, mblk_t *idmp); 941 static squeue_func_t tcp_squeue_switch(int); 942 943 static int tcp_open(queue_t *, dev_t *, int, int, cred_t *, boolean_t); 944 static int tcp_openv4(queue_t *, dev_t *, int, int, cred_t *); 945 static int tcp_openv6(queue_t *, dev_t *, int, int, cred_t *); 946 static int tcp_close(queue_t *, int); 947 static int tcpclose_accept(queue_t *); 948 949 static void tcp_squeue_add(squeue_t *); 950 static boolean_t tcp_zcopy_check(tcp_t *); 951 static void tcp_zcopy_notify(tcp_t *); 952 static mblk_t *tcp_zcopy_disable(tcp_t *, mblk_t *); 953 static mblk_t *tcp_zcopy_backoff(tcp_t *, mblk_t *, int); 954 static void tcp_ire_ill_check(tcp_t *, ire_t *, ill_t *, boolean_t); 955 956 extern void tcp_kssl_input(tcp_t *, mblk_t *); 957 958 void tcp_eager_kill(void *arg, mblk_t *mp, void *arg2); 959 void tcp_clean_death_wrapper(void *arg, mblk_t *mp, void *arg2); 960 961 /* 962 * Routines related to the TCP_IOC_ABORT_CONN ioctl command. 963 * 964 * TCP_IOC_ABORT_CONN is a non-transparent ioctl command used for aborting 965 * TCP connections. To invoke this ioctl, a tcp_ioc_abort_conn_t structure 966 * (defined in tcp.h) needs to be filled in and passed into the kernel 967 * via an I_STR ioctl command (see streamio(7I)). The tcp_ioc_abort_conn_t 968 * structure contains the four-tuple of a TCP connection and a range of TCP 969 * states (specified by ac_start and ac_end). The use of wildcard addresses 970 * and ports is allowed. Connections with a matching four tuple and a state 971 * within the specified range will be aborted. The valid states for the 972 * ac_start and ac_end fields are in the range TCPS_SYN_SENT to TCPS_TIME_WAIT, 973 * inclusive. 974 * 975 * An application which has its connection aborted by this ioctl will receive 976 * an error that is dependent on the connection state at the time of the abort. 977 * If the connection state is < TCPS_TIME_WAIT, an application should behave as 978 * though a RST packet has been received. If the connection state is equal to 979 * TCPS_TIME_WAIT, the 2MSL timeout will immediately be canceled by the kernel 980 * and all resources associated with the connection will be freed. 981 */ 982 static mblk_t *tcp_ioctl_abort_build_msg(tcp_ioc_abort_conn_t *, tcp_t *); 983 static void tcp_ioctl_abort_dump(tcp_ioc_abort_conn_t *); 984 static void tcp_ioctl_abort_handler(tcp_t *, mblk_t *); 985 static int tcp_ioctl_abort(tcp_ioc_abort_conn_t *, tcp_stack_t *tcps); 986 static void tcp_ioctl_abort_conn(queue_t *, mblk_t *); 987 static int tcp_ioctl_abort_bucket(tcp_ioc_abort_conn_t *, int, int *, 988 boolean_t, tcp_stack_t *); 989 990 static struct module_info tcp_rinfo = { 991 TCP_MOD_ID, TCP_MOD_NAME, 0, INFPSZ, TCP_RECV_HIWATER, TCP_RECV_LOWATER 992 }; 993 994 static struct module_info tcp_winfo = { 995 TCP_MOD_ID, TCP_MOD_NAME, 0, INFPSZ, 127, 16 996 }; 997 998 /* 999 * Entry points for TCP as a device. The normal case which supports 1000 * the TCP functionality. 1001 * We have separate open functions for the /dev/tcp and /dev/tcp6 devices. 1002 */ 1003 struct qinit tcp_rinitv4 = { 1004 NULL, (pfi_t)tcp_rsrv, tcp_openv4, tcp_close, NULL, &tcp_rinfo 1005 }; 1006 1007 struct qinit tcp_rinitv6 = { 1008 NULL, (pfi_t)tcp_rsrv, tcp_openv6, tcp_close, NULL, &tcp_rinfo 1009 }; 1010 1011 struct qinit tcp_winit = { 1012 (pfi_t)tcp_wput, (pfi_t)tcp_wsrv, NULL, NULL, NULL, &tcp_winfo 1013 }; 1014 1015 /* Initial entry point for TCP in socket mode. */ 1016 struct qinit tcp_sock_winit = { 1017 (pfi_t)tcp_wput_sock, (pfi_t)tcp_wsrv, NULL, NULL, NULL, &tcp_winfo 1018 }; 1019 1020 /* 1021 * Entry points for TCP as a acceptor STREAM opened by sockfs when doing 1022 * an accept. Avoid allocating data structures since eager has already 1023 * been created. 1024 */ 1025 struct qinit tcp_acceptor_rinit = { 1026 NULL, (pfi_t)tcp_rsrv, NULL, tcpclose_accept, NULL, &tcp_winfo 1027 }; 1028 1029 struct qinit tcp_acceptor_winit = { 1030 (pfi_t)tcp_wput_accept, NULL, NULL, NULL, NULL, &tcp_winfo 1031 }; 1032 1033 /* 1034 * Entry points for TCP loopback (read side only) 1035 * The open routine is only used for reopens, thus no need to 1036 * have a separate one for tcp_openv6. 1037 */ 1038 struct qinit tcp_loopback_rinit = { 1039 (pfi_t)0, (pfi_t)tcp_rsrv, tcp_openv4, tcp_close, (pfi_t)0, 1040 &tcp_rinfo, NULL, tcp_fuse_rrw, tcp_fuse_rinfop, STRUIOT_STANDARD 1041 }; 1042 1043 /* For AF_INET aka /dev/tcp */ 1044 struct streamtab tcpinfov4 = { 1045 &tcp_rinitv4, &tcp_winit 1046 }; 1047 1048 /* For AF_INET6 aka /dev/tcp6 */ 1049 struct streamtab tcpinfov6 = { 1050 &tcp_rinitv6, &tcp_winit 1051 }; 1052 1053 /* 1054 * Have to ensure that tcp_g_q_close is not done by an 1055 * interrupt thread. 1056 */ 1057 static taskq_t *tcp_taskq; 1058 1059 /* Setable only in /etc/system. Move to ndd? */ 1060 boolean_t tcp_icmp_source_quench = B_FALSE; 1061 1062 /* 1063 * Following assumes TPI alignment requirements stay along 32 bit 1064 * boundaries 1065 */ 1066 #define ROUNDUP32(x) \ 1067 (((x) + (sizeof (int32_t) - 1)) & ~(sizeof (int32_t) - 1)) 1068 1069 /* Template for response to info request. */ 1070 static struct T_info_ack tcp_g_t_info_ack = { 1071 T_INFO_ACK, /* PRIM_type */ 1072 0, /* TSDU_size */ 1073 T_INFINITE, /* ETSDU_size */ 1074 T_INVALID, /* CDATA_size */ 1075 T_INVALID, /* DDATA_size */ 1076 sizeof (sin_t), /* ADDR_size */ 1077 0, /* OPT_size - not initialized here */ 1078 TIDUSZ, /* TIDU_size */ 1079 T_COTS_ORD, /* SERV_type */ 1080 TCPS_IDLE, /* CURRENT_state */ 1081 (XPG4_1|EXPINLINE) /* PROVIDER_flag */ 1082 }; 1083 1084 static struct T_info_ack tcp_g_t_info_ack_v6 = { 1085 T_INFO_ACK, /* PRIM_type */ 1086 0, /* TSDU_size */ 1087 T_INFINITE, /* ETSDU_size */ 1088 T_INVALID, /* CDATA_size */ 1089 T_INVALID, /* DDATA_size */ 1090 sizeof (sin6_t), /* ADDR_size */ 1091 0, /* OPT_size - not initialized here */ 1092 TIDUSZ, /* TIDU_size */ 1093 T_COTS_ORD, /* SERV_type */ 1094 TCPS_IDLE, /* CURRENT_state */ 1095 (XPG4_1|EXPINLINE) /* PROVIDER_flag */ 1096 }; 1097 1098 #define MS 1L 1099 #define SECONDS (1000 * MS) 1100 #define MINUTES (60 * SECONDS) 1101 #define HOURS (60 * MINUTES) 1102 #define DAYS (24 * HOURS) 1103 1104 #define PARAM_MAX (~(uint32_t)0) 1105 1106 /* Max size IP datagram is 64k - 1 */ 1107 #define TCP_MSS_MAX_IPV4 (IP_MAXPACKET - (sizeof (ipha_t) + sizeof (tcph_t))) 1108 #define TCP_MSS_MAX_IPV6 (IP_MAXPACKET - (sizeof (ip6_t) + sizeof (tcph_t))) 1109 /* Max of the above */ 1110 #define TCP_MSS_MAX TCP_MSS_MAX_IPV4 1111 1112 /* Largest TCP port number */ 1113 #define TCP_MAX_PORT (64 * 1024 - 1) 1114 1115 /* 1116 * tcp_wroff_xtra is the extra space in front of TCP/IP header for link 1117 * layer header. It has to be a multiple of 4. 1118 */ 1119 static tcpparam_t lcl_tcp_wroff_xtra_param = { 0, 256, 32, "tcp_wroff_xtra" }; 1120 #define tcps_wroff_xtra tcps_wroff_xtra_param->tcp_param_val 1121 1122 /* 1123 * All of these are alterable, within the min/max values given, at run time. 1124 * Note that the default value of "tcp_time_wait_interval" is four minutes, 1125 * per the TCP spec. 1126 */ 1127 /* BEGIN CSTYLED */ 1128 static tcpparam_t lcl_tcp_param_arr[] = { 1129 /*min max value name */ 1130 { 1*SECONDS, 10*MINUTES, 1*MINUTES, "tcp_time_wait_interval"}, 1131 { 1, PARAM_MAX, 128, "tcp_conn_req_max_q" }, 1132 { 0, PARAM_MAX, 1024, "tcp_conn_req_max_q0" }, 1133 { 1, 1024, 1, "tcp_conn_req_min" }, 1134 { 0*MS, 20*SECONDS, 0*MS, "tcp_conn_grace_period" }, 1135 { 128, (1<<30), 1024*1024, "tcp_cwnd_max" }, 1136 { 0, 10, 0, "tcp_debug" }, 1137 { 1024, (32*1024), 1024, "tcp_smallest_nonpriv_port"}, 1138 { 1*SECONDS, PARAM_MAX, 3*MINUTES, "tcp_ip_abort_cinterval"}, 1139 { 1*SECONDS, PARAM_MAX, 3*MINUTES, "tcp_ip_abort_linterval"}, 1140 { 500*MS, PARAM_MAX, 8*MINUTES, "tcp_ip_abort_interval"}, 1141 { 1*SECONDS, PARAM_MAX, 10*SECONDS, "tcp_ip_notify_cinterval"}, 1142 { 500*MS, PARAM_MAX, 10*SECONDS, "tcp_ip_notify_interval"}, 1143 { 1, 255, 64, "tcp_ipv4_ttl"}, 1144 { 10*SECONDS, 10*DAYS, 2*HOURS, "tcp_keepalive_interval"}, 1145 { 0, 100, 10, "tcp_maxpsz_multiplier" }, 1146 { 1, TCP_MSS_MAX_IPV4, 536, "tcp_mss_def_ipv4"}, 1147 { 1, TCP_MSS_MAX_IPV4, TCP_MSS_MAX_IPV4, "tcp_mss_max_ipv4"}, 1148 { 1, TCP_MSS_MAX, 108, "tcp_mss_min"}, 1149 { 1, (64*1024)-1, (4*1024)-1, "tcp_naglim_def"}, 1150 { 1*MS, 20*SECONDS, 3*SECONDS, "tcp_rexmit_interval_initial"}, 1151 { 1*MS, 2*HOURS, 60*SECONDS, "tcp_rexmit_interval_max"}, 1152 { 1*MS, 2*HOURS, 400*MS, "tcp_rexmit_interval_min"}, 1153 { 1*MS, 1*MINUTES, 100*MS, "tcp_deferred_ack_interval" }, 1154 { 0, 16, 0, "tcp_snd_lowat_fraction" }, 1155 { 0, 128000, 0, "tcp_sth_rcv_hiwat" }, 1156 { 0, 128000, 0, "tcp_sth_rcv_lowat" }, 1157 { 1, 10000, 3, "tcp_dupack_fast_retransmit" }, 1158 { 0, 1, 0, "tcp_ignore_path_mtu" }, 1159 { 1024, TCP_MAX_PORT, 32*1024, "tcp_smallest_anon_port"}, 1160 { 1024, TCP_MAX_PORT, TCP_MAX_PORT, "tcp_largest_anon_port"}, 1161 { TCP_XMIT_LOWATER, (1<<30), TCP_XMIT_HIWATER,"tcp_xmit_hiwat"}, 1162 { TCP_XMIT_LOWATER, (1<<30), TCP_XMIT_LOWATER,"tcp_xmit_lowat"}, 1163 { TCP_RECV_LOWATER, (1<<30), TCP_RECV_HIWATER,"tcp_recv_hiwat"}, 1164 { 1, 65536, 4, "tcp_recv_hiwat_minmss"}, 1165 { 1*SECONDS, PARAM_MAX, 675*SECONDS, "tcp_fin_wait_2_flush_interval"}, 1166 { 8192, (1<<30), 1024*1024, "tcp_max_buf"}, 1167 /* 1168 * Question: What default value should I set for tcp_strong_iss? 1169 */ 1170 { 0, 2, 1, "tcp_strong_iss"}, 1171 { 0, 65536, 20, "tcp_rtt_updates"}, 1172 { 0, 1, 1, "tcp_wscale_always"}, 1173 { 0, 1, 0, "tcp_tstamp_always"}, 1174 { 0, 1, 1, "tcp_tstamp_if_wscale"}, 1175 { 0*MS, 2*HOURS, 0*MS, "tcp_rexmit_interval_extra"}, 1176 { 0, 16, 2, "tcp_deferred_acks_max"}, 1177 { 1, 16384, 4, "tcp_slow_start_after_idle"}, 1178 { 1, 4, 4, "tcp_slow_start_initial"}, 1179 { 0, 2, 2, "tcp_sack_permitted"}, 1180 { 0, 1, 1, "tcp_compression_enabled"}, 1181 { 0, IPV6_MAX_HOPS, IPV6_DEFAULT_HOPS, "tcp_ipv6_hoplimit"}, 1182 { 1, TCP_MSS_MAX_IPV6, 1220, "tcp_mss_def_ipv6"}, 1183 { 1, TCP_MSS_MAX_IPV6, TCP_MSS_MAX_IPV6, "tcp_mss_max_ipv6"}, 1184 { 0, 1, 0, "tcp_rev_src_routes"}, 1185 { 10*MS, 500*MS, 50*MS, "tcp_local_dack_interval"}, 1186 { 100*MS, 60*SECONDS, 1*SECONDS, "tcp_ndd_get_info_interval"}, 1187 { 0, 16, 8, "tcp_local_dacks_max"}, 1188 { 0, 2, 1, "tcp_ecn_permitted"}, 1189 { 0, 1, 1, "tcp_rst_sent_rate_enabled"}, 1190 { 0, PARAM_MAX, 40, "tcp_rst_sent_rate"}, 1191 { 0, 100*MS, 50*MS, "tcp_push_timer_interval"}, 1192 { 0, 1, 0, "tcp_use_smss_as_mss_opt"}, 1193 { 0, PARAM_MAX, 8*MINUTES, "tcp_keepalive_abort_interval"}, 1194 }; 1195 /* END CSTYLED */ 1196 1197 /* 1198 * tcp_mdt_hdr_{head,tail}_min are the leading and trailing spaces of 1199 * each header fragment in the header buffer. Each parameter value has 1200 * to be a multiple of 4 (32-bit aligned). 1201 */ 1202 static tcpparam_t lcl_tcp_mdt_head_param = 1203 { 32, 256, 32, "tcp_mdt_hdr_head_min" }; 1204 static tcpparam_t lcl_tcp_mdt_tail_param = 1205 { 0, 256, 32, "tcp_mdt_hdr_tail_min" }; 1206 #define tcps_mdt_hdr_head_min tcps_mdt_head_param->tcp_param_val 1207 #define tcps_mdt_hdr_tail_min tcps_mdt_tail_param->tcp_param_val 1208 1209 /* 1210 * tcp_mdt_max_pbufs is the upper limit value that tcp uses to figure out 1211 * the maximum number of payload buffers associated per Multidata. 1212 */ 1213 static tcpparam_t lcl_tcp_mdt_max_pbufs_param = 1214 { 1, MULTIDATA_MAX_PBUFS, MULTIDATA_MAX_PBUFS, "tcp_mdt_max_pbufs" }; 1215 #define tcps_mdt_max_pbufs tcps_mdt_max_pbufs_param->tcp_param_val 1216 1217 /* Round up the value to the nearest mss. */ 1218 #define MSS_ROUNDUP(value, mss) ((((value) - 1) / (mss) + 1) * (mss)) 1219 1220 /* 1221 * Set ECN capable transport (ECT) code point in IP header. 1222 * 1223 * Note that there are 2 ECT code points '01' and '10', which are called 1224 * ECT(1) and ECT(0) respectively. Here we follow the original ECT code 1225 * point ECT(0) for TCP as described in RFC 2481. 1226 */ 1227 #define SET_ECT(tcp, iph) \ 1228 if ((tcp)->tcp_ipversion == IPV4_VERSION) { \ 1229 /* We need to clear the code point first. */ \ 1230 ((ipha_t *)(iph))->ipha_type_of_service &= 0xFC; \ 1231 ((ipha_t *)(iph))->ipha_type_of_service |= IPH_ECN_ECT0; \ 1232 } else { \ 1233 ((ip6_t *)(iph))->ip6_vcf &= htonl(0xFFCFFFFF); \ 1234 ((ip6_t *)(iph))->ip6_vcf |= htonl(IPH_ECN_ECT0 << 20); \ 1235 } 1236 1237 /* 1238 * The format argument to pass to tcp_display(). 1239 * DISP_PORT_ONLY means that the returned string has only port info. 1240 * DISP_ADDR_AND_PORT means that the returned string also contains the 1241 * remote and local IP address. 1242 */ 1243 #define DISP_PORT_ONLY 1 1244 #define DISP_ADDR_AND_PORT 2 1245 1246 #define NDD_TOO_QUICK_MSG \ 1247 "ndd get info rate too high for non-privileged users, try again " \ 1248 "later.\n" 1249 #define NDD_OUT_OF_BUF_MSG "<< Out of buffer >>\n" 1250 1251 #define IS_VMLOANED_MBLK(mp) \ 1252 (((mp)->b_datap->db_struioflag & STRUIO_ZC) != 0) 1253 1254 1255 /* Enable or disable b_cont M_MULTIDATA chaining for MDT. */ 1256 boolean_t tcp_mdt_chain = B_TRUE; 1257 1258 /* 1259 * MDT threshold in the form of effective send MSS multiplier; we take 1260 * the MDT path if the amount of unsent data exceeds the threshold value 1261 * (default threshold is 1*SMSS). 1262 */ 1263 uint_t tcp_mdt_smss_threshold = 1; 1264 1265 uint32_t do_tcpzcopy = 1; /* 0: disable, 1: enable, 2: force */ 1266 1267 /* 1268 * Forces all connections to obey the value of the tcps_maxpsz_multiplier 1269 * tunable settable via NDD. Otherwise, the per-connection behavior is 1270 * determined dynamically during tcp_adapt_ire(), which is the default. 1271 */ 1272 boolean_t tcp_static_maxpsz = B_FALSE; 1273 1274 /* Setable in /etc/system */ 1275 /* If set to 0, pick ephemeral port sequentially; otherwise randomly. */ 1276 uint32_t tcp_random_anon_port = 1; 1277 1278 /* 1279 * To reach to an eager in Q0 which can be dropped due to an incoming 1280 * new SYN request when Q0 is full, a new doubly linked list is 1281 * introduced. This list allows to select an eager from Q0 in O(1) time. 1282 * This is needed to avoid spending too much time walking through the 1283 * long list of eagers in Q0 when tcp_drop_q0() is called. Each member of 1284 * this new list has to be a member of Q0. 1285 * This list is headed by listener's tcp_t. When the list is empty, 1286 * both the pointers - tcp_eager_next_drop_q0 and tcp_eager_prev_drop_q0, 1287 * of listener's tcp_t point to listener's tcp_t itself. 1288 * 1289 * Given an eager in Q0 and a listener, MAKE_DROPPABLE() puts the eager 1290 * in the list. MAKE_UNDROPPABLE() takes the eager out of the list. 1291 * These macros do not affect the eager's membership to Q0. 1292 */ 1293 1294 1295 #define MAKE_DROPPABLE(listener, eager) \ 1296 if ((eager)->tcp_eager_next_drop_q0 == NULL) { \ 1297 (listener)->tcp_eager_next_drop_q0->tcp_eager_prev_drop_q0\ 1298 = (eager); \ 1299 (eager)->tcp_eager_prev_drop_q0 = (listener); \ 1300 (eager)->tcp_eager_next_drop_q0 = \ 1301 (listener)->tcp_eager_next_drop_q0; \ 1302 (listener)->tcp_eager_next_drop_q0 = (eager); \ 1303 } 1304 1305 #define MAKE_UNDROPPABLE(eager) \ 1306 if ((eager)->tcp_eager_next_drop_q0 != NULL) { \ 1307 (eager)->tcp_eager_next_drop_q0->tcp_eager_prev_drop_q0 \ 1308 = (eager)->tcp_eager_prev_drop_q0; \ 1309 (eager)->tcp_eager_prev_drop_q0->tcp_eager_next_drop_q0 \ 1310 = (eager)->tcp_eager_next_drop_q0; \ 1311 (eager)->tcp_eager_prev_drop_q0 = NULL; \ 1312 (eager)->tcp_eager_next_drop_q0 = NULL; \ 1313 } 1314 1315 /* 1316 * If tcp_drop_ack_unsent_cnt is greater than 0, when TCP receives more 1317 * than tcp_drop_ack_unsent_cnt number of ACKs which acknowledge unsent 1318 * data, TCP will not respond with an ACK. RFC 793 requires that 1319 * TCP responds with an ACK for such a bogus ACK. By not following 1320 * the RFC, we prevent TCP from getting into an ACK storm if somehow 1321 * an attacker successfully spoofs an acceptable segment to our 1322 * peer; or when our peer is "confused." 1323 */ 1324 uint32_t tcp_drop_ack_unsent_cnt = 10; 1325 1326 /* 1327 * Hook functions to enable cluster networking 1328 * On non-clustered systems these vectors must always be NULL. 1329 */ 1330 1331 void (*cl_inet_listen)(uint8_t protocol, sa_family_t addr_family, 1332 uint8_t *laddrp, in_port_t lport) = NULL; 1333 void (*cl_inet_unlisten)(uint8_t protocol, sa_family_t addr_family, 1334 uint8_t *laddrp, in_port_t lport) = NULL; 1335 void (*cl_inet_connect)(uint8_t protocol, sa_family_t addr_family, 1336 uint8_t *laddrp, in_port_t lport, 1337 uint8_t *faddrp, in_port_t fport) = NULL; 1338 void (*cl_inet_disconnect)(uint8_t protocol, sa_family_t addr_family, 1339 uint8_t *laddrp, in_port_t lport, 1340 uint8_t *faddrp, in_port_t fport) = NULL; 1341 1342 /* 1343 * The following are defined in ip.c 1344 */ 1345 extern int (*cl_inet_isclusterwide)(uint8_t protocol, sa_family_t addr_family, 1346 uint8_t *laddrp); 1347 extern uint32_t (*cl_inet_ipident)(uint8_t protocol, sa_family_t addr_family, 1348 uint8_t *laddrp, uint8_t *faddrp); 1349 1350 #define CL_INET_CONNECT(tcp) { \ 1351 if (cl_inet_connect != NULL) { \ 1352 /* \ 1353 * Running in cluster mode - register active connection \ 1354 * information \ 1355 */ \ 1356 if ((tcp)->tcp_ipversion == IPV4_VERSION) { \ 1357 if ((tcp)->tcp_ipha->ipha_src != 0) { \ 1358 (*cl_inet_connect)(IPPROTO_TCP, AF_INET,\ 1359 (uint8_t *)(&((tcp)->tcp_ipha->ipha_src)),\ 1360 (in_port_t)(tcp)->tcp_lport, \ 1361 (uint8_t *)(&((tcp)->tcp_ipha->ipha_dst)),\ 1362 (in_port_t)(tcp)->tcp_fport); \ 1363 } \ 1364 } else { \ 1365 if (!IN6_IS_ADDR_UNSPECIFIED( \ 1366 &(tcp)->tcp_ip6h->ip6_src)) {\ 1367 (*cl_inet_connect)(IPPROTO_TCP, AF_INET6,\ 1368 (uint8_t *)(&((tcp)->tcp_ip6h->ip6_src)),\ 1369 (in_port_t)(tcp)->tcp_lport, \ 1370 (uint8_t *)(&((tcp)->tcp_ip6h->ip6_dst)),\ 1371 (in_port_t)(tcp)->tcp_fport); \ 1372 } \ 1373 } \ 1374 } \ 1375 } 1376 1377 #define CL_INET_DISCONNECT(tcp) { \ 1378 if (cl_inet_disconnect != NULL) { \ 1379 /* \ 1380 * Running in cluster mode - deregister active \ 1381 * connection information \ 1382 */ \ 1383 if ((tcp)->tcp_ipversion == IPV4_VERSION) { \ 1384 if ((tcp)->tcp_ip_src != 0) { \ 1385 (*cl_inet_disconnect)(IPPROTO_TCP, \ 1386 AF_INET, \ 1387 (uint8_t *)(&((tcp)->tcp_ip_src)),\ 1388 (in_port_t)(tcp)->tcp_lport, \ 1389 (uint8_t *) \ 1390 (&((tcp)->tcp_ipha->ipha_dst)),\ 1391 (in_port_t)(tcp)->tcp_fport); \ 1392 } \ 1393 } else { \ 1394 if (!IN6_IS_ADDR_UNSPECIFIED( \ 1395 &(tcp)->tcp_ip_src_v6)) { \ 1396 (*cl_inet_disconnect)(IPPROTO_TCP, AF_INET6,\ 1397 (uint8_t *)(&((tcp)->tcp_ip_src_v6)),\ 1398 (in_port_t)(tcp)->tcp_lport, \ 1399 (uint8_t *) \ 1400 (&((tcp)->tcp_ip6h->ip6_dst)),\ 1401 (in_port_t)(tcp)->tcp_fport); \ 1402 } \ 1403 } \ 1404 } \ 1405 } 1406 1407 /* 1408 * Cluster networking hook for traversing current connection list. 1409 * This routine is used to extract the current list of live connections 1410 * which must continue to to be dispatched to this node. 1411 */ 1412 int cl_tcp_walk_list(int (*callback)(cl_tcp_info_t *, void *), void *arg); 1413 1414 static int cl_tcp_walk_list_stack(int (*callback)(cl_tcp_info_t *, void *), 1415 void *arg, tcp_stack_t *tcps); 1416 1417 #define DTRACE_IP_FASTPATH(mp, iph, ill, ipha, ip6h) \ 1418 DTRACE_IP7(send, mblk_t *, mp, conn_t *, NULL, void_ip_t *, \ 1419 iph, __dtrace_ipsr_ill_t *, ill, ipha_t *, ipha, \ 1420 ip6_t *, ip6h, int, 0); 1421 1422 /* 1423 * Figure out the value of window scale opton. Note that the rwnd is 1424 * ASSUMED to be rounded up to the nearest MSS before the calculation. 1425 * We cannot find the scale value and then do a round up of tcp_rwnd 1426 * because the scale value may not be correct after that. 1427 * 1428 * Set the compiler flag to make this function inline. 1429 */ 1430 static void 1431 tcp_set_ws_value(tcp_t *tcp) 1432 { 1433 int i; 1434 uint32_t rwnd = tcp->tcp_rwnd; 1435 1436 for (i = 0; rwnd > TCP_MAXWIN && i < TCP_MAX_WINSHIFT; 1437 i++, rwnd >>= 1) 1438 ; 1439 tcp->tcp_rcv_ws = i; 1440 } 1441 1442 /* 1443 * Remove a connection from the list of detached TIME_WAIT connections. 1444 * It returns B_FALSE if it can't remove the connection from the list 1445 * as the connection has already been removed from the list due to an 1446 * earlier call to tcp_time_wait_remove(); otherwise it returns B_TRUE. 1447 */ 1448 static boolean_t 1449 tcp_time_wait_remove(tcp_t *tcp, tcp_squeue_priv_t *tcp_time_wait) 1450 { 1451 boolean_t locked = B_FALSE; 1452 1453 if (tcp_time_wait == NULL) { 1454 tcp_time_wait = *((tcp_squeue_priv_t **) 1455 squeue_getprivate(tcp->tcp_connp->conn_sqp, SQPRIVATE_TCP)); 1456 mutex_enter(&tcp_time_wait->tcp_time_wait_lock); 1457 locked = B_TRUE; 1458 } else { 1459 ASSERT(MUTEX_HELD(&tcp_time_wait->tcp_time_wait_lock)); 1460 } 1461 1462 if (tcp->tcp_time_wait_expire == 0) { 1463 ASSERT(tcp->tcp_time_wait_next == NULL); 1464 ASSERT(tcp->tcp_time_wait_prev == NULL); 1465 if (locked) 1466 mutex_exit(&tcp_time_wait->tcp_time_wait_lock); 1467 return (B_FALSE); 1468 } 1469 ASSERT(TCP_IS_DETACHED(tcp)); 1470 ASSERT(tcp->tcp_state == TCPS_TIME_WAIT); 1471 1472 if (tcp == tcp_time_wait->tcp_time_wait_head) { 1473 ASSERT(tcp->tcp_time_wait_prev == NULL); 1474 tcp_time_wait->tcp_time_wait_head = tcp->tcp_time_wait_next; 1475 if (tcp_time_wait->tcp_time_wait_head != NULL) { 1476 tcp_time_wait->tcp_time_wait_head->tcp_time_wait_prev = 1477 NULL; 1478 } else { 1479 tcp_time_wait->tcp_time_wait_tail = NULL; 1480 } 1481 } else if (tcp == tcp_time_wait->tcp_time_wait_tail) { 1482 ASSERT(tcp != tcp_time_wait->tcp_time_wait_head); 1483 ASSERT(tcp->tcp_time_wait_next == NULL); 1484 tcp_time_wait->tcp_time_wait_tail = tcp->tcp_time_wait_prev; 1485 ASSERT(tcp_time_wait->tcp_time_wait_tail != NULL); 1486 tcp_time_wait->tcp_time_wait_tail->tcp_time_wait_next = NULL; 1487 } else { 1488 ASSERT(tcp->tcp_time_wait_prev->tcp_time_wait_next == tcp); 1489 ASSERT(tcp->tcp_time_wait_next->tcp_time_wait_prev == tcp); 1490 tcp->tcp_time_wait_prev->tcp_time_wait_next = 1491 tcp->tcp_time_wait_next; 1492 tcp->tcp_time_wait_next->tcp_time_wait_prev = 1493 tcp->tcp_time_wait_prev; 1494 } 1495 tcp->tcp_time_wait_next = NULL; 1496 tcp->tcp_time_wait_prev = NULL; 1497 tcp->tcp_time_wait_expire = 0; 1498 1499 if (locked) 1500 mutex_exit(&tcp_time_wait->tcp_time_wait_lock); 1501 return (B_TRUE); 1502 } 1503 1504 /* 1505 * Add a connection to the list of detached TIME_WAIT connections 1506 * and set its time to expire. 1507 */ 1508 static void 1509 tcp_time_wait_append(tcp_t *tcp) 1510 { 1511 tcp_stack_t *tcps = tcp->tcp_tcps; 1512 tcp_squeue_priv_t *tcp_time_wait = 1513 *((tcp_squeue_priv_t **)squeue_getprivate(tcp->tcp_connp->conn_sqp, 1514 SQPRIVATE_TCP)); 1515 1516 tcp_timers_stop(tcp); 1517 1518 /* Freed above */ 1519 ASSERT(tcp->tcp_timer_tid == 0); 1520 ASSERT(tcp->tcp_ack_tid == 0); 1521 1522 /* must have happened at the time of detaching the tcp */ 1523 ASSERT(tcp->tcp_ptpahn == NULL); 1524 ASSERT(tcp->tcp_flow_stopped == 0); 1525 ASSERT(tcp->tcp_time_wait_next == NULL); 1526 ASSERT(tcp->tcp_time_wait_prev == NULL); 1527 ASSERT(tcp->tcp_time_wait_expire == NULL); 1528 ASSERT(tcp->tcp_listener == NULL); 1529 1530 tcp->tcp_time_wait_expire = ddi_get_lbolt(); 1531 /* 1532 * The value computed below in tcp->tcp_time_wait_expire may 1533 * appear negative or wrap around. That is ok since our 1534 * interest is only in the difference between the current lbolt 1535 * value and tcp->tcp_time_wait_expire. But the value should not 1536 * be zero, since it means the tcp is not in the TIME_WAIT list. 1537 * The corresponding comparison in tcp_time_wait_collector() uses 1538 * modular arithmetic. 1539 */ 1540 tcp->tcp_time_wait_expire += 1541 drv_usectohz(tcps->tcps_time_wait_interval * 1000); 1542 if (tcp->tcp_time_wait_expire == 0) 1543 tcp->tcp_time_wait_expire = 1; 1544 1545 ASSERT(TCP_IS_DETACHED(tcp)); 1546 ASSERT(tcp->tcp_state == TCPS_TIME_WAIT); 1547 ASSERT(tcp->tcp_time_wait_next == NULL); 1548 ASSERT(tcp->tcp_time_wait_prev == NULL); 1549 TCP_DBGSTAT(tcps, tcp_time_wait); 1550 1551 mutex_enter(&tcp_time_wait->tcp_time_wait_lock); 1552 if (tcp_time_wait->tcp_time_wait_head == NULL) { 1553 ASSERT(tcp_time_wait->tcp_time_wait_tail == NULL); 1554 tcp_time_wait->tcp_time_wait_head = tcp; 1555 } else { 1556 ASSERT(tcp_time_wait->tcp_time_wait_tail != NULL); 1557 ASSERT(tcp_time_wait->tcp_time_wait_tail->tcp_state == 1558 TCPS_TIME_WAIT); 1559 tcp_time_wait->tcp_time_wait_tail->tcp_time_wait_next = tcp; 1560 tcp->tcp_time_wait_prev = tcp_time_wait->tcp_time_wait_tail; 1561 } 1562 tcp_time_wait->tcp_time_wait_tail = tcp; 1563 mutex_exit(&tcp_time_wait->tcp_time_wait_lock); 1564 } 1565 1566 /* ARGSUSED */ 1567 void 1568 tcp_timewait_output(void *arg, mblk_t *mp, void *arg2) 1569 { 1570 conn_t *connp = (conn_t *)arg; 1571 tcp_t *tcp = connp->conn_tcp; 1572 tcp_stack_t *tcps = tcp->tcp_tcps; 1573 1574 ASSERT(tcp != NULL); 1575 if (tcp->tcp_state == TCPS_CLOSED) { 1576 return; 1577 } 1578 1579 ASSERT((tcp->tcp_family == AF_INET && 1580 tcp->tcp_ipversion == IPV4_VERSION) || 1581 (tcp->tcp_family == AF_INET6 && 1582 (tcp->tcp_ipversion == IPV4_VERSION || 1583 tcp->tcp_ipversion == IPV6_VERSION))); 1584 ASSERT(!tcp->tcp_listener); 1585 1586 TCP_STAT(tcps, tcp_time_wait_reap); 1587 ASSERT(TCP_IS_DETACHED(tcp)); 1588 1589 /* 1590 * Because they have no upstream client to rebind or tcp_close() 1591 * them later, we axe the connection here and now. 1592 */ 1593 tcp_close_detached(tcp); 1594 } 1595 1596 /* 1597 * Remove cached/latched IPsec references. 1598 */ 1599 void 1600 tcp_ipsec_cleanup(tcp_t *tcp) 1601 { 1602 conn_t *connp = tcp->tcp_connp; 1603 1604 ASSERT(connp->conn_flags & IPCL_TCPCONN); 1605 1606 if (connp->conn_latch != NULL) { 1607 IPLATCH_REFRELE(connp->conn_latch, 1608 connp->conn_netstack); 1609 connp->conn_latch = NULL; 1610 } 1611 if (connp->conn_policy != NULL) { 1612 IPPH_REFRELE(connp->conn_policy, connp->conn_netstack); 1613 connp->conn_policy = NULL; 1614 } 1615 } 1616 1617 /* 1618 * Cleaup before placing on free list. 1619 * Disassociate from the netstack/tcp_stack_t since the freelist 1620 * is per squeue and not per netstack. 1621 */ 1622 void 1623 tcp_cleanup(tcp_t *tcp) 1624 { 1625 mblk_t *mp; 1626 char *tcp_iphc; 1627 int tcp_iphc_len; 1628 int tcp_hdr_grown; 1629 tcp_sack_info_t *tcp_sack_info; 1630 conn_t *connp = tcp->tcp_connp; 1631 tcp_stack_t *tcps = tcp->tcp_tcps; 1632 netstack_t *ns = tcps->tcps_netstack; 1633 1634 tcp_bind_hash_remove(tcp); 1635 1636 /* Cleanup that which needs the netstack first */ 1637 tcp_ipsec_cleanup(tcp); 1638 1639 tcp_free(tcp); 1640 1641 /* Release any SSL context */ 1642 if (tcp->tcp_kssl_ent != NULL) { 1643 kssl_release_ent(tcp->tcp_kssl_ent, NULL, KSSL_NO_PROXY); 1644 tcp->tcp_kssl_ent = NULL; 1645 } 1646 1647 if (tcp->tcp_kssl_ctx != NULL) { 1648 kssl_release_ctx(tcp->tcp_kssl_ctx); 1649 tcp->tcp_kssl_ctx = NULL; 1650 } 1651 tcp->tcp_kssl_pending = B_FALSE; 1652 1653 conn_delete_ire(connp, NULL); 1654 1655 /* 1656 * Since we will bzero the entire structure, we need to 1657 * remove it and reinsert it in global hash list. We 1658 * know the walkers can't get to this conn because we 1659 * had set CONDEMNED flag earlier and checked reference 1660 * under conn_lock so walker won't pick it and when we 1661 * go the ipcl_globalhash_remove() below, no walker 1662 * can get to it. 1663 */ 1664 ipcl_globalhash_remove(connp); 1665 1666 /* 1667 * Now it is safe to decrement the reference counts. 1668 * This might be the last reference on the netstack and TCPS 1669 * in which case it will cause the tcp_g_q_close and 1670 * the freeing of the IP Instance. 1671 */ 1672 connp->conn_netstack = NULL; 1673 netstack_rele(ns); 1674 ASSERT(tcps != NULL); 1675 tcp->tcp_tcps = NULL; 1676 TCPS_REFRELE(tcps); 1677 1678 /* Save some state */ 1679 mp = tcp->tcp_timercache; 1680 1681 tcp_sack_info = tcp->tcp_sack_info; 1682 tcp_iphc = tcp->tcp_iphc; 1683 tcp_iphc_len = tcp->tcp_iphc_len; 1684 tcp_hdr_grown = tcp->tcp_hdr_grown; 1685 1686 if (connp->conn_cred != NULL) { 1687 crfree(connp->conn_cred); 1688 connp->conn_cred = NULL; 1689 } 1690 if (connp->conn_peercred != NULL) { 1691 crfree(connp->conn_peercred); 1692 connp->conn_peercred = NULL; 1693 } 1694 ipcl_conn_cleanup(connp); 1695 connp->conn_flags = IPCL_TCPCONN; 1696 bzero(tcp, sizeof (tcp_t)); 1697 1698 /* restore the state */ 1699 tcp->tcp_timercache = mp; 1700 1701 tcp->tcp_sack_info = tcp_sack_info; 1702 tcp->tcp_iphc = tcp_iphc; 1703 tcp->tcp_iphc_len = tcp_iphc_len; 1704 tcp->tcp_hdr_grown = tcp_hdr_grown; 1705 1706 tcp->tcp_connp = connp; 1707 1708 ASSERT(connp->conn_tcp == tcp); 1709 ASSERT(connp->conn_flags & IPCL_TCPCONN); 1710 connp->conn_state_flags = CONN_INCIPIENT; 1711 ASSERT(connp->conn_ulp == IPPROTO_TCP); 1712 ASSERT(connp->conn_ref == 1); 1713 } 1714 1715 /* 1716 * Blows away all tcps whose TIME_WAIT has expired. List traversal 1717 * is done forwards from the head. 1718 * This walks all stack instances since 1719 * tcp_time_wait remains global across all stacks. 1720 */ 1721 /* ARGSUSED */ 1722 void 1723 tcp_time_wait_collector(void *arg) 1724 { 1725 tcp_t *tcp; 1726 clock_t now; 1727 mblk_t *mp; 1728 conn_t *connp; 1729 kmutex_t *lock; 1730 boolean_t removed; 1731 1732 squeue_t *sqp = (squeue_t *)arg; 1733 tcp_squeue_priv_t *tcp_time_wait = 1734 *((tcp_squeue_priv_t **)squeue_getprivate(sqp, SQPRIVATE_TCP)); 1735 1736 mutex_enter(&tcp_time_wait->tcp_time_wait_lock); 1737 tcp_time_wait->tcp_time_wait_tid = 0; 1738 1739 if (tcp_time_wait->tcp_free_list != NULL && 1740 tcp_time_wait->tcp_free_list->tcp_in_free_list == B_TRUE) { 1741 TCP_G_STAT(tcp_freelist_cleanup); 1742 while ((tcp = tcp_time_wait->tcp_free_list) != NULL) { 1743 tcp_time_wait->tcp_free_list = tcp->tcp_time_wait_next; 1744 tcp->tcp_time_wait_next = NULL; 1745 tcp_time_wait->tcp_free_list_cnt--; 1746 ASSERT(tcp->tcp_tcps == NULL); 1747 CONN_DEC_REF(tcp->tcp_connp); 1748 } 1749 ASSERT(tcp_time_wait->tcp_free_list_cnt == 0); 1750 } 1751 1752 /* 1753 * In order to reap time waits reliably, we should use a 1754 * source of time that is not adjustable by the user -- hence 1755 * the call to ddi_get_lbolt(). 1756 */ 1757 now = ddi_get_lbolt(); 1758 while ((tcp = tcp_time_wait->tcp_time_wait_head) != NULL) { 1759 /* 1760 * Compare times using modular arithmetic, since 1761 * lbolt can wrapover. 1762 */ 1763 if ((now - tcp->tcp_time_wait_expire) < 0) { 1764 break; 1765 } 1766 1767 removed = tcp_time_wait_remove(tcp, tcp_time_wait); 1768 ASSERT(removed); 1769 1770 connp = tcp->tcp_connp; 1771 ASSERT(connp->conn_fanout != NULL); 1772 lock = &connp->conn_fanout->connf_lock; 1773 /* 1774 * This is essentially a TW reclaim fast path optimization for 1775 * performance where the timewait collector checks under the 1776 * fanout lock (so that no one else can get access to the 1777 * conn_t) that the refcnt is 2 i.e. one for TCP and one for 1778 * the classifier hash list. If ref count is indeed 2, we can 1779 * just remove the conn under the fanout lock and avoid 1780 * cleaning up the conn under the squeue, provided that 1781 * clustering callbacks are not enabled. If clustering is 1782 * enabled, we need to make the clustering callback before 1783 * setting the CONDEMNED flag and after dropping all locks and 1784 * so we forego this optimization and fall back to the slow 1785 * path. Also please see the comments in tcp_closei_local 1786 * regarding the refcnt logic. 1787 * 1788 * Since we are holding the tcp_time_wait_lock, its better 1789 * not to block on the fanout_lock because other connections 1790 * can't add themselves to time_wait list. So we do a 1791 * tryenter instead of mutex_enter. 1792 */ 1793 if (mutex_tryenter(lock)) { 1794 mutex_enter(&connp->conn_lock); 1795 if ((connp->conn_ref == 2) && 1796 (cl_inet_disconnect == NULL)) { 1797 ipcl_hash_remove_locked(connp, 1798 connp->conn_fanout); 1799 /* 1800 * Set the CONDEMNED flag now itself so that 1801 * the refcnt cannot increase due to any 1802 * walker. But we have still not cleaned up 1803 * conn_ire_cache. This is still ok since 1804 * we are going to clean it up in tcp_cleanup 1805 * immediately and any interface unplumb 1806 * thread will wait till the ire is blown away 1807 */ 1808 connp->conn_state_flags |= CONN_CONDEMNED; 1809 mutex_exit(lock); 1810 mutex_exit(&connp->conn_lock); 1811 if (tcp_time_wait->tcp_free_list_cnt < 1812 tcp_free_list_max_cnt) { 1813 /* Add to head of tcp_free_list */ 1814 mutex_exit( 1815 &tcp_time_wait->tcp_time_wait_lock); 1816 tcp_cleanup(tcp); 1817 ASSERT(connp->conn_latch == NULL); 1818 ASSERT(connp->conn_policy == NULL); 1819 ASSERT(tcp->tcp_tcps == NULL); 1820 ASSERT(connp->conn_netstack == NULL); 1821 1822 mutex_enter( 1823 &tcp_time_wait->tcp_time_wait_lock); 1824 tcp->tcp_time_wait_next = 1825 tcp_time_wait->tcp_free_list; 1826 tcp_time_wait->tcp_free_list = tcp; 1827 tcp_time_wait->tcp_free_list_cnt++; 1828 continue; 1829 } else { 1830 /* Do not add to tcp_free_list */ 1831 mutex_exit( 1832 &tcp_time_wait->tcp_time_wait_lock); 1833 tcp_bind_hash_remove(tcp); 1834 conn_delete_ire(tcp->tcp_connp, NULL); 1835 tcp_ipsec_cleanup(tcp); 1836 CONN_DEC_REF(tcp->tcp_connp); 1837 } 1838 } else { 1839 CONN_INC_REF_LOCKED(connp); 1840 mutex_exit(lock); 1841 mutex_exit(&tcp_time_wait->tcp_time_wait_lock); 1842 mutex_exit(&connp->conn_lock); 1843 /* 1844 * We can reuse the closemp here since conn has 1845 * detached (otherwise we wouldn't even be in 1846 * time_wait list). tcp_closemp_used can safely 1847 * be changed without taking a lock as no other 1848 * thread can concurrently access it at this 1849 * point in the connection lifecycle. 1850 */ 1851 1852 if (tcp->tcp_closemp.b_prev == NULL) 1853 tcp->tcp_closemp_used = B_TRUE; 1854 else 1855 cmn_err(CE_PANIC, 1856 "tcp_timewait_collector: " 1857 "concurrent use of tcp_closemp: " 1858 "connp %p tcp %p\n", (void *)connp, 1859 (void *)tcp); 1860 1861 TCP_DEBUG_GETPCSTACK(tcp->tcmp_stk, 15); 1862 mp = &tcp->tcp_closemp; 1863 squeue_fill(connp->conn_sqp, mp, 1864 tcp_timewait_output, connp, 1865 SQTAG_TCP_TIMEWAIT); 1866 } 1867 } else { 1868 mutex_enter(&connp->conn_lock); 1869 CONN_INC_REF_LOCKED(connp); 1870 mutex_exit(&tcp_time_wait->tcp_time_wait_lock); 1871 mutex_exit(&connp->conn_lock); 1872 /* 1873 * We can reuse the closemp here since conn has 1874 * detached (otherwise we wouldn't even be in 1875 * time_wait list). tcp_closemp_used can safely 1876 * be changed without taking a lock as no other 1877 * thread can concurrently access it at this 1878 * point in the connection lifecycle. 1879 */ 1880 1881 if (tcp->tcp_closemp.b_prev == NULL) 1882 tcp->tcp_closemp_used = B_TRUE; 1883 else 1884 cmn_err(CE_PANIC, "tcp_timewait_collector: " 1885 "concurrent use of tcp_closemp: " 1886 "connp %p tcp %p\n", (void *)connp, 1887 (void *)tcp); 1888 1889 TCP_DEBUG_GETPCSTACK(tcp->tcmp_stk, 15); 1890 mp = &tcp->tcp_closemp; 1891 squeue_fill(connp->conn_sqp, mp, 1892 tcp_timewait_output, connp, 0); 1893 } 1894 mutex_enter(&tcp_time_wait->tcp_time_wait_lock); 1895 } 1896 1897 if (tcp_time_wait->tcp_free_list != NULL) 1898 tcp_time_wait->tcp_free_list->tcp_in_free_list = B_TRUE; 1899 1900 tcp_time_wait->tcp_time_wait_tid = 1901 timeout(tcp_time_wait_collector, sqp, TCP_TIME_WAIT_DELAY); 1902 mutex_exit(&tcp_time_wait->tcp_time_wait_lock); 1903 } 1904 /* 1905 * Reply to a clients T_CONN_RES TPI message. This function 1906 * is used only for TLI/XTI listener. Sockfs sends T_CONN_RES 1907 * on the acceptor STREAM and processed in tcp_wput_accept(). 1908 * Read the block comment on top of tcp_conn_request(). 1909 */ 1910 static void 1911 tcp_accept(tcp_t *listener, mblk_t *mp) 1912 { 1913 tcp_t *acceptor; 1914 tcp_t *eager; 1915 tcp_t *tcp; 1916 struct T_conn_res *tcr; 1917 t_uscalar_t acceptor_id; 1918 t_scalar_t seqnum; 1919 mblk_t *opt_mp = NULL; /* T_OPTMGMT_REQ messages */ 1920 mblk_t *ok_mp; 1921 mblk_t *mp1; 1922 tcp_stack_t *tcps = listener->tcp_tcps; 1923 1924 if ((mp->b_wptr - mp->b_rptr) < sizeof (*tcr)) { 1925 tcp_err_ack(listener, mp, TPROTO, 0); 1926 return; 1927 } 1928 tcr = (struct T_conn_res *)mp->b_rptr; 1929 1930 /* 1931 * Under ILP32 the stream head points tcr->ACCEPTOR_id at the 1932 * read side queue of the streams device underneath us i.e. the 1933 * read side queue of 'ip'. Since we can't deference QUEUE_ptr we 1934 * look it up in the queue_hash. Under LP64 it sends down the 1935 * minor_t of the accepting endpoint. 1936 * 1937 * Once the acceptor/eager are modified (in tcp_accept_swap) the 1938 * fanout hash lock is held. 1939 * This prevents any thread from entering the acceptor queue from 1940 * below (since it has not been hard bound yet i.e. any inbound 1941 * packets will arrive on the listener or default tcp queue and 1942 * go through tcp_lookup). 1943 * The CONN_INC_REF will prevent the acceptor from closing. 1944 * 1945 * XXX It is still possible for a tli application to send down data 1946 * on the accepting stream while another thread calls t_accept. 1947 * This should not be a problem for well-behaved applications since 1948 * the T_OK_ACK is sent after the queue swapping is completed. 1949 * 1950 * If the accepting fd is the same as the listening fd, avoid 1951 * queue hash lookup since that will return an eager listener in a 1952 * already established state. 1953 */ 1954 acceptor_id = tcr->ACCEPTOR_id; 1955 mutex_enter(&listener->tcp_eager_lock); 1956 if (listener->tcp_acceptor_id == acceptor_id) { 1957 eager = listener->tcp_eager_next_q; 1958 /* only count how many T_CONN_INDs so don't count q0 */ 1959 if ((listener->tcp_conn_req_cnt_q != 1) || 1960 (eager->tcp_conn_req_seqnum != tcr->SEQ_number)) { 1961 mutex_exit(&listener->tcp_eager_lock); 1962 tcp_err_ack(listener, mp, TBADF, 0); 1963 return; 1964 } 1965 if (listener->tcp_conn_req_cnt_q0 != 0) { 1966 /* Throw away all the eagers on q0. */ 1967 tcp_eager_cleanup(listener, 1); 1968 } 1969 if (listener->tcp_syn_defense) { 1970 listener->tcp_syn_defense = B_FALSE; 1971 if (listener->tcp_ip_addr_cache != NULL) { 1972 kmem_free(listener->tcp_ip_addr_cache, 1973 IP_ADDR_CACHE_SIZE * sizeof (ipaddr_t)); 1974 listener->tcp_ip_addr_cache = NULL; 1975 } 1976 } 1977 /* 1978 * Transfer tcp_conn_req_max to the eager so that when 1979 * a disconnect occurs we can revert the endpoint to the 1980 * listen state. 1981 */ 1982 eager->tcp_conn_req_max = listener->tcp_conn_req_max; 1983 ASSERT(listener->tcp_conn_req_cnt_q0 == 0); 1984 /* 1985 * Get a reference on the acceptor just like the 1986 * tcp_acceptor_hash_lookup below. 1987 */ 1988 acceptor = listener; 1989 CONN_INC_REF(acceptor->tcp_connp); 1990 } else { 1991 acceptor = tcp_acceptor_hash_lookup(acceptor_id, tcps); 1992 if (acceptor == NULL) { 1993 if (listener->tcp_debug) { 1994 (void) strlog(TCP_MOD_ID, 0, 1, 1995 SL_ERROR|SL_TRACE, 1996 "tcp_accept: did not find acceptor 0x%x\n", 1997 acceptor_id); 1998 } 1999 mutex_exit(&listener->tcp_eager_lock); 2000 tcp_err_ack(listener, mp, TPROVMISMATCH, 0); 2001 return; 2002 } 2003 /* 2004 * Verify acceptor state. The acceptable states for an acceptor 2005 * include TCPS_IDLE and TCPS_BOUND. 2006 */ 2007 switch (acceptor->tcp_state) { 2008 case TCPS_IDLE: 2009 /* FALLTHRU */ 2010 case TCPS_BOUND: 2011 break; 2012 default: 2013 CONN_DEC_REF(acceptor->tcp_connp); 2014 mutex_exit(&listener->tcp_eager_lock); 2015 tcp_err_ack(listener, mp, TOUTSTATE, 0); 2016 return; 2017 } 2018 } 2019 2020 /* The listener must be in TCPS_LISTEN */ 2021 if (listener->tcp_state != TCPS_LISTEN) { 2022 CONN_DEC_REF(acceptor->tcp_connp); 2023 mutex_exit(&listener->tcp_eager_lock); 2024 tcp_err_ack(listener, mp, TOUTSTATE, 0); 2025 return; 2026 } 2027 2028 /* 2029 * Rendezvous with an eager connection request packet hanging off 2030 * 'tcp' that has the 'seqnum' tag. We tagged the detached open 2031 * tcp structure when the connection packet arrived in 2032 * tcp_conn_request(). 2033 */ 2034 seqnum = tcr->SEQ_number; 2035 eager = listener; 2036 do { 2037 eager = eager->tcp_eager_next_q; 2038 if (eager == NULL) { 2039 CONN_DEC_REF(acceptor->tcp_connp); 2040 mutex_exit(&listener->tcp_eager_lock); 2041 tcp_err_ack(listener, mp, TBADSEQ, 0); 2042 return; 2043 } 2044 } while (eager->tcp_conn_req_seqnum != seqnum); 2045 mutex_exit(&listener->tcp_eager_lock); 2046 2047 /* 2048 * At this point, both acceptor and listener have 2 ref 2049 * that they begin with. Acceptor has one additional ref 2050 * we placed in lookup while listener has 3 additional 2051 * ref for being behind the squeue (tcp_accept() is 2052 * done on listener's squeue); being in classifier hash; 2053 * and eager's ref on listener. 2054 */ 2055 ASSERT(listener->tcp_connp->conn_ref >= 5); 2056 ASSERT(acceptor->tcp_connp->conn_ref >= 3); 2057 2058 /* 2059 * The eager at this point is set in its own squeue and 2060 * could easily have been killed (tcp_accept_finish will 2061 * deal with that) because of a TH_RST so we can only 2062 * ASSERT for a single ref. 2063 */ 2064 ASSERT(eager->tcp_connp->conn_ref >= 1); 2065 2066 /* Pre allocate the stroptions mblk also */ 2067 opt_mp = allocb(sizeof (struct stroptions), BPRI_HI); 2068 if (opt_mp == NULL) { 2069 CONN_DEC_REF(acceptor->tcp_connp); 2070 CONN_DEC_REF(eager->tcp_connp); 2071 tcp_err_ack(listener, mp, TSYSERR, ENOMEM); 2072 return; 2073 } 2074 DB_TYPE(opt_mp) = M_SETOPTS; 2075 opt_mp->b_wptr += sizeof (struct stroptions); 2076 2077 /* 2078 * Prepare for inheriting IPV6_BOUND_IF and IPV6_RECVPKTINFO 2079 * from listener to acceptor. The message is chained on opt_mp 2080 * which will be sent onto eager's squeue. 2081 */ 2082 if (listener->tcp_bound_if != 0) { 2083 /* allocate optmgmt req */ 2084 mp1 = tcp_setsockopt_mp(IPPROTO_IPV6, 2085 IPV6_BOUND_IF, (char *)&listener->tcp_bound_if, 2086 sizeof (int)); 2087 if (mp1 != NULL) 2088 linkb(opt_mp, mp1); 2089 } 2090 if (listener->tcp_ipv6_recvancillary & TCP_IPV6_RECVPKTINFO) { 2091 uint_t on = 1; 2092 2093 /* allocate optmgmt req */ 2094 mp1 = tcp_setsockopt_mp(IPPROTO_IPV6, 2095 IPV6_RECVPKTINFO, (char *)&on, sizeof (on)); 2096 if (mp1 != NULL) 2097 linkb(opt_mp, mp1); 2098 } 2099 2100 /* Re-use mp1 to hold a copy of mp, in case reallocb fails */ 2101 if ((mp1 = copymsg(mp)) == NULL) { 2102 CONN_DEC_REF(acceptor->tcp_connp); 2103 CONN_DEC_REF(eager->tcp_connp); 2104 freemsg(opt_mp); 2105 tcp_err_ack(listener, mp, TSYSERR, ENOMEM); 2106 return; 2107 } 2108 2109 tcr = (struct T_conn_res *)mp1->b_rptr; 2110 2111 /* 2112 * This is an expanded version of mi_tpi_ok_ack_alloc() 2113 * which allocates a larger mblk and appends the new 2114 * local address to the ok_ack. The address is copied by 2115 * soaccept() for getsockname(). 2116 */ 2117 { 2118 int extra; 2119 2120 extra = (eager->tcp_family == AF_INET) ? 2121 sizeof (sin_t) : sizeof (sin6_t); 2122 2123 /* 2124 * Try to re-use mp, if possible. Otherwise, allocate 2125 * an mblk and return it as ok_mp. In any case, mp 2126 * is no longer usable upon return. 2127 */ 2128 if ((ok_mp = mi_tpi_ok_ack_alloc_extra(mp, extra)) == NULL) { 2129 CONN_DEC_REF(acceptor->tcp_connp); 2130 CONN_DEC_REF(eager->tcp_connp); 2131 freemsg(opt_mp); 2132 /* Original mp has been freed by now, so use mp1 */ 2133 tcp_err_ack(listener, mp1, TSYSERR, ENOMEM); 2134 return; 2135 } 2136 2137 mp = NULL; /* We should never use mp after this point */ 2138 2139 switch (extra) { 2140 case sizeof (sin_t): { 2141 sin_t *sin = (sin_t *)ok_mp->b_wptr; 2142 2143 ok_mp->b_wptr += extra; 2144 sin->sin_family = AF_INET; 2145 sin->sin_port = eager->tcp_lport; 2146 sin->sin_addr.s_addr = 2147 eager->tcp_ipha->ipha_src; 2148 break; 2149 } 2150 case sizeof (sin6_t): { 2151 sin6_t *sin6 = (sin6_t *)ok_mp->b_wptr; 2152 2153 ok_mp->b_wptr += extra; 2154 sin6->sin6_family = AF_INET6; 2155 sin6->sin6_port = eager->tcp_lport; 2156 if (eager->tcp_ipversion == IPV4_VERSION) { 2157 sin6->sin6_flowinfo = 0; 2158 IN6_IPADDR_TO_V4MAPPED( 2159 eager->tcp_ipha->ipha_src, 2160 &sin6->sin6_addr); 2161 } else { 2162 ASSERT(eager->tcp_ip6h != NULL); 2163 sin6->sin6_flowinfo = 2164 eager->tcp_ip6h->ip6_vcf & 2165 ~IPV6_VERS_AND_FLOW_MASK; 2166 sin6->sin6_addr = 2167 eager->tcp_ip6h->ip6_src; 2168 } 2169 sin6->sin6_scope_id = 0; 2170 sin6->__sin6_src_id = 0; 2171 break; 2172 } 2173 default: 2174 break; 2175 } 2176 ASSERT(ok_mp->b_wptr <= ok_mp->b_datap->db_lim); 2177 } 2178 2179 /* 2180 * If there are no options we know that the T_CONN_RES will 2181 * succeed. However, we can't send the T_OK_ACK upstream until 2182 * the tcp_accept_swap is done since it would be dangerous to 2183 * let the application start using the new fd prior to the swap. 2184 */ 2185 tcp_accept_swap(listener, acceptor, eager); 2186 2187 /* 2188 * tcp_accept_swap unlinks eager from listener but does not drop 2189 * the eager's reference on the listener. 2190 */ 2191 ASSERT(eager->tcp_listener == NULL); 2192 ASSERT(listener->tcp_connp->conn_ref >= 5); 2193 2194 /* 2195 * The eager is now associated with its own queue. Insert in 2196 * the hash so that the connection can be reused for a future 2197 * T_CONN_RES. 2198 */ 2199 tcp_acceptor_hash_insert(acceptor_id, eager); 2200 2201 /* 2202 * We now do the processing of options with T_CONN_RES. 2203 * We delay till now since we wanted to have queue to pass to 2204 * option processing routines that points back to the right 2205 * instance structure which does not happen until after 2206 * tcp_accept_swap(). 2207 * 2208 * Note: 2209 * The sanity of the logic here assumes that whatever options 2210 * are appropriate to inherit from listner=>eager are done 2211 * before this point, and whatever were to be overridden (or not) 2212 * in transfer logic from eager=>acceptor in tcp_accept_swap(). 2213 * [ Warning: acceptor endpoint can have T_OPTMGMT_REQ done to it 2214 * before its ACCEPTOR_id comes down in T_CONN_RES ] 2215 * This may not be true at this point in time but can be fixed 2216 * independently. This option processing code starts with 2217 * the instantiated acceptor instance and the final queue at 2218 * this point. 2219 */ 2220 2221 if (tcr->OPT_length != 0) { 2222 /* Options to process */ 2223 int t_error = 0; 2224 int sys_error = 0; 2225 int do_disconnect = 0; 2226 2227 if (tcp_conprim_opt_process(eager, mp1, 2228 &do_disconnect, &t_error, &sys_error) < 0) { 2229 eager->tcp_accept_error = 1; 2230 if (do_disconnect) { 2231 /* 2232 * An option failed which does not allow 2233 * connection to be accepted. 2234 * 2235 * We allow T_CONN_RES to succeed and 2236 * put a T_DISCON_IND on the eager queue. 2237 */ 2238 ASSERT(t_error == 0 && sys_error == 0); 2239 eager->tcp_send_discon_ind = 1; 2240 } else { 2241 ASSERT(t_error != 0); 2242 freemsg(ok_mp); 2243 /* 2244 * Original mp was either freed or set 2245 * to ok_mp above, so use mp1 instead. 2246 */ 2247 tcp_err_ack(listener, mp1, t_error, sys_error); 2248 goto finish; 2249 } 2250 } 2251 /* 2252 * Most likely success in setting options (except if 2253 * eager->tcp_send_discon_ind set). 2254 * mp1 option buffer represented by OPT_length/offset 2255 * potentially modified and contains results of setting 2256 * options at this point 2257 */ 2258 } 2259 2260 /* We no longer need mp1, since all options processing has passed */ 2261 freemsg(mp1); 2262 2263 putnext(listener->tcp_rq, ok_mp); 2264 2265 mutex_enter(&listener->tcp_eager_lock); 2266 if (listener->tcp_eager_prev_q0->tcp_conn_def_q0) { 2267 tcp_t *tail; 2268 mblk_t *conn_ind; 2269 2270 /* 2271 * This path should not be executed if listener and 2272 * acceptor streams are the same. 2273 */ 2274 ASSERT(listener != acceptor); 2275 2276 tcp = listener->tcp_eager_prev_q0; 2277 /* 2278 * listener->tcp_eager_prev_q0 points to the TAIL of the 2279 * deferred T_conn_ind queue. We need to get to the head of 2280 * the queue in order to send up T_conn_ind the same order as 2281 * how the 3WHS is completed. 2282 */ 2283 while (tcp != listener) { 2284 if (!tcp->tcp_eager_prev_q0->tcp_conn_def_q0) 2285 break; 2286 else 2287 tcp = tcp->tcp_eager_prev_q0; 2288 } 2289 ASSERT(tcp != listener); 2290 conn_ind = tcp->tcp_conn.tcp_eager_conn_ind; 2291 ASSERT(conn_ind != NULL); 2292 tcp->tcp_conn.tcp_eager_conn_ind = NULL; 2293 2294 /* Move from q0 to q */ 2295 ASSERT(listener->tcp_conn_req_cnt_q0 > 0); 2296 listener->tcp_conn_req_cnt_q0--; 2297 listener->tcp_conn_req_cnt_q++; 2298 tcp->tcp_eager_next_q0->tcp_eager_prev_q0 = 2299 tcp->tcp_eager_prev_q0; 2300 tcp->tcp_eager_prev_q0->tcp_eager_next_q0 = 2301 tcp->tcp_eager_next_q0; 2302 tcp->tcp_eager_prev_q0 = NULL; 2303 tcp->tcp_eager_next_q0 = NULL; 2304 tcp->tcp_conn_def_q0 = B_FALSE; 2305 2306 /* Make sure the tcp isn't in the list of droppables */ 2307 ASSERT(tcp->tcp_eager_next_drop_q0 == NULL && 2308 tcp->tcp_eager_prev_drop_q0 == NULL); 2309 2310 /* 2311 * Insert at end of the queue because sockfs sends 2312 * down T_CONN_RES in chronological order. Leaving 2313 * the older conn indications at front of the queue 2314 * helps reducing search time. 2315 */ 2316 tail = listener->tcp_eager_last_q; 2317 if (tail != NULL) 2318 tail->tcp_eager_next_q = tcp; 2319 else 2320 listener->tcp_eager_next_q = tcp; 2321 listener->tcp_eager_last_q = tcp; 2322 tcp->tcp_eager_next_q = NULL; 2323 mutex_exit(&listener->tcp_eager_lock); 2324 putnext(tcp->tcp_rq, conn_ind); 2325 } else { 2326 mutex_exit(&listener->tcp_eager_lock); 2327 } 2328 2329 /* 2330 * Done with the acceptor - free it 2331 * 2332 * Note: from this point on, no access to listener should be made 2333 * as listener can be equal to acceptor. 2334 */ 2335 finish: 2336 ASSERT(acceptor->tcp_detached); 2337 ASSERT(tcps->tcps_g_q != NULL); 2338 acceptor->tcp_rq = tcps->tcps_g_q; 2339 acceptor->tcp_wq = WR(tcps->tcps_g_q); 2340 (void) tcp_clean_death(acceptor, 0, 2); 2341 CONN_DEC_REF(acceptor->tcp_connp); 2342 2343 /* 2344 * In case we already received a FIN we have to make tcp_rput send 2345 * the ordrel_ind. This will also send up a window update if the window 2346 * has opened up. 2347 * 2348 * In the normal case of a successful connection acceptance 2349 * we give the O_T_BIND_REQ to the read side put procedure as an 2350 * indication that this was just accepted. This tells tcp_rput to 2351 * pass up any data queued in tcp_rcv_list. 2352 * 2353 * In the fringe case where options sent with T_CONN_RES failed and 2354 * we required, we would be indicating a T_DISCON_IND to blow 2355 * away this connection. 2356 */ 2357 2358 /* 2359 * XXX: we currently have a problem if XTI application closes the 2360 * acceptor stream in between. This problem exists in on10-gate also 2361 * and is well know but nothing can be done short of major rewrite 2362 * to fix it. Now it is possible to take care of it by assigning TLI/XTI 2363 * eager same squeue as listener (we can distinguish non socket 2364 * listeners at the time of handling a SYN in tcp_conn_request) 2365 * and do most of the work that tcp_accept_finish does here itself 2366 * and then get behind the acceptor squeue to access the acceptor 2367 * queue. 2368 */ 2369 /* 2370 * We already have a ref on tcp so no need to do one before squeue_fill 2371 */ 2372 squeue_fill(eager->tcp_connp->conn_sqp, opt_mp, 2373 tcp_accept_finish, eager->tcp_connp, SQTAG_TCP_ACCEPT_FINISH); 2374 } 2375 2376 /* 2377 * Swap information between the eager and acceptor for a TLI/XTI client. 2378 * The sockfs accept is done on the acceptor stream and control goes 2379 * through tcp_wput_accept() and tcp_accept()/tcp_accept_swap() is not 2380 * called. In either case, both the eager and listener are in their own 2381 * perimeter (squeue) and the code has to deal with potential race. 2382 * 2383 * See the block comment on top of tcp_accept() and tcp_wput_accept(). 2384 */ 2385 static void 2386 tcp_accept_swap(tcp_t *listener, tcp_t *acceptor, tcp_t *eager) 2387 { 2388 conn_t *econnp, *aconnp; 2389 2390 ASSERT(eager->tcp_rq == listener->tcp_rq); 2391 ASSERT(eager->tcp_detached && !acceptor->tcp_detached); 2392 ASSERT(!eager->tcp_hard_bound); 2393 ASSERT(!TCP_IS_SOCKET(acceptor)); 2394 ASSERT(!TCP_IS_SOCKET(eager)); 2395 ASSERT(!TCP_IS_SOCKET(listener)); 2396 2397 acceptor->tcp_detached = B_TRUE; 2398 /* 2399 * To permit stream re-use by TLI/XTI, the eager needs a copy of 2400 * the acceptor id. 2401 */ 2402 eager->tcp_acceptor_id = acceptor->tcp_acceptor_id; 2403 2404 /* remove eager from listen list... */ 2405 mutex_enter(&listener->tcp_eager_lock); 2406 tcp_eager_unlink(eager); 2407 ASSERT(eager->tcp_eager_next_q == NULL && 2408 eager->tcp_eager_last_q == NULL); 2409 ASSERT(eager->tcp_eager_next_q0 == NULL && 2410 eager->tcp_eager_prev_q0 == NULL); 2411 mutex_exit(&listener->tcp_eager_lock); 2412 eager->tcp_rq = acceptor->tcp_rq; 2413 eager->tcp_wq = acceptor->tcp_wq; 2414 2415 econnp = eager->tcp_connp; 2416 aconnp = acceptor->tcp_connp; 2417 2418 eager->tcp_rq->q_ptr = econnp; 2419 eager->tcp_wq->q_ptr = econnp; 2420 2421 /* 2422 * In the TLI/XTI loopback case, we are inside the listener's squeue, 2423 * which might be a different squeue from our peer TCP instance. 2424 * For TCP Fusion, the peer expects that whenever tcp_detached is 2425 * clear, our TCP queues point to the acceptor's queues. Thus, use 2426 * membar_producer() to ensure that the assignments of tcp_rq/tcp_wq 2427 * above reach global visibility prior to the clearing of tcp_detached. 2428 */ 2429 membar_producer(); 2430 eager->tcp_detached = B_FALSE; 2431 2432 ASSERT(eager->tcp_ack_tid == 0); 2433 2434 econnp->conn_dev = aconnp->conn_dev; 2435 econnp->conn_minor_arena = aconnp->conn_minor_arena; 2436 ASSERT(econnp->conn_minor_arena != NULL); 2437 if (eager->tcp_cred != NULL) 2438 crfree(eager->tcp_cred); 2439 eager->tcp_cred = econnp->conn_cred = aconnp->conn_cred; 2440 ASSERT(econnp->conn_netstack == aconnp->conn_netstack); 2441 ASSERT(eager->tcp_tcps == acceptor->tcp_tcps); 2442 2443 aconnp->conn_cred = NULL; 2444 2445 econnp->conn_zoneid = aconnp->conn_zoneid; 2446 econnp->conn_allzones = aconnp->conn_allzones; 2447 2448 econnp->conn_mac_exempt = aconnp->conn_mac_exempt; 2449 aconnp->conn_mac_exempt = B_FALSE; 2450 2451 ASSERT(aconnp->conn_peercred == NULL); 2452 2453 /* Do the IPC initialization */ 2454 CONN_INC_REF(econnp); 2455 2456 econnp->conn_multicast_loop = aconnp->conn_multicast_loop; 2457 econnp->conn_af_isv6 = aconnp->conn_af_isv6; 2458 econnp->conn_pkt_isv6 = aconnp->conn_pkt_isv6; 2459 2460 /* Done with old IPC. Drop its ref on its connp */ 2461 CONN_DEC_REF(aconnp); 2462 } 2463 2464 2465 /* 2466 * Adapt to the information, such as rtt and rtt_sd, provided from the 2467 * ire cached in conn_cache_ire. If no ire cached, do a ire lookup. 2468 * 2469 * Checks for multicast and broadcast destination address. 2470 * Returns zero on failure; non-zero if ok. 2471 * 2472 * Note that the MSS calculation here is based on the info given in 2473 * the IRE. We do not do any calculation based on TCP options. They 2474 * will be handled in tcp_rput_other() and tcp_rput_data() when TCP 2475 * knows which options to use. 2476 * 2477 * Note on how TCP gets its parameters for a connection. 2478 * 2479 * When a tcp_t structure is allocated, it gets all the default parameters. 2480 * In tcp_adapt_ire(), it gets those metric parameters, like rtt, rtt_sd, 2481 * spipe, rpipe, ... from the route metrics. Route metric overrides the 2482 * default. 2483 * 2484 * An incoming SYN with a multicast or broadcast destination address, is dropped 2485 * in 1 of 2 places. 2486 * 2487 * 1. If the packet was received over the wire it is dropped in 2488 * ip_rput_process_broadcast() 2489 * 2490 * 2. If the packet was received through internal IP loopback, i.e. the packet 2491 * was generated and received on the same machine, it is dropped in 2492 * ip_wput_local() 2493 * 2494 * An incoming SYN with a multicast or broadcast source address is always 2495 * dropped in tcp_adapt_ire. The same logic in tcp_adapt_ire also serves to 2496 * reject an attempt to connect to a broadcast or multicast (destination) 2497 * address. 2498 */ 2499 static int 2500 tcp_adapt_ire(tcp_t *tcp, mblk_t *ire_mp) 2501 { 2502 tcp_hsp_t *hsp; 2503 ire_t *ire; 2504 ire_t *sire = NULL; 2505 iulp_t *ire_uinfo = NULL; 2506 uint32_t mss_max; 2507 uint32_t mss; 2508 boolean_t tcp_detached = TCP_IS_DETACHED(tcp); 2509 conn_t *connp = tcp->tcp_connp; 2510 boolean_t ire_cacheable = B_FALSE; 2511 zoneid_t zoneid = connp->conn_zoneid; 2512 int match_flags = MATCH_IRE_RECURSIVE | MATCH_IRE_DEFAULT | 2513 MATCH_IRE_SECATTR; 2514 ts_label_t *tsl = crgetlabel(CONN_CRED(connp)); 2515 ill_t *ill = NULL; 2516 boolean_t incoming = (ire_mp == NULL); 2517 tcp_stack_t *tcps = tcp->tcp_tcps; 2518 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 2519 2520 ASSERT(connp->conn_ire_cache == NULL); 2521 2522 if (tcp->tcp_ipversion == IPV4_VERSION) { 2523 2524 if (CLASSD(tcp->tcp_connp->conn_rem)) { 2525 BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsInDiscards); 2526 return (0); 2527 } 2528 /* 2529 * If IP_NEXTHOP is set, then look for an IRE_CACHE 2530 * for the destination with the nexthop as gateway. 2531 * ire_ctable_lookup() is used because this particular 2532 * ire, if it exists, will be marked private. 2533 * If that is not available, use the interface ire 2534 * for the nexthop. 2535 * 2536 * TSol: tcp_update_label will detect label mismatches based 2537 * only on the destination's label, but that would not 2538 * detect label mismatches based on the security attributes 2539 * of routes or next hop gateway. Hence we need to pass the 2540 * label to ire_ftable_lookup below in order to locate the 2541 * right prefix (and/or) ire cache. Similarly we also need 2542 * pass the label to the ire_cache_lookup below to locate 2543 * the right ire that also matches on the label. 2544 */ 2545 if (tcp->tcp_connp->conn_nexthop_set) { 2546 ire = ire_ctable_lookup(tcp->tcp_connp->conn_rem, 2547 tcp->tcp_connp->conn_nexthop_v4, 0, NULL, zoneid, 2548 tsl, MATCH_IRE_MARK_PRIVATE_ADDR | MATCH_IRE_GW, 2549 ipst); 2550 if (ire == NULL) { 2551 ire = ire_ftable_lookup( 2552 tcp->tcp_connp->conn_nexthop_v4, 2553 0, 0, IRE_INTERFACE, NULL, NULL, zoneid, 0, 2554 tsl, match_flags, ipst); 2555 if (ire == NULL) 2556 return (0); 2557 } else { 2558 ire_uinfo = &ire->ire_uinfo; 2559 } 2560 } else { 2561 ire = ire_cache_lookup(tcp->tcp_connp->conn_rem, 2562 zoneid, tsl, ipst); 2563 if (ire != NULL) { 2564 ire_cacheable = B_TRUE; 2565 ire_uinfo = (ire_mp != NULL) ? 2566 &((ire_t *)ire_mp->b_rptr)->ire_uinfo: 2567 &ire->ire_uinfo; 2568 2569 } else { 2570 if (ire_mp == NULL) { 2571 ire = ire_ftable_lookup( 2572 tcp->tcp_connp->conn_rem, 2573 0, 0, 0, NULL, &sire, zoneid, 0, 2574 tsl, (MATCH_IRE_RECURSIVE | 2575 MATCH_IRE_DEFAULT), ipst); 2576 if (ire == NULL) 2577 return (0); 2578 ire_uinfo = (sire != NULL) ? 2579 &sire->ire_uinfo : 2580 &ire->ire_uinfo; 2581 } else { 2582 ire = (ire_t *)ire_mp->b_rptr; 2583 ire_uinfo = 2584 &((ire_t *) 2585 ire_mp->b_rptr)->ire_uinfo; 2586 } 2587 } 2588 } 2589 ASSERT(ire != NULL); 2590 2591 if ((ire->ire_src_addr == INADDR_ANY) || 2592 (ire->ire_type & IRE_BROADCAST)) { 2593 /* 2594 * ire->ire_mp is non null when ire_mp passed in is used 2595 * ire->ire_mp is set in ip_bind_insert_ire[_v6](). 2596 */ 2597 if (ire->ire_mp == NULL) 2598 ire_refrele(ire); 2599 if (sire != NULL) 2600 ire_refrele(sire); 2601 return (0); 2602 } 2603 2604 if (tcp->tcp_ipha->ipha_src == INADDR_ANY) { 2605 ipaddr_t src_addr; 2606 2607 /* 2608 * ip_bind_connected() has stored the correct source 2609 * address in conn_src. 2610 */ 2611 src_addr = tcp->tcp_connp->conn_src; 2612 tcp->tcp_ipha->ipha_src = src_addr; 2613 /* 2614 * Copy of the src addr. in tcp_t is needed 2615 * for the lookup funcs. 2616 */ 2617 IN6_IPADDR_TO_V4MAPPED(src_addr, &tcp->tcp_ip_src_v6); 2618 } 2619 /* 2620 * Set the fragment bit so that IP will tell us if the MTU 2621 * should change. IP tells us the latest setting of 2622 * ip_path_mtu_discovery through ire_frag_flag. 2623 */ 2624 if (ipst->ips_ip_path_mtu_discovery) { 2625 tcp->tcp_ipha->ipha_fragment_offset_and_flags = 2626 htons(IPH_DF); 2627 } 2628 /* 2629 * If ire_uinfo is NULL, this is the IRE_INTERFACE case 2630 * for IP_NEXTHOP. No cache ire has been found for the 2631 * destination and we are working with the nexthop's 2632 * interface ire. Since we need to forward all packets 2633 * to the nexthop first, we "blindly" set tcp_localnet 2634 * to false, eventhough the destination may also be 2635 * onlink. 2636 */ 2637 if (ire_uinfo == NULL) 2638 tcp->tcp_localnet = 0; 2639 else 2640 tcp->tcp_localnet = (ire->ire_gateway_addr == 0); 2641 } else { 2642 /* 2643 * For incoming connection ire_mp = NULL 2644 * For outgoing connection ire_mp != NULL 2645 * Technically we should check conn_incoming_ill 2646 * when ire_mp is NULL and conn_outgoing_ill when 2647 * ire_mp is non-NULL. But this is performance 2648 * critical path and for IPV*_BOUND_IF, outgoing 2649 * and incoming ill are always set to the same value. 2650 */ 2651 ill_t *dst_ill = NULL; 2652 ipif_t *dst_ipif = NULL; 2653 2654 ASSERT(connp->conn_outgoing_ill == connp->conn_incoming_ill); 2655 2656 if (connp->conn_outgoing_ill != NULL) { 2657 /* Outgoing or incoming path */ 2658 int err; 2659 2660 dst_ill = conn_get_held_ill(connp, 2661 &connp->conn_outgoing_ill, &err); 2662 if (err == ILL_LOOKUP_FAILED || dst_ill == NULL) { 2663 ip1dbg(("tcp_adapt_ire: ill_lookup failed\n")); 2664 return (0); 2665 } 2666 match_flags |= MATCH_IRE_ILL; 2667 dst_ipif = dst_ill->ill_ipif; 2668 } 2669 ire = ire_ctable_lookup_v6(&tcp->tcp_connp->conn_remv6, 2670 0, 0, dst_ipif, zoneid, tsl, match_flags, ipst); 2671 2672 if (ire != NULL) { 2673 ire_cacheable = B_TRUE; 2674 ire_uinfo = (ire_mp != NULL) ? 2675 &((ire_t *)ire_mp->b_rptr)->ire_uinfo: 2676 &ire->ire_uinfo; 2677 } else { 2678 if (ire_mp == NULL) { 2679 ire = ire_ftable_lookup_v6( 2680 &tcp->tcp_connp->conn_remv6, 2681 0, 0, 0, dst_ipif, &sire, zoneid, 2682 0, tsl, match_flags, ipst); 2683 if (ire == NULL) { 2684 if (dst_ill != NULL) 2685 ill_refrele(dst_ill); 2686 return (0); 2687 } 2688 ire_uinfo = (sire != NULL) ? &sire->ire_uinfo : 2689 &ire->ire_uinfo; 2690 } else { 2691 ire = (ire_t *)ire_mp->b_rptr; 2692 ire_uinfo = 2693 &((ire_t *)ire_mp->b_rptr)->ire_uinfo; 2694 } 2695 } 2696 if (dst_ill != NULL) 2697 ill_refrele(dst_ill); 2698 2699 ASSERT(ire != NULL); 2700 ASSERT(ire_uinfo != NULL); 2701 2702 if (IN6_IS_ADDR_UNSPECIFIED(&ire->ire_src_addr_v6) || 2703 IN6_IS_ADDR_MULTICAST(&ire->ire_addr_v6)) { 2704 /* 2705 * ire->ire_mp is non null when ire_mp passed in is used 2706 * ire->ire_mp is set in ip_bind_insert_ire[_v6](). 2707 */ 2708 if (ire->ire_mp == NULL) 2709 ire_refrele(ire); 2710 if (sire != NULL) 2711 ire_refrele(sire); 2712 return (0); 2713 } 2714 2715 if (IN6_IS_ADDR_UNSPECIFIED(&tcp->tcp_ip6h->ip6_src)) { 2716 in6_addr_t src_addr; 2717 2718 /* 2719 * ip_bind_connected_v6() has stored the correct source 2720 * address per IPv6 addr. selection policy in 2721 * conn_src_v6. 2722 */ 2723 src_addr = tcp->tcp_connp->conn_srcv6; 2724 2725 tcp->tcp_ip6h->ip6_src = src_addr; 2726 /* 2727 * Copy of the src addr. in tcp_t is needed 2728 * for the lookup funcs. 2729 */ 2730 tcp->tcp_ip_src_v6 = src_addr; 2731 ASSERT(IN6_ARE_ADDR_EQUAL(&tcp->tcp_ip6h->ip6_src, 2732 &connp->conn_srcv6)); 2733 } 2734 tcp->tcp_localnet = 2735 IN6_IS_ADDR_UNSPECIFIED(&ire->ire_gateway_addr_v6); 2736 } 2737 2738 /* 2739 * This allows applications to fail quickly when connections are made 2740 * to dead hosts. Hosts can be labeled dead by adding a reject route 2741 * with both the RTF_REJECT and RTF_PRIVATE flags set. 2742 */ 2743 if ((ire->ire_flags & RTF_REJECT) && 2744 (ire->ire_flags & RTF_PRIVATE)) 2745 goto error; 2746 2747 /* 2748 * Make use of the cached rtt and rtt_sd values to calculate the 2749 * initial RTO. Note that they are already initialized in 2750 * tcp_init_values(). 2751 * If ire_uinfo is NULL, i.e., we do not have a cache ire for 2752 * IP_NEXTHOP, but instead are using the interface ire for the 2753 * nexthop, then we do not use the ire_uinfo from that ire to 2754 * do any initializations. 2755 */ 2756 if (ire_uinfo != NULL) { 2757 if (ire_uinfo->iulp_rtt != 0) { 2758 clock_t rto; 2759 2760 tcp->tcp_rtt_sa = ire_uinfo->iulp_rtt; 2761 tcp->tcp_rtt_sd = ire_uinfo->iulp_rtt_sd; 2762 rto = (tcp->tcp_rtt_sa >> 3) + tcp->tcp_rtt_sd + 2763 tcps->tcps_rexmit_interval_extra + 2764 (tcp->tcp_rtt_sa >> 5); 2765 2766 if (rto > tcps->tcps_rexmit_interval_max) { 2767 tcp->tcp_rto = tcps->tcps_rexmit_interval_max; 2768 } else if (rto < tcps->tcps_rexmit_interval_min) { 2769 tcp->tcp_rto = tcps->tcps_rexmit_interval_min; 2770 } else { 2771 tcp->tcp_rto = rto; 2772 } 2773 } 2774 if (ire_uinfo->iulp_ssthresh != 0) 2775 tcp->tcp_cwnd_ssthresh = ire_uinfo->iulp_ssthresh; 2776 else 2777 tcp->tcp_cwnd_ssthresh = TCP_MAX_LARGEWIN; 2778 if (ire_uinfo->iulp_spipe > 0) { 2779 tcp->tcp_xmit_hiwater = MIN(ire_uinfo->iulp_spipe, 2780 tcps->tcps_max_buf); 2781 if (tcps->tcps_snd_lowat_fraction != 0) 2782 tcp->tcp_xmit_lowater = tcp->tcp_xmit_hiwater / 2783 tcps->tcps_snd_lowat_fraction; 2784 (void) tcp_maxpsz_set(tcp, B_TRUE); 2785 } 2786 /* 2787 * Note that up till now, acceptor always inherits receive 2788 * window from the listener. But if there is a metrics 2789 * associated with a host, we should use that instead of 2790 * inheriting it from listener. Thus we need to pass this 2791 * info back to the caller. 2792 */ 2793 if (ire_uinfo->iulp_rpipe > 0) { 2794 tcp->tcp_rwnd = MIN(ire_uinfo->iulp_rpipe, 2795 tcps->tcps_max_buf); 2796 } 2797 2798 if (ire_uinfo->iulp_rtomax > 0) { 2799 tcp->tcp_second_timer_threshold = 2800 ire_uinfo->iulp_rtomax; 2801 } 2802 2803 /* 2804 * Use the metric option settings, iulp_tstamp_ok and 2805 * iulp_wscale_ok, only for active open. What this means 2806 * is that if the other side uses timestamp or window 2807 * scale option, TCP will also use those options. That 2808 * is for passive open. If the application sets a 2809 * large window, window scale is enabled regardless of 2810 * the value in iulp_wscale_ok. This is the behavior 2811 * since 2.6. So we keep it. 2812 * The only case left in passive open processing is the 2813 * check for SACK. 2814 * For ECN, it should probably be like SACK. But the 2815 * current value is binary, so we treat it like the other 2816 * cases. The metric only controls active open.For passive 2817 * open, the ndd param, tcp_ecn_permitted, controls the 2818 * behavior. 2819 */ 2820 if (!tcp_detached) { 2821 /* 2822 * The if check means that the following can only 2823 * be turned on by the metrics only IRE, but not off. 2824 */ 2825 if (ire_uinfo->iulp_tstamp_ok) 2826 tcp->tcp_snd_ts_ok = B_TRUE; 2827 if (ire_uinfo->iulp_wscale_ok) 2828 tcp->tcp_snd_ws_ok = B_TRUE; 2829 if (ire_uinfo->iulp_sack == 2) 2830 tcp->tcp_snd_sack_ok = B_TRUE; 2831 if (ire_uinfo->iulp_ecn_ok) 2832 tcp->tcp_ecn_ok = B_TRUE; 2833 } else { 2834 /* 2835 * Passive open. 2836 * 2837 * As above, the if check means that SACK can only be 2838 * turned on by the metric only IRE. 2839 */ 2840 if (ire_uinfo->iulp_sack > 0) { 2841 tcp->tcp_snd_sack_ok = B_TRUE; 2842 } 2843 } 2844 } 2845 2846 2847 /* 2848 * XXX: Note that currently, ire_max_frag can be as small as 68 2849 * because of PMTUd. So tcp_mss may go to negative if combined 2850 * length of all those options exceeds 28 bytes. But because 2851 * of the tcp_mss_min check below, we may not have a problem if 2852 * tcp_mss_min is of a reasonable value. The default is 1 so 2853 * the negative problem still exists. And the check defeats PMTUd. 2854 * In fact, if PMTUd finds that the MSS should be smaller than 2855 * tcp_mss_min, TCP should turn off PMUTd and use the tcp_mss_min 2856 * value. 2857 * 2858 * We do not deal with that now. All those problems related to 2859 * PMTUd will be fixed later. 2860 */ 2861 ASSERT(ire->ire_max_frag != 0); 2862 mss = tcp->tcp_if_mtu = ire->ire_max_frag; 2863 if (tcp->tcp_ipp_fields & IPPF_USE_MIN_MTU) { 2864 if (tcp->tcp_ipp_use_min_mtu == IPV6_USE_MIN_MTU_NEVER) { 2865 mss = MIN(mss, IPV6_MIN_MTU); 2866 } 2867 } 2868 2869 /* Sanity check for MSS value. */ 2870 if (tcp->tcp_ipversion == IPV4_VERSION) 2871 mss_max = tcps->tcps_mss_max_ipv4; 2872 else 2873 mss_max = tcps->tcps_mss_max_ipv6; 2874 2875 if (tcp->tcp_ipversion == IPV6_VERSION && 2876 (ire->ire_frag_flag & IPH_FRAG_HDR)) { 2877 /* 2878 * After receiving an ICMPv6 "packet too big" message with a 2879 * MTU < 1280, and for multirouted IPv6 packets, the IP layer 2880 * will insert a 8-byte fragment header in every packet; we 2881 * reduce the MSS by that amount here. 2882 */ 2883 mss -= sizeof (ip6_frag_t); 2884 } 2885 2886 if (tcp->tcp_ipsec_overhead == 0) 2887 tcp->tcp_ipsec_overhead = conn_ipsec_length(connp); 2888 2889 mss -= tcp->tcp_ipsec_overhead; 2890 2891 if (mss < tcps->tcps_mss_min) 2892 mss = tcps->tcps_mss_min; 2893 if (mss > mss_max) 2894 mss = mss_max; 2895 2896 /* Note that this is the maximum MSS, excluding all options. */ 2897 tcp->tcp_mss = mss; 2898 2899 /* 2900 * Initialize the ISS here now that we have the full connection ID. 2901 * The RFC 1948 method of initial sequence number generation requires 2902 * knowledge of the full connection ID before setting the ISS. 2903 */ 2904 2905 tcp_iss_init(tcp); 2906 2907 if (ire->ire_type & (IRE_LOOPBACK | IRE_LOCAL)) 2908 tcp->tcp_loopback = B_TRUE; 2909 2910 if (tcp->tcp_ipversion == IPV4_VERSION) { 2911 hsp = tcp_hsp_lookup(tcp->tcp_remote, tcps); 2912 } else { 2913 hsp = tcp_hsp_lookup_ipv6(&tcp->tcp_remote_v6, tcps); 2914 } 2915 2916 if (hsp != NULL) { 2917 /* Only modify if we're going to make them bigger */ 2918 if (hsp->tcp_hsp_sendspace > tcp->tcp_xmit_hiwater) { 2919 tcp->tcp_xmit_hiwater = hsp->tcp_hsp_sendspace; 2920 if (tcps->tcps_snd_lowat_fraction != 0) 2921 tcp->tcp_xmit_lowater = tcp->tcp_xmit_hiwater / 2922 tcps->tcps_snd_lowat_fraction; 2923 } 2924 2925 if (hsp->tcp_hsp_recvspace > tcp->tcp_rwnd) { 2926 tcp->tcp_rwnd = hsp->tcp_hsp_recvspace; 2927 } 2928 2929 /* Copy timestamp flag only for active open */ 2930 if (!tcp_detached) 2931 tcp->tcp_snd_ts_ok = hsp->tcp_hsp_tstamp; 2932 } 2933 2934 if (sire != NULL) 2935 IRE_REFRELE(sire); 2936 2937 /* 2938 * If we got an IRE_CACHE and an ILL, go through their properties; 2939 * otherwise, this is deferred until later when we have an IRE_CACHE. 2940 */ 2941 if (tcp->tcp_loopback || 2942 (ire_cacheable && (ill = ire_to_ill(ire)) != NULL)) { 2943 /* 2944 * For incoming, see if this tcp may be MDT-capable. For 2945 * outgoing, this process has been taken care of through 2946 * tcp_rput_other. 2947 */ 2948 tcp_ire_ill_check(tcp, ire, ill, incoming); 2949 tcp->tcp_ire_ill_check_done = B_TRUE; 2950 } 2951 2952 mutex_enter(&connp->conn_lock); 2953 /* 2954 * Make sure that conn is not marked incipient 2955 * for incoming connections. A blind 2956 * removal of incipient flag is cheaper than 2957 * check and removal. 2958 */ 2959 connp->conn_state_flags &= ~CONN_INCIPIENT; 2960 2961 /* 2962 * Must not cache forwarding table routes 2963 * or recache an IRE after the conn_t has 2964 * had conn_ire_cache cleared and is flagged 2965 * unusable, (see the CONN_CACHE_IRE() macro). 2966 */ 2967 if (ire_cacheable && CONN_CACHE_IRE(connp)) { 2968 rw_enter(&ire->ire_bucket->irb_lock, RW_READER); 2969 if (!(ire->ire_marks & IRE_MARK_CONDEMNED)) { 2970 connp->conn_ire_cache = ire; 2971 IRE_UNTRACE_REF(ire); 2972 rw_exit(&ire->ire_bucket->irb_lock); 2973 mutex_exit(&connp->conn_lock); 2974 return (1); 2975 } 2976 rw_exit(&ire->ire_bucket->irb_lock); 2977 } 2978 mutex_exit(&connp->conn_lock); 2979 2980 if (ire->ire_mp == NULL) 2981 ire_refrele(ire); 2982 return (1); 2983 2984 error: 2985 if (ire->ire_mp == NULL) 2986 ire_refrele(ire); 2987 if (sire != NULL) 2988 ire_refrele(sire); 2989 return (0); 2990 } 2991 2992 /* 2993 * tcp_bind is called (holding the writer lock) by tcp_wput_proto to process a 2994 * O_T_BIND_REQ/T_BIND_REQ message. 2995 */ 2996 static void 2997 tcp_bind(tcp_t *tcp, mblk_t *mp) 2998 { 2999 sin_t *sin; 3000 sin6_t *sin6; 3001 mblk_t *mp1; 3002 in_port_t requested_port; 3003 in_port_t allocated_port; 3004 struct T_bind_req *tbr; 3005 boolean_t bind_to_req_port_only; 3006 boolean_t backlog_update = B_FALSE; 3007 boolean_t user_specified; 3008 in6_addr_t v6addr; 3009 ipaddr_t v4addr; 3010 uint_t origipversion; 3011 int err; 3012 queue_t *q = tcp->tcp_wq; 3013 conn_t *connp = tcp->tcp_connp; 3014 mlp_type_t addrtype, mlptype; 3015 zone_t *zone; 3016 cred_t *cr; 3017 in_port_t mlp_port; 3018 tcp_stack_t *tcps = tcp->tcp_tcps; 3019 3020 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= (uintptr_t)INT_MAX); 3021 if ((mp->b_wptr - mp->b_rptr) < sizeof (*tbr)) { 3022 if (tcp->tcp_debug) { 3023 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE, 3024 "tcp_bind: bad req, len %u", 3025 (uint_t)(mp->b_wptr - mp->b_rptr)); 3026 } 3027 tcp_err_ack(tcp, mp, TPROTO, 0); 3028 return; 3029 } 3030 /* Make sure the largest address fits */ 3031 mp1 = reallocb(mp, sizeof (struct T_bind_ack) + sizeof (sin6_t) + 1, 1); 3032 if (mp1 == NULL) { 3033 tcp_err_ack(tcp, mp, TSYSERR, ENOMEM); 3034 return; 3035 } 3036 mp = mp1; 3037 tbr = (struct T_bind_req *)mp->b_rptr; 3038 if (tcp->tcp_state >= TCPS_BOUND) { 3039 if ((tcp->tcp_state == TCPS_BOUND || 3040 tcp->tcp_state == TCPS_LISTEN) && 3041 tcp->tcp_conn_req_max != tbr->CONIND_number && 3042 tbr->CONIND_number > 0) { 3043 /* 3044 * Handle listen() increasing CONIND_number. 3045 * This is more "liberal" then what the TPI spec 3046 * requires but is needed to avoid a t_unbind 3047 * when handling listen() since the port number 3048 * might be "stolen" between the unbind and bind. 3049 */ 3050 backlog_update = B_TRUE; 3051 goto do_bind; 3052 } 3053 if (tcp->tcp_debug) { 3054 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE, 3055 "tcp_bind: bad state, %d", tcp->tcp_state); 3056 } 3057 tcp_err_ack(tcp, mp, TOUTSTATE, 0); 3058 return; 3059 } 3060 origipversion = tcp->tcp_ipversion; 3061 3062 switch (tbr->ADDR_length) { 3063 case 0: /* request for a generic port */ 3064 tbr->ADDR_offset = sizeof (struct T_bind_req); 3065 if (tcp->tcp_family == AF_INET) { 3066 tbr->ADDR_length = sizeof (sin_t); 3067 sin = (sin_t *)&tbr[1]; 3068 *sin = sin_null; 3069 sin->sin_family = AF_INET; 3070 mp->b_wptr = (uchar_t *)&sin[1]; 3071 tcp->tcp_ipversion = IPV4_VERSION; 3072 IN6_IPADDR_TO_V4MAPPED(INADDR_ANY, &v6addr); 3073 } else { 3074 ASSERT(tcp->tcp_family == AF_INET6); 3075 tbr->ADDR_length = sizeof (sin6_t); 3076 sin6 = (sin6_t *)&tbr[1]; 3077 *sin6 = sin6_null; 3078 sin6->sin6_family = AF_INET6; 3079 mp->b_wptr = (uchar_t *)&sin6[1]; 3080 tcp->tcp_ipversion = IPV6_VERSION; 3081 V6_SET_ZERO(v6addr); 3082 } 3083 requested_port = 0; 3084 break; 3085 3086 case sizeof (sin_t): /* Complete IPv4 address */ 3087 sin = (sin_t *)mi_offset_param(mp, tbr->ADDR_offset, 3088 sizeof (sin_t)); 3089 if (sin == NULL || !OK_32PTR((char *)sin)) { 3090 if (tcp->tcp_debug) { 3091 (void) strlog(TCP_MOD_ID, 0, 1, 3092 SL_ERROR|SL_TRACE, 3093 "tcp_bind: bad address parameter, " 3094 "offset %d, len %d", 3095 tbr->ADDR_offset, tbr->ADDR_length); 3096 } 3097 tcp_err_ack(tcp, mp, TPROTO, 0); 3098 return; 3099 } 3100 /* 3101 * With sockets sockfs will accept bogus sin_family in 3102 * bind() and replace it with the family used in the socket 3103 * call. 3104 */ 3105 if (sin->sin_family != AF_INET || 3106 tcp->tcp_family != AF_INET) { 3107 tcp_err_ack(tcp, mp, TSYSERR, EAFNOSUPPORT); 3108 return; 3109 } 3110 requested_port = ntohs(sin->sin_port); 3111 tcp->tcp_ipversion = IPV4_VERSION; 3112 v4addr = sin->sin_addr.s_addr; 3113 IN6_IPADDR_TO_V4MAPPED(v4addr, &v6addr); 3114 break; 3115 3116 case sizeof (sin6_t): /* Complete IPv6 address */ 3117 sin6 = (sin6_t *)mi_offset_param(mp, 3118 tbr->ADDR_offset, sizeof (sin6_t)); 3119 if (sin6 == NULL || !OK_32PTR((char *)sin6)) { 3120 if (tcp->tcp_debug) { 3121 (void) strlog(TCP_MOD_ID, 0, 1, 3122 SL_ERROR|SL_TRACE, 3123 "tcp_bind: bad IPv6 address parameter, " 3124 "offset %d, len %d", tbr->ADDR_offset, 3125 tbr->ADDR_length); 3126 } 3127 tcp_err_ack(tcp, mp, TSYSERR, EINVAL); 3128 return; 3129 } 3130 if (sin6->sin6_family != AF_INET6 || 3131 tcp->tcp_family != AF_INET6) { 3132 tcp_err_ack(tcp, mp, TSYSERR, EAFNOSUPPORT); 3133 return; 3134 } 3135 requested_port = ntohs(sin6->sin6_port); 3136 tcp->tcp_ipversion = IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr) ? 3137 IPV4_VERSION : IPV6_VERSION; 3138 v6addr = sin6->sin6_addr; 3139 break; 3140 3141 default: 3142 if (tcp->tcp_debug) { 3143 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE, 3144 "tcp_bind: bad address length, %d", 3145 tbr->ADDR_length); 3146 } 3147 tcp_err_ack(tcp, mp, TBADADDR, 0); 3148 return; 3149 } 3150 tcp->tcp_bound_source_v6 = v6addr; 3151 3152 /* Check for change in ipversion */ 3153 if (origipversion != tcp->tcp_ipversion) { 3154 ASSERT(tcp->tcp_family == AF_INET6); 3155 err = tcp->tcp_ipversion == IPV6_VERSION ? 3156 tcp_header_init_ipv6(tcp) : tcp_header_init_ipv4(tcp); 3157 if (err) { 3158 tcp_err_ack(tcp, mp, TSYSERR, ENOMEM); 3159 return; 3160 } 3161 } 3162 3163 /* 3164 * Initialize family specific fields. Copy of the src addr. 3165 * in tcp_t is needed for the lookup funcs. 3166 */ 3167 if (tcp->tcp_ipversion == IPV6_VERSION) { 3168 tcp->tcp_ip6h->ip6_src = v6addr; 3169 } else { 3170 IN6_V4MAPPED_TO_IPADDR(&v6addr, tcp->tcp_ipha->ipha_src); 3171 } 3172 tcp->tcp_ip_src_v6 = v6addr; 3173 3174 /* 3175 * For O_T_BIND_REQ: 3176 * Verify that the target port/addr is available, or choose 3177 * another. 3178 * For T_BIND_REQ: 3179 * Verify that the target port/addr is available or fail. 3180 * In both cases when it succeeds the tcp is inserted in the 3181 * bind hash table. This ensures that the operation is atomic 3182 * under the lock on the hash bucket. 3183 */ 3184 bind_to_req_port_only = requested_port != 0 && 3185 tbr->PRIM_type != O_T_BIND_REQ; 3186 /* 3187 * Get a valid port (within the anonymous range and should not 3188 * be a privileged one) to use if the user has not given a port. 3189 * If multiple threads are here, they may all start with 3190 * with the same initial port. But, it should be fine as long as 3191 * tcp_bindi will ensure that no two threads will be assigned 3192 * the same port. 3193 * 3194 * NOTE: XXX If a privileged process asks for an anonymous port, we 3195 * still check for ports only in the range > tcp_smallest_non_priv_port, 3196 * unless TCP_ANONPRIVBIND option is set. 3197 */ 3198 mlptype = mlptSingle; 3199 mlp_port = requested_port; 3200 if (requested_port == 0) { 3201 requested_port = tcp->tcp_anon_priv_bind ? 3202 tcp_get_next_priv_port(tcp) : 3203 tcp_update_next_port(tcps->tcps_next_port_to_try, 3204 tcp, B_TRUE); 3205 if (requested_port == 0) { 3206 tcp_err_ack(tcp, mp, TNOADDR, 0); 3207 return; 3208 } 3209 user_specified = B_FALSE; 3210 3211 /* 3212 * If the user went through one of the RPC interfaces to create 3213 * this socket and RPC is MLP in this zone, then give him an 3214 * anonymous MLP. 3215 */ 3216 cr = DB_CREDDEF(mp, tcp->tcp_cred); 3217 if (connp->conn_anon_mlp && is_system_labeled()) { 3218 zone = crgetzone(cr); 3219 addrtype = tsol_mlp_addr_type(zone->zone_id, 3220 IPV6_VERSION, &v6addr, 3221 tcps->tcps_netstack->netstack_ip); 3222 if (addrtype == mlptSingle) { 3223 tcp_err_ack(tcp, mp, TNOADDR, 0); 3224 return; 3225 } 3226 mlptype = tsol_mlp_port_type(zone, IPPROTO_TCP, 3227 PMAPPORT, addrtype); 3228 mlp_port = PMAPPORT; 3229 } 3230 } else { 3231 int i; 3232 boolean_t priv = B_FALSE; 3233 3234 /* 3235 * If the requested_port is in the well-known privileged range, 3236 * verify that the stream was opened by a privileged user. 3237 * Note: No locks are held when inspecting tcp_g_*epriv_ports 3238 * but instead the code relies on: 3239 * - the fact that the address of the array and its size never 3240 * changes 3241 * - the atomic assignment of the elements of the array 3242 */ 3243 cr = DB_CREDDEF(mp, tcp->tcp_cred); 3244 if (requested_port < tcps->tcps_smallest_nonpriv_port) { 3245 priv = B_TRUE; 3246 } else { 3247 for (i = 0; i < tcps->tcps_g_num_epriv_ports; i++) { 3248 if (requested_port == 3249 tcps->tcps_g_epriv_ports[i]) { 3250 priv = B_TRUE; 3251 break; 3252 } 3253 } 3254 } 3255 if (priv) { 3256 if (secpolicy_net_privaddr(cr, requested_port, 3257 IPPROTO_TCP) != 0) { 3258 if (tcp->tcp_debug) { 3259 (void) strlog(TCP_MOD_ID, 0, 1, 3260 SL_ERROR|SL_TRACE, 3261 "tcp_bind: no priv for port %d", 3262 requested_port); 3263 } 3264 tcp_err_ack(tcp, mp, TACCES, 0); 3265 return; 3266 } 3267 } 3268 user_specified = B_TRUE; 3269 3270 if (is_system_labeled()) { 3271 zone = crgetzone(cr); 3272 addrtype = tsol_mlp_addr_type(zone->zone_id, 3273 IPV6_VERSION, &v6addr, 3274 tcps->tcps_netstack->netstack_ip); 3275 if (addrtype == mlptSingle) { 3276 tcp_err_ack(tcp, mp, TNOADDR, 0); 3277 return; 3278 } 3279 mlptype = tsol_mlp_port_type(zone, IPPROTO_TCP, 3280 requested_port, addrtype); 3281 } 3282 } 3283 3284 if (mlptype != mlptSingle) { 3285 if (secpolicy_net_bindmlp(cr) != 0) { 3286 if (tcp->tcp_debug) { 3287 (void) strlog(TCP_MOD_ID, 0, 1, 3288 SL_ERROR|SL_TRACE, 3289 "tcp_bind: no priv for multilevel port %d", 3290 requested_port); 3291 } 3292 tcp_err_ack(tcp, mp, TACCES, 0); 3293 return; 3294 } 3295 3296 /* 3297 * If we're specifically binding a shared IP address and the 3298 * port is MLP on shared addresses, then check to see if this 3299 * zone actually owns the MLP. Reject if not. 3300 */ 3301 if (mlptype == mlptShared && addrtype == mlptShared) { 3302 /* 3303 * No need to handle exclusive-stack zones since 3304 * ALL_ZONES only applies to the shared stack. 3305 */ 3306 zoneid_t mlpzone; 3307 3308 mlpzone = tsol_mlp_findzone(IPPROTO_TCP, 3309 htons(mlp_port)); 3310 if (connp->conn_zoneid != mlpzone) { 3311 if (tcp->tcp_debug) { 3312 (void) strlog(TCP_MOD_ID, 0, 1, 3313 SL_ERROR|SL_TRACE, 3314 "tcp_bind: attempt to bind port " 3315 "%d on shared addr in zone %d " 3316 "(should be %d)", 3317 mlp_port, connp->conn_zoneid, 3318 mlpzone); 3319 } 3320 tcp_err_ack(tcp, mp, TACCES, 0); 3321 return; 3322 } 3323 } 3324 3325 if (!user_specified) { 3326 err = tsol_mlp_anon(zone, mlptype, connp->conn_ulp, 3327 requested_port, B_TRUE); 3328 if (err != 0) { 3329 if (tcp->tcp_debug) { 3330 (void) strlog(TCP_MOD_ID, 0, 1, 3331 SL_ERROR|SL_TRACE, 3332 "tcp_bind: cannot establish anon " 3333 "MLP for port %d", 3334 requested_port); 3335 } 3336 tcp_err_ack(tcp, mp, TSYSERR, err); 3337 return; 3338 } 3339 connp->conn_anon_port = B_TRUE; 3340 } 3341 connp->conn_mlp_type = mlptype; 3342 } 3343 3344 allocated_port = tcp_bindi(tcp, requested_port, &v6addr, 3345 tcp->tcp_reuseaddr, B_FALSE, bind_to_req_port_only, user_specified); 3346 3347 if (allocated_port == 0) { 3348 connp->conn_mlp_type = mlptSingle; 3349 if (connp->conn_anon_port) { 3350 connp->conn_anon_port = B_FALSE; 3351 (void) tsol_mlp_anon(zone, mlptype, connp->conn_ulp, 3352 requested_port, B_FALSE); 3353 } 3354 if (bind_to_req_port_only) { 3355 if (tcp->tcp_debug) { 3356 (void) strlog(TCP_MOD_ID, 0, 1, 3357 SL_ERROR|SL_TRACE, 3358 "tcp_bind: requested addr busy"); 3359 } 3360 tcp_err_ack(tcp, mp, TADDRBUSY, 0); 3361 } else { 3362 /* If we are out of ports, fail the bind. */ 3363 if (tcp->tcp_debug) { 3364 (void) strlog(TCP_MOD_ID, 0, 1, 3365 SL_ERROR|SL_TRACE, 3366 "tcp_bind: out of ports?"); 3367 } 3368 tcp_err_ack(tcp, mp, TNOADDR, 0); 3369 } 3370 return; 3371 } 3372 ASSERT(tcp->tcp_state == TCPS_BOUND); 3373 do_bind: 3374 if (!backlog_update) { 3375 if (tcp->tcp_family == AF_INET) 3376 sin->sin_port = htons(allocated_port); 3377 else 3378 sin6->sin6_port = htons(allocated_port); 3379 } 3380 if (tcp->tcp_family == AF_INET) { 3381 if (tbr->CONIND_number != 0) { 3382 mp1 = tcp_ip_bind_mp(tcp, tbr->PRIM_type, 3383 sizeof (sin_t)); 3384 } else { 3385 /* Just verify the local IP address */ 3386 mp1 = tcp_ip_bind_mp(tcp, tbr->PRIM_type, IP_ADDR_LEN); 3387 } 3388 } else { 3389 if (tbr->CONIND_number != 0) { 3390 mp1 = tcp_ip_bind_mp(tcp, tbr->PRIM_type, 3391 sizeof (sin6_t)); 3392 } else { 3393 /* Just verify the local IP address */ 3394 mp1 = tcp_ip_bind_mp(tcp, tbr->PRIM_type, 3395 IPV6_ADDR_LEN); 3396 } 3397 } 3398 if (mp1 == NULL) { 3399 if (connp->conn_anon_port) { 3400 connp->conn_anon_port = B_FALSE; 3401 (void) tsol_mlp_anon(zone, mlptype, connp->conn_ulp, 3402 requested_port, B_FALSE); 3403 } 3404 connp->conn_mlp_type = mlptSingle; 3405 tcp_err_ack(tcp, mp, TSYSERR, ENOMEM); 3406 return; 3407 } 3408 3409 tbr->PRIM_type = T_BIND_ACK; 3410 mp->b_datap->db_type = M_PCPROTO; 3411 3412 /* Chain in the reply mp for tcp_rput() */ 3413 mp1->b_cont = mp; 3414 mp = mp1; 3415 3416 tcp->tcp_conn_req_max = tbr->CONIND_number; 3417 if (tcp->tcp_conn_req_max) { 3418 if (tcp->tcp_conn_req_max < tcps->tcps_conn_req_min) 3419 tcp->tcp_conn_req_max = tcps->tcps_conn_req_min; 3420 if (tcp->tcp_conn_req_max > tcps->tcps_conn_req_max_q) 3421 tcp->tcp_conn_req_max = tcps->tcps_conn_req_max_q; 3422 /* 3423 * If this is a listener, do not reset the eager list 3424 * and other stuffs. Note that we don't check if the 3425 * existing eager list meets the new tcp_conn_req_max 3426 * requirement. 3427 */ 3428 if (tcp->tcp_state != TCPS_LISTEN) { 3429 tcp->tcp_state = TCPS_LISTEN; 3430 /* Initialize the chain. Don't need the eager_lock */ 3431 tcp->tcp_eager_next_q0 = tcp->tcp_eager_prev_q0 = tcp; 3432 tcp->tcp_eager_next_drop_q0 = tcp; 3433 tcp->tcp_eager_prev_drop_q0 = tcp; 3434 tcp->tcp_second_ctimer_threshold = 3435 tcps->tcps_ip_abort_linterval; 3436 } 3437 } 3438 3439 /* 3440 * We can call ip_bind directly which returns a T_BIND_ACK mp. The 3441 * processing continues in tcp_rput_other(). 3442 * 3443 * We need to make sure that the conn_recv is set to a non-null 3444 * value before we insert the conn into the classifier table. 3445 * This is to avoid a race with an incoming packet which does an 3446 * ipcl_classify(). 3447 */ 3448 connp->conn_recv = tcp_conn_request; 3449 if (tcp->tcp_family == AF_INET6) { 3450 ASSERT(tcp->tcp_connp->conn_af_isv6); 3451 mp = ip_bind_v6(q, mp, tcp->tcp_connp, &tcp->tcp_sticky_ipp); 3452 } else { 3453 ASSERT(!tcp->tcp_connp->conn_af_isv6); 3454 mp = ip_bind_v4(q, mp, tcp->tcp_connp); 3455 } 3456 /* 3457 * If the bind cannot complete immediately 3458 * IP will arrange to call tcp_rput_other 3459 * when the bind completes. 3460 */ 3461 if (mp != NULL) { 3462 tcp_rput_other(tcp, mp); 3463 } else { 3464 /* 3465 * Bind will be resumed later. Need to ensure 3466 * that conn doesn't disappear when that happens. 3467 * This will be decremented in ip_resume_tcp_bind(). 3468 */ 3469 CONN_INC_REF(tcp->tcp_connp); 3470 } 3471 } 3472 3473 3474 /* 3475 * If the "bind_to_req_port_only" parameter is set, if the requested port 3476 * number is available, return it, If not return 0 3477 * 3478 * If "bind_to_req_port_only" parameter is not set and 3479 * If the requested port number is available, return it. If not, return 3480 * the first anonymous port we happen across. If no anonymous ports are 3481 * available, return 0. addr is the requested local address, if any. 3482 * 3483 * In either case, when succeeding update the tcp_t to record the port number 3484 * and insert it in the bind hash table. 3485 * 3486 * Note that TCP over IPv4 and IPv6 sockets can use the same port number 3487 * without setting SO_REUSEADDR. This is needed so that they 3488 * can be viewed as two independent transport protocols. 3489 */ 3490 static in_port_t 3491 tcp_bindi(tcp_t *tcp, in_port_t port, const in6_addr_t *laddr, 3492 int reuseaddr, boolean_t quick_connect, 3493 boolean_t bind_to_req_port_only, boolean_t user_specified) 3494 { 3495 /* number of times we have run around the loop */ 3496 int count = 0; 3497 /* maximum number of times to run around the loop */ 3498 int loopmax; 3499 conn_t *connp = tcp->tcp_connp; 3500 zoneid_t zoneid = connp->conn_zoneid; 3501 tcp_stack_t *tcps = tcp->tcp_tcps; 3502 3503 /* 3504 * Lookup for free addresses is done in a loop and "loopmax" 3505 * influences how long we spin in the loop 3506 */ 3507 if (bind_to_req_port_only) { 3508 /* 3509 * If the requested port is busy, don't bother to look 3510 * for a new one. Setting loop maximum count to 1 has 3511 * that effect. 3512 */ 3513 loopmax = 1; 3514 } else { 3515 /* 3516 * If the requested port is busy, look for a free one 3517 * in the anonymous port range. 3518 * Set loopmax appropriately so that one does not look 3519 * forever in the case all of the anonymous ports are in use. 3520 */ 3521 if (tcp->tcp_anon_priv_bind) { 3522 /* 3523 * loopmax = 3524 * (IPPORT_RESERVED-1) - tcp_min_anonpriv_port + 1 3525 */ 3526 loopmax = IPPORT_RESERVED - 3527 tcps->tcps_min_anonpriv_port; 3528 } else { 3529 loopmax = (tcps->tcps_largest_anon_port - 3530 tcps->tcps_smallest_anon_port + 1); 3531 } 3532 } 3533 do { 3534 uint16_t lport; 3535 tf_t *tbf; 3536 tcp_t *ltcp; 3537 conn_t *lconnp; 3538 3539 lport = htons(port); 3540 3541 /* 3542 * Ensure that the tcp_t is not currently in the bind hash. 3543 * Hold the lock on the hash bucket to ensure that 3544 * the duplicate check plus the insertion is an atomic 3545 * operation. 3546 * 3547 * This function does an inline lookup on the bind hash list 3548 * Make sure that we access only members of tcp_t 3549 * and that we don't look at tcp_tcp, since we are not 3550 * doing a CONN_INC_REF. 3551 */ 3552 tcp_bind_hash_remove(tcp); 3553 tbf = &tcps->tcps_bind_fanout[TCP_BIND_HASH(lport)]; 3554 mutex_enter(&tbf->tf_lock); 3555 for (ltcp = tbf->tf_tcp; ltcp != NULL; 3556 ltcp = ltcp->tcp_bind_hash) { 3557 boolean_t not_socket; 3558 boolean_t exclbind; 3559 3560 if (lport != ltcp->tcp_lport) 3561 continue; 3562 3563 lconnp = ltcp->tcp_connp; 3564 3565 /* 3566 * On a labeled system, we must treat bindings to ports 3567 * on shared IP addresses by sockets with MAC exemption 3568 * privilege as being in all zones, as there's 3569 * otherwise no way to identify the right receiver. 3570 */ 3571 if (!(IPCL_ZONE_MATCH(ltcp->tcp_connp, zoneid) || 3572 IPCL_ZONE_MATCH(connp, 3573 ltcp->tcp_connp->conn_zoneid)) && 3574 !lconnp->conn_mac_exempt && 3575 !connp->conn_mac_exempt) 3576 continue; 3577 3578 /* 3579 * If TCP_EXCLBIND is set for either the bound or 3580 * binding endpoint, the semantics of bind 3581 * is changed according to the following. 3582 * 3583 * spec = specified address (v4 or v6) 3584 * unspec = unspecified address (v4 or v6) 3585 * A = specified addresses are different for endpoints 3586 * 3587 * bound bind to allowed 3588 * ------------------------------------- 3589 * unspec unspec no 3590 * unspec spec no 3591 * spec unspec no 3592 * spec spec yes if A 3593 * 3594 * For labeled systems, SO_MAC_EXEMPT behaves the same 3595 * as TCP_EXCLBIND, except that zoneid is ignored. 3596 * 3597 * Note: 3598 * 3599 * 1. Because of TLI semantics, an endpoint can go 3600 * back from, say TCP_ESTABLISHED to TCPS_LISTEN or 3601 * TCPS_BOUND, depending on whether it is originally 3602 * a listener or not. That is why we need to check 3603 * for states greater than or equal to TCPS_BOUND 3604 * here. 3605 * 3606 * 2. Ideally, we should only check for state equals 3607 * to TCPS_LISTEN. And the following check should be 3608 * added. 3609 * 3610 * if (ltcp->tcp_state == TCPS_LISTEN || 3611 * !reuseaddr || !ltcp->tcp_reuseaddr) { 3612 * ... 3613 * } 3614 * 3615 * The semantics will be changed to this. If the 3616 * endpoint on the list is in state not equal to 3617 * TCPS_LISTEN and both endpoints have SO_REUSEADDR 3618 * set, let the bind succeed. 3619 * 3620 * Because of (1), we cannot do that for TLI 3621 * endpoints. But we can do that for socket endpoints. 3622 * If in future, we can change this going back 3623 * semantics, we can use the above check for TLI also. 3624 */ 3625 not_socket = !(TCP_IS_SOCKET(ltcp) && 3626 TCP_IS_SOCKET(tcp)); 3627 exclbind = ltcp->tcp_exclbind || tcp->tcp_exclbind; 3628 3629 if (lconnp->conn_mac_exempt || connp->conn_mac_exempt || 3630 (exclbind && (not_socket || 3631 ltcp->tcp_state <= TCPS_ESTABLISHED))) { 3632 if (V6_OR_V4_INADDR_ANY( 3633 ltcp->tcp_bound_source_v6) || 3634 V6_OR_V4_INADDR_ANY(*laddr) || 3635 IN6_ARE_ADDR_EQUAL(laddr, 3636 <cp->tcp_bound_source_v6)) { 3637 break; 3638 } 3639 continue; 3640 } 3641 3642 /* 3643 * Check ipversion to allow IPv4 and IPv6 sockets to 3644 * have disjoint port number spaces, if *_EXCLBIND 3645 * is not set and only if the application binds to a 3646 * specific port. We use the same autoassigned port 3647 * number space for IPv4 and IPv6 sockets. 3648 */ 3649 if (tcp->tcp_ipversion != ltcp->tcp_ipversion && 3650 bind_to_req_port_only) 3651 continue; 3652 3653 /* 3654 * Ideally, we should make sure that the source 3655 * address, remote address, and remote port in the 3656 * four tuple for this tcp-connection is unique. 3657 * However, trying to find out the local source 3658 * address would require too much code duplication 3659 * with IP, since IP needs needs to have that code 3660 * to support userland TCP implementations. 3661 */ 3662 if (quick_connect && 3663 (ltcp->tcp_state > TCPS_LISTEN) && 3664 ((tcp->tcp_fport != ltcp->tcp_fport) || 3665 !IN6_ARE_ADDR_EQUAL(&tcp->tcp_remote_v6, 3666 <cp->tcp_remote_v6))) 3667 continue; 3668 3669 if (!reuseaddr) { 3670 /* 3671 * No socket option SO_REUSEADDR. 3672 * If existing port is bound to 3673 * a non-wildcard IP address 3674 * and the requesting stream is 3675 * bound to a distinct 3676 * different IP addresses 3677 * (non-wildcard, also), keep 3678 * going. 3679 */ 3680 if (!V6_OR_V4_INADDR_ANY(*laddr) && 3681 !V6_OR_V4_INADDR_ANY( 3682 ltcp->tcp_bound_source_v6) && 3683 !IN6_ARE_ADDR_EQUAL(laddr, 3684 <cp->tcp_bound_source_v6)) 3685 continue; 3686 if (ltcp->tcp_state >= TCPS_BOUND) { 3687 /* 3688 * This port is being used and 3689 * its state is >= TCPS_BOUND, 3690 * so we can't bind to it. 3691 */ 3692 break; 3693 } 3694 } else { 3695 /* 3696 * socket option SO_REUSEADDR is set on the 3697 * binding tcp_t. 3698 * 3699 * If two streams are bound to 3700 * same IP address or both addr 3701 * and bound source are wildcards 3702 * (INADDR_ANY), we want to stop 3703 * searching. 3704 * We have found a match of IP source 3705 * address and source port, which is 3706 * refused regardless of the 3707 * SO_REUSEADDR setting, so we break. 3708 */ 3709 if (IN6_ARE_ADDR_EQUAL(laddr, 3710 <cp->tcp_bound_source_v6) && 3711 (ltcp->tcp_state == TCPS_LISTEN || 3712 ltcp->tcp_state == TCPS_BOUND)) 3713 break; 3714 } 3715 } 3716 if (ltcp != NULL) { 3717 /* The port number is busy */ 3718 mutex_exit(&tbf->tf_lock); 3719 } else { 3720 /* 3721 * This port is ours. Insert in fanout and mark as 3722 * bound to prevent others from getting the port 3723 * number. 3724 */ 3725 tcp->tcp_state = TCPS_BOUND; 3726 tcp->tcp_lport = htons(port); 3727 *(uint16_t *)tcp->tcp_tcph->th_lport = tcp->tcp_lport; 3728 3729 ASSERT(&tcps->tcps_bind_fanout[TCP_BIND_HASH( 3730 tcp->tcp_lport)] == tbf); 3731 tcp_bind_hash_insert(tbf, tcp, 1); 3732 3733 mutex_exit(&tbf->tf_lock); 3734 3735 /* 3736 * We don't want tcp_next_port_to_try to "inherit" 3737 * a port number supplied by the user in a bind. 3738 */ 3739 if (user_specified) 3740 return (port); 3741 3742 /* 3743 * This is the only place where tcp_next_port_to_try 3744 * is updated. After the update, it may or may not 3745 * be in the valid range. 3746 */ 3747 if (!tcp->tcp_anon_priv_bind) 3748 tcps->tcps_next_port_to_try = port + 1; 3749 return (port); 3750 } 3751 3752 if (tcp->tcp_anon_priv_bind) { 3753 port = tcp_get_next_priv_port(tcp); 3754 } else { 3755 if (count == 0 && user_specified) { 3756 /* 3757 * We may have to return an anonymous port. So 3758 * get one to start with. 3759 */ 3760 port = 3761 tcp_update_next_port( 3762 tcps->tcps_next_port_to_try, 3763 tcp, B_TRUE); 3764 user_specified = B_FALSE; 3765 } else { 3766 port = tcp_update_next_port(port + 1, tcp, 3767 B_FALSE); 3768 } 3769 } 3770 if (port == 0) 3771 break; 3772 3773 /* 3774 * Don't let this loop run forever in the case where 3775 * all of the anonymous ports are in use. 3776 */ 3777 } while (++count < loopmax); 3778 return (0); 3779 } 3780 3781 /* 3782 * tcp_clean_death / tcp_close_detached must not be called more than once 3783 * on a tcp. Thus every function that potentially calls tcp_clean_death 3784 * must check for the tcp state before calling tcp_clean_death. 3785 * Eg. tcp_input, tcp_rput_data, tcp_eager_kill, tcp_clean_death_wrapper, 3786 * tcp_timer_handler, all check for the tcp state. 3787 */ 3788 /* ARGSUSED */ 3789 void 3790 tcp_clean_death_wrapper(void *arg, mblk_t *mp, void *arg2) 3791 { 3792 tcp_t *tcp = ((conn_t *)arg)->conn_tcp; 3793 3794 freemsg(mp); 3795 if (tcp->tcp_state > TCPS_BOUND) 3796 (void) tcp_clean_death(((conn_t *)arg)->conn_tcp, 3797 ETIMEDOUT, 5); 3798 } 3799 3800 /* 3801 * We are dying for some reason. Try to do it gracefully. (May be called 3802 * as writer.) 3803 * 3804 * Return -1 if the structure was not cleaned up (if the cleanup had to be 3805 * done by a service procedure). 3806 * TBD - Should the return value distinguish between the tcp_t being 3807 * freed and it being reinitialized? 3808 */ 3809 static int 3810 tcp_clean_death(tcp_t *tcp, int err, uint8_t tag) 3811 { 3812 mblk_t *mp; 3813 queue_t *q; 3814 tcp_stack_t *tcps = tcp->tcp_tcps; 3815 sodirect_t *sodp; 3816 3817 TCP_CLD_STAT(tag); 3818 3819 #if TCP_TAG_CLEAN_DEATH 3820 tcp->tcp_cleandeathtag = tag; 3821 #endif 3822 3823 if (tcp->tcp_fused) 3824 tcp_unfuse(tcp); 3825 3826 if (tcp->tcp_linger_tid != 0 && 3827 TCP_TIMER_CANCEL(tcp, tcp->tcp_linger_tid) >= 0) { 3828 tcp_stop_lingering(tcp); 3829 } 3830 3831 ASSERT(tcp != NULL); 3832 ASSERT((tcp->tcp_family == AF_INET && 3833 tcp->tcp_ipversion == IPV4_VERSION) || 3834 (tcp->tcp_family == AF_INET6 && 3835 (tcp->tcp_ipversion == IPV4_VERSION || 3836 tcp->tcp_ipversion == IPV6_VERSION))); 3837 3838 if (TCP_IS_DETACHED(tcp)) { 3839 if (tcp->tcp_hard_binding) { 3840 /* 3841 * Its an eager that we are dealing with. We close the 3842 * eager but in case a conn_ind has already gone to the 3843 * listener, let tcp_accept_finish() send a discon_ind 3844 * to the listener and drop the last reference. If the 3845 * listener doesn't even know about the eager i.e. the 3846 * conn_ind hasn't gone up, blow away the eager and drop 3847 * the last reference as well. If the conn_ind has gone 3848 * up, state should be BOUND. tcp_accept_finish 3849 * will figure out that the connection has received a 3850 * RST and will send a DISCON_IND to the application. 3851 */ 3852 tcp_closei_local(tcp); 3853 if (!tcp->tcp_tconnind_started) { 3854 CONN_DEC_REF(tcp->tcp_connp); 3855 } else { 3856 tcp->tcp_state = TCPS_BOUND; 3857 } 3858 } else { 3859 tcp_close_detached(tcp); 3860 } 3861 return (0); 3862 } 3863 3864 TCP_STAT(tcps, tcp_clean_death_nondetached); 3865 3866 /* 3867 * If T_ORDREL_IND has not been sent yet (done when service routine 3868 * is run) postpone cleaning up the endpoint until service routine 3869 * has sent up the T_ORDREL_IND. Avoid clearing out an existing 3870 * client_errno since tcp_close uses the client_errno field. 3871 */ 3872 if (tcp->tcp_fin_rcvd && !tcp->tcp_ordrel_done) { 3873 if (err != 0) 3874 tcp->tcp_client_errno = err; 3875 3876 tcp->tcp_deferred_clean_death = B_TRUE; 3877 return (-1); 3878 } 3879 3880 /* If sodirect, not anymore */ 3881 SOD_PTR_ENTER(tcp, sodp); 3882 if (sodp != NULL) { 3883 tcp->tcp_sodirect = NULL; 3884 mutex_exit(sodp->sod_lock); 3885 } 3886 3887 q = tcp->tcp_rq; 3888 3889 /* Trash all inbound data */ 3890 flushq(q, FLUSHALL); 3891 3892 /* 3893 * If we are at least part way open and there is error 3894 * (err==0 implies no error) 3895 * notify our client by a T_DISCON_IND. 3896 */ 3897 if ((tcp->tcp_state >= TCPS_SYN_SENT) && err) { 3898 if (tcp->tcp_state >= TCPS_ESTABLISHED && 3899 !TCP_IS_SOCKET(tcp)) { 3900 /* 3901 * Send M_FLUSH according to TPI. Because sockets will 3902 * (and must) ignore FLUSHR we do that only for TPI 3903 * endpoints and sockets in STREAMS mode. 3904 */ 3905 (void) putnextctl1(q, M_FLUSH, FLUSHR); 3906 } 3907 if (tcp->tcp_debug) { 3908 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE|SL_ERROR, 3909 "tcp_clean_death: discon err %d", err); 3910 } 3911 mp = mi_tpi_discon_ind(NULL, err, 0); 3912 if (mp != NULL) { 3913 putnext(q, mp); 3914 } else { 3915 if (tcp->tcp_debug) { 3916 (void) strlog(TCP_MOD_ID, 0, 1, 3917 SL_ERROR|SL_TRACE, 3918 "tcp_clean_death, sending M_ERROR"); 3919 } 3920 (void) putnextctl1(q, M_ERROR, EPROTO); 3921 } 3922 if (tcp->tcp_state <= TCPS_SYN_RCVD) { 3923 /* SYN_SENT or SYN_RCVD */ 3924 BUMP_MIB(&tcps->tcps_mib, tcpAttemptFails); 3925 } else if (tcp->tcp_state <= TCPS_CLOSE_WAIT) { 3926 /* ESTABLISHED or CLOSE_WAIT */ 3927 BUMP_MIB(&tcps->tcps_mib, tcpEstabResets); 3928 } 3929 } 3930 3931 tcp_reinit(tcp); 3932 return (-1); 3933 } 3934 3935 /* 3936 * In case tcp is in the "lingering state" and waits for the SO_LINGER timeout 3937 * to expire, stop the wait and finish the close. 3938 */ 3939 static void 3940 tcp_stop_lingering(tcp_t *tcp) 3941 { 3942 clock_t delta = 0; 3943 tcp_stack_t *tcps = tcp->tcp_tcps; 3944 3945 tcp->tcp_linger_tid = 0; 3946 if (tcp->tcp_state > TCPS_LISTEN) { 3947 tcp_acceptor_hash_remove(tcp); 3948 mutex_enter(&tcp->tcp_non_sq_lock); 3949 if (tcp->tcp_flow_stopped) { 3950 tcp_clrqfull(tcp); 3951 } 3952 mutex_exit(&tcp->tcp_non_sq_lock); 3953 3954 if (tcp->tcp_timer_tid != 0) { 3955 delta = TCP_TIMER_CANCEL(tcp, tcp->tcp_timer_tid); 3956 tcp->tcp_timer_tid = 0; 3957 } 3958 /* 3959 * Need to cancel those timers which will not be used when 3960 * TCP is detached. This has to be done before the tcp_wq 3961 * is set to the global queue. 3962 */ 3963 tcp_timers_stop(tcp); 3964 3965 3966 tcp->tcp_detached = B_TRUE; 3967 ASSERT(tcps->tcps_g_q != NULL); 3968 tcp->tcp_rq = tcps->tcps_g_q; 3969 tcp->tcp_wq = WR(tcps->tcps_g_q); 3970 3971 if (tcp->tcp_state == TCPS_TIME_WAIT) { 3972 tcp_time_wait_append(tcp); 3973 TCP_DBGSTAT(tcps, tcp_detach_time_wait); 3974 goto finish; 3975 } 3976 3977 /* 3978 * If delta is zero the timer event wasn't executed and was 3979 * successfully canceled. In this case we need to restart it 3980 * with the minimal delta possible. 3981 */ 3982 if (delta >= 0) { 3983 tcp->tcp_timer_tid = TCP_TIMER(tcp, tcp_timer, 3984 delta ? delta : 1); 3985 } 3986 } else { 3987 tcp_closei_local(tcp); 3988 CONN_DEC_REF(tcp->tcp_connp); 3989 } 3990 finish: 3991 /* Signal closing thread that it can complete close */ 3992 mutex_enter(&tcp->tcp_closelock); 3993 tcp->tcp_detached = B_TRUE; 3994 ASSERT(tcps->tcps_g_q != NULL); 3995 tcp->tcp_rq = tcps->tcps_g_q; 3996 tcp->tcp_wq = WR(tcps->tcps_g_q); 3997 tcp->tcp_closed = 1; 3998 cv_signal(&tcp->tcp_closecv); 3999 mutex_exit(&tcp->tcp_closelock); 4000 } 4001 4002 /* 4003 * Handle lingering timeouts. This function is called when the SO_LINGER timeout 4004 * expires. 4005 */ 4006 static void 4007 tcp_close_linger_timeout(void *arg) 4008 { 4009 conn_t *connp = (conn_t *)arg; 4010 tcp_t *tcp = connp->conn_tcp; 4011 4012 tcp->tcp_client_errno = ETIMEDOUT; 4013 tcp_stop_lingering(tcp); 4014 } 4015 4016 static int 4017 tcp_close(queue_t *q, int flags) 4018 { 4019 conn_t *connp = Q_TO_CONN(q); 4020 tcp_t *tcp = connp->conn_tcp; 4021 mblk_t *mp = &tcp->tcp_closemp; 4022 boolean_t conn_ioctl_cleanup_reqd = B_FALSE; 4023 mblk_t *bp; 4024 4025 ASSERT(WR(q)->q_next == NULL); 4026 ASSERT(connp->conn_ref >= 2); 4027 4028 /* 4029 * We are being closed as /dev/tcp or /dev/tcp6. 4030 * 4031 * Mark the conn as closing. ill_pending_mp_add will not 4032 * add any mp to the pending mp list, after this conn has 4033 * started closing. Same for sq_pending_mp_add 4034 */ 4035 mutex_enter(&connp->conn_lock); 4036 connp->conn_state_flags |= CONN_CLOSING; 4037 if (connp->conn_oper_pending_ill != NULL) 4038 conn_ioctl_cleanup_reqd = B_TRUE; 4039 CONN_INC_REF_LOCKED(connp); 4040 mutex_exit(&connp->conn_lock); 4041 tcp->tcp_closeflags = (uint8_t)flags; 4042 ASSERT(connp->conn_ref >= 3); 4043 4044 /* 4045 * tcp_closemp_used is used below without any protection of a lock 4046 * as we don't expect any one else to use it concurrently at this 4047 * point otherwise it would be a major defect. 4048 */ 4049 4050 if (mp->b_prev == NULL) 4051 tcp->tcp_closemp_used = B_TRUE; 4052 else 4053 cmn_err(CE_PANIC, "tcp_close: concurrent use of tcp_closemp: " 4054 "connp %p tcp %p\n", (void *)connp, (void *)tcp); 4055 4056 TCP_DEBUG_GETPCSTACK(tcp->tcmp_stk, 15); 4057 4058 (*tcp_squeue_close_proc)(connp->conn_sqp, mp, 4059 tcp_close_output, connp, SQTAG_IP_TCP_CLOSE); 4060 4061 mutex_enter(&tcp->tcp_closelock); 4062 while (!tcp->tcp_closed) { 4063 if (!cv_wait_sig(&tcp->tcp_closecv, &tcp->tcp_closelock)) { 4064 /* 4065 * The cv_wait_sig() was interrupted. We now do the 4066 * following: 4067 * 4068 * 1) If the endpoint was lingering, we allow this 4069 * to be interrupted by cancelling the linger timeout 4070 * and closing normally. 4071 * 4072 * 2) Revert to calling cv_wait() 4073 * 4074 * We revert to using cv_wait() to avoid an 4075 * infinite loop which can occur if the calling 4076 * thread is higher priority than the squeue worker 4077 * thread and is bound to the same cpu. 4078 */ 4079 if (tcp->tcp_linger && tcp->tcp_lingertime > 0) { 4080 mutex_exit(&tcp->tcp_closelock); 4081 /* Entering squeue, bump ref count. */ 4082 CONN_INC_REF(connp); 4083 bp = allocb_wait(0, BPRI_HI, STR_NOSIG, NULL); 4084 squeue_enter(connp->conn_sqp, bp, 4085 tcp_linger_interrupted, connp, 4086 SQTAG_IP_TCP_CLOSE); 4087 mutex_enter(&tcp->tcp_closelock); 4088 } 4089 break; 4090 } 4091 } 4092 while (!tcp->tcp_closed) 4093 cv_wait(&tcp->tcp_closecv, &tcp->tcp_closelock); 4094 mutex_exit(&tcp->tcp_closelock); 4095 4096 /* 4097 * In the case of listener streams that have eagers in the q or q0 4098 * we wait for the eagers to drop their reference to us. tcp_rq and 4099 * tcp_wq of the eagers point to our queues. By waiting for the 4100 * refcnt to drop to 1, we are sure that the eagers have cleaned 4101 * up their queue pointers and also dropped their references to us. 4102 */ 4103 if (tcp->tcp_wait_for_eagers) { 4104 mutex_enter(&connp->conn_lock); 4105 while (connp->conn_ref != 1) { 4106 cv_wait(&connp->conn_cv, &connp->conn_lock); 4107 } 4108 mutex_exit(&connp->conn_lock); 4109 } 4110 /* 4111 * ioctl cleanup. The mp is queued in the 4112 * ill_pending_mp or in the sq_pending_mp. 4113 */ 4114 if (conn_ioctl_cleanup_reqd) 4115 conn_ioctl_cleanup(connp); 4116 4117 qprocsoff(q); 4118 inet_minor_free(connp->conn_minor_arena, connp->conn_dev); 4119 4120 tcp->tcp_cpid = -1; 4121 4122 /* 4123 * Drop IP's reference on the conn. This is the last reference 4124 * on the connp if the state was less than established. If the 4125 * connection has gone into timewait state, then we will have 4126 * one ref for the TCP and one more ref (total of two) for the 4127 * classifier connected hash list (a timewait connections stays 4128 * in connected hash till closed). 4129 * 4130 * We can't assert the references because there might be other 4131 * transient reference places because of some walkers or queued 4132 * packets in squeue for the timewait state. 4133 */ 4134 CONN_DEC_REF(connp); 4135 q->q_ptr = WR(q)->q_ptr = NULL; 4136 return (0); 4137 } 4138 4139 static int 4140 tcpclose_accept(queue_t *q) 4141 { 4142 vmem_t *minor_arena; 4143 dev_t conn_dev; 4144 4145 ASSERT(WR(q)->q_qinfo == &tcp_acceptor_winit); 4146 4147 /* 4148 * We had opened an acceptor STREAM for sockfs which is 4149 * now being closed due to some error. 4150 */ 4151 qprocsoff(q); 4152 4153 minor_arena = (vmem_t *)WR(q)->q_ptr; 4154 conn_dev = (dev_t)RD(q)->q_ptr; 4155 ASSERT(minor_arena != NULL); 4156 ASSERT(conn_dev != 0); 4157 inet_minor_free(minor_arena, conn_dev); 4158 q->q_ptr = WR(q)->q_ptr = NULL; 4159 return (0); 4160 } 4161 4162 /* 4163 * Called by tcp_close() routine via squeue when lingering is 4164 * interrupted by a signal. 4165 */ 4166 4167 /* ARGSUSED */ 4168 static void 4169 tcp_linger_interrupted(void *arg, mblk_t *mp, void *arg2) 4170 { 4171 conn_t *connp = (conn_t *)arg; 4172 tcp_t *tcp = connp->conn_tcp; 4173 4174 freeb(mp); 4175 if (tcp->tcp_linger_tid != 0 && 4176 TCP_TIMER_CANCEL(tcp, tcp->tcp_linger_tid) >= 0) { 4177 tcp_stop_lingering(tcp); 4178 tcp->tcp_client_errno = EINTR; 4179 } 4180 } 4181 4182 /* 4183 * Called by streams close routine via squeues when our client blows off her 4184 * descriptor, we take this to mean: "close the stream state NOW, close the tcp 4185 * connection politely" When SO_LINGER is set (with a non-zero linger time and 4186 * it is not a nonblocking socket) then this routine sleeps until the FIN is 4187 * acked. 4188 * 4189 * NOTE: tcp_close potentially returns error when lingering. 4190 * However, the stream head currently does not pass these errors 4191 * to the application. 4.4BSD only returns EINTR and EWOULDBLOCK 4192 * errors to the application (from tsleep()) and not errors 4193 * like ECONNRESET caused by receiving a reset packet. 4194 */ 4195 4196 /* ARGSUSED */ 4197 static void 4198 tcp_close_output(void *arg, mblk_t *mp, void *arg2) 4199 { 4200 char *msg; 4201 conn_t *connp = (conn_t *)arg; 4202 tcp_t *tcp = connp->conn_tcp; 4203 clock_t delta = 0; 4204 tcp_stack_t *tcps = tcp->tcp_tcps; 4205 4206 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) || 4207 (connp->conn_fanout == NULL && connp->conn_ref >= 3)); 4208 4209 /* Cancel any pending timeout */ 4210 if (tcp->tcp_ordrelid != 0) { 4211 if (tcp->tcp_timeout) { 4212 (void) TCP_TIMER_CANCEL(tcp, tcp->tcp_ordrelid); 4213 } 4214 tcp->tcp_ordrelid = 0; 4215 tcp->tcp_timeout = B_FALSE; 4216 } 4217 4218 mutex_enter(&tcp->tcp_eager_lock); 4219 if (tcp->tcp_conn_req_cnt_q0 != 0 || tcp->tcp_conn_req_cnt_q != 0) { 4220 /* Cleanup for listener */ 4221 tcp_eager_cleanup(tcp, 0); 4222 tcp->tcp_wait_for_eagers = 1; 4223 } 4224 mutex_exit(&tcp->tcp_eager_lock); 4225 4226 connp->conn_mdt_ok = B_FALSE; 4227 tcp->tcp_mdt = B_FALSE; 4228 4229 connp->conn_lso_ok = B_FALSE; 4230 tcp->tcp_lso = B_FALSE; 4231 4232 msg = NULL; 4233 switch (tcp->tcp_state) { 4234 case TCPS_CLOSED: 4235 case TCPS_IDLE: 4236 case TCPS_BOUND: 4237 case TCPS_LISTEN: 4238 break; 4239 case TCPS_SYN_SENT: 4240 msg = "tcp_close, during connect"; 4241 break; 4242 case TCPS_SYN_RCVD: 4243 /* 4244 * Close during the connect 3-way handshake 4245 * but here there may or may not be pending data 4246 * already on queue. Process almost same as in 4247 * the ESTABLISHED state. 4248 */ 4249 /* FALLTHRU */ 4250 default: 4251 if (tcp->tcp_sodirect != NULL) { 4252 /* Ok, no more sodirect */ 4253 tcp->tcp_sodirect = NULL; 4254 } 4255 4256 if (tcp->tcp_fused) 4257 tcp_unfuse(tcp); 4258 4259 /* 4260 * If SO_LINGER has set a zero linger time, abort the 4261 * connection with a reset. 4262 */ 4263 if (tcp->tcp_linger && tcp->tcp_lingertime == 0) { 4264 msg = "tcp_close, zero lingertime"; 4265 break; 4266 } 4267 4268 ASSERT(tcp->tcp_hard_bound || tcp->tcp_hard_binding); 4269 /* 4270 * Abort connection if there is unread data queued. 4271 */ 4272 if (tcp->tcp_rcv_list || tcp->tcp_reass_head) { 4273 msg = "tcp_close, unread data"; 4274 break; 4275 } 4276 /* 4277 * tcp_hard_bound is now cleared thus all packets go through 4278 * tcp_lookup. This fact is used by tcp_detach below. 4279 * 4280 * We have done a qwait() above which could have possibly 4281 * drained more messages in turn causing transition to a 4282 * different state. Check whether we have to do the rest 4283 * of the processing or not. 4284 */ 4285 if (tcp->tcp_state <= TCPS_LISTEN) 4286 break; 4287 4288 /* 4289 * Transmit the FIN before detaching the tcp_t. 4290 * After tcp_detach returns this queue/perimeter 4291 * no longer owns the tcp_t thus others can modify it. 4292 */ 4293 (void) tcp_xmit_end(tcp); 4294 4295 /* 4296 * If lingering on close then wait until the fin is acked, 4297 * the SO_LINGER time passes, or a reset is sent/received. 4298 */ 4299 if (tcp->tcp_linger && tcp->tcp_lingertime > 0 && 4300 !(tcp->tcp_fin_acked) && 4301 tcp->tcp_state >= TCPS_ESTABLISHED) { 4302 if (tcp->tcp_closeflags & (FNDELAY|FNONBLOCK)) { 4303 tcp->tcp_client_errno = EWOULDBLOCK; 4304 } else if (tcp->tcp_client_errno == 0) { 4305 4306 ASSERT(tcp->tcp_linger_tid == 0); 4307 4308 tcp->tcp_linger_tid = TCP_TIMER(tcp, 4309 tcp_close_linger_timeout, 4310 tcp->tcp_lingertime * hz); 4311 4312 /* tcp_close_linger_timeout will finish close */ 4313 if (tcp->tcp_linger_tid == 0) 4314 tcp->tcp_client_errno = ENOSR; 4315 else 4316 return; 4317 } 4318 4319 /* 4320 * Check if we need to detach or just close 4321 * the instance. 4322 */ 4323 if (tcp->tcp_state <= TCPS_LISTEN) 4324 break; 4325 } 4326 4327 /* 4328 * Make sure that no other thread will access the tcp_rq of 4329 * this instance (through lookups etc.) as tcp_rq will go 4330 * away shortly. 4331 */ 4332 tcp_acceptor_hash_remove(tcp); 4333 4334 mutex_enter(&tcp->tcp_non_sq_lock); 4335 if (tcp->tcp_flow_stopped) { 4336 tcp_clrqfull(tcp); 4337 } 4338 mutex_exit(&tcp->tcp_non_sq_lock); 4339 4340 if (tcp->tcp_timer_tid != 0) { 4341 delta = TCP_TIMER_CANCEL(tcp, tcp->tcp_timer_tid); 4342 tcp->tcp_timer_tid = 0; 4343 } 4344 /* 4345 * Need to cancel those timers which will not be used when 4346 * TCP is detached. This has to be done before the tcp_wq 4347 * is set to the global queue. 4348 */ 4349 tcp_timers_stop(tcp); 4350 4351 tcp->tcp_detached = B_TRUE; 4352 if (tcp->tcp_state == TCPS_TIME_WAIT) { 4353 tcp_time_wait_append(tcp); 4354 TCP_DBGSTAT(tcps, tcp_detach_time_wait); 4355 ASSERT(connp->conn_ref >= 3); 4356 goto finish; 4357 } 4358 4359 /* 4360 * If delta is zero the timer event wasn't executed and was 4361 * successfully canceled. In this case we need to restart it 4362 * with the minimal delta possible. 4363 */ 4364 if (delta >= 0) 4365 tcp->tcp_timer_tid = TCP_TIMER(tcp, tcp_timer, 4366 delta ? delta : 1); 4367 4368 ASSERT(connp->conn_ref >= 3); 4369 goto finish; 4370 } 4371 4372 /* Detach did not complete. Still need to remove q from stream. */ 4373 if (msg) { 4374 if (tcp->tcp_state == TCPS_ESTABLISHED || 4375 tcp->tcp_state == TCPS_CLOSE_WAIT) 4376 BUMP_MIB(&tcps->tcps_mib, tcpEstabResets); 4377 if (tcp->tcp_state == TCPS_SYN_SENT || 4378 tcp->tcp_state == TCPS_SYN_RCVD) 4379 BUMP_MIB(&tcps->tcps_mib, tcpAttemptFails); 4380 tcp_xmit_ctl(msg, tcp, tcp->tcp_snxt, 0, TH_RST); 4381 } 4382 4383 tcp_closei_local(tcp); 4384 CONN_DEC_REF(connp); 4385 ASSERT(connp->conn_ref >= 2); 4386 4387 finish: 4388 /* 4389 * Although packets are always processed on the correct 4390 * tcp's perimeter and access is serialized via squeue's, 4391 * IP still needs a queue when sending packets in time_wait 4392 * state so use WR(tcps_g_q) till ip_output() can be 4393 * changed to deal with just connp. For read side, we 4394 * could have set tcp_rq to NULL but there are some cases 4395 * in tcp_rput_data() from early days of this code which 4396 * do a putnext without checking if tcp is closed. Those 4397 * need to be identified before both tcp_rq and tcp_wq 4398 * can be set to NULL and tcps_g_q can disappear forever. 4399 */ 4400 mutex_enter(&tcp->tcp_closelock); 4401 /* 4402 * Don't change the queues in the case of a listener that has 4403 * eagers in its q or q0. It could surprise the eagers. 4404 * Instead wait for the eagers outside the squeue. 4405 */ 4406 if (!tcp->tcp_wait_for_eagers) { 4407 tcp->tcp_detached = B_TRUE; 4408 /* 4409 * When default queue is closing we set tcps_g_q to NULL 4410 * after the close is done. 4411 */ 4412 ASSERT(tcps->tcps_g_q != NULL); 4413 tcp->tcp_rq = tcps->tcps_g_q; 4414 tcp->tcp_wq = WR(tcps->tcps_g_q); 4415 } 4416 4417 /* Signal tcp_close() to finish closing. */ 4418 tcp->tcp_closed = 1; 4419 cv_signal(&tcp->tcp_closecv); 4420 mutex_exit(&tcp->tcp_closelock); 4421 } 4422 4423 4424 /* 4425 * Clean up the b_next and b_prev fields of every mblk pointed at by *mpp. 4426 * Some stream heads get upset if they see these later on as anything but NULL. 4427 */ 4428 static void 4429 tcp_close_mpp(mblk_t **mpp) 4430 { 4431 mblk_t *mp; 4432 4433 if ((mp = *mpp) != NULL) { 4434 do { 4435 mp->b_next = NULL; 4436 mp->b_prev = NULL; 4437 } while ((mp = mp->b_cont) != NULL); 4438 4439 mp = *mpp; 4440 *mpp = NULL; 4441 freemsg(mp); 4442 } 4443 } 4444 4445 /* Do detached close. */ 4446 static void 4447 tcp_close_detached(tcp_t *tcp) 4448 { 4449 if (tcp->tcp_fused) 4450 tcp_unfuse(tcp); 4451 4452 /* 4453 * Clustering code serializes TCP disconnect callbacks and 4454 * cluster tcp list walks by blocking a TCP disconnect callback 4455 * if a cluster tcp list walk is in progress. This ensures 4456 * accurate accounting of TCPs in the cluster code even though 4457 * the TCP list walk itself is not atomic. 4458 */ 4459 tcp_closei_local(tcp); 4460 CONN_DEC_REF(tcp->tcp_connp); 4461 } 4462 4463 /* 4464 * Stop all TCP timers, and free the timer mblks if requested. 4465 */ 4466 void 4467 tcp_timers_stop(tcp_t *tcp) 4468 { 4469 if (tcp->tcp_timer_tid != 0) { 4470 (void) TCP_TIMER_CANCEL(tcp, tcp->tcp_timer_tid); 4471 tcp->tcp_timer_tid = 0; 4472 } 4473 if (tcp->tcp_ka_tid != 0) { 4474 (void) TCP_TIMER_CANCEL(tcp, tcp->tcp_ka_tid); 4475 tcp->tcp_ka_tid = 0; 4476 } 4477 if (tcp->tcp_ack_tid != 0) { 4478 (void) TCP_TIMER_CANCEL(tcp, tcp->tcp_ack_tid); 4479 tcp->tcp_ack_tid = 0; 4480 } 4481 if (tcp->tcp_push_tid != 0) { 4482 (void) TCP_TIMER_CANCEL(tcp, tcp->tcp_push_tid); 4483 tcp->tcp_push_tid = 0; 4484 } 4485 } 4486 4487 /* 4488 * The tcp_t is going away. Remove it from all lists and set it 4489 * to TCPS_CLOSED. The freeing up of memory is deferred until 4490 * tcp_inactive. This is needed since a thread in tcp_rput might have 4491 * done a CONN_INC_REF on this structure before it was removed from the 4492 * hashes. 4493 */ 4494 static void 4495 tcp_closei_local(tcp_t *tcp) 4496 { 4497 ire_t *ire; 4498 conn_t *connp = tcp->tcp_connp; 4499 tcp_stack_t *tcps = tcp->tcp_tcps; 4500 4501 if (!TCP_IS_SOCKET(tcp)) 4502 tcp_acceptor_hash_remove(tcp); 4503 4504 UPDATE_MIB(&tcps->tcps_mib, tcpHCInSegs, tcp->tcp_ibsegs); 4505 tcp->tcp_ibsegs = 0; 4506 UPDATE_MIB(&tcps->tcps_mib, tcpHCOutSegs, tcp->tcp_obsegs); 4507 tcp->tcp_obsegs = 0; 4508 4509 /* 4510 * If we are an eager connection hanging off a listener that 4511 * hasn't formally accepted the connection yet, get off his 4512 * list and blow off any data that we have accumulated. 4513 */ 4514 if (tcp->tcp_listener != NULL) { 4515 tcp_t *listener = tcp->tcp_listener; 4516 mutex_enter(&listener->tcp_eager_lock); 4517 /* 4518 * tcp_tconnind_started == B_TRUE means that the 4519 * conn_ind has already gone to listener. At 4520 * this point, eager will be closed but we 4521 * leave it in listeners eager list so that 4522 * if listener decides to close without doing 4523 * accept, we can clean this up. In tcp_wput_accept 4524 * we take care of the case of accept on closed 4525 * eager. 4526 */ 4527 if (!tcp->tcp_tconnind_started) { 4528 tcp_eager_unlink(tcp); 4529 mutex_exit(&listener->tcp_eager_lock); 4530 /* 4531 * We don't want to have any pointers to the 4532 * listener queue, after we have released our 4533 * reference on the listener 4534 */ 4535 ASSERT(tcps->tcps_g_q != NULL); 4536 tcp->tcp_rq = tcps->tcps_g_q; 4537 tcp->tcp_wq = WR(tcps->tcps_g_q); 4538 CONN_DEC_REF(listener->tcp_connp); 4539 } else { 4540 mutex_exit(&listener->tcp_eager_lock); 4541 } 4542 } 4543 4544 /* Stop all the timers */ 4545 tcp_timers_stop(tcp); 4546 4547 if (tcp->tcp_state == TCPS_LISTEN) { 4548 if (tcp->tcp_ip_addr_cache) { 4549 kmem_free((void *)tcp->tcp_ip_addr_cache, 4550 IP_ADDR_CACHE_SIZE * sizeof (ipaddr_t)); 4551 tcp->tcp_ip_addr_cache = NULL; 4552 } 4553 } 4554 mutex_enter(&tcp->tcp_non_sq_lock); 4555 if (tcp->tcp_flow_stopped) 4556 tcp_clrqfull(tcp); 4557 mutex_exit(&tcp->tcp_non_sq_lock); 4558 4559 tcp_bind_hash_remove(tcp); 4560 /* 4561 * If the tcp_time_wait_collector (which runs outside the squeue) 4562 * is trying to remove this tcp from the time wait list, we will 4563 * block in tcp_time_wait_remove while trying to acquire the 4564 * tcp_time_wait_lock. The logic in tcp_time_wait_collector also 4565 * requires the ipcl_hash_remove to be ordered after the 4566 * tcp_time_wait_remove for the refcnt checks to work correctly. 4567 */ 4568 if (tcp->tcp_state == TCPS_TIME_WAIT) 4569 (void) tcp_time_wait_remove(tcp, NULL); 4570 CL_INET_DISCONNECT(tcp); 4571 ipcl_hash_remove(connp); 4572 4573 /* 4574 * Delete the cached ire in conn_ire_cache and also mark 4575 * the conn as CONDEMNED 4576 */ 4577 mutex_enter(&connp->conn_lock); 4578 connp->conn_state_flags |= CONN_CONDEMNED; 4579 ire = connp->conn_ire_cache; 4580 connp->conn_ire_cache = NULL; 4581 mutex_exit(&connp->conn_lock); 4582 if (ire != NULL) 4583 IRE_REFRELE_NOTR(ire); 4584 4585 /* Need to cleanup any pending ioctls */ 4586 ASSERT(tcp->tcp_time_wait_next == NULL); 4587 ASSERT(tcp->tcp_time_wait_prev == NULL); 4588 ASSERT(tcp->tcp_time_wait_expire == 0); 4589 tcp->tcp_state = TCPS_CLOSED; 4590 4591 /* Release any SSL context */ 4592 if (tcp->tcp_kssl_ent != NULL) { 4593 kssl_release_ent(tcp->tcp_kssl_ent, NULL, KSSL_NO_PROXY); 4594 tcp->tcp_kssl_ent = NULL; 4595 } 4596 if (tcp->tcp_kssl_ctx != NULL) { 4597 kssl_release_ctx(tcp->tcp_kssl_ctx); 4598 tcp->tcp_kssl_ctx = NULL; 4599 } 4600 tcp->tcp_kssl_pending = B_FALSE; 4601 4602 tcp_ipsec_cleanup(tcp); 4603 } 4604 4605 /* 4606 * tcp is dying (called from ipcl_conn_destroy and error cases). 4607 * Free the tcp_t in either case. 4608 */ 4609 void 4610 tcp_free(tcp_t *tcp) 4611 { 4612 mblk_t *mp; 4613 ip6_pkt_t *ipp; 4614 4615 ASSERT(tcp != NULL); 4616 ASSERT(tcp->tcp_ptpahn == NULL && tcp->tcp_acceptor_hash == NULL); 4617 4618 tcp->tcp_rq = NULL; 4619 tcp->tcp_wq = NULL; 4620 4621 tcp_close_mpp(&tcp->tcp_xmit_head); 4622 tcp_close_mpp(&tcp->tcp_reass_head); 4623 if (tcp->tcp_rcv_list != NULL) { 4624 /* Free b_next chain */ 4625 tcp_close_mpp(&tcp->tcp_rcv_list); 4626 } 4627 if ((mp = tcp->tcp_urp_mp) != NULL) { 4628 freemsg(mp); 4629 } 4630 if ((mp = tcp->tcp_urp_mark_mp) != NULL) { 4631 freemsg(mp); 4632 } 4633 4634 if (tcp->tcp_fused_sigurg_mp != NULL) { 4635 freeb(tcp->tcp_fused_sigurg_mp); 4636 tcp->tcp_fused_sigurg_mp = NULL; 4637 } 4638 4639 if (tcp->tcp_sack_info != NULL) { 4640 if (tcp->tcp_notsack_list != NULL) { 4641 TCP_NOTSACK_REMOVE_ALL(tcp->tcp_notsack_list); 4642 } 4643 bzero(tcp->tcp_sack_info, sizeof (tcp_sack_info_t)); 4644 } 4645 4646 if (tcp->tcp_hopopts != NULL) { 4647 mi_free(tcp->tcp_hopopts); 4648 tcp->tcp_hopopts = NULL; 4649 tcp->tcp_hopoptslen = 0; 4650 } 4651 ASSERT(tcp->tcp_hopoptslen == 0); 4652 if (tcp->tcp_dstopts != NULL) { 4653 mi_free(tcp->tcp_dstopts); 4654 tcp->tcp_dstopts = NULL; 4655 tcp->tcp_dstoptslen = 0; 4656 } 4657 ASSERT(tcp->tcp_dstoptslen == 0); 4658 if (tcp->tcp_rtdstopts != NULL) { 4659 mi_free(tcp->tcp_rtdstopts); 4660 tcp->tcp_rtdstopts = NULL; 4661 tcp->tcp_rtdstoptslen = 0; 4662 } 4663 ASSERT(tcp->tcp_rtdstoptslen == 0); 4664 if (tcp->tcp_rthdr != NULL) { 4665 mi_free(tcp->tcp_rthdr); 4666 tcp->tcp_rthdr = NULL; 4667 tcp->tcp_rthdrlen = 0; 4668 } 4669 ASSERT(tcp->tcp_rthdrlen == 0); 4670 4671 ipp = &tcp->tcp_sticky_ipp; 4672 if (ipp->ipp_fields & (IPPF_HOPOPTS | IPPF_RTDSTOPTS | IPPF_DSTOPTS | 4673 IPPF_RTHDR)) 4674 ip6_pkt_free(ipp); 4675 4676 /* 4677 * Free memory associated with the tcp/ip header template. 4678 */ 4679 4680 if (tcp->tcp_iphc != NULL) 4681 bzero(tcp->tcp_iphc, tcp->tcp_iphc_len); 4682 4683 /* 4684 * Following is really a blowing away a union. 4685 * It happens to have exactly two members of identical size 4686 * the following code is enough. 4687 */ 4688 tcp_close_mpp(&tcp->tcp_conn.tcp_eager_conn_ind); 4689 } 4690 4691 4692 /* 4693 * Put a connection confirmation message upstream built from the 4694 * address information within 'iph' and 'tcph'. Report our success or failure. 4695 */ 4696 static boolean_t 4697 tcp_conn_con(tcp_t *tcp, uchar_t *iphdr, tcph_t *tcph, mblk_t *idmp, 4698 mblk_t **defermp) 4699 { 4700 sin_t sin; 4701 sin6_t sin6; 4702 mblk_t *mp; 4703 char *optp = NULL; 4704 int optlen = 0; 4705 cred_t *cr; 4706 4707 if (defermp != NULL) 4708 *defermp = NULL; 4709 4710 if (tcp->tcp_conn.tcp_opts_conn_req != NULL) { 4711 /* 4712 * Return in T_CONN_CON results of option negotiation through 4713 * the T_CONN_REQ. Note: If there is an real end-to-end option 4714 * negotiation, then what is received from remote end needs 4715 * to be taken into account but there is no such thing (yet?) 4716 * in our TCP/IP. 4717 * Note: We do not use mi_offset_param() here as 4718 * tcp_opts_conn_req contents do not directly come from 4719 * an application and are either generated in kernel or 4720 * from user input that was already verified. 4721 */ 4722 mp = tcp->tcp_conn.tcp_opts_conn_req; 4723 optp = (char *)(mp->b_rptr + 4724 ((struct T_conn_req *)mp->b_rptr)->OPT_offset); 4725 optlen = (int) 4726 ((struct T_conn_req *)mp->b_rptr)->OPT_length; 4727 } 4728 4729 if (IPH_HDR_VERSION(iphdr) == IPV4_VERSION) { 4730 ipha_t *ipha = (ipha_t *)iphdr; 4731 4732 /* packet is IPv4 */ 4733 if (tcp->tcp_family == AF_INET) { 4734 sin = sin_null; 4735 sin.sin_addr.s_addr = ipha->ipha_src; 4736 sin.sin_port = *(uint16_t *)tcph->th_lport; 4737 sin.sin_family = AF_INET; 4738 mp = mi_tpi_conn_con(NULL, (char *)&sin, 4739 (int)sizeof (sin_t), optp, optlen); 4740 } else { 4741 sin6 = sin6_null; 4742 IN6_IPADDR_TO_V4MAPPED(ipha->ipha_src, &sin6.sin6_addr); 4743 sin6.sin6_port = *(uint16_t *)tcph->th_lport; 4744 sin6.sin6_family = AF_INET6; 4745 mp = mi_tpi_conn_con(NULL, (char *)&sin6, 4746 (int)sizeof (sin6_t), optp, optlen); 4747 4748 } 4749 } else { 4750 ip6_t *ip6h = (ip6_t *)iphdr; 4751 4752 ASSERT(IPH_HDR_VERSION(iphdr) == IPV6_VERSION); 4753 ASSERT(tcp->tcp_family == AF_INET6); 4754 sin6 = sin6_null; 4755 sin6.sin6_addr = ip6h->ip6_src; 4756 sin6.sin6_port = *(uint16_t *)tcph->th_lport; 4757 sin6.sin6_family = AF_INET6; 4758 sin6.sin6_flowinfo = ip6h->ip6_vcf & ~IPV6_VERS_AND_FLOW_MASK; 4759 mp = mi_tpi_conn_con(NULL, (char *)&sin6, 4760 (int)sizeof (sin6_t), optp, optlen); 4761 } 4762 4763 if (!mp) 4764 return (B_FALSE); 4765 4766 if ((cr = DB_CRED(idmp)) != NULL) { 4767 mblk_setcred(mp, cr); 4768 DB_CPID(mp) = DB_CPID(idmp); 4769 } 4770 4771 if (defermp == NULL) 4772 putnext(tcp->tcp_rq, mp); 4773 else 4774 *defermp = mp; 4775 4776 if (tcp->tcp_conn.tcp_opts_conn_req != NULL) 4777 tcp_close_mpp(&tcp->tcp_conn.tcp_opts_conn_req); 4778 return (B_TRUE); 4779 } 4780 4781 /* 4782 * Defense for the SYN attack - 4783 * 1. When q0 is full, drop from the tail (tcp_eager_prev_drop_q0) the oldest 4784 * one from the list of droppable eagers. This list is a subset of q0. 4785 * see comments before the definition of MAKE_DROPPABLE(). 4786 * 2. Don't drop a SYN request before its first timeout. This gives every 4787 * request at least til the first timeout to complete its 3-way handshake. 4788 * 3. Maintain tcp_syn_rcvd_timeout as an accurate count of how many 4789 * requests currently on the queue that has timed out. This will be used 4790 * as an indicator of whether an attack is under way, so that appropriate 4791 * actions can be taken. (It's incremented in tcp_timer() and decremented 4792 * either when eager goes into ESTABLISHED, or gets freed up.) 4793 * 4. The current threshold is - # of timeout > q0len/4 => SYN alert on 4794 * # of timeout drops back to <= q0len/32 => SYN alert off 4795 */ 4796 static boolean_t 4797 tcp_drop_q0(tcp_t *tcp) 4798 { 4799 tcp_t *eager; 4800 mblk_t *mp; 4801 tcp_stack_t *tcps = tcp->tcp_tcps; 4802 4803 ASSERT(MUTEX_HELD(&tcp->tcp_eager_lock)); 4804 ASSERT(tcp->tcp_eager_next_q0 != tcp->tcp_eager_prev_q0); 4805 4806 /* Pick oldest eager from the list of droppable eagers */ 4807 eager = tcp->tcp_eager_prev_drop_q0; 4808 4809 /* If list is empty. return B_FALSE */ 4810 if (eager == tcp) { 4811 return (B_FALSE); 4812 } 4813 4814 /* If allocated, the mp will be freed in tcp_clean_death_wrapper() */ 4815 if ((mp = allocb(0, BPRI_HI)) == NULL) 4816 return (B_FALSE); 4817 4818 /* 4819 * Take this eager out from the list of droppable eagers since we are 4820 * going to drop it. 4821 */ 4822 MAKE_UNDROPPABLE(eager); 4823 4824 if (tcp->tcp_debug) { 4825 (void) strlog(TCP_MOD_ID, 0, 3, SL_TRACE, 4826 "tcp_drop_q0: listen half-open queue (max=%d) overflow" 4827 " (%d pending) on %s, drop one", tcps->tcps_conn_req_max_q0, 4828 tcp->tcp_conn_req_cnt_q0, 4829 tcp_display(tcp, NULL, DISP_PORT_ONLY)); 4830 } 4831 4832 BUMP_MIB(&tcps->tcps_mib, tcpHalfOpenDrop); 4833 4834 /* Put a reference on the conn as we are enqueueing it in the sqeue */ 4835 CONN_INC_REF(eager->tcp_connp); 4836 4837 /* Mark the IRE created for this SYN request temporary */ 4838 tcp_ip_ire_mark_advice(eager); 4839 squeue_fill(eager->tcp_connp->conn_sqp, mp, 4840 tcp_clean_death_wrapper, eager->tcp_connp, SQTAG_TCP_DROP_Q0); 4841 4842 return (B_TRUE); 4843 } 4844 4845 int 4846 tcp_conn_create_v6(conn_t *lconnp, conn_t *connp, mblk_t *mp, 4847 tcph_t *tcph, uint_t ipvers, mblk_t *idmp) 4848 { 4849 tcp_t *ltcp = lconnp->conn_tcp; 4850 tcp_t *tcp = connp->conn_tcp; 4851 mblk_t *tpi_mp; 4852 ipha_t *ipha; 4853 ip6_t *ip6h; 4854 sin6_t sin6; 4855 in6_addr_t v6dst; 4856 int err; 4857 int ifindex = 0; 4858 cred_t *cr; 4859 tcp_stack_t *tcps = tcp->tcp_tcps; 4860 4861 if (ipvers == IPV4_VERSION) { 4862 ipha = (ipha_t *)mp->b_rptr; 4863 4864 connp->conn_send = ip_output; 4865 connp->conn_recv = tcp_input; 4866 4867 IN6_IPADDR_TO_V4MAPPED(ipha->ipha_dst, &connp->conn_srcv6); 4868 IN6_IPADDR_TO_V4MAPPED(ipha->ipha_src, &connp->conn_remv6); 4869 4870 sin6 = sin6_null; 4871 IN6_IPADDR_TO_V4MAPPED(ipha->ipha_src, &sin6.sin6_addr); 4872 IN6_IPADDR_TO_V4MAPPED(ipha->ipha_dst, &v6dst); 4873 sin6.sin6_port = *(uint16_t *)tcph->th_lport; 4874 sin6.sin6_family = AF_INET6; 4875 sin6.__sin6_src_id = ip_srcid_find_addr(&v6dst, 4876 lconnp->conn_zoneid, tcps->tcps_netstack); 4877 if (tcp->tcp_recvdstaddr) { 4878 sin6_t sin6d; 4879 4880 sin6d = sin6_null; 4881 IN6_IPADDR_TO_V4MAPPED(ipha->ipha_dst, 4882 &sin6d.sin6_addr); 4883 sin6d.sin6_port = *(uint16_t *)tcph->th_fport; 4884 sin6d.sin6_family = AF_INET; 4885 tpi_mp = mi_tpi_extconn_ind(NULL, 4886 (char *)&sin6d, sizeof (sin6_t), 4887 (char *)&tcp, 4888 (t_scalar_t)sizeof (intptr_t), 4889 (char *)&sin6d, sizeof (sin6_t), 4890 (t_scalar_t)ltcp->tcp_conn_req_seqnum); 4891 } else { 4892 tpi_mp = mi_tpi_conn_ind(NULL, 4893 (char *)&sin6, sizeof (sin6_t), 4894 (char *)&tcp, (t_scalar_t)sizeof (intptr_t), 4895 (t_scalar_t)ltcp->tcp_conn_req_seqnum); 4896 } 4897 } else { 4898 ip6h = (ip6_t *)mp->b_rptr; 4899 4900 connp->conn_send = ip_output_v6; 4901 connp->conn_recv = tcp_input; 4902 4903 connp->conn_srcv6 = ip6h->ip6_dst; 4904 connp->conn_remv6 = ip6h->ip6_src; 4905 4906 /* db_cksumstuff is set at ip_fanout_tcp_v6 */ 4907 ifindex = (int)DB_CKSUMSTUFF(mp); 4908 DB_CKSUMSTUFF(mp) = 0; 4909 4910 sin6 = sin6_null; 4911 sin6.sin6_addr = ip6h->ip6_src; 4912 sin6.sin6_port = *(uint16_t *)tcph->th_lport; 4913 sin6.sin6_family = AF_INET6; 4914 sin6.sin6_flowinfo = ip6h->ip6_vcf & ~IPV6_VERS_AND_FLOW_MASK; 4915 sin6.__sin6_src_id = ip_srcid_find_addr(&ip6h->ip6_dst, 4916 lconnp->conn_zoneid, tcps->tcps_netstack); 4917 4918 if (IN6_IS_ADDR_LINKSCOPE(&ip6h->ip6_src)) { 4919 /* Pass up the scope_id of remote addr */ 4920 sin6.sin6_scope_id = ifindex; 4921 } else { 4922 sin6.sin6_scope_id = 0; 4923 } 4924 if (tcp->tcp_recvdstaddr) { 4925 sin6_t sin6d; 4926 4927 sin6d = sin6_null; 4928 sin6.sin6_addr = ip6h->ip6_dst; 4929 sin6d.sin6_port = *(uint16_t *)tcph->th_fport; 4930 sin6d.sin6_family = AF_INET; 4931 tpi_mp = mi_tpi_extconn_ind(NULL, 4932 (char *)&sin6d, sizeof (sin6_t), 4933 (char *)&tcp, (t_scalar_t)sizeof (intptr_t), 4934 (char *)&sin6d, sizeof (sin6_t), 4935 (t_scalar_t)ltcp->tcp_conn_req_seqnum); 4936 } else { 4937 tpi_mp = mi_tpi_conn_ind(NULL, 4938 (char *)&sin6, sizeof (sin6_t), 4939 (char *)&tcp, (t_scalar_t)sizeof (intptr_t), 4940 (t_scalar_t)ltcp->tcp_conn_req_seqnum); 4941 } 4942 } 4943 4944 if (tpi_mp == NULL) 4945 return (ENOMEM); 4946 4947 connp->conn_fport = *(uint16_t *)tcph->th_lport; 4948 connp->conn_lport = *(uint16_t *)tcph->th_fport; 4949 connp->conn_flags |= (IPCL_TCP6|IPCL_EAGER); 4950 connp->conn_fully_bound = B_FALSE; 4951 4952 /* Inherit information from the "parent" */ 4953 tcp->tcp_ipversion = ltcp->tcp_ipversion; 4954 tcp->tcp_family = ltcp->tcp_family; 4955 tcp->tcp_wq = ltcp->tcp_wq; 4956 tcp->tcp_rq = ltcp->tcp_rq; 4957 tcp->tcp_mss = tcps->tcps_mss_def_ipv6; 4958 tcp->tcp_detached = B_TRUE; 4959 if ((err = tcp_init_values(tcp)) != 0) { 4960 freemsg(tpi_mp); 4961 return (err); 4962 } 4963 4964 if (ipvers == IPV4_VERSION) { 4965 if ((err = tcp_header_init_ipv4(tcp)) != 0) { 4966 freemsg(tpi_mp); 4967 return (err); 4968 } 4969 ASSERT(tcp->tcp_ipha != NULL); 4970 } else { 4971 /* ifindex must be already set */ 4972 ASSERT(ifindex != 0); 4973 4974 if (ltcp->tcp_bound_if != 0) { 4975 /* 4976 * Set newtcp's bound_if equal to 4977 * listener's value. If ifindex is 4978 * not the same as ltcp->tcp_bound_if, 4979 * it must be a packet for the ipmp group 4980 * of interfaces 4981 */ 4982 tcp->tcp_bound_if = ltcp->tcp_bound_if; 4983 } else if (IN6_IS_ADDR_LINKSCOPE(&ip6h->ip6_src)) { 4984 tcp->tcp_bound_if = ifindex; 4985 } 4986 4987 tcp->tcp_ipv6_recvancillary = ltcp->tcp_ipv6_recvancillary; 4988 tcp->tcp_recvifindex = 0; 4989 tcp->tcp_recvhops = 0xffffffffU; 4990 ASSERT(tcp->tcp_ip6h != NULL); 4991 } 4992 4993 tcp->tcp_lport = ltcp->tcp_lport; 4994 4995 if (ltcp->tcp_ipversion == tcp->tcp_ipversion) { 4996 if (tcp->tcp_iphc_len != ltcp->tcp_iphc_len) { 4997 /* 4998 * Listener had options of some sort; eager inherits. 4999 * Free up the eager template and allocate one 5000 * of the right size. 5001 */ 5002 if (tcp->tcp_hdr_grown) { 5003 kmem_free(tcp->tcp_iphc, tcp->tcp_iphc_len); 5004 } else { 5005 bzero(tcp->tcp_iphc, tcp->tcp_iphc_len); 5006 kmem_cache_free(tcp_iphc_cache, tcp->tcp_iphc); 5007 } 5008 tcp->tcp_iphc = kmem_zalloc(ltcp->tcp_iphc_len, 5009 KM_NOSLEEP); 5010 if (tcp->tcp_iphc == NULL) { 5011 tcp->tcp_iphc_len = 0; 5012 freemsg(tpi_mp); 5013 return (ENOMEM); 5014 } 5015 tcp->tcp_iphc_len = ltcp->tcp_iphc_len; 5016 tcp->tcp_hdr_grown = B_TRUE; 5017 } 5018 tcp->tcp_hdr_len = ltcp->tcp_hdr_len; 5019 tcp->tcp_ip_hdr_len = ltcp->tcp_ip_hdr_len; 5020 tcp->tcp_tcp_hdr_len = ltcp->tcp_tcp_hdr_len; 5021 tcp->tcp_ip6_hops = ltcp->tcp_ip6_hops; 5022 tcp->tcp_ip6_vcf = ltcp->tcp_ip6_vcf; 5023 5024 /* 5025 * Copy the IP+TCP header template from listener to eager 5026 */ 5027 bcopy(ltcp->tcp_iphc, tcp->tcp_iphc, ltcp->tcp_hdr_len); 5028 if (tcp->tcp_ipversion == IPV6_VERSION) { 5029 if (((ip6i_t *)(tcp->tcp_iphc))->ip6i_nxt == 5030 IPPROTO_RAW) { 5031 tcp->tcp_ip6h = 5032 (ip6_t *)(tcp->tcp_iphc + 5033 sizeof (ip6i_t)); 5034 } else { 5035 tcp->tcp_ip6h = 5036 (ip6_t *)(tcp->tcp_iphc); 5037 } 5038 tcp->tcp_ipha = NULL; 5039 } else { 5040 tcp->tcp_ipha = (ipha_t *)tcp->tcp_iphc; 5041 tcp->tcp_ip6h = NULL; 5042 } 5043 tcp->tcp_tcph = (tcph_t *)(tcp->tcp_iphc + 5044 tcp->tcp_ip_hdr_len); 5045 } else { 5046 /* 5047 * only valid case when ipversion of listener and 5048 * eager differ is when listener is IPv6 and 5049 * eager is IPv4. 5050 * Eager header template has been initialized to the 5051 * maximum v4 header sizes, which includes space for 5052 * TCP and IP options. 5053 */ 5054 ASSERT((ltcp->tcp_ipversion == IPV6_VERSION) && 5055 (tcp->tcp_ipversion == IPV4_VERSION)); 5056 ASSERT(tcp->tcp_iphc_len >= 5057 TCP_MAX_COMBINED_HEADER_LENGTH); 5058 tcp->tcp_tcp_hdr_len = ltcp->tcp_tcp_hdr_len; 5059 /* copy IP header fields individually */ 5060 tcp->tcp_ipha->ipha_ttl = 5061 ltcp->tcp_ip6h->ip6_hops; 5062 bcopy(ltcp->tcp_tcph->th_lport, 5063 tcp->tcp_tcph->th_lport, sizeof (ushort_t)); 5064 } 5065 5066 bcopy(tcph->th_lport, tcp->tcp_tcph->th_fport, sizeof (in_port_t)); 5067 bcopy(tcp->tcp_tcph->th_fport, &tcp->tcp_fport, 5068 sizeof (in_port_t)); 5069 5070 if (ltcp->tcp_lport == 0) { 5071 tcp->tcp_lport = *(in_port_t *)tcph->th_fport; 5072 bcopy(tcph->th_fport, tcp->tcp_tcph->th_lport, 5073 sizeof (in_port_t)); 5074 } 5075 5076 if (tcp->tcp_ipversion == IPV4_VERSION) { 5077 ASSERT(ipha != NULL); 5078 tcp->tcp_ipha->ipha_dst = ipha->ipha_src; 5079 tcp->tcp_ipha->ipha_src = ipha->ipha_dst; 5080 5081 /* Source routing option copyover (reverse it) */ 5082 if (tcps->tcps_rev_src_routes) 5083 tcp_opt_reverse(tcp, ipha); 5084 } else { 5085 ASSERT(ip6h != NULL); 5086 tcp->tcp_ip6h->ip6_dst = ip6h->ip6_src; 5087 tcp->tcp_ip6h->ip6_src = ip6h->ip6_dst; 5088 } 5089 5090 ASSERT(tcp->tcp_conn.tcp_eager_conn_ind == NULL); 5091 ASSERT(!tcp->tcp_tconnind_started); 5092 /* 5093 * If the SYN contains a credential, it's a loopback packet; attach 5094 * the credential to the TPI message. 5095 */ 5096 if ((cr = DB_CRED(idmp)) != NULL) { 5097 mblk_setcred(tpi_mp, cr); 5098 DB_CPID(tpi_mp) = DB_CPID(idmp); 5099 } 5100 tcp->tcp_conn.tcp_eager_conn_ind = tpi_mp; 5101 5102 /* Inherit the listener's SSL protection state */ 5103 5104 if ((tcp->tcp_kssl_ent = ltcp->tcp_kssl_ent) != NULL) { 5105 kssl_hold_ent(tcp->tcp_kssl_ent); 5106 tcp->tcp_kssl_pending = B_TRUE; 5107 } 5108 5109 return (0); 5110 } 5111 5112 5113 int 5114 tcp_conn_create_v4(conn_t *lconnp, conn_t *connp, ipha_t *ipha, 5115 tcph_t *tcph, mblk_t *idmp) 5116 { 5117 tcp_t *ltcp = lconnp->conn_tcp; 5118 tcp_t *tcp = connp->conn_tcp; 5119 sin_t sin; 5120 mblk_t *tpi_mp = NULL; 5121 int err; 5122 cred_t *cr; 5123 tcp_stack_t *tcps = tcp->tcp_tcps; 5124 5125 sin = sin_null; 5126 sin.sin_addr.s_addr = ipha->ipha_src; 5127 sin.sin_port = *(uint16_t *)tcph->th_lport; 5128 sin.sin_family = AF_INET; 5129 if (ltcp->tcp_recvdstaddr) { 5130 sin_t sind; 5131 5132 sind = sin_null; 5133 sind.sin_addr.s_addr = ipha->ipha_dst; 5134 sind.sin_port = *(uint16_t *)tcph->th_fport; 5135 sind.sin_family = AF_INET; 5136 tpi_mp = mi_tpi_extconn_ind(NULL, 5137 (char *)&sind, sizeof (sin_t), (char *)&tcp, 5138 (t_scalar_t)sizeof (intptr_t), (char *)&sind, 5139 sizeof (sin_t), (t_scalar_t)ltcp->tcp_conn_req_seqnum); 5140 } else { 5141 tpi_mp = mi_tpi_conn_ind(NULL, 5142 (char *)&sin, sizeof (sin_t), 5143 (char *)&tcp, (t_scalar_t)sizeof (intptr_t), 5144 (t_scalar_t)ltcp->tcp_conn_req_seqnum); 5145 } 5146 5147 if (tpi_mp == NULL) { 5148 return (ENOMEM); 5149 } 5150 5151 connp->conn_flags |= (IPCL_TCP4|IPCL_EAGER); 5152 connp->conn_send = ip_output; 5153 connp->conn_recv = tcp_input; 5154 connp->conn_fully_bound = B_FALSE; 5155 5156 IN6_IPADDR_TO_V4MAPPED(ipha->ipha_dst, &connp->conn_srcv6); 5157 IN6_IPADDR_TO_V4MAPPED(ipha->ipha_src, &connp->conn_remv6); 5158 connp->conn_fport = *(uint16_t *)tcph->th_lport; 5159 connp->conn_lport = *(uint16_t *)tcph->th_fport; 5160 5161 /* Inherit information from the "parent" */ 5162 tcp->tcp_ipversion = ltcp->tcp_ipversion; 5163 tcp->tcp_family = ltcp->tcp_family; 5164 tcp->tcp_wq = ltcp->tcp_wq; 5165 tcp->tcp_rq = ltcp->tcp_rq; 5166 tcp->tcp_mss = tcps->tcps_mss_def_ipv4; 5167 tcp->tcp_detached = B_TRUE; 5168 if ((err = tcp_init_values(tcp)) != 0) { 5169 freemsg(tpi_mp); 5170 return (err); 5171 } 5172 5173 /* 5174 * Let's make sure that eager tcp template has enough space to 5175 * copy IPv4 listener's tcp template. Since the conn_t structure is 5176 * preserved and tcp_iphc_len is also preserved, an eager conn_t may 5177 * have a tcp_template of total len TCP_MAX_COMBINED_HEADER_LENGTH or 5178 * more (in case of re-allocation of conn_t with tcp-IPv6 template with 5179 * extension headers or with ip6i_t struct). Note that bcopy() below 5180 * copies listener tcp's hdr_len which cannot be greater than TCP_MAX_ 5181 * COMBINED_HEADER_LENGTH as this listener must be a IPv4 listener. 5182 */ 5183 ASSERT(tcp->tcp_iphc_len >= TCP_MAX_COMBINED_HEADER_LENGTH); 5184 ASSERT(ltcp->tcp_hdr_len <= TCP_MAX_COMBINED_HEADER_LENGTH); 5185 5186 tcp->tcp_hdr_len = ltcp->tcp_hdr_len; 5187 tcp->tcp_ip_hdr_len = ltcp->tcp_ip_hdr_len; 5188 tcp->tcp_tcp_hdr_len = ltcp->tcp_tcp_hdr_len; 5189 tcp->tcp_ttl = ltcp->tcp_ttl; 5190 tcp->tcp_tos = ltcp->tcp_tos; 5191 5192 /* Copy the IP+TCP header template from listener to eager */ 5193 bcopy(ltcp->tcp_iphc, tcp->tcp_iphc, ltcp->tcp_hdr_len); 5194 tcp->tcp_ipha = (ipha_t *)tcp->tcp_iphc; 5195 tcp->tcp_ip6h = NULL; 5196 tcp->tcp_tcph = (tcph_t *)(tcp->tcp_iphc + 5197 tcp->tcp_ip_hdr_len); 5198 5199 /* Initialize the IP addresses and Ports */ 5200 tcp->tcp_ipha->ipha_dst = ipha->ipha_src; 5201 tcp->tcp_ipha->ipha_src = ipha->ipha_dst; 5202 bcopy(tcph->th_lport, tcp->tcp_tcph->th_fport, sizeof (in_port_t)); 5203 bcopy(tcph->th_fport, tcp->tcp_tcph->th_lport, sizeof (in_port_t)); 5204 5205 /* Source routing option copyover (reverse it) */ 5206 if (tcps->tcps_rev_src_routes) 5207 tcp_opt_reverse(tcp, ipha); 5208 5209 ASSERT(tcp->tcp_conn.tcp_eager_conn_ind == NULL); 5210 ASSERT(!tcp->tcp_tconnind_started); 5211 5212 /* 5213 * If the SYN contains a credential, it's a loopback packet; attach 5214 * the credential to the TPI message. 5215 */ 5216 if ((cr = DB_CRED(idmp)) != NULL) { 5217 mblk_setcred(tpi_mp, cr); 5218 DB_CPID(tpi_mp) = DB_CPID(idmp); 5219 } 5220 tcp->tcp_conn.tcp_eager_conn_ind = tpi_mp; 5221 5222 /* Inherit the listener's SSL protection state */ 5223 if ((tcp->tcp_kssl_ent = ltcp->tcp_kssl_ent) != NULL) { 5224 kssl_hold_ent(tcp->tcp_kssl_ent); 5225 tcp->tcp_kssl_pending = B_TRUE; 5226 } 5227 5228 return (0); 5229 } 5230 5231 /* 5232 * sets up conn for ipsec. 5233 * if the first mblk is M_CTL it is consumed and mpp is updated. 5234 * in case of error mpp is freed. 5235 */ 5236 conn_t * 5237 tcp_get_ipsec_conn(tcp_t *tcp, squeue_t *sqp, mblk_t **mpp) 5238 { 5239 conn_t *connp = tcp->tcp_connp; 5240 conn_t *econnp; 5241 squeue_t *new_sqp; 5242 mblk_t *first_mp = *mpp; 5243 mblk_t *mp = *mpp; 5244 boolean_t mctl_present = B_FALSE; 5245 uint_t ipvers; 5246 5247 econnp = tcp_get_conn(sqp, tcp->tcp_tcps); 5248 if (econnp == NULL) { 5249 freemsg(first_mp); 5250 return (NULL); 5251 } 5252 if (DB_TYPE(mp) == M_CTL) { 5253 if (mp->b_cont == NULL || 5254 mp->b_cont->b_datap->db_type != M_DATA) { 5255 freemsg(first_mp); 5256 return (NULL); 5257 } 5258 mp = mp->b_cont; 5259 if ((mp->b_datap->db_struioflag & STRUIO_EAGER) == 0) { 5260 freemsg(first_mp); 5261 return (NULL); 5262 } 5263 5264 mp->b_datap->db_struioflag &= ~STRUIO_EAGER; 5265 first_mp->b_datap->db_struioflag &= ~STRUIO_POLICY; 5266 mctl_present = B_TRUE; 5267 } else { 5268 ASSERT(mp->b_datap->db_struioflag & STRUIO_POLICY); 5269 mp->b_datap->db_struioflag &= ~STRUIO_POLICY; 5270 } 5271 5272 new_sqp = (squeue_t *)DB_CKSUMSTART(mp); 5273 DB_CKSUMSTART(mp) = 0; 5274 5275 ASSERT(OK_32PTR(mp->b_rptr)); 5276 ipvers = IPH_HDR_VERSION(mp->b_rptr); 5277 if (ipvers == IPV4_VERSION) { 5278 uint16_t *up; 5279 uint32_t ports; 5280 ipha_t *ipha; 5281 5282 ipha = (ipha_t *)mp->b_rptr; 5283 up = (uint16_t *)((uchar_t *)ipha + 5284 IPH_HDR_LENGTH(ipha) + TCP_PORTS_OFFSET); 5285 ports = *(uint32_t *)up; 5286 IPCL_TCP_EAGER_INIT(econnp, IPPROTO_TCP, 5287 ipha->ipha_dst, ipha->ipha_src, ports); 5288 } else { 5289 uint16_t *up; 5290 uint32_t ports; 5291 uint16_t ip_hdr_len; 5292 uint8_t *nexthdrp; 5293 ip6_t *ip6h; 5294 tcph_t *tcph; 5295 5296 ip6h = (ip6_t *)mp->b_rptr; 5297 if (ip6h->ip6_nxt == IPPROTO_TCP) { 5298 ip_hdr_len = IPV6_HDR_LEN; 5299 } else if (!ip_hdr_length_nexthdr_v6(mp, ip6h, &ip_hdr_len, 5300 &nexthdrp) || *nexthdrp != IPPROTO_TCP) { 5301 CONN_DEC_REF(econnp); 5302 freemsg(first_mp); 5303 return (NULL); 5304 } 5305 tcph = (tcph_t *)&mp->b_rptr[ip_hdr_len]; 5306 up = (uint16_t *)tcph->th_lport; 5307 ports = *(uint32_t *)up; 5308 IPCL_TCP_EAGER_INIT_V6(econnp, IPPROTO_TCP, 5309 ip6h->ip6_dst, ip6h->ip6_src, ports); 5310 } 5311 5312 /* 5313 * The caller already ensured that there is a sqp present. 5314 */ 5315 econnp->conn_sqp = new_sqp; 5316 5317 if (connp->conn_policy != NULL) { 5318 ipsec_in_t *ii; 5319 ii = (ipsec_in_t *)(first_mp->b_rptr); 5320 ASSERT(ii->ipsec_in_policy == NULL); 5321 IPPH_REFHOLD(connp->conn_policy); 5322 ii->ipsec_in_policy = connp->conn_policy; 5323 5324 first_mp->b_datap->db_type = IPSEC_POLICY_SET; 5325 if (!ip_bind_ipsec_policy_set(econnp, first_mp)) { 5326 CONN_DEC_REF(econnp); 5327 freemsg(first_mp); 5328 return (NULL); 5329 } 5330 } 5331 5332 if (ipsec_conn_cache_policy(econnp, ipvers == IPV4_VERSION) != 0) { 5333 CONN_DEC_REF(econnp); 5334 freemsg(first_mp); 5335 return (NULL); 5336 } 5337 5338 /* 5339 * If we know we have some policy, pass the "IPSEC" 5340 * options size TCP uses this adjust the MSS. 5341 */ 5342 econnp->conn_tcp->tcp_ipsec_overhead = conn_ipsec_length(econnp); 5343 if (mctl_present) { 5344 freeb(first_mp); 5345 *mpp = mp; 5346 } 5347 5348 return (econnp); 5349 } 5350 5351 /* 5352 * tcp_get_conn/tcp_free_conn 5353 * 5354 * tcp_get_conn is used to get a clean tcp connection structure. 5355 * It tries to reuse the connections put on the freelist by the 5356 * time_wait_collector failing which it goes to kmem_cache. This 5357 * way has two benefits compared to just allocating from and 5358 * freeing to kmem_cache. 5359 * 1) The time_wait_collector can free (which includes the cleanup) 5360 * outside the squeue. So when the interrupt comes, we have a clean 5361 * connection sitting in the freelist. Obviously, this buys us 5362 * performance. 5363 * 5364 * 2) Defence against DOS attack. Allocating a tcp/conn in tcp_conn_request 5365 * has multiple disadvantages - tying up the squeue during alloc, and the 5366 * fact that IPSec policy initialization has to happen here which 5367 * requires us sending a M_CTL and checking for it i.e. real ugliness. 5368 * But allocating the conn/tcp in IP land is also not the best since 5369 * we can't check the 'q' and 'q0' which are protected by squeue and 5370 * blindly allocate memory which might have to be freed here if we are 5371 * not allowed to accept the connection. By using the freelist and 5372 * putting the conn/tcp back in freelist, we don't pay a penalty for 5373 * allocating memory without checking 'q/q0' and freeing it if we can't 5374 * accept the connection. 5375 * 5376 * Care should be taken to put the conn back in the same squeue's freelist 5377 * from which it was allocated. Best results are obtained if conn is 5378 * allocated from listener's squeue and freed to the same. Time wait 5379 * collector will free up the freelist is the connection ends up sitting 5380 * there for too long. 5381 */ 5382 void * 5383 tcp_get_conn(void *arg, tcp_stack_t *tcps) 5384 { 5385 tcp_t *tcp = NULL; 5386 conn_t *connp = NULL; 5387 squeue_t *sqp = (squeue_t *)arg; 5388 tcp_squeue_priv_t *tcp_time_wait; 5389 netstack_t *ns; 5390 5391 tcp_time_wait = 5392 *((tcp_squeue_priv_t **)squeue_getprivate(sqp, SQPRIVATE_TCP)); 5393 5394 mutex_enter(&tcp_time_wait->tcp_time_wait_lock); 5395 tcp = tcp_time_wait->tcp_free_list; 5396 ASSERT((tcp != NULL) ^ (tcp_time_wait->tcp_free_list_cnt == 0)); 5397 if (tcp != NULL) { 5398 tcp_time_wait->tcp_free_list = tcp->tcp_time_wait_next; 5399 tcp_time_wait->tcp_free_list_cnt--; 5400 mutex_exit(&tcp_time_wait->tcp_time_wait_lock); 5401 tcp->tcp_time_wait_next = NULL; 5402 connp = tcp->tcp_connp; 5403 connp->conn_flags |= IPCL_REUSED; 5404 5405 ASSERT(tcp->tcp_tcps == NULL); 5406 ASSERT(connp->conn_netstack == NULL); 5407 ns = tcps->tcps_netstack; 5408 netstack_hold(ns); 5409 connp->conn_netstack = ns; 5410 tcp->tcp_tcps = tcps; 5411 TCPS_REFHOLD(tcps); 5412 ipcl_globalhash_insert(connp); 5413 return ((void *)connp); 5414 } 5415 mutex_exit(&tcp_time_wait->tcp_time_wait_lock); 5416 if ((connp = ipcl_conn_create(IPCL_TCPCONN, KM_NOSLEEP, 5417 tcps->tcps_netstack)) == NULL) 5418 return (NULL); 5419 tcp = connp->conn_tcp; 5420 tcp->tcp_tcps = tcps; 5421 TCPS_REFHOLD(tcps); 5422 return ((void *)connp); 5423 } 5424 5425 /* 5426 * Update the cached label for the given tcp_t. This should be called once per 5427 * connection, and before any packets are sent or tcp_process_options is 5428 * invoked. Returns B_FALSE if the correct label could not be constructed. 5429 */ 5430 static boolean_t 5431 tcp_update_label(tcp_t *tcp, const cred_t *cr) 5432 { 5433 conn_t *connp = tcp->tcp_connp; 5434 5435 if (tcp->tcp_ipversion == IPV4_VERSION) { 5436 uchar_t optbuf[IP_MAX_OPT_LENGTH]; 5437 int added; 5438 5439 if (tsol_compute_label(cr, tcp->tcp_remote, optbuf, 5440 connp->conn_mac_exempt, 5441 tcp->tcp_tcps->tcps_netstack->netstack_ip) != 0) 5442 return (B_FALSE); 5443 5444 added = tsol_remove_secopt(tcp->tcp_ipha, tcp->tcp_hdr_len); 5445 if (added == -1) 5446 return (B_FALSE); 5447 tcp->tcp_hdr_len += added; 5448 tcp->tcp_tcph = (tcph_t *)((uchar_t *)tcp->tcp_tcph + added); 5449 tcp->tcp_ip_hdr_len += added; 5450 if ((tcp->tcp_label_len = optbuf[IPOPT_OLEN]) != 0) { 5451 tcp->tcp_label_len = (tcp->tcp_label_len + 3) & ~3; 5452 added = tsol_prepend_option(optbuf, tcp->tcp_ipha, 5453 tcp->tcp_hdr_len); 5454 if (added == -1) 5455 return (B_FALSE); 5456 tcp->tcp_hdr_len += added; 5457 tcp->tcp_tcph = (tcph_t *) 5458 ((uchar_t *)tcp->tcp_tcph + added); 5459 tcp->tcp_ip_hdr_len += added; 5460 } 5461 } else { 5462 uchar_t optbuf[TSOL_MAX_IPV6_OPTION]; 5463 5464 if (tsol_compute_label_v6(cr, &tcp->tcp_remote_v6, optbuf, 5465 connp->conn_mac_exempt, 5466 tcp->tcp_tcps->tcps_netstack->netstack_ip) != 0) 5467 return (B_FALSE); 5468 if (tsol_update_sticky(&tcp->tcp_sticky_ipp, 5469 &tcp->tcp_label_len, optbuf) != 0) 5470 return (B_FALSE); 5471 if (tcp_build_hdrs(tcp->tcp_rq, tcp) != 0) 5472 return (B_FALSE); 5473 } 5474 5475 connp->conn_ulp_labeled = 1; 5476 5477 return (B_TRUE); 5478 } 5479 5480 /* BEGIN CSTYLED */ 5481 /* 5482 * 5483 * The sockfs ACCEPT path: 5484 * ======================= 5485 * 5486 * The eager is now established in its own perimeter as soon as SYN is 5487 * received in tcp_conn_request(). When sockfs receives conn_ind, it 5488 * completes the accept processing on the acceptor STREAM. The sending 5489 * of conn_ind part is common for both sockfs listener and a TLI/XTI 5490 * listener but a TLI/XTI listener completes the accept processing 5491 * on the listener perimeter. 5492 * 5493 * Common control flow for 3 way handshake: 5494 * ---------------------------------------- 5495 * 5496 * incoming SYN (listener perimeter) -> tcp_rput_data() 5497 * -> tcp_conn_request() 5498 * 5499 * incoming SYN-ACK-ACK (eager perim) -> tcp_rput_data() 5500 * send T_CONN_IND (listener perim) -> tcp_send_conn_ind() 5501 * 5502 * Sockfs ACCEPT Path: 5503 * ------------------- 5504 * 5505 * open acceptor stream (tcp_open allocates tcp_wput_accept() 5506 * as STREAM entry point) 5507 * 5508 * soaccept() sends T_CONN_RES on the acceptor STREAM to tcp_wput_accept() 5509 * 5510 * tcp_wput_accept() extracts the eager and makes the q->q_ptr <-> eager 5511 * association (we are not behind eager's squeue but sockfs is protecting us 5512 * and no one knows about this stream yet. The STREAMS entry point q->q_info 5513 * is changed to point at tcp_wput(). 5514 * 5515 * tcp_wput_accept() sends any deferred eagers via tcp_send_pending() to 5516 * listener (done on listener's perimeter). 5517 * 5518 * tcp_wput_accept() calls tcp_accept_finish() on eagers perimeter to finish 5519 * accept. 5520 * 5521 * TLI/XTI client ACCEPT path: 5522 * --------------------------- 5523 * 5524 * soaccept() sends T_CONN_RES on the listener STREAM. 5525 * 5526 * tcp_accept() -> tcp_accept_swap() complete the processing and send 5527 * the bind_mp to eager perimeter to finish accept (tcp_rput_other()). 5528 * 5529 * Locks: 5530 * ====== 5531 * 5532 * listener->tcp_eager_lock protects the listeners->tcp_eager_next_q0 and 5533 * and listeners->tcp_eager_next_q. 5534 * 5535 * Referencing: 5536 * ============ 5537 * 5538 * 1) We start out in tcp_conn_request by eager placing a ref on 5539 * listener and listener adding eager to listeners->tcp_eager_next_q0. 5540 * 5541 * 2) When a SYN-ACK-ACK arrives, we send the conn_ind to listener. Before 5542 * doing so we place a ref on the eager. This ref is finally dropped at the 5543 * end of tcp_accept_finish() while unwinding from the squeue, i.e. the 5544 * reference is dropped by the squeue framework. 5545 * 5546 * 3) The ref on listener placed in 1 above is dropped in tcp_accept_finish 5547 * 5548 * The reference must be released by the same entity that added the reference 5549 * In the above scheme, the eager is the entity that adds and releases the 5550 * references. Note that tcp_accept_finish executes in the squeue of the eager 5551 * (albeit after it is attached to the acceptor stream). Though 1. executes 5552 * in the listener's squeue, the eager is nascent at this point and the 5553 * reference can be considered to have been added on behalf of the eager. 5554 * 5555 * Eager getting a Reset or listener closing: 5556 * ========================================== 5557 * 5558 * Once the listener and eager are linked, the listener never does the unlink. 5559 * If the listener needs to close, tcp_eager_cleanup() is called which queues 5560 * a message on all eager perimeter. The eager then does the unlink, clears 5561 * any pointers to the listener's queue and drops the reference to the 5562 * listener. The listener waits in tcp_close outside the squeue until its 5563 * refcount has dropped to 1. This ensures that the listener has waited for 5564 * all eagers to clear their association with the listener. 5565 * 5566 * Similarly, if eager decides to go away, it can unlink itself and close. 5567 * When the T_CONN_RES comes down, we check if eager has closed. Note that 5568 * the reference to eager is still valid because of the extra ref we put 5569 * in tcp_send_conn_ind. 5570 * 5571 * Listener can always locate the eager under the protection 5572 * of the listener->tcp_eager_lock, and then do a refhold 5573 * on the eager during the accept processing. 5574 * 5575 * The acceptor stream accesses the eager in the accept processing 5576 * based on the ref placed on eager before sending T_conn_ind. 5577 * The only entity that can negate this refhold is a listener close 5578 * which is mutually exclusive with an active acceptor stream. 5579 * 5580 * Eager's reference on the listener 5581 * =================================== 5582 * 5583 * If the accept happens (even on a closed eager) the eager drops its 5584 * reference on the listener at the start of tcp_accept_finish. If the 5585 * eager is killed due to an incoming RST before the T_conn_ind is sent up, 5586 * the reference is dropped in tcp_closei_local. If the listener closes, 5587 * the reference is dropped in tcp_eager_kill. In all cases the reference 5588 * is dropped while executing in the eager's context (squeue). 5589 */ 5590 /* END CSTYLED */ 5591 5592 /* Process the SYN packet, mp, directed at the listener 'tcp' */ 5593 5594 /* 5595 * THIS FUNCTION IS DIRECTLY CALLED BY IP VIA SQUEUE FOR SYN. 5596 * tcp_rput_data will not see any SYN packets. 5597 */ 5598 /* ARGSUSED */ 5599 void 5600 tcp_conn_request(void *arg, mblk_t *mp, void *arg2) 5601 { 5602 tcph_t *tcph; 5603 uint32_t seg_seq; 5604 tcp_t *eager; 5605 uint_t ipvers; 5606 ipha_t *ipha; 5607 ip6_t *ip6h; 5608 int err; 5609 conn_t *econnp = NULL; 5610 squeue_t *new_sqp; 5611 mblk_t *mp1; 5612 uint_t ip_hdr_len; 5613 conn_t *connp = (conn_t *)arg; 5614 tcp_t *tcp = connp->conn_tcp; 5615 cred_t *credp; 5616 tcp_stack_t *tcps = tcp->tcp_tcps; 5617 ip_stack_t *ipst; 5618 5619 if (tcp->tcp_state != TCPS_LISTEN) 5620 goto error2; 5621 5622 ASSERT((tcp->tcp_connp->conn_flags & IPCL_BOUND) != 0); 5623 5624 mutex_enter(&tcp->tcp_eager_lock); 5625 if (tcp->tcp_conn_req_cnt_q >= tcp->tcp_conn_req_max) { 5626 mutex_exit(&tcp->tcp_eager_lock); 5627 TCP_STAT(tcps, tcp_listendrop); 5628 BUMP_MIB(&tcps->tcps_mib, tcpListenDrop); 5629 if (tcp->tcp_debug) { 5630 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE|SL_ERROR, 5631 "tcp_conn_request: listen backlog (max=%d) " 5632 "overflow (%d pending) on %s", 5633 tcp->tcp_conn_req_max, tcp->tcp_conn_req_cnt_q, 5634 tcp_display(tcp, NULL, DISP_PORT_ONLY)); 5635 } 5636 goto error2; 5637 } 5638 5639 if (tcp->tcp_conn_req_cnt_q0 >= 5640 tcp->tcp_conn_req_max + tcps->tcps_conn_req_max_q0) { 5641 /* 5642 * Q0 is full. Drop a pending half-open req from the queue 5643 * to make room for the new SYN req. Also mark the time we 5644 * drop a SYN. 5645 * 5646 * A more aggressive defense against SYN attack will 5647 * be to set the "tcp_syn_defense" flag now. 5648 */ 5649 TCP_STAT(tcps, tcp_listendropq0); 5650 tcp->tcp_last_rcv_lbolt = lbolt64; 5651 if (!tcp_drop_q0(tcp)) { 5652 mutex_exit(&tcp->tcp_eager_lock); 5653 BUMP_MIB(&tcps->tcps_mib, tcpListenDropQ0); 5654 if (tcp->tcp_debug) { 5655 (void) strlog(TCP_MOD_ID, 0, 3, SL_TRACE, 5656 "tcp_conn_request: listen half-open queue " 5657 "(max=%d) full (%d pending) on %s", 5658 tcps->tcps_conn_req_max_q0, 5659 tcp->tcp_conn_req_cnt_q0, 5660 tcp_display(tcp, NULL, 5661 DISP_PORT_ONLY)); 5662 } 5663 goto error2; 5664 } 5665 } 5666 mutex_exit(&tcp->tcp_eager_lock); 5667 5668 /* 5669 * IP adds STRUIO_EAGER and ensures that the received packet is 5670 * M_DATA even if conn_ipv6_recvpktinfo is enabled or for ip6 5671 * link local address. If IPSec is enabled, db_struioflag has 5672 * STRUIO_POLICY set (mutually exclusive from STRUIO_EAGER); 5673 * otherwise an error case if neither of them is set. 5674 */ 5675 if ((mp->b_datap->db_struioflag & STRUIO_EAGER) != 0) { 5676 new_sqp = (squeue_t *)DB_CKSUMSTART(mp); 5677 DB_CKSUMSTART(mp) = 0; 5678 mp->b_datap->db_struioflag &= ~STRUIO_EAGER; 5679 econnp = (conn_t *)tcp_get_conn(arg2, tcps); 5680 if (econnp == NULL) 5681 goto error2; 5682 ASSERT(econnp->conn_netstack == connp->conn_netstack); 5683 econnp->conn_sqp = new_sqp; 5684 } else if ((mp->b_datap->db_struioflag & STRUIO_POLICY) != 0) { 5685 /* 5686 * mp is updated in tcp_get_ipsec_conn(). 5687 */ 5688 econnp = tcp_get_ipsec_conn(tcp, arg2, &mp); 5689 if (econnp == NULL) { 5690 /* 5691 * mp freed by tcp_get_ipsec_conn. 5692 */ 5693 return; 5694 } 5695 ASSERT(econnp->conn_netstack == connp->conn_netstack); 5696 } else { 5697 goto error2; 5698 } 5699 5700 ASSERT(DB_TYPE(mp) == M_DATA); 5701 5702 ipvers = IPH_HDR_VERSION(mp->b_rptr); 5703 ASSERT(ipvers == IPV6_VERSION || ipvers == IPV4_VERSION); 5704 ASSERT(OK_32PTR(mp->b_rptr)); 5705 if (ipvers == IPV4_VERSION) { 5706 ipha = (ipha_t *)mp->b_rptr; 5707 ip_hdr_len = IPH_HDR_LENGTH(ipha); 5708 tcph = (tcph_t *)&mp->b_rptr[ip_hdr_len]; 5709 } else { 5710 ip6h = (ip6_t *)mp->b_rptr; 5711 ip_hdr_len = ip_hdr_length_v6(mp, ip6h); 5712 tcph = (tcph_t *)&mp->b_rptr[ip_hdr_len]; 5713 } 5714 5715 if (tcp->tcp_family == AF_INET) { 5716 ASSERT(ipvers == IPV4_VERSION); 5717 err = tcp_conn_create_v4(connp, econnp, ipha, tcph, mp); 5718 } else { 5719 err = tcp_conn_create_v6(connp, econnp, mp, tcph, ipvers, mp); 5720 } 5721 5722 if (err) 5723 goto error3; 5724 5725 eager = econnp->conn_tcp; 5726 5727 /* Inherit various TCP parameters from the listener */ 5728 eager->tcp_naglim = tcp->tcp_naglim; 5729 eager->tcp_first_timer_threshold = 5730 tcp->tcp_first_timer_threshold; 5731 eager->tcp_second_timer_threshold = 5732 tcp->tcp_second_timer_threshold; 5733 5734 eager->tcp_first_ctimer_threshold = 5735 tcp->tcp_first_ctimer_threshold; 5736 eager->tcp_second_ctimer_threshold = 5737 tcp->tcp_second_ctimer_threshold; 5738 5739 /* 5740 * tcp_adapt_ire() may change tcp_rwnd according to the ire metrics. 5741 * If it does not, the eager's receive window will be set to the 5742 * listener's receive window later in this function. 5743 */ 5744 eager->tcp_rwnd = 0; 5745 5746 /* 5747 * Inherit listener's tcp_init_cwnd. Need to do this before 5748 * calling tcp_process_options() where tcp_mss_set() is called 5749 * to set the initial cwnd. 5750 */ 5751 eager->tcp_init_cwnd = tcp->tcp_init_cwnd; 5752 5753 /* 5754 * Zones: tcp_adapt_ire() and tcp_send_data() both need the 5755 * zone id before the accept is completed in tcp_wput_accept(). 5756 */ 5757 econnp->conn_zoneid = connp->conn_zoneid; 5758 econnp->conn_allzones = connp->conn_allzones; 5759 5760 /* Copy nexthop information from listener to eager */ 5761 if (connp->conn_nexthop_set) { 5762 econnp->conn_nexthop_set = connp->conn_nexthop_set; 5763 econnp->conn_nexthop_v4 = connp->conn_nexthop_v4; 5764 } 5765 5766 /* 5767 * TSOL: tsol_input_proc() needs the eager's cred before the 5768 * eager is accepted 5769 */ 5770 econnp->conn_cred = eager->tcp_cred = credp = connp->conn_cred; 5771 crhold(credp); 5772 5773 /* 5774 * If the caller has the process-wide flag set, then default to MAC 5775 * exempt mode. This allows read-down to unlabeled hosts. 5776 */ 5777 if (getpflags(NET_MAC_AWARE, credp) != 0) 5778 econnp->conn_mac_exempt = B_TRUE; 5779 5780 if (is_system_labeled()) { 5781 cred_t *cr; 5782 5783 if (connp->conn_mlp_type != mlptSingle) { 5784 cr = econnp->conn_peercred = DB_CRED(mp); 5785 if (cr != NULL) 5786 crhold(cr); 5787 else 5788 cr = econnp->conn_cred; 5789 DTRACE_PROBE2(mlp_syn_accept, conn_t *, 5790 econnp, cred_t *, cr) 5791 } else { 5792 cr = econnp->conn_cred; 5793 DTRACE_PROBE2(syn_accept, conn_t *, 5794 econnp, cred_t *, cr) 5795 } 5796 5797 if (!tcp_update_label(eager, cr)) { 5798 DTRACE_PROBE3( 5799 tx__ip__log__error__connrequest__tcp, 5800 char *, "eager connp(1) label on SYN mp(2) failed", 5801 conn_t *, econnp, mblk_t *, mp); 5802 goto error3; 5803 } 5804 } 5805 5806 eager->tcp_hard_binding = B_TRUE; 5807 5808 tcp_bind_hash_insert(&tcps->tcps_bind_fanout[ 5809 TCP_BIND_HASH(eager->tcp_lport)], eager, 0); 5810 5811 CL_INET_CONNECT(eager); 5812 5813 /* 5814 * No need to check for multicast destination since ip will only pass 5815 * up multicasts to those that have expressed interest 5816 * TODO: what about rejecting broadcasts? 5817 * Also check that source is not a multicast or broadcast address. 5818 */ 5819 eager->tcp_state = TCPS_SYN_RCVD; 5820 5821 5822 /* 5823 * There should be no ire in the mp as we are being called after 5824 * receiving the SYN. 5825 */ 5826 ASSERT(tcp_ire_mp(mp) == NULL); 5827 5828 /* 5829 * Adapt our mss, ttl, ... according to information provided in IRE. 5830 */ 5831 5832 if (tcp_adapt_ire(eager, NULL) == 0) { 5833 /* Undo the bind_hash_insert */ 5834 tcp_bind_hash_remove(eager); 5835 goto error3; 5836 } 5837 5838 /* Process all TCP options. */ 5839 tcp_process_options(eager, tcph); 5840 5841 /* Is the other end ECN capable? */ 5842 if (tcps->tcps_ecn_permitted >= 1 && 5843 (tcph->th_flags[0] & (TH_ECE|TH_CWR)) == (TH_ECE|TH_CWR)) { 5844 eager->tcp_ecn_ok = B_TRUE; 5845 } 5846 5847 /* 5848 * listener->tcp_rq->q_hiwat should be the default window size or a 5849 * window size changed via SO_RCVBUF option. First round up the 5850 * eager's tcp_rwnd to the nearest MSS. Then find out the window 5851 * scale option value if needed. Call tcp_rwnd_set() to finish the 5852 * setting. 5853 * 5854 * Note if there is a rpipe metric associated with the remote host, 5855 * we should not inherit receive window size from listener. 5856 */ 5857 eager->tcp_rwnd = MSS_ROUNDUP( 5858 (eager->tcp_rwnd == 0 ? tcp->tcp_rq->q_hiwat : 5859 eager->tcp_rwnd), eager->tcp_mss); 5860 if (eager->tcp_snd_ws_ok) 5861 tcp_set_ws_value(eager); 5862 /* 5863 * Note that this is the only place tcp_rwnd_set() is called for 5864 * accepting a connection. We need to call it here instead of 5865 * after the 3-way handshake because we need to tell the other 5866 * side our rwnd in the SYN-ACK segment. 5867 */ 5868 (void) tcp_rwnd_set(eager, eager->tcp_rwnd); 5869 5870 /* 5871 * We eliminate the need for sockfs to send down a T_SVR4_OPTMGMT_REQ 5872 * via soaccept()->soinheritoptions() which essentially applies 5873 * all the listener options to the new STREAM. The options that we 5874 * need to take care of are: 5875 * SO_DEBUG, SO_REUSEADDR, SO_KEEPALIVE, SO_DONTROUTE, SO_BROADCAST, 5876 * SO_USELOOPBACK, SO_OOBINLINE, SO_DGRAM_ERRIND, SO_LINGER, 5877 * SO_SNDBUF, SO_RCVBUF. 5878 * 5879 * SO_RCVBUF: tcp_rwnd_set() above takes care of it. 5880 * SO_SNDBUF: Set the tcp_xmit_hiwater for the eager. When 5881 * tcp_maxpsz_set() gets called later from 5882 * tcp_accept_finish(), the option takes effect. 5883 * 5884 */ 5885 /* Set the TCP options */ 5886 eager->tcp_xmit_hiwater = tcp->tcp_xmit_hiwater; 5887 eager->tcp_dgram_errind = tcp->tcp_dgram_errind; 5888 eager->tcp_oobinline = tcp->tcp_oobinline; 5889 eager->tcp_reuseaddr = tcp->tcp_reuseaddr; 5890 eager->tcp_broadcast = tcp->tcp_broadcast; 5891 eager->tcp_useloopback = tcp->tcp_useloopback; 5892 eager->tcp_dontroute = tcp->tcp_dontroute; 5893 eager->tcp_linger = tcp->tcp_linger; 5894 eager->tcp_lingertime = tcp->tcp_lingertime; 5895 if (tcp->tcp_ka_enabled) 5896 eager->tcp_ka_enabled = 1; 5897 5898 /* Set the IP options */ 5899 econnp->conn_broadcast = connp->conn_broadcast; 5900 econnp->conn_loopback = connp->conn_loopback; 5901 econnp->conn_dontroute = connp->conn_dontroute; 5902 econnp->conn_reuseaddr = connp->conn_reuseaddr; 5903 5904 /* Put a ref on the listener for the eager. */ 5905 CONN_INC_REF(connp); 5906 mutex_enter(&tcp->tcp_eager_lock); 5907 tcp->tcp_eager_next_q0->tcp_eager_prev_q0 = eager; 5908 eager->tcp_eager_next_q0 = tcp->tcp_eager_next_q0; 5909 tcp->tcp_eager_next_q0 = eager; 5910 eager->tcp_eager_prev_q0 = tcp; 5911 5912 /* Set tcp_listener before adding it to tcp_conn_fanout */ 5913 eager->tcp_listener = tcp; 5914 eager->tcp_saved_listener = tcp; 5915 5916 /* 5917 * Tag this detached tcp vector for later retrieval 5918 * by our listener client in tcp_accept(). 5919 */ 5920 eager->tcp_conn_req_seqnum = tcp->tcp_conn_req_seqnum; 5921 tcp->tcp_conn_req_cnt_q0++; 5922 if (++tcp->tcp_conn_req_seqnum == -1) { 5923 /* 5924 * -1 is "special" and defined in TPI as something 5925 * that should never be used in T_CONN_IND 5926 */ 5927 ++tcp->tcp_conn_req_seqnum; 5928 } 5929 mutex_exit(&tcp->tcp_eager_lock); 5930 5931 if (tcp->tcp_syn_defense) { 5932 /* Don't drop the SYN that comes from a good IP source */ 5933 ipaddr_t *addr_cache = (ipaddr_t *)(tcp->tcp_ip_addr_cache); 5934 if (addr_cache != NULL && eager->tcp_remote == 5935 addr_cache[IP_ADDR_CACHE_HASH(eager->tcp_remote)]) { 5936 eager->tcp_dontdrop = B_TRUE; 5937 } 5938 } 5939 5940 /* 5941 * We need to insert the eager in its own perimeter but as soon 5942 * as we do that, we expose the eager to the classifier and 5943 * should not touch any field outside the eager's perimeter. 5944 * So do all the work necessary before inserting the eager 5945 * in its own perimeter. Be optimistic that ipcl_conn_insert() 5946 * will succeed but undo everything if it fails. 5947 */ 5948 seg_seq = ABE32_TO_U32(tcph->th_seq); 5949 eager->tcp_irs = seg_seq; 5950 eager->tcp_rack = seg_seq; 5951 eager->tcp_rnxt = seg_seq + 1; 5952 U32_TO_ABE32(eager->tcp_rnxt, eager->tcp_tcph->th_ack); 5953 BUMP_MIB(&tcps->tcps_mib, tcpPassiveOpens); 5954 eager->tcp_state = TCPS_SYN_RCVD; 5955 mp1 = tcp_xmit_mp(eager, eager->tcp_xmit_head, eager->tcp_mss, 5956 NULL, NULL, eager->tcp_iss, B_FALSE, NULL, B_FALSE); 5957 if (mp1 == NULL) { 5958 /* 5959 * Increment the ref count as we are going to 5960 * enqueueing an mp in squeue 5961 */ 5962 CONN_INC_REF(econnp); 5963 goto error; 5964 } 5965 DB_CPID(mp1) = tcp->tcp_cpid; 5966 eager->tcp_cpid = tcp->tcp_cpid; 5967 eager->tcp_open_time = lbolt64; 5968 5969 /* 5970 * We need to start the rto timer. In normal case, we start 5971 * the timer after sending the packet on the wire (or at 5972 * least believing that packet was sent by waiting for 5973 * CALL_IP_WPUT() to return). Since this is the first packet 5974 * being sent on the wire for the eager, our initial tcp_rto 5975 * is at least tcp_rexmit_interval_min which is a fairly 5976 * large value to allow the algorithm to adjust slowly to large 5977 * fluctuations of RTT during first few transmissions. 5978 * 5979 * Starting the timer first and then sending the packet in this 5980 * case shouldn't make much difference since tcp_rexmit_interval_min 5981 * is of the order of several 100ms and starting the timer 5982 * first and then sending the packet will result in difference 5983 * of few micro seconds. 5984 * 5985 * Without this optimization, we are forced to hold the fanout 5986 * lock across the ipcl_bind_insert() and sending the packet 5987 * so that we don't race against an incoming packet (maybe RST) 5988 * for this eager. 5989 * 5990 * It is necessary to acquire an extra reference on the eager 5991 * at this point and hold it until after tcp_send_data() to 5992 * ensure against an eager close race. 5993 */ 5994 5995 CONN_INC_REF(eager->tcp_connp); 5996 5997 TCP_TIMER_RESTART(eager, eager->tcp_rto); 5998 5999 /* 6000 * Insert the eager in its own perimeter now. We are ready to deal 6001 * with any packets on eager. 6002 */ 6003 if (eager->tcp_ipversion == IPV4_VERSION) { 6004 if (ipcl_conn_insert(econnp, IPPROTO_TCP, 0, 0, 0) != 0) { 6005 goto error; 6006 } 6007 } else { 6008 if (ipcl_conn_insert_v6(econnp, IPPROTO_TCP, 0, 0, 0, 0) != 0) { 6009 goto error; 6010 } 6011 } 6012 6013 /* mark conn as fully-bound */ 6014 econnp->conn_fully_bound = B_TRUE; 6015 6016 /* Send the SYN-ACK */ 6017 tcp_send_data(eager, eager->tcp_wq, mp1); 6018 CONN_DEC_REF(eager->tcp_connp); 6019 freemsg(mp); 6020 6021 return; 6022 error: 6023 freemsg(mp1); 6024 eager->tcp_closemp_used = B_TRUE; 6025 TCP_DEBUG_GETPCSTACK(eager->tcmp_stk, 15); 6026 squeue_fill(econnp->conn_sqp, &eager->tcp_closemp, tcp_eager_kill, 6027 econnp, SQTAG_TCP_CONN_REQ_2); 6028 6029 /* 6030 * If a connection already exists, send the mp to that connections so 6031 * that it can be appropriately dealt with. 6032 */ 6033 ipst = tcps->tcps_netstack->netstack_ip; 6034 6035 if ((econnp = ipcl_classify(mp, connp->conn_zoneid, ipst)) != NULL) { 6036 if (!IPCL_IS_CONNECTED(econnp)) { 6037 /* 6038 * Something bad happened. ipcl_conn_insert() 6039 * failed because a connection already existed 6040 * in connected hash but we can't find it 6041 * anymore (someone blew it away). Just 6042 * free this message and hopefully remote 6043 * will retransmit at which time the SYN can be 6044 * treated as a new connection or dealth with 6045 * a TH_RST if a connection already exists. 6046 */ 6047 CONN_DEC_REF(econnp); 6048 freemsg(mp); 6049 } else { 6050 squeue_fill(econnp->conn_sqp, mp, tcp_input, 6051 econnp, SQTAG_TCP_CONN_REQ_1); 6052 } 6053 } else { 6054 /* Nobody wants this packet */ 6055 freemsg(mp); 6056 } 6057 return; 6058 error3: 6059 CONN_DEC_REF(econnp); 6060 error2: 6061 freemsg(mp); 6062 } 6063 6064 /* 6065 * In an ideal case of vertical partition in NUMA architecture, its 6066 * beneficial to have the listener and all the incoming connections 6067 * tied to the same squeue. The other constraint is that incoming 6068 * connections should be tied to the squeue attached to interrupted 6069 * CPU for obvious locality reason so this leaves the listener to 6070 * be tied to the same squeue. Our only problem is that when listener 6071 * is binding, the CPU that will get interrupted by the NIC whose 6072 * IP address the listener is binding to is not even known. So 6073 * the code below allows us to change that binding at the time the 6074 * CPU is interrupted by virtue of incoming connection's squeue. 6075 * 6076 * This is usefull only in case of a listener bound to a specific IP 6077 * address. For other kind of listeners, they get bound the 6078 * very first time and there is no attempt to rebind them. 6079 */ 6080 void 6081 tcp_conn_request_unbound(void *arg, mblk_t *mp, void *arg2) 6082 { 6083 conn_t *connp = (conn_t *)arg; 6084 squeue_t *sqp = (squeue_t *)arg2; 6085 squeue_t *new_sqp; 6086 uint32_t conn_flags; 6087 6088 if ((mp->b_datap->db_struioflag & STRUIO_EAGER) != 0) { 6089 new_sqp = (squeue_t *)DB_CKSUMSTART(mp); 6090 } else { 6091 goto done; 6092 } 6093 6094 if (connp->conn_fanout == NULL) 6095 goto done; 6096 6097 if (!(connp->conn_flags & IPCL_FULLY_BOUND)) { 6098 mutex_enter(&connp->conn_fanout->connf_lock); 6099 mutex_enter(&connp->conn_lock); 6100 /* 6101 * No one from read or write side can access us now 6102 * except for already queued packets on this squeue. 6103 * But since we haven't changed the squeue yet, they 6104 * can't execute. If they are processed after we have 6105 * changed the squeue, they are sent back to the 6106 * correct squeue down below. 6107 * But a listner close can race with processing of 6108 * incoming SYN. If incoming SYN processing changes 6109 * the squeue then the listener close which is waiting 6110 * to enter the squeue would operate on the wrong 6111 * squeue. Hence we don't change the squeue here unless 6112 * the refcount is exactly the minimum refcount. The 6113 * minimum refcount of 4 is counted as - 1 each for 6114 * TCP and IP, 1 for being in the classifier hash, and 6115 * 1 for the mblk being processed. 6116 */ 6117 6118 if (connp->conn_ref != 4 || 6119 connp->conn_tcp->tcp_state != TCPS_LISTEN) { 6120 mutex_exit(&connp->conn_lock); 6121 mutex_exit(&connp->conn_fanout->connf_lock); 6122 goto done; 6123 } 6124 if (connp->conn_sqp != new_sqp) { 6125 while (connp->conn_sqp != new_sqp) 6126 (void) casptr(&connp->conn_sqp, sqp, new_sqp); 6127 } 6128 6129 do { 6130 conn_flags = connp->conn_flags; 6131 conn_flags |= IPCL_FULLY_BOUND; 6132 (void) cas32(&connp->conn_flags, connp->conn_flags, 6133 conn_flags); 6134 } while (!(connp->conn_flags & IPCL_FULLY_BOUND)); 6135 6136 mutex_exit(&connp->conn_fanout->connf_lock); 6137 mutex_exit(&connp->conn_lock); 6138 } 6139 6140 done: 6141 if (connp->conn_sqp != sqp) { 6142 CONN_INC_REF(connp); 6143 squeue_fill(connp->conn_sqp, mp, 6144 connp->conn_recv, connp, SQTAG_TCP_CONN_REQ_UNBOUND); 6145 } else { 6146 tcp_conn_request(connp, mp, sqp); 6147 } 6148 } 6149 6150 /* 6151 * Successful connect request processing begins when our client passes 6152 * a T_CONN_REQ message into tcp_wput() and ends when tcp_rput() passes 6153 * our T_OK_ACK reply message upstream. The control flow looks like this: 6154 * upstream -> tcp_wput() -> tcp_wput_proto() -> tcp_connect() -> IP 6155 * upstream <- tcp_rput() <- IP 6156 * After various error checks are completed, tcp_connect() lays 6157 * the target address and port into the composite header template, 6158 * preallocates the T_OK_ACK reply message, construct a full 12 byte bind 6159 * request followed by an IRE request, and passes the three mblk message 6160 * down to IP looking like this: 6161 * O_T_BIND_REQ for IP --> IRE req --> T_OK_ACK for our client 6162 * Processing continues in tcp_rput() when we receive the following message: 6163 * T_BIND_ACK from IP --> IRE ack --> T_OK_ACK for our client 6164 * After consuming the first two mblks, tcp_rput() calls tcp_timer(), 6165 * to fire off the connection request, and then passes the T_OK_ACK mblk 6166 * upstream that we filled in below. There are, of course, numerous 6167 * error conditions along the way which truncate the processing described 6168 * above. 6169 */ 6170 static void 6171 tcp_connect(tcp_t *tcp, mblk_t *mp) 6172 { 6173 sin_t *sin; 6174 sin6_t *sin6; 6175 queue_t *q = tcp->tcp_wq; 6176 struct T_conn_req *tcr; 6177 ipaddr_t *dstaddrp; 6178 in_port_t dstport; 6179 uint_t srcid; 6180 6181 tcr = (struct T_conn_req *)mp->b_rptr; 6182 6183 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= (uintptr_t)INT_MAX); 6184 if ((mp->b_wptr - mp->b_rptr) < sizeof (*tcr)) { 6185 tcp_err_ack(tcp, mp, TPROTO, 0); 6186 return; 6187 } 6188 6189 /* 6190 * Determine packet type based on type of address passed in 6191 * the request should contain an IPv4 or IPv6 address. 6192 * Make sure that address family matches the type of 6193 * family of the the address passed down 6194 */ 6195 switch (tcr->DEST_length) { 6196 default: 6197 tcp_err_ack(tcp, mp, TBADADDR, 0); 6198 return; 6199 6200 case (sizeof (sin_t) - sizeof (sin->sin_zero)): { 6201 /* 6202 * XXX: The check for valid DEST_length was not there 6203 * in earlier releases and some buggy 6204 * TLI apps (e.g Sybase) got away with not feeding 6205 * in sin_zero part of address. 6206 * We allow that bug to keep those buggy apps humming. 6207 * Test suites require the check on DEST_length. 6208 * We construct a new mblk with valid DEST_length 6209 * free the original so the rest of the code does 6210 * not have to keep track of this special shorter 6211 * length address case. 6212 */ 6213 mblk_t *nmp; 6214 struct T_conn_req *ntcr; 6215 sin_t *nsin; 6216 6217 nmp = allocb(sizeof (struct T_conn_req) + sizeof (sin_t) + 6218 tcr->OPT_length, BPRI_HI); 6219 if (nmp == NULL) { 6220 tcp_err_ack(tcp, mp, TSYSERR, ENOMEM); 6221 return; 6222 } 6223 ntcr = (struct T_conn_req *)nmp->b_rptr; 6224 bzero(ntcr, sizeof (struct T_conn_req)); /* zero fill */ 6225 ntcr->PRIM_type = T_CONN_REQ; 6226 ntcr->DEST_length = sizeof (sin_t); 6227 ntcr->DEST_offset = sizeof (struct T_conn_req); 6228 6229 nsin = (sin_t *)((uchar_t *)ntcr + ntcr->DEST_offset); 6230 *nsin = sin_null; 6231 /* Get pointer to shorter address to copy from original mp */ 6232 sin = (sin_t *)mi_offset_param(mp, tcr->DEST_offset, 6233 tcr->DEST_length); /* extract DEST_length worth of sin_t */ 6234 if (sin == NULL || !OK_32PTR((char *)sin)) { 6235 freemsg(nmp); 6236 tcp_err_ack(tcp, mp, TSYSERR, EINVAL); 6237 return; 6238 } 6239 nsin->sin_family = sin->sin_family; 6240 nsin->sin_port = sin->sin_port; 6241 nsin->sin_addr = sin->sin_addr; 6242 /* Note:nsin->sin_zero zero-fill with sin_null assign above */ 6243 nmp->b_wptr = (uchar_t *)&nsin[1]; 6244 if (tcr->OPT_length != 0) { 6245 ntcr->OPT_length = tcr->OPT_length; 6246 ntcr->OPT_offset = nmp->b_wptr - nmp->b_rptr; 6247 bcopy((uchar_t *)tcr + tcr->OPT_offset, 6248 (uchar_t *)ntcr + ntcr->OPT_offset, 6249 tcr->OPT_length); 6250 nmp->b_wptr += tcr->OPT_length; 6251 } 6252 freemsg(mp); /* original mp freed */ 6253 mp = nmp; /* re-initialize original variables */ 6254 tcr = ntcr; 6255 } 6256 /* FALLTHRU */ 6257 6258 case sizeof (sin_t): 6259 sin = (sin_t *)mi_offset_param(mp, tcr->DEST_offset, 6260 sizeof (sin_t)); 6261 if (sin == NULL || !OK_32PTR((char *)sin)) { 6262 tcp_err_ack(tcp, mp, TSYSERR, EINVAL); 6263 return; 6264 } 6265 if (tcp->tcp_family != AF_INET || 6266 sin->sin_family != AF_INET) { 6267 tcp_err_ack(tcp, mp, TSYSERR, EAFNOSUPPORT); 6268 return; 6269 } 6270 if (sin->sin_port == 0) { 6271 tcp_err_ack(tcp, mp, TBADADDR, 0); 6272 return; 6273 } 6274 if (tcp->tcp_connp && tcp->tcp_connp->conn_ipv6_v6only) { 6275 tcp_err_ack(tcp, mp, TSYSERR, EAFNOSUPPORT); 6276 return; 6277 } 6278 6279 break; 6280 6281 case sizeof (sin6_t): 6282 sin6 = (sin6_t *)mi_offset_param(mp, tcr->DEST_offset, 6283 sizeof (sin6_t)); 6284 if (sin6 == NULL || !OK_32PTR((char *)sin6)) { 6285 tcp_err_ack(tcp, mp, TSYSERR, EINVAL); 6286 return; 6287 } 6288 if (tcp->tcp_family != AF_INET6 || 6289 sin6->sin6_family != AF_INET6) { 6290 tcp_err_ack(tcp, mp, TSYSERR, EAFNOSUPPORT); 6291 return; 6292 } 6293 if (sin6->sin6_port == 0) { 6294 tcp_err_ack(tcp, mp, TBADADDR, 0); 6295 return; 6296 } 6297 break; 6298 } 6299 /* 6300 * TODO: If someone in TCPS_TIME_WAIT has this dst/port we 6301 * should key on their sequence number and cut them loose. 6302 */ 6303 6304 /* 6305 * If options passed in, feed it for verification and handling 6306 */ 6307 if (tcr->OPT_length != 0) { 6308 mblk_t *ok_mp; 6309 mblk_t *discon_mp; 6310 mblk_t *conn_opts_mp; 6311 int t_error, sys_error, do_disconnect; 6312 6313 conn_opts_mp = NULL; 6314 6315 if (tcp_conprim_opt_process(tcp, mp, 6316 &do_disconnect, &t_error, &sys_error) < 0) { 6317 if (do_disconnect) { 6318 ASSERT(t_error == 0 && sys_error == 0); 6319 discon_mp = mi_tpi_discon_ind(NULL, 6320 ECONNREFUSED, 0); 6321 if (!discon_mp) { 6322 tcp_err_ack_prim(tcp, mp, T_CONN_REQ, 6323 TSYSERR, ENOMEM); 6324 return; 6325 } 6326 ok_mp = mi_tpi_ok_ack_alloc(mp); 6327 if (!ok_mp) { 6328 tcp_err_ack_prim(tcp, NULL, T_CONN_REQ, 6329 TSYSERR, ENOMEM); 6330 return; 6331 } 6332 qreply(q, ok_mp); 6333 qreply(q, discon_mp); /* no flush! */ 6334 } else { 6335 ASSERT(t_error != 0); 6336 tcp_err_ack_prim(tcp, mp, T_CONN_REQ, t_error, 6337 sys_error); 6338 } 6339 return; 6340 } 6341 /* 6342 * Success in setting options, the mp option buffer represented 6343 * by OPT_length/offset has been potentially modified and 6344 * contains results of option processing. We copy it in 6345 * another mp to save it for potentially influencing returning 6346 * it in T_CONN_CONN. 6347 */ 6348 if (tcr->OPT_length != 0) { /* there are resulting options */ 6349 conn_opts_mp = copyb(mp); 6350 if (!conn_opts_mp) { 6351 tcp_err_ack_prim(tcp, mp, T_CONN_REQ, 6352 TSYSERR, ENOMEM); 6353 return; 6354 } 6355 ASSERT(tcp->tcp_conn.tcp_opts_conn_req == NULL); 6356 tcp->tcp_conn.tcp_opts_conn_req = conn_opts_mp; 6357 /* 6358 * Note: 6359 * These resulting option negotiation can include any 6360 * end-to-end negotiation options but there no such 6361 * thing (yet?) in our TCP/IP. 6362 */ 6363 } 6364 } 6365 6366 /* 6367 * If we're connecting to an IPv4-mapped IPv6 address, we need to 6368 * make sure that the template IP header in the tcp structure is an 6369 * IPv4 header, and that the tcp_ipversion is IPV4_VERSION. We 6370 * need to this before we call tcp_bindi() so that the port lookup 6371 * code will look for ports in the correct port space (IPv4 and 6372 * IPv6 have separate port spaces). 6373 */ 6374 if (tcp->tcp_family == AF_INET6 && tcp->tcp_ipversion == IPV6_VERSION && 6375 IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) { 6376 int err = 0; 6377 6378 err = tcp_header_init_ipv4(tcp); 6379 if (err != 0) { 6380 mp = mi_tpi_err_ack_alloc(mp, TSYSERR, ENOMEM); 6381 goto connect_failed; 6382 } 6383 if (tcp->tcp_lport != 0) 6384 *(uint16_t *)tcp->tcp_tcph->th_lport = tcp->tcp_lport; 6385 } 6386 6387 if (tcp->tcp_issocket) { 6388 /* 6389 * TCP is _D_SODIRECT and sockfs is directly above so save 6390 * the shared sonode sodirect_t pointer (if any) to enable 6391 * TCP sodirect. 6392 */ 6393 tcp->tcp_sodirect = SOD_QTOSODP(tcp->tcp_rq); 6394 } 6395 6396 switch (tcp->tcp_state) { 6397 case TCPS_IDLE: 6398 /* 6399 * We support quick connect, refer to comments in 6400 * tcp_connect_*() 6401 */ 6402 /* FALLTHRU */ 6403 case TCPS_BOUND: 6404 case TCPS_LISTEN: 6405 if (tcp->tcp_family == AF_INET6) { 6406 if (!IN6_IS_ADDR_V4MAPPED(&sin6->sin6_addr)) { 6407 tcp_connect_ipv6(tcp, mp, 6408 &sin6->sin6_addr, 6409 sin6->sin6_port, sin6->sin6_flowinfo, 6410 sin6->__sin6_src_id, sin6->sin6_scope_id); 6411 return; 6412 } 6413 /* 6414 * Destination adress is mapped IPv6 address. 6415 * Source bound address should be unspecified or 6416 * IPv6 mapped address as well. 6417 */ 6418 if (!IN6_IS_ADDR_UNSPECIFIED( 6419 &tcp->tcp_bound_source_v6) && 6420 !IN6_IS_ADDR_V4MAPPED(&tcp->tcp_bound_source_v6)) { 6421 mp = mi_tpi_err_ack_alloc(mp, TSYSERR, 6422 EADDRNOTAVAIL); 6423 break; 6424 } 6425 dstaddrp = &V4_PART_OF_V6((sin6->sin6_addr)); 6426 dstport = sin6->sin6_port; 6427 srcid = sin6->__sin6_src_id; 6428 } else { 6429 dstaddrp = &sin->sin_addr.s_addr; 6430 dstport = sin->sin_port; 6431 srcid = 0; 6432 } 6433 6434 tcp_connect_ipv4(tcp, mp, dstaddrp, dstport, srcid); 6435 return; 6436 default: 6437 mp = mi_tpi_err_ack_alloc(mp, TOUTSTATE, 0); 6438 break; 6439 } 6440 /* 6441 * Note: Code below is the "failure" case 6442 */ 6443 /* return error ack and blow away saved option results if any */ 6444 connect_failed: 6445 if (mp != NULL) 6446 putnext(tcp->tcp_rq, mp); 6447 else { 6448 tcp_err_ack_prim(tcp, NULL, T_CONN_REQ, 6449 TSYSERR, ENOMEM); 6450 } 6451 if (tcp->tcp_conn.tcp_opts_conn_req != NULL) 6452 tcp_close_mpp(&tcp->tcp_conn.tcp_opts_conn_req); 6453 } 6454 6455 /* 6456 * Handle connect to IPv4 destinations, including connections for AF_INET6 6457 * sockets connecting to IPv4 mapped IPv6 destinations. 6458 */ 6459 static void 6460 tcp_connect_ipv4(tcp_t *tcp, mblk_t *mp, ipaddr_t *dstaddrp, in_port_t dstport, 6461 uint_t srcid) 6462 { 6463 tcph_t *tcph; 6464 mblk_t *mp1; 6465 ipaddr_t dstaddr = *dstaddrp; 6466 int32_t oldstate; 6467 uint16_t lport; 6468 tcp_stack_t *tcps = tcp->tcp_tcps; 6469 6470 ASSERT(tcp->tcp_ipversion == IPV4_VERSION); 6471 6472 /* Check for attempt to connect to INADDR_ANY */ 6473 if (dstaddr == INADDR_ANY) { 6474 /* 6475 * SunOS 4.x and 4.3 BSD allow an application 6476 * to connect a TCP socket to INADDR_ANY. 6477 * When they do this, the kernel picks the 6478 * address of one interface and uses it 6479 * instead. The kernel usually ends up 6480 * picking the address of the loopback 6481 * interface. This is an undocumented feature. 6482 * However, we provide the same thing here 6483 * in order to have source and binary 6484 * compatibility with SunOS 4.x. 6485 * Update the T_CONN_REQ (sin/sin6) since it is used to 6486 * generate the T_CONN_CON. 6487 */ 6488 dstaddr = htonl(INADDR_LOOPBACK); 6489 *dstaddrp = dstaddr; 6490 } 6491 6492 /* Handle __sin6_src_id if socket not bound to an IP address */ 6493 if (srcid != 0 && tcp->tcp_ipha->ipha_src == INADDR_ANY) { 6494 ip_srcid_find_id(srcid, &tcp->tcp_ip_src_v6, 6495 tcp->tcp_connp->conn_zoneid, tcps->tcps_netstack); 6496 IN6_V4MAPPED_TO_IPADDR(&tcp->tcp_ip_src_v6, 6497 tcp->tcp_ipha->ipha_src); 6498 } 6499 6500 /* 6501 * Don't let an endpoint connect to itself. Note that 6502 * the test here does not catch the case where the 6503 * source IP addr was left unspecified by the user. In 6504 * this case, the source addr is set in tcp_adapt_ire() 6505 * using the reply to the T_BIND message that we send 6506 * down to IP here and the check is repeated in tcp_rput_other. 6507 */ 6508 if (dstaddr == tcp->tcp_ipha->ipha_src && 6509 dstport == tcp->tcp_lport) { 6510 mp = mi_tpi_err_ack_alloc(mp, TBADADDR, 0); 6511 goto failed; 6512 } 6513 6514 tcp->tcp_ipha->ipha_dst = dstaddr; 6515 IN6_IPADDR_TO_V4MAPPED(dstaddr, &tcp->tcp_remote_v6); 6516 6517 /* 6518 * Massage a source route if any putting the first hop 6519 * in iph_dst. Compute a starting value for the checksum which 6520 * takes into account that the original iph_dst should be 6521 * included in the checksum but that ip will include the 6522 * first hop in the source route in the tcp checksum. 6523 */ 6524 tcp->tcp_sum = ip_massage_options(tcp->tcp_ipha, tcps->tcps_netstack); 6525 tcp->tcp_sum = (tcp->tcp_sum & 0xFFFF) + (tcp->tcp_sum >> 16); 6526 tcp->tcp_sum -= ((tcp->tcp_ipha->ipha_dst >> 16) + 6527 (tcp->tcp_ipha->ipha_dst & 0xffff)); 6528 if ((int)tcp->tcp_sum < 0) 6529 tcp->tcp_sum--; 6530 tcp->tcp_sum = (tcp->tcp_sum & 0xFFFF) + (tcp->tcp_sum >> 16); 6531 tcp->tcp_sum = ntohs((tcp->tcp_sum & 0xFFFF) + 6532 (tcp->tcp_sum >> 16)); 6533 tcph = tcp->tcp_tcph; 6534 *(uint16_t *)tcph->th_fport = dstport; 6535 tcp->tcp_fport = dstport; 6536 6537 oldstate = tcp->tcp_state; 6538 /* 6539 * At this point the remote destination address and remote port fields 6540 * in the tcp-four-tuple have been filled in the tcp structure. Now we 6541 * have to see which state tcp was in so we can take apropriate action. 6542 */ 6543 if (oldstate == TCPS_IDLE) { 6544 /* 6545 * We support a quick connect capability here, allowing 6546 * clients to transition directly from IDLE to SYN_SENT 6547 * tcp_bindi will pick an unused port, insert the connection 6548 * in the bind hash and transition to BOUND state. 6549 */ 6550 lport = tcp_update_next_port(tcps->tcps_next_port_to_try, 6551 tcp, B_TRUE); 6552 lport = tcp_bindi(tcp, lport, &tcp->tcp_ip_src_v6, 0, B_TRUE, 6553 B_FALSE, B_FALSE); 6554 if (lport == 0) { 6555 mp = mi_tpi_err_ack_alloc(mp, TNOADDR, 0); 6556 goto failed; 6557 } 6558 } 6559 tcp->tcp_state = TCPS_SYN_SENT; 6560 6561 /* 6562 * TODO: allow data with connect requests 6563 * by unlinking M_DATA trailers here and 6564 * linking them in behind the T_OK_ACK mblk. 6565 * The tcp_rput() bind ack handler would then 6566 * feed them to tcp_wput_data() rather than call 6567 * tcp_timer(). 6568 */ 6569 mp = mi_tpi_ok_ack_alloc(mp); 6570 if (!mp) { 6571 tcp->tcp_state = oldstate; 6572 goto failed; 6573 } 6574 if (tcp->tcp_family == AF_INET) { 6575 mp1 = tcp_ip_bind_mp(tcp, O_T_BIND_REQ, 6576 sizeof (ipa_conn_t)); 6577 } else { 6578 mp1 = tcp_ip_bind_mp(tcp, O_T_BIND_REQ, 6579 sizeof (ipa6_conn_t)); 6580 } 6581 if (mp1) { 6582 /* 6583 * We need to make sure that the conn_recv is set to a non-null 6584 * value before we insert the conn_t into the classifier table. 6585 * This is to avoid a race with an incoming packet which does 6586 * an ipcl_classify(). 6587 */ 6588 tcp->tcp_connp->conn_recv = tcp_input; 6589 6590 /* Hang onto the T_OK_ACK for later. */ 6591 linkb(mp1, mp); 6592 mblk_setcred(mp1, tcp->tcp_cred); 6593 if (tcp->tcp_family == AF_INET) 6594 mp1 = ip_bind_v4(tcp->tcp_wq, mp1, tcp->tcp_connp); 6595 else { 6596 mp1 = ip_bind_v6(tcp->tcp_wq, mp1, tcp->tcp_connp, 6597 &tcp->tcp_sticky_ipp); 6598 } 6599 BUMP_MIB(&tcps->tcps_mib, tcpActiveOpens); 6600 tcp->tcp_active_open = 1; 6601 /* 6602 * If the bind cannot complete immediately 6603 * IP will arrange to call tcp_rput_other 6604 * when the bind completes. 6605 */ 6606 if (mp1 != NULL) 6607 tcp_rput_other(tcp, mp1); 6608 return; 6609 } 6610 /* Error case */ 6611 tcp->tcp_state = oldstate; 6612 mp = mi_tpi_err_ack_alloc(mp, TSYSERR, ENOMEM); 6613 6614 failed: 6615 /* return error ack and blow away saved option results if any */ 6616 if (mp != NULL) 6617 putnext(tcp->tcp_rq, mp); 6618 else { 6619 tcp_err_ack_prim(tcp, NULL, T_CONN_REQ, 6620 TSYSERR, ENOMEM); 6621 } 6622 if (tcp->tcp_conn.tcp_opts_conn_req != NULL) 6623 tcp_close_mpp(&tcp->tcp_conn.tcp_opts_conn_req); 6624 6625 } 6626 6627 /* 6628 * Handle connect to IPv6 destinations. 6629 */ 6630 static void 6631 tcp_connect_ipv6(tcp_t *tcp, mblk_t *mp, in6_addr_t *dstaddrp, 6632 in_port_t dstport, uint32_t flowinfo, uint_t srcid, uint32_t scope_id) 6633 { 6634 tcph_t *tcph; 6635 mblk_t *mp1; 6636 ip6_rthdr_t *rth; 6637 int32_t oldstate; 6638 uint16_t lport; 6639 tcp_stack_t *tcps = tcp->tcp_tcps; 6640 6641 ASSERT(tcp->tcp_family == AF_INET6); 6642 6643 /* 6644 * If we're here, it means that the destination address is a native 6645 * IPv6 address. Return an error if tcp_ipversion is not IPv6. A 6646 * reason why it might not be IPv6 is if the socket was bound to an 6647 * IPv4-mapped IPv6 address. 6648 */ 6649 if (tcp->tcp_ipversion != IPV6_VERSION) { 6650 mp = mi_tpi_err_ack_alloc(mp, TBADADDR, 0); 6651 goto failed; 6652 } 6653 6654 /* 6655 * Interpret a zero destination to mean loopback. 6656 * Update the T_CONN_REQ (sin/sin6) since it is used to 6657 * generate the T_CONN_CON. 6658 */ 6659 if (IN6_IS_ADDR_UNSPECIFIED(dstaddrp)) { 6660 *dstaddrp = ipv6_loopback; 6661 } 6662 6663 /* Handle __sin6_src_id if socket not bound to an IP address */ 6664 if (srcid != 0 && IN6_IS_ADDR_UNSPECIFIED(&tcp->tcp_ip6h->ip6_src)) { 6665 ip_srcid_find_id(srcid, &tcp->tcp_ip6h->ip6_src, 6666 tcp->tcp_connp->conn_zoneid, tcps->tcps_netstack); 6667 tcp->tcp_ip_src_v6 = tcp->tcp_ip6h->ip6_src; 6668 } 6669 6670 /* 6671 * Take care of the scope_id now and add ip6i_t 6672 * if ip6i_t is not already allocated through TCP 6673 * sticky options. At this point tcp_ip6h does not 6674 * have dst info, thus use dstaddrp. 6675 */ 6676 if (scope_id != 0 && 6677 IN6_IS_ADDR_LINKSCOPE(dstaddrp)) { 6678 ip6_pkt_t *ipp = &tcp->tcp_sticky_ipp; 6679 ip6i_t *ip6i; 6680 6681 ipp->ipp_ifindex = scope_id; 6682 ip6i = (ip6i_t *)tcp->tcp_iphc; 6683 6684 if ((ipp->ipp_fields & IPPF_HAS_IP6I) && 6685 ip6i != NULL && (ip6i->ip6i_nxt == IPPROTO_RAW)) { 6686 /* Already allocated */ 6687 ip6i->ip6i_flags |= IP6I_IFINDEX; 6688 ip6i->ip6i_ifindex = ipp->ipp_ifindex; 6689 ipp->ipp_fields |= IPPF_SCOPE_ID; 6690 } else { 6691 int reterr; 6692 6693 ipp->ipp_fields |= IPPF_SCOPE_ID; 6694 if (ipp->ipp_fields & IPPF_HAS_IP6I) 6695 ip2dbg(("tcp_connect_v6: SCOPE_ID set\n")); 6696 reterr = tcp_build_hdrs(tcp->tcp_rq, tcp); 6697 if (reterr != 0) 6698 goto failed; 6699 ip1dbg(("tcp_connect_ipv6: tcp_bld_hdrs returned\n")); 6700 } 6701 } 6702 6703 /* 6704 * Don't let an endpoint connect to itself. Note that 6705 * the test here does not catch the case where the 6706 * source IP addr was left unspecified by the user. In 6707 * this case, the source addr is set in tcp_adapt_ire() 6708 * using the reply to the T_BIND message that we send 6709 * down to IP here and the check is repeated in tcp_rput_other. 6710 */ 6711 if (IN6_ARE_ADDR_EQUAL(dstaddrp, &tcp->tcp_ip6h->ip6_src) && 6712 (dstport == tcp->tcp_lport)) { 6713 mp = mi_tpi_err_ack_alloc(mp, TBADADDR, 0); 6714 goto failed; 6715 } 6716 6717 tcp->tcp_ip6h->ip6_dst = *dstaddrp; 6718 tcp->tcp_remote_v6 = *dstaddrp; 6719 tcp->tcp_ip6h->ip6_vcf = 6720 (IPV6_DEFAULT_VERS_AND_FLOW & IPV6_VERS_AND_FLOW_MASK) | 6721 (flowinfo & ~IPV6_VERS_AND_FLOW_MASK); 6722 6723 6724 /* 6725 * Massage a routing header (if present) putting the first hop 6726 * in ip6_dst. Compute a starting value for the checksum which 6727 * takes into account that the original ip6_dst should be 6728 * included in the checksum but that ip will include the 6729 * first hop in the source route in the tcp checksum. 6730 */ 6731 rth = ip_find_rthdr_v6(tcp->tcp_ip6h, (uint8_t *)tcp->tcp_tcph); 6732 if (rth != NULL) { 6733 tcp->tcp_sum = ip_massage_options_v6(tcp->tcp_ip6h, rth, 6734 tcps->tcps_netstack); 6735 tcp->tcp_sum = ntohs((tcp->tcp_sum & 0xFFFF) + 6736 (tcp->tcp_sum >> 16)); 6737 } else { 6738 tcp->tcp_sum = 0; 6739 } 6740 6741 tcph = tcp->tcp_tcph; 6742 *(uint16_t *)tcph->th_fport = dstport; 6743 tcp->tcp_fport = dstport; 6744 6745 oldstate = tcp->tcp_state; 6746 /* 6747 * At this point the remote destination address and remote port fields 6748 * in the tcp-four-tuple have been filled in the tcp structure. Now we 6749 * have to see which state tcp was in so we can take apropriate action. 6750 */ 6751 if (oldstate == TCPS_IDLE) { 6752 /* 6753 * We support a quick connect capability here, allowing 6754 * clients to transition directly from IDLE to SYN_SENT 6755 * tcp_bindi will pick an unused port, insert the connection 6756 * in the bind hash and transition to BOUND state. 6757 */ 6758 lport = tcp_update_next_port(tcps->tcps_next_port_to_try, 6759 tcp, B_TRUE); 6760 lport = tcp_bindi(tcp, lport, &tcp->tcp_ip_src_v6, 0, B_TRUE, 6761 B_FALSE, B_FALSE); 6762 if (lport == 0) { 6763 mp = mi_tpi_err_ack_alloc(mp, TNOADDR, 0); 6764 goto failed; 6765 } 6766 } 6767 tcp->tcp_state = TCPS_SYN_SENT; 6768 /* 6769 * TODO: allow data with connect requests 6770 * by unlinking M_DATA trailers here and 6771 * linking them in behind the T_OK_ACK mblk. 6772 * The tcp_rput() bind ack handler would then 6773 * feed them to tcp_wput_data() rather than call 6774 * tcp_timer(). 6775 */ 6776 mp = mi_tpi_ok_ack_alloc(mp); 6777 if (!mp) { 6778 tcp->tcp_state = oldstate; 6779 goto failed; 6780 } 6781 mp1 = tcp_ip_bind_mp(tcp, O_T_BIND_REQ, sizeof (ipa6_conn_t)); 6782 if (mp1) { 6783 /* 6784 * We need to make sure that the conn_recv is set to a non-null 6785 * value before we insert the conn_t into the classifier table. 6786 * This is to avoid a race with an incoming packet which does 6787 * an ipcl_classify(). 6788 */ 6789 tcp->tcp_connp->conn_recv = tcp_input; 6790 6791 /* Hang onto the T_OK_ACK for later. */ 6792 linkb(mp1, mp); 6793 mblk_setcred(mp1, tcp->tcp_cred); 6794 mp1 = ip_bind_v6(tcp->tcp_wq, mp1, tcp->tcp_connp, 6795 &tcp->tcp_sticky_ipp); 6796 BUMP_MIB(&tcps->tcps_mib, tcpActiveOpens); 6797 tcp->tcp_active_open = 1; 6798 /* ip_bind_v6() may return ACK or ERROR */ 6799 if (mp1 != NULL) 6800 tcp_rput_other(tcp, mp1); 6801 return; 6802 } 6803 /* Error case */ 6804 tcp->tcp_state = oldstate; 6805 mp = mi_tpi_err_ack_alloc(mp, TSYSERR, ENOMEM); 6806 6807 failed: 6808 /* return error ack and blow away saved option results if any */ 6809 if (mp != NULL) 6810 putnext(tcp->tcp_rq, mp); 6811 else { 6812 tcp_err_ack_prim(tcp, NULL, T_CONN_REQ, 6813 TSYSERR, ENOMEM); 6814 } 6815 if (tcp->tcp_conn.tcp_opts_conn_req != NULL) 6816 tcp_close_mpp(&tcp->tcp_conn.tcp_opts_conn_req); 6817 } 6818 6819 /* 6820 * We need a stream q for detached closing tcp connections 6821 * to use. Our client hereby indicates that this q is the 6822 * one to use. 6823 */ 6824 static void 6825 tcp_def_q_set(tcp_t *tcp, mblk_t *mp) 6826 { 6827 struct iocblk *iocp = (struct iocblk *)mp->b_rptr; 6828 queue_t *q = tcp->tcp_wq; 6829 tcp_stack_t *tcps = tcp->tcp_tcps; 6830 6831 #ifdef NS_DEBUG 6832 (void) printf("TCP_IOC_DEFAULT_Q for stack %d\n", 6833 tcps->tcps_netstack->netstack_stackid); 6834 #endif 6835 mp->b_datap->db_type = M_IOCACK; 6836 iocp->ioc_count = 0; 6837 mutex_enter(&tcps->tcps_g_q_lock); 6838 if (tcps->tcps_g_q != NULL) { 6839 mutex_exit(&tcps->tcps_g_q_lock); 6840 iocp->ioc_error = EALREADY; 6841 } else { 6842 mblk_t *mp1; 6843 6844 mp1 = tcp_ip_bind_mp(tcp, O_T_BIND_REQ, 0); 6845 if (mp1 == NULL) { 6846 mutex_exit(&tcps->tcps_g_q_lock); 6847 iocp->ioc_error = ENOMEM; 6848 } else { 6849 tcps->tcps_g_q = tcp->tcp_rq; 6850 mutex_exit(&tcps->tcps_g_q_lock); 6851 iocp->ioc_error = 0; 6852 iocp->ioc_rval = 0; 6853 /* 6854 * We are passing tcp_sticky_ipp as NULL 6855 * as it is not useful for tcp_default queue 6856 * 6857 * Set conn_recv just in case. 6858 */ 6859 tcp->tcp_connp->conn_recv = tcp_conn_request; 6860 6861 mp1 = ip_bind_v6(q, mp1, tcp->tcp_connp, NULL); 6862 if (mp1 != NULL) 6863 tcp_rput_other(tcp, mp1); 6864 } 6865 } 6866 qreply(q, mp); 6867 } 6868 6869 /* 6870 * Our client hereby directs us to reject the connection request 6871 * that tcp_conn_request() marked with 'seqnum'. Rejection consists 6872 * of sending the appropriate RST, not an ICMP error. 6873 */ 6874 static void 6875 tcp_disconnect(tcp_t *tcp, mblk_t *mp) 6876 { 6877 tcp_t *ltcp = NULL; 6878 t_scalar_t seqnum; 6879 conn_t *connp; 6880 tcp_stack_t *tcps = tcp->tcp_tcps; 6881 6882 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= (uintptr_t)INT_MAX); 6883 if ((mp->b_wptr - mp->b_rptr) < sizeof (struct T_discon_req)) { 6884 tcp_err_ack(tcp, mp, TPROTO, 0); 6885 return; 6886 } 6887 6888 /* 6889 * Right now, upper modules pass down a T_DISCON_REQ to TCP, 6890 * when the stream is in BOUND state. Do not send a reset, 6891 * since the destination IP address is not valid, and it can 6892 * be the initialized value of all zeros (broadcast address). 6893 * 6894 * If TCP has sent down a bind request to IP and has not 6895 * received the reply, reject the request. Otherwise, TCP 6896 * will be confused. 6897 */ 6898 if (tcp->tcp_state <= TCPS_BOUND || tcp->tcp_hard_binding) { 6899 if (tcp->tcp_debug) { 6900 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE, 6901 "tcp_disconnect: bad state, %d", tcp->tcp_state); 6902 } 6903 tcp_err_ack(tcp, mp, TOUTSTATE, 0); 6904 return; 6905 } 6906 6907 seqnum = ((struct T_discon_req *)mp->b_rptr)->SEQ_number; 6908 6909 if (seqnum == -1 || tcp->tcp_conn_req_max == 0) { 6910 6911 /* 6912 * According to TPI, for non-listeners, ignore seqnum 6913 * and disconnect. 6914 * Following interpretation of -1 seqnum is historical 6915 * and implied TPI ? (TPI only states that for T_CONN_IND, 6916 * a valid seqnum should not be -1). 6917 * 6918 * -1 means disconnect everything 6919 * regardless even on a listener. 6920 */ 6921 6922 int old_state = tcp->tcp_state; 6923 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 6924 6925 /* 6926 * The connection can't be on the tcp_time_wait_head list 6927 * since it is not detached. 6928 */ 6929 ASSERT(tcp->tcp_time_wait_next == NULL); 6930 ASSERT(tcp->tcp_time_wait_prev == NULL); 6931 ASSERT(tcp->tcp_time_wait_expire == 0); 6932 ltcp = NULL; 6933 /* 6934 * If it used to be a listener, check to make sure no one else 6935 * has taken the port before switching back to LISTEN state. 6936 */ 6937 if (tcp->tcp_ipversion == IPV4_VERSION) { 6938 connp = ipcl_lookup_listener_v4(tcp->tcp_lport, 6939 tcp->tcp_ipha->ipha_src, 6940 tcp->tcp_connp->conn_zoneid, ipst); 6941 if (connp != NULL) 6942 ltcp = connp->conn_tcp; 6943 } else { 6944 /* Allow tcp_bound_if listeners? */ 6945 connp = ipcl_lookup_listener_v6(tcp->tcp_lport, 6946 &tcp->tcp_ip6h->ip6_src, 0, 6947 tcp->tcp_connp->conn_zoneid, ipst); 6948 if (connp != NULL) 6949 ltcp = connp->conn_tcp; 6950 } 6951 if (tcp->tcp_conn_req_max && ltcp == NULL) { 6952 tcp->tcp_state = TCPS_LISTEN; 6953 } else if (old_state > TCPS_BOUND) { 6954 tcp->tcp_conn_req_max = 0; 6955 tcp->tcp_state = TCPS_BOUND; 6956 } 6957 if (ltcp != NULL) 6958 CONN_DEC_REF(ltcp->tcp_connp); 6959 if (old_state == TCPS_SYN_SENT || old_state == TCPS_SYN_RCVD) { 6960 BUMP_MIB(&tcps->tcps_mib, tcpAttemptFails); 6961 } else if (old_state == TCPS_ESTABLISHED || 6962 old_state == TCPS_CLOSE_WAIT) { 6963 BUMP_MIB(&tcps->tcps_mib, tcpEstabResets); 6964 } 6965 6966 if (tcp->tcp_fused) 6967 tcp_unfuse(tcp); 6968 6969 mutex_enter(&tcp->tcp_eager_lock); 6970 if ((tcp->tcp_conn_req_cnt_q0 != 0) || 6971 (tcp->tcp_conn_req_cnt_q != 0)) { 6972 tcp_eager_cleanup(tcp, 0); 6973 } 6974 mutex_exit(&tcp->tcp_eager_lock); 6975 6976 tcp_xmit_ctl("tcp_disconnect", tcp, tcp->tcp_snxt, 6977 tcp->tcp_rnxt, TH_RST | TH_ACK); 6978 6979 tcp_reinit(tcp); 6980 6981 if (old_state >= TCPS_ESTABLISHED) { 6982 /* Send M_FLUSH according to TPI */ 6983 (void) putnextctl1(tcp->tcp_rq, M_FLUSH, FLUSHRW); 6984 } 6985 mp = mi_tpi_ok_ack_alloc(mp); 6986 if (mp) 6987 putnext(tcp->tcp_rq, mp); 6988 return; 6989 } else if (!tcp_eager_blowoff(tcp, seqnum)) { 6990 tcp_err_ack(tcp, mp, TBADSEQ, 0); 6991 return; 6992 } 6993 if (tcp->tcp_state >= TCPS_ESTABLISHED) { 6994 /* Send M_FLUSH according to TPI */ 6995 (void) putnextctl1(tcp->tcp_rq, M_FLUSH, FLUSHRW); 6996 } 6997 mp = mi_tpi_ok_ack_alloc(mp); 6998 if (mp) 6999 putnext(tcp->tcp_rq, mp); 7000 } 7001 7002 /* 7003 * Diagnostic routine used to return a string associated with the tcp state. 7004 * Note that if the caller does not supply a buffer, it will use an internal 7005 * static string. This means that if multiple threads call this function at 7006 * the same time, output can be corrupted... Note also that this function 7007 * does not check the size of the supplied buffer. The caller has to make 7008 * sure that it is big enough. 7009 */ 7010 static char * 7011 tcp_display(tcp_t *tcp, char *sup_buf, char format) 7012 { 7013 char buf1[30]; 7014 static char priv_buf[INET6_ADDRSTRLEN * 2 + 80]; 7015 char *buf; 7016 char *cp; 7017 in6_addr_t local, remote; 7018 char local_addrbuf[INET6_ADDRSTRLEN]; 7019 char remote_addrbuf[INET6_ADDRSTRLEN]; 7020 7021 if (sup_buf != NULL) 7022 buf = sup_buf; 7023 else 7024 buf = priv_buf; 7025 7026 if (tcp == NULL) 7027 return ("NULL_TCP"); 7028 switch (tcp->tcp_state) { 7029 case TCPS_CLOSED: 7030 cp = "TCP_CLOSED"; 7031 break; 7032 case TCPS_IDLE: 7033 cp = "TCP_IDLE"; 7034 break; 7035 case TCPS_BOUND: 7036 cp = "TCP_BOUND"; 7037 break; 7038 case TCPS_LISTEN: 7039 cp = "TCP_LISTEN"; 7040 break; 7041 case TCPS_SYN_SENT: 7042 cp = "TCP_SYN_SENT"; 7043 break; 7044 case TCPS_SYN_RCVD: 7045 cp = "TCP_SYN_RCVD"; 7046 break; 7047 case TCPS_ESTABLISHED: 7048 cp = "TCP_ESTABLISHED"; 7049 break; 7050 case TCPS_CLOSE_WAIT: 7051 cp = "TCP_CLOSE_WAIT"; 7052 break; 7053 case TCPS_FIN_WAIT_1: 7054 cp = "TCP_FIN_WAIT_1"; 7055 break; 7056 case TCPS_CLOSING: 7057 cp = "TCP_CLOSING"; 7058 break; 7059 case TCPS_LAST_ACK: 7060 cp = "TCP_LAST_ACK"; 7061 break; 7062 case TCPS_FIN_WAIT_2: 7063 cp = "TCP_FIN_WAIT_2"; 7064 break; 7065 case TCPS_TIME_WAIT: 7066 cp = "TCP_TIME_WAIT"; 7067 break; 7068 default: 7069 (void) mi_sprintf(buf1, "TCPUnkState(%d)", tcp->tcp_state); 7070 cp = buf1; 7071 break; 7072 } 7073 switch (format) { 7074 case DISP_ADDR_AND_PORT: 7075 if (tcp->tcp_ipversion == IPV4_VERSION) { 7076 /* 7077 * Note that we use the remote address in the tcp_b 7078 * structure. This means that it will print out 7079 * the real destination address, not the next hop's 7080 * address if source routing is used. 7081 */ 7082 IN6_IPADDR_TO_V4MAPPED(tcp->tcp_ip_src, &local); 7083 IN6_IPADDR_TO_V4MAPPED(tcp->tcp_remote, &remote); 7084 7085 } else { 7086 local = tcp->tcp_ip_src_v6; 7087 remote = tcp->tcp_remote_v6; 7088 } 7089 (void) inet_ntop(AF_INET6, &local, local_addrbuf, 7090 sizeof (local_addrbuf)); 7091 (void) inet_ntop(AF_INET6, &remote, remote_addrbuf, 7092 sizeof (remote_addrbuf)); 7093 (void) mi_sprintf(buf, "[%s.%u, %s.%u] %s", 7094 local_addrbuf, ntohs(tcp->tcp_lport), remote_addrbuf, 7095 ntohs(tcp->tcp_fport), cp); 7096 break; 7097 case DISP_PORT_ONLY: 7098 default: 7099 (void) mi_sprintf(buf, "[%u, %u] %s", 7100 ntohs(tcp->tcp_lport), ntohs(tcp->tcp_fport), cp); 7101 break; 7102 } 7103 7104 return (buf); 7105 } 7106 7107 /* 7108 * Called via squeue to get on to eager's perimeter. It sends a 7109 * TH_RST if eager is in the fanout table. The listener wants the 7110 * eager to disappear either by means of tcp_eager_blowoff() or 7111 * tcp_eager_cleanup() being called. tcp_eager_kill() can also be 7112 * called (via squeue) if the eager cannot be inserted in the 7113 * fanout table in tcp_conn_request(). 7114 */ 7115 /* ARGSUSED */ 7116 void 7117 tcp_eager_kill(void *arg, mblk_t *mp, void *arg2) 7118 { 7119 conn_t *econnp = (conn_t *)arg; 7120 tcp_t *eager = econnp->conn_tcp; 7121 tcp_t *listener = eager->tcp_listener; 7122 tcp_stack_t *tcps = eager->tcp_tcps; 7123 7124 /* 7125 * We could be called because listener is closing. Since 7126 * the eager is using listener's queue's, its not safe. 7127 * Better use the default queue just to send the TH_RST 7128 * out. 7129 */ 7130 ASSERT(tcps->tcps_g_q != NULL); 7131 eager->tcp_rq = tcps->tcps_g_q; 7132 eager->tcp_wq = WR(tcps->tcps_g_q); 7133 7134 /* 7135 * An eager's conn_fanout will be NULL if it's a duplicate 7136 * for an existing 4-tuples in the conn fanout table. 7137 * We don't want to send an RST out in such case. 7138 */ 7139 if (econnp->conn_fanout != NULL && eager->tcp_state > TCPS_LISTEN) { 7140 tcp_xmit_ctl("tcp_eager_kill, can't wait", 7141 eager, eager->tcp_snxt, 0, TH_RST); 7142 } 7143 7144 /* We are here because listener wants this eager gone */ 7145 if (listener != NULL) { 7146 mutex_enter(&listener->tcp_eager_lock); 7147 tcp_eager_unlink(eager); 7148 if (eager->tcp_tconnind_started) { 7149 /* 7150 * The eager has sent a conn_ind up to the 7151 * listener but listener decides to close 7152 * instead. We need to drop the extra ref 7153 * placed on eager in tcp_rput_data() before 7154 * sending the conn_ind to listener. 7155 */ 7156 CONN_DEC_REF(econnp); 7157 } 7158 mutex_exit(&listener->tcp_eager_lock); 7159 CONN_DEC_REF(listener->tcp_connp); 7160 } 7161 7162 if (eager->tcp_state > TCPS_BOUND) 7163 tcp_close_detached(eager); 7164 } 7165 7166 /* 7167 * Reset any eager connection hanging off this listener marked 7168 * with 'seqnum' and then reclaim it's resources. 7169 */ 7170 static boolean_t 7171 tcp_eager_blowoff(tcp_t *listener, t_scalar_t seqnum) 7172 { 7173 tcp_t *eager; 7174 mblk_t *mp; 7175 tcp_stack_t *tcps = listener->tcp_tcps; 7176 7177 TCP_STAT(tcps, tcp_eager_blowoff_calls); 7178 eager = listener; 7179 mutex_enter(&listener->tcp_eager_lock); 7180 do { 7181 eager = eager->tcp_eager_next_q; 7182 if (eager == NULL) { 7183 mutex_exit(&listener->tcp_eager_lock); 7184 return (B_FALSE); 7185 } 7186 } while (eager->tcp_conn_req_seqnum != seqnum); 7187 7188 if (eager->tcp_closemp_used) { 7189 mutex_exit(&listener->tcp_eager_lock); 7190 return (B_TRUE); 7191 } 7192 eager->tcp_closemp_used = B_TRUE; 7193 TCP_DEBUG_GETPCSTACK(eager->tcmp_stk, 15); 7194 CONN_INC_REF(eager->tcp_connp); 7195 mutex_exit(&listener->tcp_eager_lock); 7196 mp = &eager->tcp_closemp; 7197 squeue_fill(eager->tcp_connp->conn_sqp, mp, tcp_eager_kill, 7198 eager->tcp_connp, SQTAG_TCP_EAGER_BLOWOFF); 7199 return (B_TRUE); 7200 } 7201 7202 /* 7203 * Reset any eager connection hanging off this listener 7204 * and then reclaim it's resources. 7205 */ 7206 static void 7207 tcp_eager_cleanup(tcp_t *listener, boolean_t q0_only) 7208 { 7209 tcp_t *eager; 7210 mblk_t *mp; 7211 tcp_stack_t *tcps = listener->tcp_tcps; 7212 7213 ASSERT(MUTEX_HELD(&listener->tcp_eager_lock)); 7214 7215 if (!q0_only) { 7216 /* First cleanup q */ 7217 TCP_STAT(tcps, tcp_eager_blowoff_q); 7218 eager = listener->tcp_eager_next_q; 7219 while (eager != NULL) { 7220 if (!eager->tcp_closemp_used) { 7221 eager->tcp_closemp_used = B_TRUE; 7222 TCP_DEBUG_GETPCSTACK(eager->tcmp_stk, 15); 7223 CONN_INC_REF(eager->tcp_connp); 7224 mp = &eager->tcp_closemp; 7225 squeue_fill(eager->tcp_connp->conn_sqp, mp, 7226 tcp_eager_kill, eager->tcp_connp, 7227 SQTAG_TCP_EAGER_CLEANUP); 7228 } 7229 eager = eager->tcp_eager_next_q; 7230 } 7231 } 7232 /* Then cleanup q0 */ 7233 TCP_STAT(tcps, tcp_eager_blowoff_q0); 7234 eager = listener->tcp_eager_next_q0; 7235 while (eager != listener) { 7236 if (!eager->tcp_closemp_used) { 7237 eager->tcp_closemp_used = B_TRUE; 7238 TCP_DEBUG_GETPCSTACK(eager->tcmp_stk, 15); 7239 CONN_INC_REF(eager->tcp_connp); 7240 mp = &eager->tcp_closemp; 7241 squeue_fill(eager->tcp_connp->conn_sqp, mp, 7242 tcp_eager_kill, eager->tcp_connp, 7243 SQTAG_TCP_EAGER_CLEANUP_Q0); 7244 } 7245 eager = eager->tcp_eager_next_q0; 7246 } 7247 } 7248 7249 /* 7250 * If we are an eager connection hanging off a listener that hasn't 7251 * formally accepted the connection yet, get off his list and blow off 7252 * any data that we have accumulated. 7253 */ 7254 static void 7255 tcp_eager_unlink(tcp_t *tcp) 7256 { 7257 tcp_t *listener = tcp->tcp_listener; 7258 7259 ASSERT(MUTEX_HELD(&listener->tcp_eager_lock)); 7260 ASSERT(listener != NULL); 7261 if (tcp->tcp_eager_next_q0 != NULL) { 7262 ASSERT(tcp->tcp_eager_prev_q0 != NULL); 7263 7264 /* Remove the eager tcp from q0 */ 7265 tcp->tcp_eager_next_q0->tcp_eager_prev_q0 = 7266 tcp->tcp_eager_prev_q0; 7267 tcp->tcp_eager_prev_q0->tcp_eager_next_q0 = 7268 tcp->tcp_eager_next_q0; 7269 ASSERT(listener->tcp_conn_req_cnt_q0 > 0); 7270 listener->tcp_conn_req_cnt_q0--; 7271 7272 tcp->tcp_eager_next_q0 = NULL; 7273 tcp->tcp_eager_prev_q0 = NULL; 7274 7275 /* 7276 * Take the eager out, if it is in the list of droppable 7277 * eagers. 7278 */ 7279 MAKE_UNDROPPABLE(tcp); 7280 7281 if (tcp->tcp_syn_rcvd_timeout != 0) { 7282 /* we have timed out before */ 7283 ASSERT(listener->tcp_syn_rcvd_timeout > 0); 7284 listener->tcp_syn_rcvd_timeout--; 7285 } 7286 } else { 7287 tcp_t **tcpp = &listener->tcp_eager_next_q; 7288 tcp_t *prev = NULL; 7289 7290 for (; tcpp[0]; tcpp = &tcpp[0]->tcp_eager_next_q) { 7291 if (tcpp[0] == tcp) { 7292 if (listener->tcp_eager_last_q == tcp) { 7293 /* 7294 * If we are unlinking the last 7295 * element on the list, adjust 7296 * tail pointer. Set tail pointer 7297 * to nil when list is empty. 7298 */ 7299 ASSERT(tcp->tcp_eager_next_q == NULL); 7300 if (listener->tcp_eager_last_q == 7301 listener->tcp_eager_next_q) { 7302 listener->tcp_eager_last_q = 7303 NULL; 7304 } else { 7305 /* 7306 * We won't get here if there 7307 * is only one eager in the 7308 * list. 7309 */ 7310 ASSERT(prev != NULL); 7311 listener->tcp_eager_last_q = 7312 prev; 7313 } 7314 } 7315 tcpp[0] = tcp->tcp_eager_next_q; 7316 tcp->tcp_eager_next_q = NULL; 7317 tcp->tcp_eager_last_q = NULL; 7318 ASSERT(listener->tcp_conn_req_cnt_q > 0); 7319 listener->tcp_conn_req_cnt_q--; 7320 break; 7321 } 7322 prev = tcpp[0]; 7323 } 7324 } 7325 tcp->tcp_listener = NULL; 7326 } 7327 7328 /* Shorthand to generate and send TPI error acks to our client */ 7329 static void 7330 tcp_err_ack(tcp_t *tcp, mblk_t *mp, int t_error, int sys_error) 7331 { 7332 if ((mp = mi_tpi_err_ack_alloc(mp, t_error, sys_error)) != NULL) 7333 putnext(tcp->tcp_rq, mp); 7334 } 7335 7336 /* Shorthand to generate and send TPI error acks to our client */ 7337 static void 7338 tcp_err_ack_prim(tcp_t *tcp, mblk_t *mp, int primitive, 7339 int t_error, int sys_error) 7340 { 7341 struct T_error_ack *teackp; 7342 7343 if ((mp = tpi_ack_alloc(mp, sizeof (struct T_error_ack), 7344 M_PCPROTO, T_ERROR_ACK)) != NULL) { 7345 teackp = (struct T_error_ack *)mp->b_rptr; 7346 teackp->ERROR_prim = primitive; 7347 teackp->TLI_error = t_error; 7348 teackp->UNIX_error = sys_error; 7349 putnext(tcp->tcp_rq, mp); 7350 } 7351 } 7352 7353 /* 7354 * Note: No locks are held when inspecting tcp_g_*epriv_ports 7355 * but instead the code relies on: 7356 * - the fact that the address of the array and its size never changes 7357 * - the atomic assignment of the elements of the array 7358 */ 7359 /* ARGSUSED */ 7360 static int 7361 tcp_extra_priv_ports_get(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr) 7362 { 7363 int i; 7364 tcp_stack_t *tcps = Q_TO_TCP(q)->tcp_tcps; 7365 7366 for (i = 0; i < tcps->tcps_g_num_epriv_ports; i++) { 7367 if (tcps->tcps_g_epriv_ports[i] != 0) 7368 (void) mi_mpprintf(mp, "%d ", 7369 tcps->tcps_g_epriv_ports[i]); 7370 } 7371 return (0); 7372 } 7373 7374 /* 7375 * Hold a lock while changing tcp_g_epriv_ports to prevent multiple 7376 * threads from changing it at the same time. 7377 */ 7378 /* ARGSUSED */ 7379 static int 7380 tcp_extra_priv_ports_add(queue_t *q, mblk_t *mp, char *value, caddr_t cp, 7381 cred_t *cr) 7382 { 7383 long new_value; 7384 int i; 7385 tcp_stack_t *tcps = Q_TO_TCP(q)->tcp_tcps; 7386 7387 /* 7388 * Fail the request if the new value does not lie within the 7389 * port number limits. 7390 */ 7391 if (ddi_strtol(value, NULL, 10, &new_value) != 0 || 7392 new_value <= 0 || new_value >= 65536) { 7393 return (EINVAL); 7394 } 7395 7396 mutex_enter(&tcps->tcps_epriv_port_lock); 7397 /* Check if the value is already in the list */ 7398 for (i = 0; i < tcps->tcps_g_num_epriv_ports; i++) { 7399 if (new_value == tcps->tcps_g_epriv_ports[i]) { 7400 mutex_exit(&tcps->tcps_epriv_port_lock); 7401 return (EEXIST); 7402 } 7403 } 7404 /* Find an empty slot */ 7405 for (i = 0; i < tcps->tcps_g_num_epriv_ports; i++) { 7406 if (tcps->tcps_g_epriv_ports[i] == 0) 7407 break; 7408 } 7409 if (i == tcps->tcps_g_num_epriv_ports) { 7410 mutex_exit(&tcps->tcps_epriv_port_lock); 7411 return (EOVERFLOW); 7412 } 7413 /* Set the new value */ 7414 tcps->tcps_g_epriv_ports[i] = (uint16_t)new_value; 7415 mutex_exit(&tcps->tcps_epriv_port_lock); 7416 return (0); 7417 } 7418 7419 /* 7420 * Hold a lock while changing tcp_g_epriv_ports to prevent multiple 7421 * threads from changing it at the same time. 7422 */ 7423 /* ARGSUSED */ 7424 static int 7425 tcp_extra_priv_ports_del(queue_t *q, mblk_t *mp, char *value, caddr_t cp, 7426 cred_t *cr) 7427 { 7428 long new_value; 7429 int i; 7430 tcp_stack_t *tcps = Q_TO_TCP(q)->tcp_tcps; 7431 7432 /* 7433 * Fail the request if the new value does not lie within the 7434 * port number limits. 7435 */ 7436 if (ddi_strtol(value, NULL, 10, &new_value) != 0 || new_value <= 0 || 7437 new_value >= 65536) { 7438 return (EINVAL); 7439 } 7440 7441 mutex_enter(&tcps->tcps_epriv_port_lock); 7442 /* Check that the value is already in the list */ 7443 for (i = 0; i < tcps->tcps_g_num_epriv_ports; i++) { 7444 if (tcps->tcps_g_epriv_ports[i] == new_value) 7445 break; 7446 } 7447 if (i == tcps->tcps_g_num_epriv_ports) { 7448 mutex_exit(&tcps->tcps_epriv_port_lock); 7449 return (ESRCH); 7450 } 7451 /* Clear the value */ 7452 tcps->tcps_g_epriv_ports[i] = 0; 7453 mutex_exit(&tcps->tcps_epriv_port_lock); 7454 return (0); 7455 } 7456 7457 /* Return the TPI/TLI equivalent of our current tcp_state */ 7458 static int 7459 tcp_tpistate(tcp_t *tcp) 7460 { 7461 switch (tcp->tcp_state) { 7462 case TCPS_IDLE: 7463 return (TS_UNBND); 7464 case TCPS_LISTEN: 7465 /* 7466 * Return whether there are outstanding T_CONN_IND waiting 7467 * for the matching T_CONN_RES. Therefore don't count q0. 7468 */ 7469 if (tcp->tcp_conn_req_cnt_q > 0) 7470 return (TS_WRES_CIND); 7471 else 7472 return (TS_IDLE); 7473 case TCPS_BOUND: 7474 return (TS_IDLE); 7475 case TCPS_SYN_SENT: 7476 return (TS_WCON_CREQ); 7477 case TCPS_SYN_RCVD: 7478 /* 7479 * Note: assumption: this has to the active open SYN_RCVD. 7480 * The passive instance is detached in SYN_RCVD stage of 7481 * incoming connection processing so we cannot get request 7482 * for T_info_ack on it. 7483 */ 7484 return (TS_WACK_CRES); 7485 case TCPS_ESTABLISHED: 7486 return (TS_DATA_XFER); 7487 case TCPS_CLOSE_WAIT: 7488 return (TS_WREQ_ORDREL); 7489 case TCPS_FIN_WAIT_1: 7490 return (TS_WIND_ORDREL); 7491 case TCPS_FIN_WAIT_2: 7492 return (TS_WIND_ORDREL); 7493 7494 case TCPS_CLOSING: 7495 case TCPS_LAST_ACK: 7496 case TCPS_TIME_WAIT: 7497 case TCPS_CLOSED: 7498 /* 7499 * Following TS_WACK_DREQ7 is a rendition of "not 7500 * yet TS_IDLE" TPI state. There is no best match to any 7501 * TPI state for TCPS_{CLOSING, LAST_ACK, TIME_WAIT} but we 7502 * choose a value chosen that will map to TLI/XTI level 7503 * state of TSTATECHNG (state is process of changing) which 7504 * captures what this dummy state represents. 7505 */ 7506 return (TS_WACK_DREQ7); 7507 default: 7508 cmn_err(CE_WARN, "tcp_tpistate: strange state (%d) %s", 7509 tcp->tcp_state, tcp_display(tcp, NULL, 7510 DISP_PORT_ONLY)); 7511 return (TS_UNBND); 7512 } 7513 } 7514 7515 static void 7516 tcp_copy_info(struct T_info_ack *tia, tcp_t *tcp) 7517 { 7518 tcp_stack_t *tcps = tcp->tcp_tcps; 7519 7520 if (tcp->tcp_family == AF_INET6) 7521 *tia = tcp_g_t_info_ack_v6; 7522 else 7523 *tia = tcp_g_t_info_ack; 7524 tia->CURRENT_state = tcp_tpistate(tcp); 7525 tia->OPT_size = tcp_max_optsize; 7526 if (tcp->tcp_mss == 0) { 7527 /* Not yet set - tcp_open does not set mss */ 7528 if (tcp->tcp_ipversion == IPV4_VERSION) 7529 tia->TIDU_size = tcps->tcps_mss_def_ipv4; 7530 else 7531 tia->TIDU_size = tcps->tcps_mss_def_ipv6; 7532 } else { 7533 tia->TIDU_size = tcp->tcp_mss; 7534 } 7535 /* TODO: Default ETSDU is 1. Is that correct for tcp? */ 7536 } 7537 7538 /* 7539 * This routine responds to T_CAPABILITY_REQ messages. It is called by 7540 * tcp_wput. Much of the T_CAPABILITY_ACK information is copied from 7541 * tcp_g_t_info_ack. The current state of the stream is copied from 7542 * tcp_state. 7543 */ 7544 static void 7545 tcp_capability_req(tcp_t *tcp, mblk_t *mp) 7546 { 7547 t_uscalar_t cap_bits1; 7548 struct T_capability_ack *tcap; 7549 7550 if (MBLKL(mp) < sizeof (struct T_capability_req)) { 7551 freemsg(mp); 7552 return; 7553 } 7554 7555 cap_bits1 = ((struct T_capability_req *)mp->b_rptr)->CAP_bits1; 7556 7557 mp = tpi_ack_alloc(mp, sizeof (struct T_capability_ack), 7558 mp->b_datap->db_type, T_CAPABILITY_ACK); 7559 if (mp == NULL) 7560 return; 7561 7562 tcap = (struct T_capability_ack *)mp->b_rptr; 7563 tcap->CAP_bits1 = 0; 7564 7565 if (cap_bits1 & TC1_INFO) { 7566 tcp_copy_info(&tcap->INFO_ack, tcp); 7567 tcap->CAP_bits1 |= TC1_INFO; 7568 } 7569 7570 if (cap_bits1 & TC1_ACCEPTOR_ID) { 7571 tcap->ACCEPTOR_id = tcp->tcp_acceptor_id; 7572 tcap->CAP_bits1 |= TC1_ACCEPTOR_ID; 7573 } 7574 7575 putnext(tcp->tcp_rq, mp); 7576 } 7577 7578 /* 7579 * This routine responds to T_INFO_REQ messages. It is called by tcp_wput. 7580 * Most of the T_INFO_ACK information is copied from tcp_g_t_info_ack. 7581 * The current state of the stream is copied from tcp_state. 7582 */ 7583 static void 7584 tcp_info_req(tcp_t *tcp, mblk_t *mp) 7585 { 7586 mp = tpi_ack_alloc(mp, sizeof (struct T_info_ack), M_PCPROTO, 7587 T_INFO_ACK); 7588 if (!mp) { 7589 tcp_err_ack(tcp, mp, TSYSERR, ENOMEM); 7590 return; 7591 } 7592 tcp_copy_info((struct T_info_ack *)mp->b_rptr, tcp); 7593 putnext(tcp->tcp_rq, mp); 7594 } 7595 7596 /* Respond to the TPI addr request */ 7597 static void 7598 tcp_addr_req(tcp_t *tcp, mblk_t *mp) 7599 { 7600 sin_t *sin; 7601 mblk_t *ackmp; 7602 struct T_addr_ack *taa; 7603 7604 /* Make it large enough for worst case */ 7605 ackmp = reallocb(mp, sizeof (struct T_addr_ack) + 7606 2 * sizeof (sin6_t), 1); 7607 if (ackmp == NULL) { 7608 tcp_err_ack(tcp, mp, TSYSERR, ENOMEM); 7609 return; 7610 } 7611 7612 if (tcp->tcp_ipversion == IPV6_VERSION) { 7613 tcp_addr_req_ipv6(tcp, ackmp); 7614 return; 7615 } 7616 taa = (struct T_addr_ack *)ackmp->b_rptr; 7617 7618 bzero(taa, sizeof (struct T_addr_ack)); 7619 ackmp->b_wptr = (uchar_t *)&taa[1]; 7620 7621 taa->PRIM_type = T_ADDR_ACK; 7622 ackmp->b_datap->db_type = M_PCPROTO; 7623 7624 /* 7625 * Note: Following code assumes 32 bit alignment of basic 7626 * data structures like sin_t and struct T_addr_ack. 7627 */ 7628 if (tcp->tcp_state >= TCPS_BOUND) { 7629 /* 7630 * Fill in local address 7631 */ 7632 taa->LOCADDR_length = sizeof (sin_t); 7633 taa->LOCADDR_offset = sizeof (*taa); 7634 7635 sin = (sin_t *)&taa[1]; 7636 7637 /* Fill zeroes and then intialize non-zero fields */ 7638 *sin = sin_null; 7639 7640 sin->sin_family = AF_INET; 7641 7642 sin->sin_addr.s_addr = tcp->tcp_ipha->ipha_src; 7643 sin->sin_port = *(uint16_t *)tcp->tcp_tcph->th_lport; 7644 7645 ackmp->b_wptr = (uchar_t *)&sin[1]; 7646 7647 if (tcp->tcp_state >= TCPS_SYN_RCVD) { 7648 /* 7649 * Fill in Remote address 7650 */ 7651 taa->REMADDR_length = sizeof (sin_t); 7652 taa->REMADDR_offset = ROUNDUP32(taa->LOCADDR_offset + 7653 taa->LOCADDR_length); 7654 7655 sin = (sin_t *)(ackmp->b_rptr + taa->REMADDR_offset); 7656 *sin = sin_null; 7657 sin->sin_family = AF_INET; 7658 sin->sin_addr.s_addr = tcp->tcp_remote; 7659 sin->sin_port = tcp->tcp_fport; 7660 7661 ackmp->b_wptr = (uchar_t *)&sin[1]; 7662 } 7663 } 7664 putnext(tcp->tcp_rq, ackmp); 7665 } 7666 7667 /* Assumes that tcp_addr_req gets enough space and alignment */ 7668 static void 7669 tcp_addr_req_ipv6(tcp_t *tcp, mblk_t *ackmp) 7670 { 7671 sin6_t *sin6; 7672 struct T_addr_ack *taa; 7673 7674 ASSERT(tcp->tcp_ipversion == IPV6_VERSION); 7675 ASSERT(OK_32PTR(ackmp->b_rptr)); 7676 ASSERT(ackmp->b_wptr - ackmp->b_rptr >= sizeof (struct T_addr_ack) + 7677 2 * sizeof (sin6_t)); 7678 7679 taa = (struct T_addr_ack *)ackmp->b_rptr; 7680 7681 bzero(taa, sizeof (struct T_addr_ack)); 7682 ackmp->b_wptr = (uchar_t *)&taa[1]; 7683 7684 taa->PRIM_type = T_ADDR_ACK; 7685 ackmp->b_datap->db_type = M_PCPROTO; 7686 7687 /* 7688 * Note: Following code assumes 32 bit alignment of basic 7689 * data structures like sin6_t and struct T_addr_ack. 7690 */ 7691 if (tcp->tcp_state >= TCPS_BOUND) { 7692 /* 7693 * Fill in local address 7694 */ 7695 taa->LOCADDR_length = sizeof (sin6_t); 7696 taa->LOCADDR_offset = sizeof (*taa); 7697 7698 sin6 = (sin6_t *)&taa[1]; 7699 *sin6 = sin6_null; 7700 7701 sin6->sin6_family = AF_INET6; 7702 sin6->sin6_addr = tcp->tcp_ip6h->ip6_src; 7703 sin6->sin6_port = tcp->tcp_lport; 7704 7705 ackmp->b_wptr = (uchar_t *)&sin6[1]; 7706 7707 if (tcp->tcp_state >= TCPS_SYN_RCVD) { 7708 /* 7709 * Fill in Remote address 7710 */ 7711 taa->REMADDR_length = sizeof (sin6_t); 7712 taa->REMADDR_offset = ROUNDUP32(taa->LOCADDR_offset + 7713 taa->LOCADDR_length); 7714 7715 sin6 = (sin6_t *)(ackmp->b_rptr + taa->REMADDR_offset); 7716 *sin6 = sin6_null; 7717 sin6->sin6_family = AF_INET6; 7718 sin6->sin6_flowinfo = 7719 tcp->tcp_ip6h->ip6_vcf & 7720 ~IPV6_VERS_AND_FLOW_MASK; 7721 sin6->sin6_addr = tcp->tcp_remote_v6; 7722 sin6->sin6_port = tcp->tcp_fport; 7723 7724 ackmp->b_wptr = (uchar_t *)&sin6[1]; 7725 } 7726 } 7727 putnext(tcp->tcp_rq, ackmp); 7728 } 7729 7730 /* 7731 * Handle reinitialization of a tcp structure. 7732 * Maintain "binding state" resetting the state to BOUND, LISTEN, or IDLE. 7733 */ 7734 static void 7735 tcp_reinit(tcp_t *tcp) 7736 { 7737 mblk_t *mp; 7738 int err; 7739 tcp_stack_t *tcps = tcp->tcp_tcps; 7740 7741 TCP_STAT(tcps, tcp_reinit_calls); 7742 7743 /* tcp_reinit should never be called for detached tcp_t's */ 7744 ASSERT(tcp->tcp_listener == NULL); 7745 ASSERT((tcp->tcp_family == AF_INET && 7746 tcp->tcp_ipversion == IPV4_VERSION) || 7747 (tcp->tcp_family == AF_INET6 && 7748 (tcp->tcp_ipversion == IPV4_VERSION || 7749 tcp->tcp_ipversion == IPV6_VERSION))); 7750 7751 /* Cancel outstanding timers */ 7752 tcp_timers_stop(tcp); 7753 7754 /* 7755 * Reset everything in the state vector, after updating global 7756 * MIB data from instance counters. 7757 */ 7758 UPDATE_MIB(&tcps->tcps_mib, tcpHCInSegs, tcp->tcp_ibsegs); 7759 tcp->tcp_ibsegs = 0; 7760 UPDATE_MIB(&tcps->tcps_mib, tcpHCOutSegs, tcp->tcp_obsegs); 7761 tcp->tcp_obsegs = 0; 7762 7763 tcp_close_mpp(&tcp->tcp_xmit_head); 7764 if (tcp->tcp_snd_zcopy_aware) 7765 tcp_zcopy_notify(tcp); 7766 tcp->tcp_xmit_last = tcp->tcp_xmit_tail = NULL; 7767 tcp->tcp_unsent = tcp->tcp_xmit_tail_unsent = 0; 7768 mutex_enter(&tcp->tcp_non_sq_lock); 7769 if (tcp->tcp_flow_stopped && 7770 TCP_UNSENT_BYTES(tcp) <= tcp->tcp_xmit_lowater) { 7771 tcp_clrqfull(tcp); 7772 } 7773 mutex_exit(&tcp->tcp_non_sq_lock); 7774 tcp_close_mpp(&tcp->tcp_reass_head); 7775 tcp->tcp_reass_tail = NULL; 7776 if (tcp->tcp_rcv_list != NULL) { 7777 /* Free b_next chain */ 7778 tcp_close_mpp(&tcp->tcp_rcv_list); 7779 tcp->tcp_rcv_last_head = NULL; 7780 tcp->tcp_rcv_last_tail = NULL; 7781 tcp->tcp_rcv_cnt = 0; 7782 } 7783 tcp->tcp_rcv_last_tail = NULL; 7784 7785 if ((mp = tcp->tcp_urp_mp) != NULL) { 7786 freemsg(mp); 7787 tcp->tcp_urp_mp = NULL; 7788 } 7789 if ((mp = tcp->tcp_urp_mark_mp) != NULL) { 7790 freemsg(mp); 7791 tcp->tcp_urp_mark_mp = NULL; 7792 } 7793 if (tcp->tcp_fused_sigurg_mp != NULL) { 7794 freeb(tcp->tcp_fused_sigurg_mp); 7795 tcp->tcp_fused_sigurg_mp = NULL; 7796 } 7797 7798 /* 7799 * Following is a union with two members which are 7800 * identical types and size so the following cleanup 7801 * is enough. 7802 */ 7803 tcp_close_mpp(&tcp->tcp_conn.tcp_eager_conn_ind); 7804 7805 CL_INET_DISCONNECT(tcp); 7806 7807 /* 7808 * The connection can't be on the tcp_time_wait_head list 7809 * since it is not detached. 7810 */ 7811 ASSERT(tcp->tcp_time_wait_next == NULL); 7812 ASSERT(tcp->tcp_time_wait_prev == NULL); 7813 ASSERT(tcp->tcp_time_wait_expire == 0); 7814 7815 if (tcp->tcp_kssl_pending) { 7816 tcp->tcp_kssl_pending = B_FALSE; 7817 7818 /* Don't reset if the initialized by bind. */ 7819 if (tcp->tcp_kssl_ent != NULL) { 7820 kssl_release_ent(tcp->tcp_kssl_ent, NULL, 7821 KSSL_NO_PROXY); 7822 } 7823 } 7824 if (tcp->tcp_kssl_ctx != NULL) { 7825 kssl_release_ctx(tcp->tcp_kssl_ctx); 7826 tcp->tcp_kssl_ctx = NULL; 7827 } 7828 7829 /* 7830 * Reset/preserve other values 7831 */ 7832 tcp_reinit_values(tcp); 7833 ipcl_hash_remove(tcp->tcp_connp); 7834 conn_delete_ire(tcp->tcp_connp, NULL); 7835 tcp_ipsec_cleanup(tcp); 7836 7837 if (tcp->tcp_conn_req_max != 0) { 7838 /* 7839 * This is the case when a TLI program uses the same 7840 * transport end point to accept a connection. This 7841 * makes the TCP both a listener and acceptor. When 7842 * this connection is closed, we need to set the state 7843 * back to TCPS_LISTEN. Make sure that the eager list 7844 * is reinitialized. 7845 * 7846 * Note that this stream is still bound to the four 7847 * tuples of the previous connection in IP. If a new 7848 * SYN with different foreign address comes in, IP will 7849 * not find it and will send it to the global queue. In 7850 * the global queue, TCP will do a tcp_lookup_listener() 7851 * to find this stream. This works because this stream 7852 * is only removed from connected hash. 7853 * 7854 */ 7855 tcp->tcp_state = TCPS_LISTEN; 7856 tcp->tcp_eager_next_q0 = tcp->tcp_eager_prev_q0 = tcp; 7857 tcp->tcp_eager_next_drop_q0 = tcp; 7858 tcp->tcp_eager_prev_drop_q0 = tcp; 7859 tcp->tcp_connp->conn_recv = tcp_conn_request; 7860 if (tcp->tcp_family == AF_INET6) { 7861 ASSERT(tcp->tcp_connp->conn_af_isv6); 7862 (void) ipcl_bind_insert_v6(tcp->tcp_connp, IPPROTO_TCP, 7863 &tcp->tcp_ip6h->ip6_src, tcp->tcp_lport); 7864 } else { 7865 ASSERT(!tcp->tcp_connp->conn_af_isv6); 7866 (void) ipcl_bind_insert(tcp->tcp_connp, IPPROTO_TCP, 7867 tcp->tcp_ipha->ipha_src, tcp->tcp_lport); 7868 } 7869 } else { 7870 tcp->tcp_state = TCPS_BOUND; 7871 } 7872 7873 /* 7874 * Initialize to default values 7875 * Can't fail since enough header template space already allocated 7876 * at open(). 7877 */ 7878 err = tcp_init_values(tcp); 7879 ASSERT(err == 0); 7880 /* Restore state in tcp_tcph */ 7881 bcopy(&tcp->tcp_lport, tcp->tcp_tcph->th_lport, TCP_PORT_LEN); 7882 if (tcp->tcp_ipversion == IPV4_VERSION) 7883 tcp->tcp_ipha->ipha_src = tcp->tcp_bound_source; 7884 else 7885 tcp->tcp_ip6h->ip6_src = tcp->tcp_bound_source_v6; 7886 /* 7887 * Copy of the src addr. in tcp_t is needed in tcp_t 7888 * since the lookup funcs can only lookup on tcp_t 7889 */ 7890 tcp->tcp_ip_src_v6 = tcp->tcp_bound_source_v6; 7891 7892 ASSERT(tcp->tcp_ptpbhn != NULL); 7893 tcp->tcp_rq->q_hiwat = tcps->tcps_recv_hiwat; 7894 tcp->tcp_rwnd = tcps->tcps_recv_hiwat; 7895 tcp->tcp_mss = tcp->tcp_ipversion != IPV4_VERSION ? 7896 tcps->tcps_mss_def_ipv6 : tcps->tcps_mss_def_ipv4; 7897 } 7898 7899 /* 7900 * Force values to zero that need be zero. 7901 * Do not touch values asociated with the BOUND or LISTEN state 7902 * since the connection will end up in that state after the reinit. 7903 * NOTE: tcp_reinit_values MUST have a line for each field in the tcp_t 7904 * structure! 7905 */ 7906 static void 7907 tcp_reinit_values(tcp) 7908 tcp_t *tcp; 7909 { 7910 tcp_stack_t *tcps = tcp->tcp_tcps; 7911 7912 #ifndef lint 7913 #define DONTCARE(x) 7914 #define PRESERVE(x) 7915 #else 7916 #define DONTCARE(x) ((x) = (x)) 7917 #define PRESERVE(x) ((x) = (x)) 7918 #endif /* lint */ 7919 7920 PRESERVE(tcp->tcp_bind_hash); 7921 PRESERVE(tcp->tcp_ptpbhn); 7922 PRESERVE(tcp->tcp_acceptor_hash); 7923 PRESERVE(tcp->tcp_ptpahn); 7924 7925 /* Should be ASSERT NULL on these with new code! */ 7926 ASSERT(tcp->tcp_time_wait_next == NULL); 7927 ASSERT(tcp->tcp_time_wait_prev == NULL); 7928 ASSERT(tcp->tcp_time_wait_expire == 0); 7929 PRESERVE(tcp->tcp_state); 7930 PRESERVE(tcp->tcp_rq); 7931 PRESERVE(tcp->tcp_wq); 7932 7933 ASSERT(tcp->tcp_xmit_head == NULL); 7934 ASSERT(tcp->tcp_xmit_last == NULL); 7935 ASSERT(tcp->tcp_unsent == 0); 7936 ASSERT(tcp->tcp_xmit_tail == NULL); 7937 ASSERT(tcp->tcp_xmit_tail_unsent == 0); 7938 7939 tcp->tcp_snxt = 0; /* Displayed in mib */ 7940 tcp->tcp_suna = 0; /* Displayed in mib */ 7941 tcp->tcp_swnd = 0; 7942 DONTCARE(tcp->tcp_cwnd); /* Init in tcp_mss_set */ 7943 7944 ASSERT(tcp->tcp_ibsegs == 0); 7945 ASSERT(tcp->tcp_obsegs == 0); 7946 7947 if (tcp->tcp_iphc != NULL) { 7948 ASSERT(tcp->tcp_iphc_len >= TCP_MAX_COMBINED_HEADER_LENGTH); 7949 bzero(tcp->tcp_iphc, tcp->tcp_iphc_len); 7950 } 7951 7952 DONTCARE(tcp->tcp_naglim); /* Init in tcp_init_values */ 7953 DONTCARE(tcp->tcp_hdr_len); /* Init in tcp_init_values */ 7954 DONTCARE(tcp->tcp_ipha); 7955 DONTCARE(tcp->tcp_ip6h); 7956 DONTCARE(tcp->tcp_ip_hdr_len); 7957 DONTCARE(tcp->tcp_tcph); 7958 DONTCARE(tcp->tcp_tcp_hdr_len); /* Init in tcp_init_values */ 7959 tcp->tcp_valid_bits = 0; 7960 7961 DONTCARE(tcp->tcp_xmit_hiwater); /* Init in tcp_init_values */ 7962 DONTCARE(tcp->tcp_timer_backoff); /* Init in tcp_init_values */ 7963 DONTCARE(tcp->tcp_last_recv_time); /* Init in tcp_init_values */ 7964 tcp->tcp_last_rcv_lbolt = 0; 7965 7966 tcp->tcp_init_cwnd = 0; 7967 7968 tcp->tcp_urp_last_valid = 0; 7969 tcp->tcp_hard_binding = 0; 7970 tcp->tcp_hard_bound = 0; 7971 PRESERVE(tcp->tcp_cred); 7972 PRESERVE(tcp->tcp_cpid); 7973 PRESERVE(tcp->tcp_open_time); 7974 PRESERVE(tcp->tcp_exclbind); 7975 7976 tcp->tcp_fin_acked = 0; 7977 tcp->tcp_fin_rcvd = 0; 7978 tcp->tcp_fin_sent = 0; 7979 tcp->tcp_ordrel_done = 0; 7980 7981 tcp->tcp_debug = 0; 7982 tcp->tcp_dontroute = 0; 7983 tcp->tcp_broadcast = 0; 7984 7985 tcp->tcp_useloopback = 0; 7986 tcp->tcp_reuseaddr = 0; 7987 tcp->tcp_oobinline = 0; 7988 tcp->tcp_dgram_errind = 0; 7989 7990 tcp->tcp_detached = 0; 7991 tcp->tcp_bind_pending = 0; 7992 tcp->tcp_unbind_pending = 0; 7993 tcp->tcp_deferred_clean_death = 0; 7994 7995 tcp->tcp_snd_ws_ok = B_FALSE; 7996 tcp->tcp_snd_ts_ok = B_FALSE; 7997 tcp->tcp_linger = 0; 7998 tcp->tcp_ka_enabled = 0; 7999 tcp->tcp_zero_win_probe = 0; 8000 8001 tcp->tcp_loopback = 0; 8002 tcp->tcp_localnet = 0; 8003 tcp->tcp_syn_defense = 0; 8004 tcp->tcp_set_timer = 0; 8005 8006 tcp->tcp_active_open = 0; 8007 ASSERT(tcp->tcp_timeout == B_FALSE); 8008 tcp->tcp_rexmit = B_FALSE; 8009 tcp->tcp_xmit_zc_clean = B_FALSE; 8010 8011 tcp->tcp_snd_sack_ok = B_FALSE; 8012 PRESERVE(tcp->tcp_recvdstaddr); 8013 tcp->tcp_hwcksum = B_FALSE; 8014 8015 tcp->tcp_ire_ill_check_done = B_FALSE; 8016 DONTCARE(tcp->tcp_maxpsz); /* Init in tcp_init_values */ 8017 8018 tcp->tcp_mdt = B_FALSE; 8019 tcp->tcp_mdt_hdr_head = 0; 8020 tcp->tcp_mdt_hdr_tail = 0; 8021 8022 tcp->tcp_conn_def_q0 = 0; 8023 tcp->tcp_ip_forward_progress = B_FALSE; 8024 tcp->tcp_anon_priv_bind = 0; 8025 tcp->tcp_ecn_ok = B_FALSE; 8026 8027 tcp->tcp_cwr = B_FALSE; 8028 tcp->tcp_ecn_echo_on = B_FALSE; 8029 8030 if (tcp->tcp_sack_info != NULL) { 8031 if (tcp->tcp_notsack_list != NULL) { 8032 TCP_NOTSACK_REMOVE_ALL(tcp->tcp_notsack_list); 8033 } 8034 kmem_cache_free(tcp_sack_info_cache, tcp->tcp_sack_info); 8035 tcp->tcp_sack_info = NULL; 8036 } 8037 8038 tcp->tcp_rcv_ws = 0; 8039 tcp->tcp_snd_ws = 0; 8040 tcp->tcp_ts_recent = 0; 8041 tcp->tcp_rnxt = 0; /* Displayed in mib */ 8042 DONTCARE(tcp->tcp_rwnd); /* Set in tcp_reinit() */ 8043 tcp->tcp_if_mtu = 0; 8044 8045 ASSERT(tcp->tcp_reass_head == NULL); 8046 ASSERT(tcp->tcp_reass_tail == NULL); 8047 8048 tcp->tcp_cwnd_cnt = 0; 8049 8050 ASSERT(tcp->tcp_rcv_list == NULL); 8051 ASSERT(tcp->tcp_rcv_last_head == NULL); 8052 ASSERT(tcp->tcp_rcv_last_tail == NULL); 8053 ASSERT(tcp->tcp_rcv_cnt == 0); 8054 8055 DONTCARE(tcp->tcp_cwnd_ssthresh); /* Init in tcp_adapt_ire */ 8056 DONTCARE(tcp->tcp_cwnd_max); /* Init in tcp_init_values */ 8057 tcp->tcp_csuna = 0; 8058 8059 tcp->tcp_rto = 0; /* Displayed in MIB */ 8060 DONTCARE(tcp->tcp_rtt_sa); /* Init in tcp_init_values */ 8061 DONTCARE(tcp->tcp_rtt_sd); /* Init in tcp_init_values */ 8062 tcp->tcp_rtt_update = 0; 8063 8064 DONTCARE(tcp->tcp_swl1); /* Init in case TCPS_LISTEN/TCPS_SYN_SENT */ 8065 DONTCARE(tcp->tcp_swl2); /* Init in case TCPS_LISTEN/TCPS_SYN_SENT */ 8066 8067 tcp->tcp_rack = 0; /* Displayed in mib */ 8068 tcp->tcp_rack_cnt = 0; 8069 tcp->tcp_rack_cur_max = 0; 8070 tcp->tcp_rack_abs_max = 0; 8071 8072 tcp->tcp_max_swnd = 0; 8073 8074 ASSERT(tcp->tcp_listener == NULL); 8075 8076 DONTCARE(tcp->tcp_xmit_lowater); /* Init in tcp_init_values */ 8077 8078 DONTCARE(tcp->tcp_irs); /* tcp_valid_bits cleared */ 8079 DONTCARE(tcp->tcp_iss); /* tcp_valid_bits cleared */ 8080 DONTCARE(tcp->tcp_fss); /* tcp_valid_bits cleared */ 8081 DONTCARE(tcp->tcp_urg); /* tcp_valid_bits cleared */ 8082 8083 ASSERT(tcp->tcp_conn_req_cnt_q == 0); 8084 ASSERT(tcp->tcp_conn_req_cnt_q0 == 0); 8085 PRESERVE(tcp->tcp_conn_req_max); 8086 PRESERVE(tcp->tcp_conn_req_seqnum); 8087 8088 DONTCARE(tcp->tcp_ip_hdr_len); /* Init in tcp_init_values */ 8089 DONTCARE(tcp->tcp_first_timer_threshold); /* Init in tcp_init_values */ 8090 DONTCARE(tcp->tcp_second_timer_threshold); /* Init in tcp_init_values */ 8091 DONTCARE(tcp->tcp_first_ctimer_threshold); /* Init in tcp_init_values */ 8092 DONTCARE(tcp->tcp_second_ctimer_threshold); /* in tcp_init_values */ 8093 8094 tcp->tcp_lingertime = 0; 8095 8096 DONTCARE(tcp->tcp_urp_last); /* tcp_urp_last_valid is cleared */ 8097 ASSERT(tcp->tcp_urp_mp == NULL); 8098 ASSERT(tcp->tcp_urp_mark_mp == NULL); 8099 ASSERT(tcp->tcp_fused_sigurg_mp == NULL); 8100 8101 ASSERT(tcp->tcp_eager_next_q == NULL); 8102 ASSERT(tcp->tcp_eager_last_q == NULL); 8103 ASSERT((tcp->tcp_eager_next_q0 == NULL && 8104 tcp->tcp_eager_prev_q0 == NULL) || 8105 tcp->tcp_eager_next_q0 == tcp->tcp_eager_prev_q0); 8106 ASSERT(tcp->tcp_conn.tcp_eager_conn_ind == NULL); 8107 8108 ASSERT((tcp->tcp_eager_next_drop_q0 == NULL && 8109 tcp->tcp_eager_prev_drop_q0 == NULL) || 8110 tcp->tcp_eager_next_drop_q0 == tcp->tcp_eager_prev_drop_q0); 8111 8112 tcp->tcp_client_errno = 0; 8113 8114 DONTCARE(tcp->tcp_sum); /* Init in tcp_init_values */ 8115 8116 tcp->tcp_remote_v6 = ipv6_all_zeros; /* Displayed in MIB */ 8117 8118 PRESERVE(tcp->tcp_bound_source_v6); 8119 tcp->tcp_last_sent_len = 0; 8120 tcp->tcp_dupack_cnt = 0; 8121 8122 tcp->tcp_fport = 0; /* Displayed in MIB */ 8123 PRESERVE(tcp->tcp_lport); 8124 8125 PRESERVE(tcp->tcp_acceptor_lockp); 8126 8127 ASSERT(tcp->tcp_ordrelid == 0); 8128 PRESERVE(tcp->tcp_acceptor_id); 8129 DONTCARE(tcp->tcp_ipsec_overhead); 8130 8131 PRESERVE(tcp->tcp_family); 8132 if (tcp->tcp_family == AF_INET6) { 8133 tcp->tcp_ipversion = IPV6_VERSION; 8134 tcp->tcp_mss = tcps->tcps_mss_def_ipv6; 8135 } else { 8136 tcp->tcp_ipversion = IPV4_VERSION; 8137 tcp->tcp_mss = tcps->tcps_mss_def_ipv4; 8138 } 8139 8140 tcp->tcp_bound_if = 0; 8141 tcp->tcp_ipv6_recvancillary = 0; 8142 tcp->tcp_recvifindex = 0; 8143 tcp->tcp_recvhops = 0; 8144 tcp->tcp_closed = 0; 8145 tcp->tcp_cleandeathtag = 0; 8146 if (tcp->tcp_hopopts != NULL) { 8147 mi_free(tcp->tcp_hopopts); 8148 tcp->tcp_hopopts = NULL; 8149 tcp->tcp_hopoptslen = 0; 8150 } 8151 ASSERT(tcp->tcp_hopoptslen == 0); 8152 if (tcp->tcp_dstopts != NULL) { 8153 mi_free(tcp->tcp_dstopts); 8154 tcp->tcp_dstopts = NULL; 8155 tcp->tcp_dstoptslen = 0; 8156 } 8157 ASSERT(tcp->tcp_dstoptslen == 0); 8158 if (tcp->tcp_rtdstopts != NULL) { 8159 mi_free(tcp->tcp_rtdstopts); 8160 tcp->tcp_rtdstopts = NULL; 8161 tcp->tcp_rtdstoptslen = 0; 8162 } 8163 ASSERT(tcp->tcp_rtdstoptslen == 0); 8164 if (tcp->tcp_rthdr != NULL) { 8165 mi_free(tcp->tcp_rthdr); 8166 tcp->tcp_rthdr = NULL; 8167 tcp->tcp_rthdrlen = 0; 8168 } 8169 ASSERT(tcp->tcp_rthdrlen == 0); 8170 PRESERVE(tcp->tcp_drop_opt_ack_cnt); 8171 8172 /* Reset fusion-related fields */ 8173 tcp->tcp_fused = B_FALSE; 8174 tcp->tcp_unfusable = B_FALSE; 8175 tcp->tcp_fused_sigurg = B_FALSE; 8176 tcp->tcp_direct_sockfs = B_FALSE; 8177 tcp->tcp_fuse_syncstr_stopped = B_FALSE; 8178 tcp->tcp_fuse_syncstr_plugged = B_FALSE; 8179 tcp->tcp_loopback_peer = NULL; 8180 tcp->tcp_fuse_rcv_hiwater = 0; 8181 tcp->tcp_fuse_rcv_unread_hiwater = 0; 8182 tcp->tcp_fuse_rcv_unread_cnt = 0; 8183 8184 tcp->tcp_lso = B_FALSE; 8185 8186 tcp->tcp_in_ack_unsent = 0; 8187 tcp->tcp_cork = B_FALSE; 8188 tcp->tcp_tconnind_started = B_FALSE; 8189 8190 PRESERVE(tcp->tcp_squeue_bytes); 8191 8192 ASSERT(tcp->tcp_kssl_ctx == NULL); 8193 ASSERT(!tcp->tcp_kssl_pending); 8194 PRESERVE(tcp->tcp_kssl_ent); 8195 8196 /* Sodirect */ 8197 tcp->tcp_sodirect = NULL; 8198 8199 tcp->tcp_closemp_used = B_FALSE; 8200 8201 #ifdef DEBUG 8202 DONTCARE(tcp->tcmp_stk[0]); 8203 #endif 8204 8205 8206 #undef DONTCARE 8207 #undef PRESERVE 8208 } 8209 8210 /* 8211 * Allocate necessary resources and initialize state vector. 8212 * Guaranteed not to fail so that when an error is returned, 8213 * the caller doesn't need to do any additional cleanup. 8214 */ 8215 int 8216 tcp_init(tcp_t *tcp, queue_t *q) 8217 { 8218 int err; 8219 8220 tcp->tcp_rq = q; 8221 tcp->tcp_wq = WR(q); 8222 tcp->tcp_state = TCPS_IDLE; 8223 if ((err = tcp_init_values(tcp)) != 0) 8224 tcp_timers_stop(tcp); 8225 return (err); 8226 } 8227 8228 static int 8229 tcp_init_values(tcp_t *tcp) 8230 { 8231 int err; 8232 tcp_stack_t *tcps = tcp->tcp_tcps; 8233 8234 ASSERT((tcp->tcp_family == AF_INET && 8235 tcp->tcp_ipversion == IPV4_VERSION) || 8236 (tcp->tcp_family == AF_INET6 && 8237 (tcp->tcp_ipversion == IPV4_VERSION || 8238 tcp->tcp_ipversion == IPV6_VERSION))); 8239 8240 /* 8241 * Initialize tcp_rtt_sa and tcp_rtt_sd so that the calculated RTO 8242 * will be close to tcp_rexmit_interval_initial. By doing this, we 8243 * allow the algorithm to adjust slowly to large fluctuations of RTT 8244 * during first few transmissions of a connection as seen in slow 8245 * links. 8246 */ 8247 tcp->tcp_rtt_sa = tcps->tcps_rexmit_interval_initial << 2; 8248 tcp->tcp_rtt_sd = tcps->tcps_rexmit_interval_initial >> 1; 8249 tcp->tcp_rto = (tcp->tcp_rtt_sa >> 3) + tcp->tcp_rtt_sd + 8250 tcps->tcps_rexmit_interval_extra + (tcp->tcp_rtt_sa >> 5) + 8251 tcps->tcps_conn_grace_period; 8252 if (tcp->tcp_rto < tcps->tcps_rexmit_interval_min) 8253 tcp->tcp_rto = tcps->tcps_rexmit_interval_min; 8254 tcp->tcp_timer_backoff = 0; 8255 tcp->tcp_ms_we_have_waited = 0; 8256 tcp->tcp_last_recv_time = lbolt; 8257 tcp->tcp_cwnd_max = tcps->tcps_cwnd_max_; 8258 tcp->tcp_cwnd_ssthresh = TCP_MAX_LARGEWIN; 8259 tcp->tcp_snd_burst = TCP_CWND_INFINITE; 8260 8261 tcp->tcp_maxpsz = tcps->tcps_maxpsz_multiplier; 8262 8263 tcp->tcp_first_timer_threshold = tcps->tcps_ip_notify_interval; 8264 tcp->tcp_first_ctimer_threshold = tcps->tcps_ip_notify_cinterval; 8265 tcp->tcp_second_timer_threshold = tcps->tcps_ip_abort_interval; 8266 /* 8267 * Fix it to tcp_ip_abort_linterval later if it turns out to be a 8268 * passive open. 8269 */ 8270 tcp->tcp_second_ctimer_threshold = tcps->tcps_ip_abort_cinterval; 8271 8272 tcp->tcp_naglim = tcps->tcps_naglim_def; 8273 8274 /* NOTE: ISS is now set in tcp_adapt_ire(). */ 8275 8276 tcp->tcp_mdt_hdr_head = 0; 8277 tcp->tcp_mdt_hdr_tail = 0; 8278 8279 /* Reset fusion-related fields */ 8280 tcp->tcp_fused = B_FALSE; 8281 tcp->tcp_unfusable = B_FALSE; 8282 tcp->tcp_fused_sigurg = B_FALSE; 8283 tcp->tcp_direct_sockfs = B_FALSE; 8284 tcp->tcp_fuse_syncstr_stopped = B_FALSE; 8285 tcp->tcp_fuse_syncstr_plugged = B_FALSE; 8286 tcp->tcp_loopback_peer = NULL; 8287 tcp->tcp_fuse_rcv_hiwater = 0; 8288 tcp->tcp_fuse_rcv_unread_hiwater = 0; 8289 tcp->tcp_fuse_rcv_unread_cnt = 0; 8290 8291 /* Sodirect */ 8292 tcp->tcp_sodirect = NULL; 8293 8294 /* Initialize the header template */ 8295 if (tcp->tcp_ipversion == IPV4_VERSION) { 8296 err = tcp_header_init_ipv4(tcp); 8297 } else { 8298 err = tcp_header_init_ipv6(tcp); 8299 } 8300 if (err) 8301 return (err); 8302 8303 /* 8304 * Init the window scale to the max so tcp_rwnd_set() won't pare 8305 * down tcp_rwnd. tcp_adapt_ire() will set the right value later. 8306 */ 8307 tcp->tcp_rcv_ws = TCP_MAX_WINSHIFT; 8308 tcp->tcp_xmit_lowater = tcps->tcps_xmit_lowat; 8309 tcp->tcp_xmit_hiwater = tcps->tcps_xmit_hiwat; 8310 8311 tcp->tcp_cork = B_FALSE; 8312 /* 8313 * Init the tcp_debug option. This value determines whether TCP 8314 * calls strlog() to print out debug messages. Doing this 8315 * initialization here means that this value is not inherited thru 8316 * tcp_reinit(). 8317 */ 8318 tcp->tcp_debug = tcps->tcps_dbg; 8319 8320 tcp->tcp_ka_interval = tcps->tcps_keepalive_interval; 8321 tcp->tcp_ka_abort_thres = tcps->tcps_keepalive_abort_interval; 8322 8323 return (0); 8324 } 8325 8326 /* 8327 * Initialize the IPv4 header. Loses any record of any IP options. 8328 */ 8329 static int 8330 tcp_header_init_ipv4(tcp_t *tcp) 8331 { 8332 tcph_t *tcph; 8333 uint32_t sum; 8334 conn_t *connp; 8335 tcp_stack_t *tcps = tcp->tcp_tcps; 8336 8337 /* 8338 * This is a simple initialization. If there's 8339 * already a template, it should never be too small, 8340 * so reuse it. Otherwise, allocate space for the new one. 8341 */ 8342 if (tcp->tcp_iphc == NULL) { 8343 ASSERT(tcp->tcp_iphc_len == 0); 8344 tcp->tcp_iphc_len = TCP_MAX_COMBINED_HEADER_LENGTH; 8345 tcp->tcp_iphc = kmem_cache_alloc(tcp_iphc_cache, KM_NOSLEEP); 8346 if (tcp->tcp_iphc == NULL) { 8347 tcp->tcp_iphc_len = 0; 8348 return (ENOMEM); 8349 } 8350 } 8351 8352 /* options are gone; may need a new label */ 8353 connp = tcp->tcp_connp; 8354 connp->conn_mlp_type = mlptSingle; 8355 connp->conn_ulp_labeled = !is_system_labeled(); 8356 ASSERT(tcp->tcp_iphc_len >= TCP_MAX_COMBINED_HEADER_LENGTH); 8357 tcp->tcp_ipha = (ipha_t *)tcp->tcp_iphc; 8358 tcp->tcp_ip6h = NULL; 8359 tcp->tcp_ipversion = IPV4_VERSION; 8360 tcp->tcp_hdr_len = sizeof (ipha_t) + sizeof (tcph_t); 8361 tcp->tcp_tcp_hdr_len = sizeof (tcph_t); 8362 tcp->tcp_ip_hdr_len = sizeof (ipha_t); 8363 tcp->tcp_ipha->ipha_length = htons(sizeof (ipha_t) + sizeof (tcph_t)); 8364 tcp->tcp_ipha->ipha_version_and_hdr_length 8365 = (IP_VERSION << 4) | IP_SIMPLE_HDR_LENGTH_IN_WORDS; 8366 tcp->tcp_ipha->ipha_ident = 0; 8367 8368 tcp->tcp_ttl = (uchar_t)tcps->tcps_ipv4_ttl; 8369 tcp->tcp_tos = 0; 8370 tcp->tcp_ipha->ipha_fragment_offset_and_flags = 0; 8371 tcp->tcp_ipha->ipha_ttl = (uchar_t)tcps->tcps_ipv4_ttl; 8372 tcp->tcp_ipha->ipha_protocol = IPPROTO_TCP; 8373 8374 tcph = (tcph_t *)(tcp->tcp_iphc + sizeof (ipha_t)); 8375 tcp->tcp_tcph = tcph; 8376 tcph->th_offset_and_rsrvd[0] = (5 << 4); 8377 /* 8378 * IP wants our header length in the checksum field to 8379 * allow it to perform a single pseudo-header+checksum 8380 * calculation on behalf of TCP. 8381 * Include the adjustment for a source route once IP_OPTIONS is set. 8382 */ 8383 sum = sizeof (tcph_t) + tcp->tcp_sum; 8384 sum = (sum >> 16) + (sum & 0xFFFF); 8385 U16_TO_ABE16(sum, tcph->th_sum); 8386 return (0); 8387 } 8388 8389 /* 8390 * Initialize the IPv6 header. Loses any record of any IPv6 extension headers. 8391 */ 8392 static int 8393 tcp_header_init_ipv6(tcp_t *tcp) 8394 { 8395 tcph_t *tcph; 8396 uint32_t sum; 8397 conn_t *connp; 8398 tcp_stack_t *tcps = tcp->tcp_tcps; 8399 8400 /* 8401 * This is a simple initialization. If there's 8402 * already a template, it should never be too small, 8403 * so reuse it. Otherwise, allocate space for the new one. 8404 * Ensure that there is enough space to "downgrade" the tcp_t 8405 * to an IPv4 tcp_t. This requires having space for a full load 8406 * of IPv4 options, as well as a full load of TCP options 8407 * (TCP_MAX_COMBINED_HEADER_LENGTH, 120 bytes); this is more space 8408 * than a v6 header and a TCP header with a full load of TCP options 8409 * (IPV6_HDR_LEN is 40 bytes; TCP_MAX_HDR_LENGTH is 60 bytes). 8410 * We want to avoid reallocation in the "downgraded" case when 8411 * processing outbound IPv4 options. 8412 */ 8413 if (tcp->tcp_iphc == NULL) { 8414 ASSERT(tcp->tcp_iphc_len == 0); 8415 tcp->tcp_iphc_len = TCP_MAX_COMBINED_HEADER_LENGTH; 8416 tcp->tcp_iphc = kmem_cache_alloc(tcp_iphc_cache, KM_NOSLEEP); 8417 if (tcp->tcp_iphc == NULL) { 8418 tcp->tcp_iphc_len = 0; 8419 return (ENOMEM); 8420 } 8421 } 8422 8423 /* options are gone; may need a new label */ 8424 connp = tcp->tcp_connp; 8425 connp->conn_mlp_type = mlptSingle; 8426 connp->conn_ulp_labeled = !is_system_labeled(); 8427 8428 ASSERT(tcp->tcp_iphc_len >= TCP_MAX_COMBINED_HEADER_LENGTH); 8429 tcp->tcp_ipversion = IPV6_VERSION; 8430 tcp->tcp_hdr_len = IPV6_HDR_LEN + sizeof (tcph_t); 8431 tcp->tcp_tcp_hdr_len = sizeof (tcph_t); 8432 tcp->tcp_ip_hdr_len = IPV6_HDR_LEN; 8433 tcp->tcp_ip6h = (ip6_t *)tcp->tcp_iphc; 8434 tcp->tcp_ipha = NULL; 8435 8436 /* Initialize the header template */ 8437 8438 tcp->tcp_ip6h->ip6_vcf = IPV6_DEFAULT_VERS_AND_FLOW; 8439 tcp->tcp_ip6h->ip6_plen = ntohs(sizeof (tcph_t)); 8440 tcp->tcp_ip6h->ip6_nxt = IPPROTO_TCP; 8441 tcp->tcp_ip6h->ip6_hops = (uint8_t)tcps->tcps_ipv6_hoplimit; 8442 8443 tcph = (tcph_t *)(tcp->tcp_iphc + IPV6_HDR_LEN); 8444 tcp->tcp_tcph = tcph; 8445 tcph->th_offset_and_rsrvd[0] = (5 << 4); 8446 /* 8447 * IP wants our header length in the checksum field to 8448 * allow it to perform a single psuedo-header+checksum 8449 * calculation on behalf of TCP. 8450 * Include the adjustment for a source route when IPV6_RTHDR is set. 8451 */ 8452 sum = sizeof (tcph_t) + tcp->tcp_sum; 8453 sum = (sum >> 16) + (sum & 0xFFFF); 8454 U16_TO_ABE16(sum, tcph->th_sum); 8455 return (0); 8456 } 8457 8458 /* At minimum we need 8 bytes in the TCP header for the lookup */ 8459 #define ICMP_MIN_TCP_HDR 8 8460 8461 /* 8462 * tcp_icmp_error is called by tcp_rput_other to process ICMP error messages 8463 * passed up by IP. The message is always received on the correct tcp_t. 8464 * Assumes that IP has pulled up everything up to and including the ICMP header. 8465 */ 8466 void 8467 tcp_icmp_error(tcp_t *tcp, mblk_t *mp) 8468 { 8469 icmph_t *icmph; 8470 ipha_t *ipha; 8471 int iph_hdr_length; 8472 tcph_t *tcph; 8473 boolean_t ipsec_mctl = B_FALSE; 8474 boolean_t secure; 8475 mblk_t *first_mp = mp; 8476 uint32_t new_mss; 8477 uint32_t ratio; 8478 size_t mp_size = MBLKL(mp); 8479 uint32_t seg_seq; 8480 tcp_stack_t *tcps = tcp->tcp_tcps; 8481 8482 /* Assume IP provides aligned packets - otherwise toss */ 8483 if (!OK_32PTR(mp->b_rptr)) { 8484 freemsg(mp); 8485 return; 8486 } 8487 8488 /* 8489 * Since ICMP errors are normal data marked with M_CTL when sent 8490 * to TCP or UDP, we have to look for a IPSEC_IN value to identify 8491 * packets starting with an ipsec_info_t, see ipsec_info.h. 8492 */ 8493 if ((mp_size == sizeof (ipsec_info_t)) && 8494 (((ipsec_info_t *)mp->b_rptr)->ipsec_info_type == IPSEC_IN)) { 8495 ASSERT(mp->b_cont != NULL); 8496 mp = mp->b_cont; 8497 /* IP should have done this */ 8498 ASSERT(OK_32PTR(mp->b_rptr)); 8499 mp_size = MBLKL(mp); 8500 ipsec_mctl = B_TRUE; 8501 } 8502 8503 /* 8504 * Verify that we have a complete outer IP header. If not, drop it. 8505 */ 8506 if (mp_size < sizeof (ipha_t)) { 8507 noticmpv4: 8508 freemsg(first_mp); 8509 return; 8510 } 8511 8512 ipha = (ipha_t *)mp->b_rptr; 8513 /* 8514 * Verify IP version. Anything other than IPv4 or IPv6 packet is sent 8515 * upstream. ICMPv6 is handled in tcp_icmp_error_ipv6. 8516 */ 8517 switch (IPH_HDR_VERSION(ipha)) { 8518 case IPV6_VERSION: 8519 tcp_icmp_error_ipv6(tcp, first_mp, ipsec_mctl); 8520 return; 8521 case IPV4_VERSION: 8522 break; 8523 default: 8524 goto noticmpv4; 8525 } 8526 8527 /* Skip past the outer IP and ICMP headers */ 8528 iph_hdr_length = IPH_HDR_LENGTH(ipha); 8529 icmph = (icmph_t *)&mp->b_rptr[iph_hdr_length]; 8530 /* 8531 * If we don't have the correct outer IP header length or if the ULP 8532 * is not IPPROTO_ICMP or if we don't have a complete inner IP header 8533 * send it upstream. 8534 */ 8535 if (iph_hdr_length < sizeof (ipha_t) || 8536 ipha->ipha_protocol != IPPROTO_ICMP || 8537 (ipha_t *)&icmph[1] + 1 > (ipha_t *)mp->b_wptr) { 8538 goto noticmpv4; 8539 } 8540 ipha = (ipha_t *)&icmph[1]; 8541 8542 /* Skip past the inner IP and find the ULP header */ 8543 iph_hdr_length = IPH_HDR_LENGTH(ipha); 8544 tcph = (tcph_t *)((char *)ipha + iph_hdr_length); 8545 /* 8546 * If we don't have the correct inner IP header length or if the ULP 8547 * is not IPPROTO_TCP or if we don't have at least ICMP_MIN_TCP_HDR 8548 * bytes of TCP header, drop it. 8549 */ 8550 if (iph_hdr_length < sizeof (ipha_t) || 8551 ipha->ipha_protocol != IPPROTO_TCP || 8552 (uchar_t *)tcph + ICMP_MIN_TCP_HDR > mp->b_wptr) { 8553 goto noticmpv4; 8554 } 8555 8556 if (TCP_IS_DETACHED_NONEAGER(tcp)) { 8557 if (ipsec_mctl) { 8558 secure = ipsec_in_is_secure(first_mp); 8559 } else { 8560 secure = B_FALSE; 8561 } 8562 if (secure) { 8563 /* 8564 * If we are willing to accept this in clear 8565 * we don't have to verify policy. 8566 */ 8567 if (!ipsec_inbound_accept_clear(mp, ipha, NULL)) { 8568 if (!tcp_check_policy(tcp, first_mp, 8569 ipha, NULL, secure, ipsec_mctl)) { 8570 /* 8571 * tcp_check_policy called 8572 * ip_drop_packet() on failure. 8573 */ 8574 return; 8575 } 8576 } 8577 } 8578 } else if (ipsec_mctl) { 8579 /* 8580 * This is a hard_bound connection. IP has already 8581 * verified policy. We don't have to do it again. 8582 */ 8583 freeb(first_mp); 8584 first_mp = mp; 8585 ipsec_mctl = B_FALSE; 8586 } 8587 8588 seg_seq = ABE32_TO_U32(tcph->th_seq); 8589 /* 8590 * TCP SHOULD check that the TCP sequence number contained in 8591 * payload of the ICMP error message is within the range 8592 * SND.UNA <= SEG.SEQ < SND.NXT. 8593 */ 8594 if (SEQ_LT(seg_seq, tcp->tcp_suna) || SEQ_GEQ(seg_seq, tcp->tcp_snxt)) { 8595 /* 8596 * If the ICMP message is bogus, should we kill the 8597 * connection, or should we just drop the bogus ICMP 8598 * message? It would probably make more sense to just 8599 * drop the message so that if this one managed to get 8600 * in, the real connection should not suffer. 8601 */ 8602 goto noticmpv4; 8603 } 8604 8605 switch (icmph->icmph_type) { 8606 case ICMP_DEST_UNREACHABLE: 8607 switch (icmph->icmph_code) { 8608 case ICMP_FRAGMENTATION_NEEDED: 8609 /* 8610 * Reduce the MSS based on the new MTU. This will 8611 * eliminate any fragmentation locally. 8612 * N.B. There may well be some funny side-effects on 8613 * the local send policy and the remote receive policy. 8614 * Pending further research, we provide 8615 * tcp_ignore_path_mtu just in case this proves 8616 * disastrous somewhere. 8617 * 8618 * After updating the MSS, retransmit part of the 8619 * dropped segment using the new mss by calling 8620 * tcp_wput_data(). Need to adjust all those 8621 * params to make sure tcp_wput_data() work properly. 8622 */ 8623 if (tcps->tcps_ignore_path_mtu) 8624 break; 8625 8626 /* 8627 * Decrease the MSS by time stamp options 8628 * IP options and IPSEC options. tcp_hdr_len 8629 * includes time stamp option and IP option 8630 * length. 8631 */ 8632 8633 new_mss = ntohs(icmph->icmph_du_mtu) - 8634 tcp->tcp_hdr_len - tcp->tcp_ipsec_overhead; 8635 8636 /* 8637 * Only update the MSS if the new one is 8638 * smaller than the previous one. This is 8639 * to avoid problems when getting multiple 8640 * ICMP errors for the same MTU. 8641 */ 8642 if (new_mss >= tcp->tcp_mss) 8643 break; 8644 8645 /* 8646 * Stop doing PMTU if new_mss is less than 68 8647 * or less than tcp_mss_min. 8648 * The value 68 comes from rfc 1191. 8649 */ 8650 if (new_mss < MAX(68, tcps->tcps_mss_min)) 8651 tcp->tcp_ipha->ipha_fragment_offset_and_flags = 8652 0; 8653 8654 ratio = tcp->tcp_cwnd / tcp->tcp_mss; 8655 ASSERT(ratio >= 1); 8656 tcp_mss_set(tcp, new_mss, B_TRUE); 8657 8658 /* 8659 * Make sure we have something to 8660 * send. 8661 */ 8662 if (SEQ_LT(tcp->tcp_suna, tcp->tcp_snxt) && 8663 (tcp->tcp_xmit_head != NULL)) { 8664 /* 8665 * Shrink tcp_cwnd in 8666 * proportion to the old MSS/new MSS. 8667 */ 8668 tcp->tcp_cwnd = ratio * tcp->tcp_mss; 8669 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && 8670 (tcp->tcp_unsent == 0)) { 8671 tcp->tcp_rexmit_max = tcp->tcp_fss; 8672 } else { 8673 tcp->tcp_rexmit_max = tcp->tcp_snxt; 8674 } 8675 tcp->tcp_rexmit_nxt = tcp->tcp_suna; 8676 tcp->tcp_rexmit = B_TRUE; 8677 tcp->tcp_dupack_cnt = 0; 8678 tcp->tcp_snd_burst = TCP_CWND_SS; 8679 tcp_ss_rexmit(tcp); 8680 } 8681 break; 8682 case ICMP_PORT_UNREACHABLE: 8683 case ICMP_PROTOCOL_UNREACHABLE: 8684 switch (tcp->tcp_state) { 8685 case TCPS_SYN_SENT: 8686 case TCPS_SYN_RCVD: 8687 /* 8688 * ICMP can snipe away incipient 8689 * TCP connections as long as 8690 * seq number is same as initial 8691 * send seq number. 8692 */ 8693 if (seg_seq == tcp->tcp_iss) { 8694 (void) tcp_clean_death(tcp, 8695 ECONNREFUSED, 6); 8696 } 8697 break; 8698 } 8699 break; 8700 case ICMP_HOST_UNREACHABLE: 8701 case ICMP_NET_UNREACHABLE: 8702 /* Record the error in case we finally time out. */ 8703 if (icmph->icmph_code == ICMP_HOST_UNREACHABLE) 8704 tcp->tcp_client_errno = EHOSTUNREACH; 8705 else 8706 tcp->tcp_client_errno = ENETUNREACH; 8707 if (tcp->tcp_state == TCPS_SYN_RCVD) { 8708 if (tcp->tcp_listener != NULL && 8709 tcp->tcp_listener->tcp_syn_defense) { 8710 /* 8711 * Ditch the half-open connection if we 8712 * suspect a SYN attack is under way. 8713 */ 8714 tcp_ip_ire_mark_advice(tcp); 8715 (void) tcp_clean_death(tcp, 8716 tcp->tcp_client_errno, 7); 8717 } 8718 } 8719 break; 8720 default: 8721 break; 8722 } 8723 break; 8724 case ICMP_SOURCE_QUENCH: { 8725 /* 8726 * use a global boolean to control 8727 * whether TCP should respond to ICMP_SOURCE_QUENCH. 8728 * The default is false. 8729 */ 8730 if (tcp_icmp_source_quench) { 8731 /* 8732 * Reduce the sending rate as if we got a 8733 * retransmit timeout 8734 */ 8735 uint32_t npkt; 8736 8737 npkt = ((tcp->tcp_snxt - tcp->tcp_suna) >> 1) / 8738 tcp->tcp_mss; 8739 tcp->tcp_cwnd_ssthresh = MAX(npkt, 2) * tcp->tcp_mss; 8740 tcp->tcp_cwnd = tcp->tcp_mss; 8741 tcp->tcp_cwnd_cnt = 0; 8742 } 8743 break; 8744 } 8745 } 8746 freemsg(first_mp); 8747 } 8748 8749 /* 8750 * tcp_icmp_error_ipv6 is called by tcp_rput_other to process ICMPv6 8751 * error messages passed up by IP. 8752 * Assumes that IP has pulled up all the extension headers as well 8753 * as the ICMPv6 header. 8754 */ 8755 static void 8756 tcp_icmp_error_ipv6(tcp_t *tcp, mblk_t *mp, boolean_t ipsec_mctl) 8757 { 8758 icmp6_t *icmp6; 8759 ip6_t *ip6h; 8760 uint16_t iph_hdr_length; 8761 tcpha_t *tcpha; 8762 uint8_t *nexthdrp; 8763 uint32_t new_mss; 8764 uint32_t ratio; 8765 boolean_t secure; 8766 mblk_t *first_mp = mp; 8767 size_t mp_size; 8768 uint32_t seg_seq; 8769 tcp_stack_t *tcps = tcp->tcp_tcps; 8770 8771 /* 8772 * The caller has determined if this is an IPSEC_IN packet and 8773 * set ipsec_mctl appropriately (see tcp_icmp_error). 8774 */ 8775 if (ipsec_mctl) 8776 mp = mp->b_cont; 8777 8778 mp_size = MBLKL(mp); 8779 8780 /* 8781 * Verify that we have a complete IP header. If not, send it upstream. 8782 */ 8783 if (mp_size < sizeof (ip6_t)) { 8784 noticmpv6: 8785 freemsg(first_mp); 8786 return; 8787 } 8788 8789 /* 8790 * Verify this is an ICMPV6 packet, else send it upstream. 8791 */ 8792 ip6h = (ip6_t *)mp->b_rptr; 8793 if (ip6h->ip6_nxt == IPPROTO_ICMPV6) { 8794 iph_hdr_length = IPV6_HDR_LEN; 8795 } else if (!ip_hdr_length_nexthdr_v6(mp, ip6h, &iph_hdr_length, 8796 &nexthdrp) || 8797 *nexthdrp != IPPROTO_ICMPV6) { 8798 goto noticmpv6; 8799 } 8800 icmp6 = (icmp6_t *)&mp->b_rptr[iph_hdr_length]; 8801 ip6h = (ip6_t *)&icmp6[1]; 8802 /* 8803 * Verify if we have a complete ICMP and inner IP header. 8804 */ 8805 if ((uchar_t *)&ip6h[1] > mp->b_wptr) 8806 goto noticmpv6; 8807 8808 if (!ip_hdr_length_nexthdr_v6(mp, ip6h, &iph_hdr_length, &nexthdrp)) 8809 goto noticmpv6; 8810 tcpha = (tcpha_t *)((char *)ip6h + iph_hdr_length); 8811 /* 8812 * Validate inner header. If the ULP is not IPPROTO_TCP or if we don't 8813 * have at least ICMP_MIN_TCP_HDR bytes of TCP header drop the 8814 * packet. 8815 */ 8816 if ((*nexthdrp != IPPROTO_TCP) || 8817 ((uchar_t *)tcpha + ICMP_MIN_TCP_HDR) > mp->b_wptr) { 8818 goto noticmpv6; 8819 } 8820 8821 /* 8822 * ICMP errors come on the right queue or come on 8823 * listener/global queue for detached connections and 8824 * get switched to the right queue. If it comes on the 8825 * right queue, policy check has already been done by IP 8826 * and thus free the first_mp without verifying the policy. 8827 * If it has come for a non-hard bound connection, we need 8828 * to verify policy as IP may not have done it. 8829 */ 8830 if (!tcp->tcp_hard_bound) { 8831 if (ipsec_mctl) { 8832 secure = ipsec_in_is_secure(first_mp); 8833 } else { 8834 secure = B_FALSE; 8835 } 8836 if (secure) { 8837 /* 8838 * If we are willing to accept this in clear 8839 * we don't have to verify policy. 8840 */ 8841 if (!ipsec_inbound_accept_clear(mp, NULL, ip6h)) { 8842 if (!tcp_check_policy(tcp, first_mp, 8843 NULL, ip6h, secure, ipsec_mctl)) { 8844 /* 8845 * tcp_check_policy called 8846 * ip_drop_packet() on failure. 8847 */ 8848 return; 8849 } 8850 } 8851 } 8852 } else if (ipsec_mctl) { 8853 /* 8854 * This is a hard_bound connection. IP has already 8855 * verified policy. We don't have to do it again. 8856 */ 8857 freeb(first_mp); 8858 first_mp = mp; 8859 ipsec_mctl = B_FALSE; 8860 } 8861 8862 seg_seq = ntohl(tcpha->tha_seq); 8863 /* 8864 * TCP SHOULD check that the TCP sequence number contained in 8865 * payload of the ICMP error message is within the range 8866 * SND.UNA <= SEG.SEQ < SND.NXT. 8867 */ 8868 if (SEQ_LT(seg_seq, tcp->tcp_suna) || SEQ_GEQ(seg_seq, tcp->tcp_snxt)) { 8869 /* 8870 * If the ICMP message is bogus, should we kill the 8871 * connection, or should we just drop the bogus ICMP 8872 * message? It would probably make more sense to just 8873 * drop the message so that if this one managed to get 8874 * in, the real connection should not suffer. 8875 */ 8876 goto noticmpv6; 8877 } 8878 8879 switch (icmp6->icmp6_type) { 8880 case ICMP6_PACKET_TOO_BIG: 8881 /* 8882 * Reduce the MSS based on the new MTU. This will 8883 * eliminate any fragmentation locally. 8884 * N.B. There may well be some funny side-effects on 8885 * the local send policy and the remote receive policy. 8886 * Pending further research, we provide 8887 * tcp_ignore_path_mtu just in case this proves 8888 * disastrous somewhere. 8889 * 8890 * After updating the MSS, retransmit part of the 8891 * dropped segment using the new mss by calling 8892 * tcp_wput_data(). Need to adjust all those 8893 * params to make sure tcp_wput_data() work properly. 8894 */ 8895 if (tcps->tcps_ignore_path_mtu) 8896 break; 8897 8898 /* 8899 * Decrease the MSS by time stamp options 8900 * IP options and IPSEC options. tcp_hdr_len 8901 * includes time stamp option and IP option 8902 * length. 8903 */ 8904 new_mss = ntohs(icmp6->icmp6_mtu) - tcp->tcp_hdr_len - 8905 tcp->tcp_ipsec_overhead; 8906 8907 /* 8908 * Only update the MSS if the new one is 8909 * smaller than the previous one. This is 8910 * to avoid problems when getting multiple 8911 * ICMP errors for the same MTU. 8912 */ 8913 if (new_mss >= tcp->tcp_mss) 8914 break; 8915 8916 ratio = tcp->tcp_cwnd / tcp->tcp_mss; 8917 ASSERT(ratio >= 1); 8918 tcp_mss_set(tcp, new_mss, B_TRUE); 8919 8920 /* 8921 * Make sure we have something to 8922 * send. 8923 */ 8924 if (SEQ_LT(tcp->tcp_suna, tcp->tcp_snxt) && 8925 (tcp->tcp_xmit_head != NULL)) { 8926 /* 8927 * Shrink tcp_cwnd in 8928 * proportion to the old MSS/new MSS. 8929 */ 8930 tcp->tcp_cwnd = ratio * tcp->tcp_mss; 8931 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && 8932 (tcp->tcp_unsent == 0)) { 8933 tcp->tcp_rexmit_max = tcp->tcp_fss; 8934 } else { 8935 tcp->tcp_rexmit_max = tcp->tcp_snxt; 8936 } 8937 tcp->tcp_rexmit_nxt = tcp->tcp_suna; 8938 tcp->tcp_rexmit = B_TRUE; 8939 tcp->tcp_dupack_cnt = 0; 8940 tcp->tcp_snd_burst = TCP_CWND_SS; 8941 tcp_ss_rexmit(tcp); 8942 } 8943 break; 8944 8945 case ICMP6_DST_UNREACH: 8946 switch (icmp6->icmp6_code) { 8947 case ICMP6_DST_UNREACH_NOPORT: 8948 if (((tcp->tcp_state == TCPS_SYN_SENT) || 8949 (tcp->tcp_state == TCPS_SYN_RCVD)) && 8950 (seg_seq == tcp->tcp_iss)) { 8951 (void) tcp_clean_death(tcp, 8952 ECONNREFUSED, 8); 8953 } 8954 break; 8955 8956 case ICMP6_DST_UNREACH_ADMIN: 8957 case ICMP6_DST_UNREACH_NOROUTE: 8958 case ICMP6_DST_UNREACH_BEYONDSCOPE: 8959 case ICMP6_DST_UNREACH_ADDR: 8960 /* Record the error in case we finally time out. */ 8961 tcp->tcp_client_errno = EHOSTUNREACH; 8962 if (((tcp->tcp_state == TCPS_SYN_SENT) || 8963 (tcp->tcp_state == TCPS_SYN_RCVD)) && 8964 (seg_seq == tcp->tcp_iss)) { 8965 if (tcp->tcp_listener != NULL && 8966 tcp->tcp_listener->tcp_syn_defense) { 8967 /* 8968 * Ditch the half-open connection if we 8969 * suspect a SYN attack is under way. 8970 */ 8971 tcp_ip_ire_mark_advice(tcp); 8972 (void) tcp_clean_death(tcp, 8973 tcp->tcp_client_errno, 9); 8974 } 8975 } 8976 8977 8978 break; 8979 default: 8980 break; 8981 } 8982 break; 8983 8984 case ICMP6_PARAM_PROB: 8985 /* If this corresponds to an ICMP_PROTOCOL_UNREACHABLE */ 8986 if (icmp6->icmp6_code == ICMP6_PARAMPROB_NEXTHEADER && 8987 (uchar_t *)ip6h + icmp6->icmp6_pptr == 8988 (uchar_t *)nexthdrp) { 8989 if (tcp->tcp_state == TCPS_SYN_SENT || 8990 tcp->tcp_state == TCPS_SYN_RCVD) { 8991 (void) tcp_clean_death(tcp, 8992 ECONNREFUSED, 10); 8993 } 8994 break; 8995 } 8996 break; 8997 8998 case ICMP6_TIME_EXCEEDED: 8999 default: 9000 break; 9001 } 9002 freemsg(first_mp); 9003 } 9004 9005 /* 9006 * IP recognizes seven kinds of bind requests: 9007 * 9008 * - A zero-length address binds only to the protocol number. 9009 * 9010 * - A 4-byte address is treated as a request to 9011 * validate that the address is a valid local IPv4 9012 * address, appropriate for an application to bind to. 9013 * IP does the verification, but does not make any note 9014 * of the address at this time. 9015 * 9016 * - A 16-byte address contains is treated as a request 9017 * to validate a local IPv6 address, as the 4-byte 9018 * address case above. 9019 * 9020 * - A 16-byte sockaddr_in to validate the local IPv4 address and also 9021 * use it for the inbound fanout of packets. 9022 * 9023 * - A 24-byte sockaddr_in6 to validate the local IPv6 address and also 9024 * use it for the inbound fanout of packets. 9025 * 9026 * - A 12-byte address (ipa_conn_t) containing complete IPv4 fanout 9027 * information consisting of local and remote addresses 9028 * and ports. In this case, the addresses are both 9029 * validated as appropriate for this operation, and, if 9030 * so, the information is retained for use in the 9031 * inbound fanout. 9032 * 9033 * - A 36-byte address address (ipa6_conn_t) containing complete IPv6 9034 * fanout information, like the 12-byte case above. 9035 * 9036 * IP will also fill in the IRE request mblk with information 9037 * regarding our peer. In all cases, we notify IP of our protocol 9038 * type by appending a single protocol byte to the bind request. 9039 */ 9040 static mblk_t * 9041 tcp_ip_bind_mp(tcp_t *tcp, t_scalar_t bind_prim, t_scalar_t addr_length) 9042 { 9043 char *cp; 9044 mblk_t *mp; 9045 struct T_bind_req *tbr; 9046 ipa_conn_t *ac; 9047 ipa6_conn_t *ac6; 9048 sin_t *sin; 9049 sin6_t *sin6; 9050 9051 ASSERT(bind_prim == O_T_BIND_REQ || bind_prim == T_BIND_REQ); 9052 ASSERT((tcp->tcp_family == AF_INET && 9053 tcp->tcp_ipversion == IPV4_VERSION) || 9054 (tcp->tcp_family == AF_INET6 && 9055 (tcp->tcp_ipversion == IPV4_VERSION || 9056 tcp->tcp_ipversion == IPV6_VERSION))); 9057 9058 mp = allocb(sizeof (*tbr) + addr_length + 1, BPRI_HI); 9059 if (!mp) 9060 return (mp); 9061 mp->b_datap->db_type = M_PROTO; 9062 tbr = (struct T_bind_req *)mp->b_rptr; 9063 tbr->PRIM_type = bind_prim; 9064 tbr->ADDR_offset = sizeof (*tbr); 9065 tbr->CONIND_number = 0; 9066 tbr->ADDR_length = addr_length; 9067 cp = (char *)&tbr[1]; 9068 switch (addr_length) { 9069 case sizeof (ipa_conn_t): 9070 ASSERT(tcp->tcp_family == AF_INET); 9071 ASSERT(tcp->tcp_ipversion == IPV4_VERSION); 9072 9073 mp->b_cont = allocb(sizeof (ire_t), BPRI_HI); 9074 if (mp->b_cont == NULL) { 9075 freemsg(mp); 9076 return (NULL); 9077 } 9078 mp->b_cont->b_wptr += sizeof (ire_t); 9079 mp->b_cont->b_datap->db_type = IRE_DB_REQ_TYPE; 9080 9081 /* cp known to be 32 bit aligned */ 9082 ac = (ipa_conn_t *)cp; 9083 ac->ac_laddr = tcp->tcp_ipha->ipha_src; 9084 ac->ac_faddr = tcp->tcp_remote; 9085 ac->ac_fport = tcp->tcp_fport; 9086 ac->ac_lport = tcp->tcp_lport; 9087 tcp->tcp_hard_binding = 1; 9088 break; 9089 9090 case sizeof (ipa6_conn_t): 9091 ASSERT(tcp->tcp_family == AF_INET6); 9092 9093 mp->b_cont = allocb(sizeof (ire_t), BPRI_HI); 9094 if (mp->b_cont == NULL) { 9095 freemsg(mp); 9096 return (NULL); 9097 } 9098 mp->b_cont->b_wptr += sizeof (ire_t); 9099 mp->b_cont->b_datap->db_type = IRE_DB_REQ_TYPE; 9100 9101 /* cp known to be 32 bit aligned */ 9102 ac6 = (ipa6_conn_t *)cp; 9103 if (tcp->tcp_ipversion == IPV4_VERSION) { 9104 IN6_IPADDR_TO_V4MAPPED(tcp->tcp_ipha->ipha_src, 9105 &ac6->ac6_laddr); 9106 } else { 9107 ac6->ac6_laddr = tcp->tcp_ip6h->ip6_src; 9108 } 9109 ac6->ac6_faddr = tcp->tcp_remote_v6; 9110 ac6->ac6_fport = tcp->tcp_fport; 9111 ac6->ac6_lport = tcp->tcp_lport; 9112 tcp->tcp_hard_binding = 1; 9113 break; 9114 9115 case sizeof (sin_t): 9116 /* 9117 * NOTE: IPV6_ADDR_LEN also has same size. 9118 * Use family to discriminate. 9119 */ 9120 if (tcp->tcp_family == AF_INET) { 9121 sin = (sin_t *)cp; 9122 9123 *sin = sin_null; 9124 sin->sin_family = AF_INET; 9125 sin->sin_addr.s_addr = tcp->tcp_bound_source; 9126 sin->sin_port = tcp->tcp_lport; 9127 break; 9128 } else { 9129 *(in6_addr_t *)cp = tcp->tcp_bound_source_v6; 9130 } 9131 break; 9132 9133 case sizeof (sin6_t): 9134 ASSERT(tcp->tcp_family == AF_INET6); 9135 sin6 = (sin6_t *)cp; 9136 9137 *sin6 = sin6_null; 9138 sin6->sin6_family = AF_INET6; 9139 sin6->sin6_addr = tcp->tcp_bound_source_v6; 9140 sin6->sin6_port = tcp->tcp_lport; 9141 break; 9142 9143 case IP_ADDR_LEN: 9144 ASSERT(tcp->tcp_ipversion == IPV4_VERSION); 9145 *(uint32_t *)cp = tcp->tcp_ipha->ipha_src; 9146 break; 9147 9148 } 9149 /* Add protocol number to end */ 9150 cp[addr_length] = (char)IPPROTO_TCP; 9151 mp->b_wptr = (uchar_t *)&cp[addr_length + 1]; 9152 return (mp); 9153 } 9154 9155 /* 9156 * Notify IP that we are having trouble with this connection. IP should 9157 * blow the IRE away and start over. 9158 */ 9159 static void 9160 tcp_ip_notify(tcp_t *tcp) 9161 { 9162 struct iocblk *iocp; 9163 ipid_t *ipid; 9164 mblk_t *mp; 9165 9166 /* IPv6 has NUD thus notification to delete the IRE is not needed */ 9167 if (tcp->tcp_ipversion == IPV6_VERSION) 9168 return; 9169 9170 mp = mkiocb(IP_IOCTL); 9171 if (mp == NULL) 9172 return; 9173 9174 iocp = (struct iocblk *)mp->b_rptr; 9175 iocp->ioc_count = sizeof (ipid_t) + sizeof (tcp->tcp_ipha->ipha_dst); 9176 9177 mp->b_cont = allocb(iocp->ioc_count, BPRI_HI); 9178 if (!mp->b_cont) { 9179 freeb(mp); 9180 return; 9181 } 9182 9183 ipid = (ipid_t *)mp->b_cont->b_rptr; 9184 mp->b_cont->b_wptr += iocp->ioc_count; 9185 bzero(ipid, sizeof (*ipid)); 9186 ipid->ipid_cmd = IP_IOC_IRE_DELETE_NO_REPLY; 9187 ipid->ipid_ire_type = IRE_CACHE; 9188 ipid->ipid_addr_offset = sizeof (ipid_t); 9189 ipid->ipid_addr_length = sizeof (tcp->tcp_ipha->ipha_dst); 9190 /* 9191 * Note: in the case of source routing we want to blow away the 9192 * route to the first source route hop. 9193 */ 9194 bcopy(&tcp->tcp_ipha->ipha_dst, &ipid[1], 9195 sizeof (tcp->tcp_ipha->ipha_dst)); 9196 9197 CALL_IP_WPUT(tcp->tcp_connp, tcp->tcp_wq, mp); 9198 } 9199 9200 /* Unlink and return any mblk that looks like it contains an ire */ 9201 static mblk_t * 9202 tcp_ire_mp(mblk_t *mp) 9203 { 9204 mblk_t *prev_mp; 9205 9206 for (;;) { 9207 prev_mp = mp; 9208 mp = mp->b_cont; 9209 if (mp == NULL) 9210 break; 9211 switch (DB_TYPE(mp)) { 9212 case IRE_DB_TYPE: 9213 case IRE_DB_REQ_TYPE: 9214 if (prev_mp != NULL) 9215 prev_mp->b_cont = mp->b_cont; 9216 mp->b_cont = NULL; 9217 return (mp); 9218 default: 9219 break; 9220 } 9221 } 9222 return (mp); 9223 } 9224 9225 /* 9226 * Timer callback routine for keepalive probe. We do a fake resend of 9227 * last ACKed byte. Then set a timer using RTO. When the timer expires, 9228 * check to see if we have heard anything from the other end for the last 9229 * RTO period. If we have, set the timer to expire for another 9230 * tcp_keepalive_intrvl and check again. If we have not, set a timer using 9231 * RTO << 1 and check again when it expires. Keep exponentially increasing 9232 * the timeout if we have not heard from the other side. If for more than 9233 * (tcp_ka_interval + tcp_ka_abort_thres) we have not heard anything, 9234 * kill the connection unless the keepalive abort threshold is 0. In 9235 * that case, we will probe "forever." 9236 */ 9237 static void 9238 tcp_keepalive_killer(void *arg) 9239 { 9240 mblk_t *mp; 9241 conn_t *connp = (conn_t *)arg; 9242 tcp_t *tcp = connp->conn_tcp; 9243 int32_t firetime; 9244 int32_t idletime; 9245 int32_t ka_intrvl; 9246 tcp_stack_t *tcps = tcp->tcp_tcps; 9247 9248 tcp->tcp_ka_tid = 0; 9249 9250 if (tcp->tcp_fused) 9251 return; 9252 9253 BUMP_MIB(&tcps->tcps_mib, tcpTimKeepalive); 9254 ka_intrvl = tcp->tcp_ka_interval; 9255 9256 /* 9257 * Keepalive probe should only be sent if the application has not 9258 * done a close on the connection. 9259 */ 9260 if (tcp->tcp_state > TCPS_CLOSE_WAIT) { 9261 return; 9262 } 9263 /* Timer fired too early, restart it. */ 9264 if (tcp->tcp_state < TCPS_ESTABLISHED) { 9265 tcp->tcp_ka_tid = TCP_TIMER(tcp, tcp_keepalive_killer, 9266 MSEC_TO_TICK(ka_intrvl)); 9267 return; 9268 } 9269 9270 idletime = TICK_TO_MSEC(lbolt - tcp->tcp_last_recv_time); 9271 /* 9272 * If we have not heard from the other side for a long 9273 * time, kill the connection unless the keepalive abort 9274 * threshold is 0. In that case, we will probe "forever." 9275 */ 9276 if (tcp->tcp_ka_abort_thres != 0 && 9277 idletime > (ka_intrvl + tcp->tcp_ka_abort_thres)) { 9278 BUMP_MIB(&tcps->tcps_mib, tcpTimKeepaliveDrop); 9279 (void) tcp_clean_death(tcp, tcp->tcp_client_errno ? 9280 tcp->tcp_client_errno : ETIMEDOUT, 11); 9281 return; 9282 } 9283 9284 if (tcp->tcp_snxt == tcp->tcp_suna && 9285 idletime >= ka_intrvl) { 9286 /* Fake resend of last ACKed byte. */ 9287 mblk_t *mp1 = allocb(1, BPRI_LO); 9288 9289 if (mp1 != NULL) { 9290 *mp1->b_wptr++ = '\0'; 9291 mp = tcp_xmit_mp(tcp, mp1, 1, NULL, NULL, 9292 tcp->tcp_suna - 1, B_FALSE, NULL, B_TRUE); 9293 freeb(mp1); 9294 /* 9295 * if allocation failed, fall through to start the 9296 * timer back. 9297 */ 9298 if (mp != NULL) { 9299 tcp_send_data(tcp, tcp->tcp_wq, mp); 9300 BUMP_MIB(&tcps->tcps_mib, 9301 tcpTimKeepaliveProbe); 9302 if (tcp->tcp_ka_last_intrvl != 0) { 9303 int max; 9304 /* 9305 * We should probe again at least 9306 * in ka_intrvl, but not more than 9307 * tcp_rexmit_interval_max. 9308 */ 9309 max = tcps->tcps_rexmit_interval_max; 9310 firetime = MIN(ka_intrvl - 1, 9311 tcp->tcp_ka_last_intrvl << 1); 9312 if (firetime > max) 9313 firetime = max; 9314 } else { 9315 firetime = tcp->tcp_rto; 9316 } 9317 tcp->tcp_ka_tid = TCP_TIMER(tcp, 9318 tcp_keepalive_killer, 9319 MSEC_TO_TICK(firetime)); 9320 tcp->tcp_ka_last_intrvl = firetime; 9321 return; 9322 } 9323 } 9324 } else { 9325 tcp->tcp_ka_last_intrvl = 0; 9326 } 9327 9328 /* firetime can be negative if (mp1 == NULL || mp == NULL) */ 9329 if ((firetime = ka_intrvl - idletime) < 0) { 9330 firetime = ka_intrvl; 9331 } 9332 tcp->tcp_ka_tid = TCP_TIMER(tcp, tcp_keepalive_killer, 9333 MSEC_TO_TICK(firetime)); 9334 } 9335 9336 int 9337 tcp_maxpsz_set(tcp_t *tcp, boolean_t set_maxblk) 9338 { 9339 queue_t *q = tcp->tcp_rq; 9340 int32_t mss = tcp->tcp_mss; 9341 int maxpsz; 9342 9343 if (TCP_IS_DETACHED(tcp)) 9344 return (mss); 9345 9346 if (tcp->tcp_fused) { 9347 maxpsz = tcp_fuse_maxpsz_set(tcp); 9348 mss = INFPSZ; 9349 } else if (tcp->tcp_mdt || tcp->tcp_lso || tcp->tcp_maxpsz == 0) { 9350 /* 9351 * Set the sd_qn_maxpsz according to the socket send buffer 9352 * size, and sd_maxblk to INFPSZ (-1). This will essentially 9353 * instruct the stream head to copyin user data into contiguous 9354 * kernel-allocated buffers without breaking it up into smaller 9355 * chunks. We round up the buffer size to the nearest SMSS. 9356 */ 9357 maxpsz = MSS_ROUNDUP(tcp->tcp_xmit_hiwater, mss); 9358 if (tcp->tcp_kssl_ctx == NULL) 9359 mss = INFPSZ; 9360 else 9361 mss = SSL3_MAX_RECORD_LEN; 9362 } else { 9363 /* 9364 * Set sd_qn_maxpsz to approx half the (receivers) buffer 9365 * (and a multiple of the mss). This instructs the stream 9366 * head to break down larger than SMSS writes into SMSS- 9367 * size mblks, up to tcp_maxpsz_multiplier mblks at a time. 9368 */ 9369 maxpsz = tcp->tcp_maxpsz * mss; 9370 if (maxpsz > tcp->tcp_xmit_hiwater/2) { 9371 maxpsz = tcp->tcp_xmit_hiwater/2; 9372 /* Round up to nearest mss */ 9373 maxpsz = MSS_ROUNDUP(maxpsz, mss); 9374 } 9375 } 9376 (void) setmaxps(q, maxpsz); 9377 tcp->tcp_wq->q_maxpsz = maxpsz; 9378 9379 if (set_maxblk) 9380 (void) mi_set_sth_maxblk(q, mss); 9381 9382 return (mss); 9383 } 9384 9385 /* 9386 * Extract option values from a tcp header. We put any found values into the 9387 * tcpopt struct and return a bitmask saying which options were found. 9388 */ 9389 static int 9390 tcp_parse_options(tcph_t *tcph, tcp_opt_t *tcpopt) 9391 { 9392 uchar_t *endp; 9393 int len; 9394 uint32_t mss; 9395 uchar_t *up = (uchar_t *)tcph; 9396 int found = 0; 9397 int32_t sack_len; 9398 tcp_seq sack_begin, sack_end; 9399 tcp_t *tcp; 9400 9401 endp = up + TCP_HDR_LENGTH(tcph); 9402 up += TCP_MIN_HEADER_LENGTH; 9403 while (up < endp) { 9404 len = endp - up; 9405 switch (*up) { 9406 case TCPOPT_EOL: 9407 break; 9408 9409 case TCPOPT_NOP: 9410 up++; 9411 continue; 9412 9413 case TCPOPT_MAXSEG: 9414 if (len < TCPOPT_MAXSEG_LEN || 9415 up[1] != TCPOPT_MAXSEG_LEN) 9416 break; 9417 9418 mss = BE16_TO_U16(up+2); 9419 /* Caller must handle tcp_mss_min and tcp_mss_max_* */ 9420 tcpopt->tcp_opt_mss = mss; 9421 found |= TCP_OPT_MSS_PRESENT; 9422 9423 up += TCPOPT_MAXSEG_LEN; 9424 continue; 9425 9426 case TCPOPT_WSCALE: 9427 if (len < TCPOPT_WS_LEN || up[1] != TCPOPT_WS_LEN) 9428 break; 9429 9430 if (up[2] > TCP_MAX_WINSHIFT) 9431 tcpopt->tcp_opt_wscale = TCP_MAX_WINSHIFT; 9432 else 9433 tcpopt->tcp_opt_wscale = up[2]; 9434 found |= TCP_OPT_WSCALE_PRESENT; 9435 9436 up += TCPOPT_WS_LEN; 9437 continue; 9438 9439 case TCPOPT_SACK_PERMITTED: 9440 if (len < TCPOPT_SACK_OK_LEN || 9441 up[1] != TCPOPT_SACK_OK_LEN) 9442 break; 9443 found |= TCP_OPT_SACK_OK_PRESENT; 9444 up += TCPOPT_SACK_OK_LEN; 9445 continue; 9446 9447 case TCPOPT_SACK: 9448 if (len <= 2 || up[1] <= 2 || len < up[1]) 9449 break; 9450 9451 /* If TCP is not interested in SACK blks... */ 9452 if ((tcp = tcpopt->tcp) == NULL) { 9453 up += up[1]; 9454 continue; 9455 } 9456 sack_len = up[1] - TCPOPT_HEADER_LEN; 9457 up += TCPOPT_HEADER_LEN; 9458 9459 /* 9460 * If the list is empty, allocate one and assume 9461 * nothing is sack'ed. 9462 */ 9463 ASSERT(tcp->tcp_sack_info != NULL); 9464 if (tcp->tcp_notsack_list == NULL) { 9465 tcp_notsack_update(&(tcp->tcp_notsack_list), 9466 tcp->tcp_suna, tcp->tcp_snxt, 9467 &(tcp->tcp_num_notsack_blk), 9468 &(tcp->tcp_cnt_notsack_list)); 9469 9470 /* 9471 * Make sure tcp_notsack_list is not NULL. 9472 * This happens when kmem_alloc(KM_NOSLEEP) 9473 * returns NULL. 9474 */ 9475 if (tcp->tcp_notsack_list == NULL) { 9476 up += sack_len; 9477 continue; 9478 } 9479 tcp->tcp_fack = tcp->tcp_suna; 9480 } 9481 9482 while (sack_len > 0) { 9483 if (up + 8 > endp) { 9484 up = endp; 9485 break; 9486 } 9487 sack_begin = BE32_TO_U32(up); 9488 up += 4; 9489 sack_end = BE32_TO_U32(up); 9490 up += 4; 9491 sack_len -= 8; 9492 /* 9493 * Bounds checking. Make sure the SACK 9494 * info is within tcp_suna and tcp_snxt. 9495 * If this SACK blk is out of bound, ignore 9496 * it but continue to parse the following 9497 * blks. 9498 */ 9499 if (SEQ_LEQ(sack_end, sack_begin) || 9500 SEQ_LT(sack_begin, tcp->tcp_suna) || 9501 SEQ_GT(sack_end, tcp->tcp_snxt)) { 9502 continue; 9503 } 9504 tcp_notsack_insert(&(tcp->tcp_notsack_list), 9505 sack_begin, sack_end, 9506 &(tcp->tcp_num_notsack_blk), 9507 &(tcp->tcp_cnt_notsack_list)); 9508 if (SEQ_GT(sack_end, tcp->tcp_fack)) { 9509 tcp->tcp_fack = sack_end; 9510 } 9511 } 9512 found |= TCP_OPT_SACK_PRESENT; 9513 continue; 9514 9515 case TCPOPT_TSTAMP: 9516 if (len < TCPOPT_TSTAMP_LEN || 9517 up[1] != TCPOPT_TSTAMP_LEN) 9518 break; 9519 9520 tcpopt->tcp_opt_ts_val = BE32_TO_U32(up+2); 9521 tcpopt->tcp_opt_ts_ecr = BE32_TO_U32(up+6); 9522 9523 found |= TCP_OPT_TSTAMP_PRESENT; 9524 9525 up += TCPOPT_TSTAMP_LEN; 9526 continue; 9527 9528 default: 9529 if (len <= 1 || len < (int)up[1] || up[1] == 0) 9530 break; 9531 up += up[1]; 9532 continue; 9533 } 9534 break; 9535 } 9536 return (found); 9537 } 9538 9539 /* 9540 * Set the mss associated with a particular tcp based on its current value, 9541 * and a new one passed in. Observe minimums and maximums, and reset 9542 * other state variables that we want to view as multiples of mss. 9543 * 9544 * This function is called mainly because values like tcp_mss, tcp_cwnd, 9545 * highwater marks etc. need to be initialized or adjusted. 9546 * 1) From tcp_process_options() when the other side's SYN/SYN-ACK 9547 * packet arrives. 9548 * 2) We need to set a new MSS when ICMP_FRAGMENTATION_NEEDED or 9549 * ICMP6_PACKET_TOO_BIG arrives. 9550 * 3) From tcp_paws_check() if the other side stops sending the timestamp, 9551 * to increase the MSS to use the extra bytes available. 9552 * 9553 * Callers except tcp_paws_check() ensure that they only reduce mss. 9554 */ 9555 static void 9556 tcp_mss_set(tcp_t *tcp, uint32_t mss, boolean_t do_ss) 9557 { 9558 uint32_t mss_max; 9559 tcp_stack_t *tcps = tcp->tcp_tcps; 9560 9561 if (tcp->tcp_ipversion == IPV4_VERSION) 9562 mss_max = tcps->tcps_mss_max_ipv4; 9563 else 9564 mss_max = tcps->tcps_mss_max_ipv6; 9565 9566 if (mss < tcps->tcps_mss_min) 9567 mss = tcps->tcps_mss_min; 9568 if (mss > mss_max) 9569 mss = mss_max; 9570 /* 9571 * Unless naglim has been set by our client to 9572 * a non-mss value, force naglim to track mss. 9573 * This can help to aggregate small writes. 9574 */ 9575 if (mss < tcp->tcp_naglim || tcp->tcp_mss == tcp->tcp_naglim) 9576 tcp->tcp_naglim = mss; 9577 /* 9578 * TCP should be able to buffer at least 4 MSS data for obvious 9579 * performance reason. 9580 */ 9581 if ((mss << 2) > tcp->tcp_xmit_hiwater) 9582 tcp->tcp_xmit_hiwater = mss << 2; 9583 9584 if (do_ss) { 9585 /* 9586 * Either the tcp_cwnd is as yet uninitialized, or mss is 9587 * changing due to a reduction in MTU, presumably as a 9588 * result of a new path component, reset cwnd to its 9589 * "initial" value, as a multiple of the new mss. 9590 */ 9591 SET_TCP_INIT_CWND(tcp, mss, tcps->tcps_slow_start_initial); 9592 } else { 9593 /* 9594 * Called by tcp_paws_check(), the mss increased 9595 * marginally to allow use of space previously taken 9596 * by the timestamp option. It would be inappropriate 9597 * to apply slow start or tcp_init_cwnd values to 9598 * tcp_cwnd, simply adjust to a multiple of the new mss. 9599 */ 9600 tcp->tcp_cwnd = (tcp->tcp_cwnd / tcp->tcp_mss) * mss; 9601 tcp->tcp_cwnd_cnt = 0; 9602 } 9603 tcp->tcp_mss = mss; 9604 (void) tcp_maxpsz_set(tcp, B_TRUE); 9605 } 9606 9607 /* For /dev/tcp aka AF_INET open */ 9608 static int 9609 tcp_openv4(queue_t *q, dev_t *devp, int flag, int sflag, cred_t *credp) 9610 { 9611 return (tcp_open(q, devp, flag, sflag, credp, B_FALSE)); 9612 } 9613 9614 /* For /dev/tcp6 aka AF_INET6 open */ 9615 static int 9616 tcp_openv6(queue_t *q, dev_t *devp, int flag, int sflag, cred_t *credp) 9617 { 9618 return (tcp_open(q, devp, flag, sflag, credp, B_TRUE)); 9619 } 9620 9621 static int 9622 tcp_open(queue_t *q, dev_t *devp, int flag, int sflag, cred_t *credp, 9623 boolean_t isv6) 9624 { 9625 tcp_t *tcp = NULL; 9626 conn_t *connp; 9627 int err; 9628 vmem_t *minor_arena = NULL; 9629 dev_t conn_dev; 9630 zoneid_t zoneid; 9631 tcp_stack_t *tcps = NULL; 9632 9633 if (q->q_ptr != NULL) 9634 return (0); 9635 9636 if (sflag == MODOPEN) 9637 return (EINVAL); 9638 9639 if (!(flag & SO_ACCEPTOR)) { 9640 /* 9641 * Special case for install: miniroot needs to be able to 9642 * access files via NFS as though it were always in the 9643 * global zone. 9644 */ 9645 if (credp == kcred && nfs_global_client_only != 0) { 9646 zoneid = GLOBAL_ZONEID; 9647 tcps = netstack_find_by_stackid(GLOBAL_NETSTACKID)-> 9648 netstack_tcp; 9649 ASSERT(tcps != NULL); 9650 } else { 9651 netstack_t *ns; 9652 9653 ns = netstack_find_by_cred(credp); 9654 ASSERT(ns != NULL); 9655 tcps = ns->netstack_tcp; 9656 ASSERT(tcps != NULL); 9657 9658 /* 9659 * For exclusive stacks we set the zoneid to zero 9660 * to make TCP operate as if in the global zone. 9661 */ 9662 if (tcps->tcps_netstack->netstack_stackid != 9663 GLOBAL_NETSTACKID) 9664 zoneid = GLOBAL_ZONEID; 9665 else 9666 zoneid = crgetzoneid(credp); 9667 } 9668 /* 9669 * For stackid zero this is done from strplumb.c, but 9670 * non-zero stackids are handled here. 9671 */ 9672 if (tcps->tcps_g_q == NULL && 9673 tcps->tcps_netstack->netstack_stackid != 9674 GLOBAL_NETSTACKID) { 9675 tcp_g_q_setup(tcps); 9676 } 9677 } 9678 9679 if ((ip_minor_arena_la != NULL) && (flag & SO_SOCKSTR) && 9680 ((conn_dev = inet_minor_alloc(ip_minor_arena_la)) != 0)) { 9681 minor_arena = ip_minor_arena_la; 9682 } else { 9683 /* 9684 * Either minor numbers in the large arena were exhausted 9685 * or a non socket application is doing the open. 9686 * Try to allocate from the small arena. 9687 */ 9688 if ((conn_dev = inet_minor_alloc(ip_minor_arena_sa)) == 0) { 9689 if (tcps != NULL) 9690 netstack_rele(tcps->tcps_netstack); 9691 return (EBUSY); 9692 } 9693 minor_arena = ip_minor_arena_sa; 9694 } 9695 ASSERT(minor_arena != NULL); 9696 9697 *devp = makedevice(getemajor(*devp), (minor_t)conn_dev); 9698 9699 if (flag & SO_ACCEPTOR) { 9700 /* No netstack_find_by_cred, hence no netstack_rele needed */ 9701 ASSERT(tcps == NULL); 9702 q->q_qinfo = &tcp_acceptor_rinit; 9703 /* 9704 * the conn_dev and minor_arena will be subsequently used by 9705 * tcp_wput_accept() and tcpclose_accept() to figure out the 9706 * minor device number for this connection from the q_ptr. 9707 */ 9708 RD(q)->q_ptr = (void *)conn_dev; 9709 WR(q)->q_qinfo = &tcp_acceptor_winit; 9710 WR(q)->q_ptr = (void *)minor_arena; 9711 qprocson(q); 9712 return (0); 9713 } 9714 9715 connp = (conn_t *)tcp_get_conn(IP_SQUEUE_GET(lbolt), tcps); 9716 /* 9717 * Both tcp_get_conn and netstack_find_by_cred incremented refcnt, 9718 * so we drop it by one. 9719 */ 9720 netstack_rele(tcps->tcps_netstack); 9721 if (connp == NULL) { 9722 inet_minor_free(minor_arena, conn_dev); 9723 q->q_ptr = NULL; 9724 return (ENOSR); 9725 } 9726 connp->conn_sqp = IP_SQUEUE_GET(lbolt); 9727 tcp = connp->conn_tcp; 9728 9729 q->q_ptr = WR(q)->q_ptr = connp; 9730 if (isv6) { 9731 connp->conn_flags |= (IPCL_TCP6|IPCL_ISV6); 9732 connp->conn_send = ip_output_v6; 9733 connp->conn_af_isv6 = B_TRUE; 9734 connp->conn_pkt_isv6 = B_TRUE; 9735 connp->conn_src_preferences = IPV6_PREFER_SRC_DEFAULT; 9736 tcp->tcp_ipversion = IPV6_VERSION; 9737 tcp->tcp_family = AF_INET6; 9738 tcp->tcp_mss = tcps->tcps_mss_def_ipv6; 9739 } else { 9740 connp->conn_flags |= IPCL_TCP4; 9741 connp->conn_send = ip_output; 9742 connp->conn_af_isv6 = B_FALSE; 9743 connp->conn_pkt_isv6 = B_FALSE; 9744 tcp->tcp_ipversion = IPV4_VERSION; 9745 tcp->tcp_family = AF_INET; 9746 tcp->tcp_mss = tcps->tcps_mss_def_ipv4; 9747 } 9748 9749 /* 9750 * TCP keeps a copy of cred for cache locality reasons but 9751 * we put a reference only once. If connp->conn_cred 9752 * becomes invalid, tcp_cred should also be set to NULL. 9753 */ 9754 tcp->tcp_cred = connp->conn_cred = credp; 9755 crhold(connp->conn_cred); 9756 tcp->tcp_cpid = curproc->p_pid; 9757 tcp->tcp_open_time = lbolt64; 9758 connp->conn_zoneid = zoneid; 9759 connp->conn_mlp_type = mlptSingle; 9760 connp->conn_ulp_labeled = !is_system_labeled(); 9761 ASSERT(connp->conn_netstack == tcps->tcps_netstack); 9762 ASSERT(tcp->tcp_tcps == tcps); 9763 9764 /* 9765 * If the caller has the process-wide flag set, then default to MAC 9766 * exempt mode. This allows read-down to unlabeled hosts. 9767 */ 9768 if (getpflags(NET_MAC_AWARE, credp) != 0) 9769 connp->conn_mac_exempt = B_TRUE; 9770 9771 connp->conn_dev = conn_dev; 9772 connp->conn_minor_arena = minor_arena; 9773 9774 ASSERT(q->q_qinfo == &tcp_rinitv4 || q->q_qinfo == &tcp_rinitv6); 9775 ASSERT(WR(q)->q_qinfo == &tcp_winit); 9776 9777 if (flag & SO_SOCKSTR) { 9778 /* 9779 * No need to insert a socket in tcp acceptor hash. 9780 * If it was a socket acceptor stream, we dealt with 9781 * it above. A socket listener can never accept a 9782 * connection and doesn't need acceptor_id. 9783 */ 9784 connp->conn_flags |= IPCL_SOCKET; 9785 tcp->tcp_issocket = 1; 9786 WR(q)->q_qinfo = &tcp_sock_winit; 9787 } else { 9788 #ifdef _ILP32 9789 tcp->tcp_acceptor_id = (t_uscalar_t)RD(q); 9790 #else 9791 tcp->tcp_acceptor_id = conn_dev; 9792 #endif /* _ILP32 */ 9793 tcp_acceptor_hash_insert(tcp->tcp_acceptor_id, tcp); 9794 } 9795 9796 err = tcp_init(tcp, q); 9797 if (err != 0) { 9798 inet_minor_free(connp->conn_minor_arena, connp->conn_dev); 9799 tcp_acceptor_hash_remove(tcp); 9800 CONN_DEC_REF(connp); 9801 q->q_ptr = WR(q)->q_ptr = NULL; 9802 return (err); 9803 } 9804 9805 RD(q)->q_hiwat = tcps->tcps_recv_hiwat; 9806 tcp->tcp_rwnd = tcps->tcps_recv_hiwat; 9807 9808 /* Non-zero default values */ 9809 connp->conn_multicast_loop = IP_DEFAULT_MULTICAST_LOOP; 9810 /* 9811 * Put the ref for TCP. Ref for IP was already put 9812 * by ipcl_conn_create. Also Make the conn_t globally 9813 * visible to walkers 9814 */ 9815 mutex_enter(&connp->conn_lock); 9816 CONN_INC_REF_LOCKED(connp); 9817 ASSERT(connp->conn_ref == 2); 9818 connp->conn_state_flags &= ~CONN_INCIPIENT; 9819 mutex_exit(&connp->conn_lock); 9820 9821 qprocson(q); 9822 return (0); 9823 } 9824 9825 /* 9826 * Some TCP options can be "set" by requesting them in the option 9827 * buffer. This is needed for XTI feature test though we do not 9828 * allow it in general. We interpret that this mechanism is more 9829 * applicable to OSI protocols and need not be allowed in general. 9830 * This routine filters out options for which it is not allowed (most) 9831 * and lets through those (few) for which it is. [ The XTI interface 9832 * test suite specifics will imply that any XTI_GENERIC level XTI_* if 9833 * ever implemented will have to be allowed here ]. 9834 */ 9835 static boolean_t 9836 tcp_allow_connopt_set(int level, int name) 9837 { 9838 9839 switch (level) { 9840 case IPPROTO_TCP: 9841 switch (name) { 9842 case TCP_NODELAY: 9843 return (B_TRUE); 9844 default: 9845 return (B_FALSE); 9846 } 9847 /*NOTREACHED*/ 9848 default: 9849 return (B_FALSE); 9850 } 9851 /*NOTREACHED*/ 9852 } 9853 9854 /* 9855 * This routine gets default values of certain options whose default 9856 * values are maintained by protocol specific code 9857 */ 9858 /* ARGSUSED */ 9859 int 9860 tcp_opt_default(queue_t *q, int level, int name, uchar_t *ptr) 9861 { 9862 int32_t *i1 = (int32_t *)ptr; 9863 tcp_stack_t *tcps = Q_TO_TCP(q)->tcp_tcps; 9864 9865 switch (level) { 9866 case IPPROTO_TCP: 9867 switch (name) { 9868 case TCP_NOTIFY_THRESHOLD: 9869 *i1 = tcps->tcps_ip_notify_interval; 9870 break; 9871 case TCP_ABORT_THRESHOLD: 9872 *i1 = tcps->tcps_ip_abort_interval; 9873 break; 9874 case TCP_CONN_NOTIFY_THRESHOLD: 9875 *i1 = tcps->tcps_ip_notify_cinterval; 9876 break; 9877 case TCP_CONN_ABORT_THRESHOLD: 9878 *i1 = tcps->tcps_ip_abort_cinterval; 9879 break; 9880 default: 9881 return (-1); 9882 } 9883 break; 9884 case IPPROTO_IP: 9885 switch (name) { 9886 case IP_TTL: 9887 *i1 = tcps->tcps_ipv4_ttl; 9888 break; 9889 default: 9890 return (-1); 9891 } 9892 break; 9893 case IPPROTO_IPV6: 9894 switch (name) { 9895 case IPV6_UNICAST_HOPS: 9896 *i1 = tcps->tcps_ipv6_hoplimit; 9897 break; 9898 default: 9899 return (-1); 9900 } 9901 break; 9902 default: 9903 return (-1); 9904 } 9905 return (sizeof (int)); 9906 } 9907 9908 9909 /* 9910 * TCP routine to get the values of options. 9911 */ 9912 int 9913 tcp_opt_get(queue_t *q, int level, int name, uchar_t *ptr) 9914 { 9915 int *i1 = (int *)ptr; 9916 conn_t *connp = Q_TO_CONN(q); 9917 tcp_t *tcp = connp->conn_tcp; 9918 ip6_pkt_t *ipp = &tcp->tcp_sticky_ipp; 9919 9920 switch (level) { 9921 case SOL_SOCKET: 9922 switch (name) { 9923 case SO_LINGER: { 9924 struct linger *lgr = (struct linger *)ptr; 9925 9926 lgr->l_onoff = tcp->tcp_linger ? SO_LINGER : 0; 9927 lgr->l_linger = tcp->tcp_lingertime; 9928 } 9929 return (sizeof (struct linger)); 9930 case SO_DEBUG: 9931 *i1 = tcp->tcp_debug ? SO_DEBUG : 0; 9932 break; 9933 case SO_KEEPALIVE: 9934 *i1 = tcp->tcp_ka_enabled ? SO_KEEPALIVE : 0; 9935 break; 9936 case SO_DONTROUTE: 9937 *i1 = tcp->tcp_dontroute ? SO_DONTROUTE : 0; 9938 break; 9939 case SO_USELOOPBACK: 9940 *i1 = tcp->tcp_useloopback ? SO_USELOOPBACK : 0; 9941 break; 9942 case SO_BROADCAST: 9943 *i1 = tcp->tcp_broadcast ? SO_BROADCAST : 0; 9944 break; 9945 case SO_REUSEADDR: 9946 *i1 = tcp->tcp_reuseaddr ? SO_REUSEADDR : 0; 9947 break; 9948 case SO_OOBINLINE: 9949 *i1 = tcp->tcp_oobinline ? SO_OOBINLINE : 0; 9950 break; 9951 case SO_DGRAM_ERRIND: 9952 *i1 = tcp->tcp_dgram_errind ? SO_DGRAM_ERRIND : 0; 9953 break; 9954 case SO_TYPE: 9955 *i1 = SOCK_STREAM; 9956 break; 9957 case SO_SNDBUF: 9958 *i1 = tcp->tcp_xmit_hiwater; 9959 break; 9960 case SO_RCVBUF: 9961 *i1 = RD(q)->q_hiwat; 9962 break; 9963 case SO_SND_COPYAVOID: 9964 *i1 = tcp->tcp_snd_zcopy_on ? 9965 SO_SND_COPYAVOID : 0; 9966 break; 9967 case SO_ALLZONES: 9968 *i1 = connp->conn_allzones ? 1 : 0; 9969 break; 9970 case SO_ANON_MLP: 9971 *i1 = connp->conn_anon_mlp; 9972 break; 9973 case SO_MAC_EXEMPT: 9974 *i1 = connp->conn_mac_exempt; 9975 break; 9976 case SO_EXCLBIND: 9977 *i1 = tcp->tcp_exclbind ? SO_EXCLBIND : 0; 9978 break; 9979 case SO_PROTOTYPE: 9980 *i1 = IPPROTO_TCP; 9981 break; 9982 case SO_DOMAIN: 9983 *i1 = tcp->tcp_family; 9984 break; 9985 default: 9986 return (-1); 9987 } 9988 break; 9989 case IPPROTO_TCP: 9990 switch (name) { 9991 case TCP_NODELAY: 9992 *i1 = (tcp->tcp_naglim == 1) ? TCP_NODELAY : 0; 9993 break; 9994 case TCP_MAXSEG: 9995 *i1 = tcp->tcp_mss; 9996 break; 9997 case TCP_NOTIFY_THRESHOLD: 9998 *i1 = (int)tcp->tcp_first_timer_threshold; 9999 break; 10000 case TCP_ABORT_THRESHOLD: 10001 *i1 = tcp->tcp_second_timer_threshold; 10002 break; 10003 case TCP_CONN_NOTIFY_THRESHOLD: 10004 *i1 = tcp->tcp_first_ctimer_threshold; 10005 break; 10006 case TCP_CONN_ABORT_THRESHOLD: 10007 *i1 = tcp->tcp_second_ctimer_threshold; 10008 break; 10009 case TCP_RECVDSTADDR: 10010 *i1 = tcp->tcp_recvdstaddr; 10011 break; 10012 case TCP_ANONPRIVBIND: 10013 *i1 = tcp->tcp_anon_priv_bind; 10014 break; 10015 case TCP_EXCLBIND: 10016 *i1 = tcp->tcp_exclbind ? TCP_EXCLBIND : 0; 10017 break; 10018 case TCP_INIT_CWND: 10019 *i1 = tcp->tcp_init_cwnd; 10020 break; 10021 case TCP_KEEPALIVE_THRESHOLD: 10022 *i1 = tcp->tcp_ka_interval; 10023 break; 10024 case TCP_KEEPALIVE_ABORT_THRESHOLD: 10025 *i1 = tcp->tcp_ka_abort_thres; 10026 break; 10027 case TCP_CORK: 10028 *i1 = tcp->tcp_cork; 10029 break; 10030 default: 10031 return (-1); 10032 } 10033 break; 10034 case IPPROTO_IP: 10035 if (tcp->tcp_family != AF_INET) 10036 return (-1); 10037 switch (name) { 10038 case IP_OPTIONS: 10039 case T_IP_OPTIONS: { 10040 /* 10041 * This is compatible with BSD in that in only return 10042 * the reverse source route with the final destination 10043 * as the last entry. The first 4 bytes of the option 10044 * will contain the final destination. 10045 */ 10046 int opt_len; 10047 10048 opt_len = (char *)tcp->tcp_tcph - (char *)tcp->tcp_ipha; 10049 opt_len -= tcp->tcp_label_len + IP_SIMPLE_HDR_LENGTH; 10050 ASSERT(opt_len >= 0); 10051 /* Caller ensures enough space */ 10052 if (opt_len > 0) { 10053 /* 10054 * TODO: Do we have to handle getsockopt on an 10055 * initiator as well? 10056 */ 10057 return (ip_opt_get_user(tcp->tcp_ipha, ptr)); 10058 } 10059 return (0); 10060 } 10061 case IP_TOS: 10062 case T_IP_TOS: 10063 *i1 = (int)tcp->tcp_ipha->ipha_type_of_service; 10064 break; 10065 case IP_TTL: 10066 *i1 = (int)tcp->tcp_ipha->ipha_ttl; 10067 break; 10068 case IP_NEXTHOP: 10069 /* Handled at IP level */ 10070 return (-EINVAL); 10071 default: 10072 return (-1); 10073 } 10074 break; 10075 case IPPROTO_IPV6: 10076 /* 10077 * IPPROTO_IPV6 options are only supported for sockets 10078 * that are using IPv6 on the wire. 10079 */ 10080 if (tcp->tcp_ipversion != IPV6_VERSION) { 10081 return (-1); 10082 } 10083 switch (name) { 10084 case IPV6_UNICAST_HOPS: 10085 *i1 = (unsigned int) tcp->tcp_ip6h->ip6_hops; 10086 break; /* goto sizeof (int) option return */ 10087 case IPV6_BOUND_IF: 10088 /* Zero if not set */ 10089 *i1 = tcp->tcp_bound_if; 10090 break; /* goto sizeof (int) option return */ 10091 case IPV6_RECVPKTINFO: 10092 if (tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVPKTINFO) 10093 *i1 = 1; 10094 else 10095 *i1 = 0; 10096 break; /* goto sizeof (int) option return */ 10097 case IPV6_RECVTCLASS: 10098 if (tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVTCLASS) 10099 *i1 = 1; 10100 else 10101 *i1 = 0; 10102 break; /* goto sizeof (int) option return */ 10103 case IPV6_RECVHOPLIMIT: 10104 if (tcp->tcp_ipv6_recvancillary & 10105 TCP_IPV6_RECVHOPLIMIT) 10106 *i1 = 1; 10107 else 10108 *i1 = 0; 10109 break; /* goto sizeof (int) option return */ 10110 case IPV6_RECVHOPOPTS: 10111 if (tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVHOPOPTS) 10112 *i1 = 1; 10113 else 10114 *i1 = 0; 10115 break; /* goto sizeof (int) option return */ 10116 case IPV6_RECVDSTOPTS: 10117 if (tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVDSTOPTS) 10118 *i1 = 1; 10119 else 10120 *i1 = 0; 10121 break; /* goto sizeof (int) option return */ 10122 case _OLD_IPV6_RECVDSTOPTS: 10123 if (tcp->tcp_ipv6_recvancillary & 10124 TCP_OLD_IPV6_RECVDSTOPTS) 10125 *i1 = 1; 10126 else 10127 *i1 = 0; 10128 break; /* goto sizeof (int) option return */ 10129 case IPV6_RECVRTHDR: 10130 if (tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVRTHDR) 10131 *i1 = 1; 10132 else 10133 *i1 = 0; 10134 break; /* goto sizeof (int) option return */ 10135 case IPV6_RECVRTHDRDSTOPTS: 10136 if (tcp->tcp_ipv6_recvancillary & 10137 TCP_IPV6_RECVRTDSTOPTS) 10138 *i1 = 1; 10139 else 10140 *i1 = 0; 10141 break; /* goto sizeof (int) option return */ 10142 case IPV6_PKTINFO: { 10143 /* XXX assumes that caller has room for max size! */ 10144 struct in6_pktinfo *pkti; 10145 10146 pkti = (struct in6_pktinfo *)ptr; 10147 if (ipp->ipp_fields & IPPF_IFINDEX) 10148 pkti->ipi6_ifindex = ipp->ipp_ifindex; 10149 else 10150 pkti->ipi6_ifindex = 0; 10151 if (ipp->ipp_fields & IPPF_ADDR) 10152 pkti->ipi6_addr = ipp->ipp_addr; 10153 else 10154 pkti->ipi6_addr = ipv6_all_zeros; 10155 return (sizeof (struct in6_pktinfo)); 10156 } 10157 case IPV6_TCLASS: 10158 if (ipp->ipp_fields & IPPF_TCLASS) 10159 *i1 = ipp->ipp_tclass; 10160 else 10161 *i1 = IPV6_FLOW_TCLASS( 10162 IPV6_DEFAULT_VERS_AND_FLOW); 10163 break; /* goto sizeof (int) option return */ 10164 case IPV6_NEXTHOP: { 10165 sin6_t *sin6 = (sin6_t *)ptr; 10166 10167 if (!(ipp->ipp_fields & IPPF_NEXTHOP)) 10168 return (0); 10169 *sin6 = sin6_null; 10170 sin6->sin6_family = AF_INET6; 10171 sin6->sin6_addr = ipp->ipp_nexthop; 10172 return (sizeof (sin6_t)); 10173 } 10174 case IPV6_HOPOPTS: 10175 if (!(ipp->ipp_fields & IPPF_HOPOPTS)) 10176 return (0); 10177 if (ipp->ipp_hopoptslen <= tcp->tcp_label_len) 10178 return (0); 10179 bcopy((char *)ipp->ipp_hopopts + tcp->tcp_label_len, 10180 ptr, ipp->ipp_hopoptslen - tcp->tcp_label_len); 10181 if (tcp->tcp_label_len > 0) { 10182 ptr[0] = ((char *)ipp->ipp_hopopts)[0]; 10183 ptr[1] = (ipp->ipp_hopoptslen - 10184 tcp->tcp_label_len + 7) / 8 - 1; 10185 } 10186 return (ipp->ipp_hopoptslen - tcp->tcp_label_len); 10187 case IPV6_RTHDRDSTOPTS: 10188 if (!(ipp->ipp_fields & IPPF_RTDSTOPTS)) 10189 return (0); 10190 bcopy(ipp->ipp_rtdstopts, ptr, ipp->ipp_rtdstoptslen); 10191 return (ipp->ipp_rtdstoptslen); 10192 case IPV6_RTHDR: 10193 if (!(ipp->ipp_fields & IPPF_RTHDR)) 10194 return (0); 10195 bcopy(ipp->ipp_rthdr, ptr, ipp->ipp_rthdrlen); 10196 return (ipp->ipp_rthdrlen); 10197 case IPV6_DSTOPTS: 10198 if (!(ipp->ipp_fields & IPPF_DSTOPTS)) 10199 return (0); 10200 bcopy(ipp->ipp_dstopts, ptr, ipp->ipp_dstoptslen); 10201 return (ipp->ipp_dstoptslen); 10202 case IPV6_SRC_PREFERENCES: 10203 return (ip6_get_src_preferences(connp, 10204 (uint32_t *)ptr)); 10205 case IPV6_PATHMTU: { 10206 struct ip6_mtuinfo *mtuinfo = (struct ip6_mtuinfo *)ptr; 10207 10208 if (tcp->tcp_state < TCPS_ESTABLISHED) 10209 return (-1); 10210 10211 return (ip_fill_mtuinfo(&connp->conn_remv6, 10212 connp->conn_fport, mtuinfo, 10213 connp->conn_netstack)); 10214 } 10215 default: 10216 return (-1); 10217 } 10218 break; 10219 default: 10220 return (-1); 10221 } 10222 return (sizeof (int)); 10223 } 10224 10225 /* 10226 * We declare as 'int' rather than 'void' to satisfy pfi_t arg requirements. 10227 * Parameters are assumed to be verified by the caller. 10228 */ 10229 /* ARGSUSED */ 10230 int 10231 tcp_opt_set(queue_t *q, uint_t optset_context, int level, int name, 10232 uint_t inlen, uchar_t *invalp, uint_t *outlenp, uchar_t *outvalp, 10233 void *thisdg_attrs, cred_t *cr, mblk_t *mblk) 10234 { 10235 conn_t *connp = Q_TO_CONN(q); 10236 tcp_t *tcp = connp->conn_tcp; 10237 int *i1 = (int *)invalp; 10238 boolean_t onoff = (*i1 == 0) ? 0 : 1; 10239 boolean_t checkonly; 10240 int reterr; 10241 tcp_stack_t *tcps = Q_TO_TCP(q)->tcp_tcps; 10242 10243 switch (optset_context) { 10244 case SETFN_OPTCOM_CHECKONLY: 10245 checkonly = B_TRUE; 10246 /* 10247 * Note: Implies T_CHECK semantics for T_OPTCOM_REQ 10248 * inlen != 0 implies value supplied and 10249 * we have to "pretend" to set it. 10250 * inlen == 0 implies that there is no 10251 * value part in T_CHECK request and just validation 10252 * done elsewhere should be enough, we just return here. 10253 */ 10254 if (inlen == 0) { 10255 *outlenp = 0; 10256 return (0); 10257 } 10258 break; 10259 case SETFN_OPTCOM_NEGOTIATE: 10260 checkonly = B_FALSE; 10261 break; 10262 case SETFN_UD_NEGOTIATE: /* error on conn-oriented transports ? */ 10263 case SETFN_CONN_NEGOTIATE: 10264 checkonly = B_FALSE; 10265 /* 10266 * Negotiating local and "association-related" options 10267 * from other (T_CONN_REQ, T_CONN_RES,T_UNITDATA_REQ) 10268 * primitives is allowed by XTI, but we choose 10269 * to not implement this style negotiation for Internet 10270 * protocols (We interpret it is a must for OSI world but 10271 * optional for Internet protocols) for all options. 10272 * [ Will do only for the few options that enable test 10273 * suites that our XTI implementation of this feature 10274 * works for transports that do allow it ] 10275 */ 10276 if (!tcp_allow_connopt_set(level, name)) { 10277 *outlenp = 0; 10278 return (EINVAL); 10279 } 10280 break; 10281 default: 10282 /* 10283 * We should never get here 10284 */ 10285 *outlenp = 0; 10286 return (EINVAL); 10287 } 10288 10289 ASSERT((optset_context != SETFN_OPTCOM_CHECKONLY) || 10290 (optset_context == SETFN_OPTCOM_CHECKONLY && inlen != 0)); 10291 10292 /* 10293 * For TCP, we should have no ancillary data sent down 10294 * (sendmsg isn't supported for SOCK_STREAM), so thisdg_attrs 10295 * has to be zero. 10296 */ 10297 ASSERT(thisdg_attrs == NULL); 10298 10299 /* 10300 * For fixed length options, no sanity check 10301 * of passed in length is done. It is assumed *_optcom_req() 10302 * routines do the right thing. 10303 */ 10304 10305 switch (level) { 10306 case SOL_SOCKET: 10307 switch (name) { 10308 case SO_LINGER: { 10309 struct linger *lgr = (struct linger *)invalp; 10310 10311 if (!checkonly) { 10312 if (lgr->l_onoff) { 10313 tcp->tcp_linger = 1; 10314 tcp->tcp_lingertime = lgr->l_linger; 10315 } else { 10316 tcp->tcp_linger = 0; 10317 tcp->tcp_lingertime = 0; 10318 } 10319 /* struct copy */ 10320 *(struct linger *)outvalp = *lgr; 10321 } else { 10322 if (!lgr->l_onoff) { 10323 ((struct linger *) 10324 outvalp)->l_onoff = 0; 10325 ((struct linger *) 10326 outvalp)->l_linger = 0; 10327 } else { 10328 /* struct copy */ 10329 *(struct linger *)outvalp = *lgr; 10330 } 10331 } 10332 *outlenp = sizeof (struct linger); 10333 return (0); 10334 } 10335 case SO_DEBUG: 10336 if (!checkonly) 10337 tcp->tcp_debug = onoff; 10338 break; 10339 case SO_KEEPALIVE: 10340 if (checkonly) { 10341 /* T_CHECK case */ 10342 break; 10343 } 10344 10345 if (!onoff) { 10346 if (tcp->tcp_ka_enabled) { 10347 if (tcp->tcp_ka_tid != 0) { 10348 (void) TCP_TIMER_CANCEL(tcp, 10349 tcp->tcp_ka_tid); 10350 tcp->tcp_ka_tid = 0; 10351 } 10352 tcp->tcp_ka_enabled = 0; 10353 } 10354 break; 10355 } 10356 if (!tcp->tcp_ka_enabled) { 10357 /* Crank up the keepalive timer */ 10358 tcp->tcp_ka_last_intrvl = 0; 10359 tcp->tcp_ka_tid = TCP_TIMER(tcp, 10360 tcp_keepalive_killer, 10361 MSEC_TO_TICK(tcp->tcp_ka_interval)); 10362 tcp->tcp_ka_enabled = 1; 10363 } 10364 break; 10365 case SO_DONTROUTE: 10366 /* 10367 * SO_DONTROUTE, SO_USELOOPBACK, and SO_BROADCAST are 10368 * only of interest to IP. We track them here only so 10369 * that we can report their current value. 10370 */ 10371 if (!checkonly) { 10372 tcp->tcp_dontroute = onoff; 10373 tcp->tcp_connp->conn_dontroute = onoff; 10374 } 10375 break; 10376 case SO_USELOOPBACK: 10377 if (!checkonly) { 10378 tcp->tcp_useloopback = onoff; 10379 tcp->tcp_connp->conn_loopback = onoff; 10380 } 10381 break; 10382 case SO_BROADCAST: 10383 if (!checkonly) { 10384 tcp->tcp_broadcast = onoff; 10385 tcp->tcp_connp->conn_broadcast = onoff; 10386 } 10387 break; 10388 case SO_REUSEADDR: 10389 if (!checkonly) { 10390 tcp->tcp_reuseaddr = onoff; 10391 tcp->tcp_connp->conn_reuseaddr = onoff; 10392 } 10393 break; 10394 case SO_OOBINLINE: 10395 if (!checkonly) 10396 tcp->tcp_oobinline = onoff; 10397 break; 10398 case SO_DGRAM_ERRIND: 10399 if (!checkonly) 10400 tcp->tcp_dgram_errind = onoff; 10401 break; 10402 case SO_SNDBUF: { 10403 if (*i1 > tcps->tcps_max_buf) { 10404 *outlenp = 0; 10405 return (ENOBUFS); 10406 } 10407 if (checkonly) 10408 break; 10409 10410 tcp->tcp_xmit_hiwater = *i1; 10411 if (tcps->tcps_snd_lowat_fraction != 0) 10412 tcp->tcp_xmit_lowater = 10413 tcp->tcp_xmit_hiwater / 10414 tcps->tcps_snd_lowat_fraction; 10415 (void) tcp_maxpsz_set(tcp, B_TRUE); 10416 /* 10417 * If we are flow-controlled, recheck the condition. 10418 * There are apps that increase SO_SNDBUF size when 10419 * flow-controlled (EWOULDBLOCK), and expect the flow 10420 * control condition to be lifted right away. 10421 */ 10422 mutex_enter(&tcp->tcp_non_sq_lock); 10423 if (tcp->tcp_flow_stopped && 10424 TCP_UNSENT_BYTES(tcp) < tcp->tcp_xmit_hiwater) { 10425 tcp_clrqfull(tcp); 10426 } 10427 mutex_exit(&tcp->tcp_non_sq_lock); 10428 break; 10429 } 10430 case SO_RCVBUF: 10431 if (*i1 > tcps->tcps_max_buf) { 10432 *outlenp = 0; 10433 return (ENOBUFS); 10434 } 10435 /* Silently ignore zero */ 10436 if (!checkonly && *i1 != 0) { 10437 *i1 = MSS_ROUNDUP(*i1, tcp->tcp_mss); 10438 (void) tcp_rwnd_set(tcp, *i1); 10439 } 10440 /* 10441 * XXX should we return the rwnd here 10442 * and tcp_opt_get ? 10443 */ 10444 break; 10445 case SO_SND_COPYAVOID: 10446 if (!checkonly) { 10447 /* we only allow enable at most once for now */ 10448 if (tcp->tcp_loopback || 10449 (tcp->tcp_kssl_ctx != NULL) || 10450 (!tcp->tcp_snd_zcopy_aware && 10451 (onoff != 1 || !tcp_zcopy_check(tcp)))) { 10452 *outlenp = 0; 10453 return (EOPNOTSUPP); 10454 } 10455 tcp->tcp_snd_zcopy_aware = 1; 10456 } 10457 break; 10458 case SO_ALLZONES: 10459 /* Pass option along to IP level for handling */ 10460 return (-EINVAL); 10461 case SO_ANON_MLP: 10462 /* Pass option along to IP level for handling */ 10463 return (-EINVAL); 10464 case SO_MAC_EXEMPT: 10465 /* Pass option along to IP level for handling */ 10466 return (-EINVAL); 10467 case SO_EXCLBIND: 10468 if (!checkonly) 10469 tcp->tcp_exclbind = onoff; 10470 break; 10471 default: 10472 *outlenp = 0; 10473 return (EINVAL); 10474 } 10475 break; 10476 case IPPROTO_TCP: 10477 switch (name) { 10478 case TCP_NODELAY: 10479 if (!checkonly) 10480 tcp->tcp_naglim = *i1 ? 1 : tcp->tcp_mss; 10481 break; 10482 case TCP_NOTIFY_THRESHOLD: 10483 if (!checkonly) 10484 tcp->tcp_first_timer_threshold = *i1; 10485 break; 10486 case TCP_ABORT_THRESHOLD: 10487 if (!checkonly) 10488 tcp->tcp_second_timer_threshold = *i1; 10489 break; 10490 case TCP_CONN_NOTIFY_THRESHOLD: 10491 if (!checkonly) 10492 tcp->tcp_first_ctimer_threshold = *i1; 10493 break; 10494 case TCP_CONN_ABORT_THRESHOLD: 10495 if (!checkonly) 10496 tcp->tcp_second_ctimer_threshold = *i1; 10497 break; 10498 case TCP_RECVDSTADDR: 10499 if (tcp->tcp_state > TCPS_LISTEN) 10500 return (EOPNOTSUPP); 10501 if (!checkonly) 10502 tcp->tcp_recvdstaddr = onoff; 10503 break; 10504 case TCP_ANONPRIVBIND: 10505 if ((reterr = secpolicy_net_privaddr(cr, 0, 10506 IPPROTO_TCP)) != 0) { 10507 *outlenp = 0; 10508 return (reterr); 10509 } 10510 if (!checkonly) { 10511 tcp->tcp_anon_priv_bind = onoff; 10512 } 10513 break; 10514 case TCP_EXCLBIND: 10515 if (!checkonly) 10516 tcp->tcp_exclbind = onoff; 10517 break; /* goto sizeof (int) option return */ 10518 case TCP_INIT_CWND: { 10519 uint32_t init_cwnd = *((uint32_t *)invalp); 10520 10521 if (checkonly) 10522 break; 10523 10524 /* 10525 * Only allow socket with network configuration 10526 * privilege to set the initial cwnd to be larger 10527 * than allowed by RFC 3390. 10528 */ 10529 if (init_cwnd <= MIN(4, MAX(2, 4380 / tcp->tcp_mss))) { 10530 tcp->tcp_init_cwnd = init_cwnd; 10531 break; 10532 } 10533 if ((reterr = secpolicy_ip_config(cr, B_TRUE)) != 0) { 10534 *outlenp = 0; 10535 return (reterr); 10536 } 10537 if (init_cwnd > TCP_MAX_INIT_CWND) { 10538 *outlenp = 0; 10539 return (EINVAL); 10540 } 10541 tcp->tcp_init_cwnd = init_cwnd; 10542 break; 10543 } 10544 case TCP_KEEPALIVE_THRESHOLD: 10545 if (checkonly) 10546 break; 10547 10548 if (*i1 < tcps->tcps_keepalive_interval_low || 10549 *i1 > tcps->tcps_keepalive_interval_high) { 10550 *outlenp = 0; 10551 return (EINVAL); 10552 } 10553 if (*i1 != tcp->tcp_ka_interval) { 10554 tcp->tcp_ka_interval = *i1; 10555 /* 10556 * Check if we need to restart the 10557 * keepalive timer. 10558 */ 10559 if (tcp->tcp_ka_tid != 0) { 10560 ASSERT(tcp->tcp_ka_enabled); 10561 (void) TCP_TIMER_CANCEL(tcp, 10562 tcp->tcp_ka_tid); 10563 tcp->tcp_ka_last_intrvl = 0; 10564 tcp->tcp_ka_tid = TCP_TIMER(tcp, 10565 tcp_keepalive_killer, 10566 MSEC_TO_TICK(tcp->tcp_ka_interval)); 10567 } 10568 } 10569 break; 10570 case TCP_KEEPALIVE_ABORT_THRESHOLD: 10571 if (!checkonly) { 10572 if (*i1 < 10573 tcps->tcps_keepalive_abort_interval_low || 10574 *i1 > 10575 tcps->tcps_keepalive_abort_interval_high) { 10576 *outlenp = 0; 10577 return (EINVAL); 10578 } 10579 tcp->tcp_ka_abort_thres = *i1; 10580 } 10581 break; 10582 case TCP_CORK: 10583 if (!checkonly) { 10584 /* 10585 * if tcp->tcp_cork was set and is now 10586 * being unset, we have to make sure that 10587 * the remaining data gets sent out. Also 10588 * unset tcp->tcp_cork so that tcp_wput_data() 10589 * can send data even if it is less than mss 10590 */ 10591 if (tcp->tcp_cork && onoff == 0 && 10592 tcp->tcp_unsent > 0) { 10593 tcp->tcp_cork = B_FALSE; 10594 tcp_wput_data(tcp, NULL, B_FALSE); 10595 } 10596 tcp->tcp_cork = onoff; 10597 } 10598 break; 10599 default: 10600 *outlenp = 0; 10601 return (EINVAL); 10602 } 10603 break; 10604 case IPPROTO_IP: 10605 if (tcp->tcp_family != AF_INET) { 10606 *outlenp = 0; 10607 return (ENOPROTOOPT); 10608 } 10609 switch (name) { 10610 case IP_OPTIONS: 10611 case T_IP_OPTIONS: 10612 reterr = tcp_opt_set_header(tcp, checkonly, 10613 invalp, inlen); 10614 if (reterr) { 10615 *outlenp = 0; 10616 return (reterr); 10617 } 10618 /* OK return - copy input buffer into output buffer */ 10619 if (invalp != outvalp) { 10620 /* don't trust bcopy for identical src/dst */ 10621 bcopy(invalp, outvalp, inlen); 10622 } 10623 *outlenp = inlen; 10624 return (0); 10625 case IP_TOS: 10626 case T_IP_TOS: 10627 if (!checkonly) { 10628 tcp->tcp_ipha->ipha_type_of_service = 10629 (uchar_t)*i1; 10630 tcp->tcp_tos = (uchar_t)*i1; 10631 } 10632 break; 10633 case IP_TTL: 10634 if (!checkonly) { 10635 tcp->tcp_ipha->ipha_ttl = (uchar_t)*i1; 10636 tcp->tcp_ttl = (uchar_t)*i1; 10637 } 10638 break; 10639 case IP_BOUND_IF: 10640 case IP_NEXTHOP: 10641 /* Handled at the IP level */ 10642 return (-EINVAL); 10643 case IP_SEC_OPT: 10644 /* 10645 * We should not allow policy setting after 10646 * we start listening for connections. 10647 */ 10648 if (tcp->tcp_state == TCPS_LISTEN) { 10649 return (EINVAL); 10650 } else { 10651 /* Handled at the IP level */ 10652 return (-EINVAL); 10653 } 10654 default: 10655 *outlenp = 0; 10656 return (EINVAL); 10657 } 10658 break; 10659 case IPPROTO_IPV6: { 10660 ip6_pkt_t *ipp; 10661 10662 /* 10663 * IPPROTO_IPV6 options are only supported for sockets 10664 * that are using IPv6 on the wire. 10665 */ 10666 if (tcp->tcp_ipversion != IPV6_VERSION) { 10667 *outlenp = 0; 10668 return (ENOPROTOOPT); 10669 } 10670 /* 10671 * Only sticky options; no ancillary data 10672 */ 10673 ASSERT(thisdg_attrs == NULL); 10674 ipp = &tcp->tcp_sticky_ipp; 10675 10676 switch (name) { 10677 case IPV6_UNICAST_HOPS: 10678 /* -1 means use default */ 10679 if (*i1 < -1 || *i1 > IPV6_MAX_HOPS) { 10680 *outlenp = 0; 10681 return (EINVAL); 10682 } 10683 if (!checkonly) { 10684 if (*i1 == -1) { 10685 tcp->tcp_ip6h->ip6_hops = 10686 ipp->ipp_unicast_hops = 10687 (uint8_t)tcps->tcps_ipv6_hoplimit; 10688 ipp->ipp_fields &= ~IPPF_UNICAST_HOPS; 10689 /* Pass modified value to IP. */ 10690 *i1 = tcp->tcp_ip6h->ip6_hops; 10691 } else { 10692 tcp->tcp_ip6h->ip6_hops = 10693 ipp->ipp_unicast_hops = 10694 (uint8_t)*i1; 10695 ipp->ipp_fields |= IPPF_UNICAST_HOPS; 10696 } 10697 reterr = tcp_build_hdrs(q, tcp); 10698 if (reterr != 0) 10699 return (reterr); 10700 } 10701 break; 10702 case IPV6_BOUND_IF: 10703 if (!checkonly) { 10704 int error = 0; 10705 10706 tcp->tcp_bound_if = *i1; 10707 error = ip_opt_set_ill(tcp->tcp_connp, *i1, 10708 B_TRUE, checkonly, level, name, mblk); 10709 if (error != 0) { 10710 *outlenp = 0; 10711 return (error); 10712 } 10713 } 10714 break; 10715 /* 10716 * Set boolean switches for ancillary data delivery 10717 */ 10718 case IPV6_RECVPKTINFO: 10719 if (!checkonly) { 10720 if (onoff) 10721 tcp->tcp_ipv6_recvancillary |= 10722 TCP_IPV6_RECVPKTINFO; 10723 else 10724 tcp->tcp_ipv6_recvancillary &= 10725 ~TCP_IPV6_RECVPKTINFO; 10726 /* Force it to be sent up with the next msg */ 10727 tcp->tcp_recvifindex = 0; 10728 } 10729 break; 10730 case IPV6_RECVTCLASS: 10731 if (!checkonly) { 10732 if (onoff) 10733 tcp->tcp_ipv6_recvancillary |= 10734 TCP_IPV6_RECVTCLASS; 10735 else 10736 tcp->tcp_ipv6_recvancillary &= 10737 ~TCP_IPV6_RECVTCLASS; 10738 } 10739 break; 10740 case IPV6_RECVHOPLIMIT: 10741 if (!checkonly) { 10742 if (onoff) 10743 tcp->tcp_ipv6_recvancillary |= 10744 TCP_IPV6_RECVHOPLIMIT; 10745 else 10746 tcp->tcp_ipv6_recvancillary &= 10747 ~TCP_IPV6_RECVHOPLIMIT; 10748 /* Force it to be sent up with the next msg */ 10749 tcp->tcp_recvhops = 0xffffffffU; 10750 } 10751 break; 10752 case IPV6_RECVHOPOPTS: 10753 if (!checkonly) { 10754 if (onoff) 10755 tcp->tcp_ipv6_recvancillary |= 10756 TCP_IPV6_RECVHOPOPTS; 10757 else 10758 tcp->tcp_ipv6_recvancillary &= 10759 ~TCP_IPV6_RECVHOPOPTS; 10760 } 10761 break; 10762 case IPV6_RECVDSTOPTS: 10763 if (!checkonly) { 10764 if (onoff) 10765 tcp->tcp_ipv6_recvancillary |= 10766 TCP_IPV6_RECVDSTOPTS; 10767 else 10768 tcp->tcp_ipv6_recvancillary &= 10769 ~TCP_IPV6_RECVDSTOPTS; 10770 } 10771 break; 10772 case _OLD_IPV6_RECVDSTOPTS: 10773 if (!checkonly) { 10774 if (onoff) 10775 tcp->tcp_ipv6_recvancillary |= 10776 TCP_OLD_IPV6_RECVDSTOPTS; 10777 else 10778 tcp->tcp_ipv6_recvancillary &= 10779 ~TCP_OLD_IPV6_RECVDSTOPTS; 10780 } 10781 break; 10782 case IPV6_RECVRTHDR: 10783 if (!checkonly) { 10784 if (onoff) 10785 tcp->tcp_ipv6_recvancillary |= 10786 TCP_IPV6_RECVRTHDR; 10787 else 10788 tcp->tcp_ipv6_recvancillary &= 10789 ~TCP_IPV6_RECVRTHDR; 10790 } 10791 break; 10792 case IPV6_RECVRTHDRDSTOPTS: 10793 if (!checkonly) { 10794 if (onoff) 10795 tcp->tcp_ipv6_recvancillary |= 10796 TCP_IPV6_RECVRTDSTOPTS; 10797 else 10798 tcp->tcp_ipv6_recvancillary &= 10799 ~TCP_IPV6_RECVRTDSTOPTS; 10800 } 10801 break; 10802 case IPV6_PKTINFO: 10803 if (inlen != 0 && inlen != sizeof (struct in6_pktinfo)) 10804 return (EINVAL); 10805 if (checkonly) 10806 break; 10807 10808 if (inlen == 0) { 10809 ipp->ipp_fields &= ~(IPPF_IFINDEX|IPPF_ADDR); 10810 } else { 10811 struct in6_pktinfo *pkti; 10812 10813 pkti = (struct in6_pktinfo *)invalp; 10814 /* 10815 * RFC 3542 states that ipi6_addr must be 10816 * the unspecified address when setting the 10817 * IPV6_PKTINFO sticky socket option on a 10818 * TCP socket. 10819 */ 10820 if (!IN6_IS_ADDR_UNSPECIFIED(&pkti->ipi6_addr)) 10821 return (EINVAL); 10822 /* 10823 * ip6_set_pktinfo() validates the source 10824 * address and interface index. 10825 */ 10826 reterr = ip6_set_pktinfo(cr, tcp->tcp_connp, 10827 pkti, mblk); 10828 if (reterr != 0) 10829 return (reterr); 10830 ipp->ipp_ifindex = pkti->ipi6_ifindex; 10831 ipp->ipp_addr = pkti->ipi6_addr; 10832 if (ipp->ipp_ifindex != 0) 10833 ipp->ipp_fields |= IPPF_IFINDEX; 10834 else 10835 ipp->ipp_fields &= ~IPPF_IFINDEX; 10836 if (!IN6_IS_ADDR_UNSPECIFIED(&ipp->ipp_addr)) 10837 ipp->ipp_fields |= IPPF_ADDR; 10838 else 10839 ipp->ipp_fields &= ~IPPF_ADDR; 10840 } 10841 reterr = tcp_build_hdrs(q, tcp); 10842 if (reterr != 0) 10843 return (reterr); 10844 break; 10845 case IPV6_TCLASS: 10846 if (inlen != 0 && inlen != sizeof (int)) 10847 return (EINVAL); 10848 if (checkonly) 10849 break; 10850 10851 if (inlen == 0) { 10852 ipp->ipp_fields &= ~IPPF_TCLASS; 10853 } else { 10854 if (*i1 > 255 || *i1 < -1) 10855 return (EINVAL); 10856 if (*i1 == -1) { 10857 ipp->ipp_tclass = 0; 10858 *i1 = 0; 10859 } else { 10860 ipp->ipp_tclass = *i1; 10861 } 10862 ipp->ipp_fields |= IPPF_TCLASS; 10863 } 10864 reterr = tcp_build_hdrs(q, tcp); 10865 if (reterr != 0) 10866 return (reterr); 10867 break; 10868 case IPV6_NEXTHOP: 10869 /* 10870 * IP will verify that the nexthop is reachable 10871 * and fail for sticky options. 10872 */ 10873 if (inlen != 0 && inlen != sizeof (sin6_t)) 10874 return (EINVAL); 10875 if (checkonly) 10876 break; 10877 10878 if (inlen == 0) { 10879 ipp->ipp_fields &= ~IPPF_NEXTHOP; 10880 } else { 10881 sin6_t *sin6 = (sin6_t *)invalp; 10882 10883 if (sin6->sin6_family != AF_INET6) 10884 return (EAFNOSUPPORT); 10885 if (IN6_IS_ADDR_V4MAPPED( 10886 &sin6->sin6_addr)) 10887 return (EADDRNOTAVAIL); 10888 ipp->ipp_nexthop = sin6->sin6_addr; 10889 if (!IN6_IS_ADDR_UNSPECIFIED( 10890 &ipp->ipp_nexthop)) 10891 ipp->ipp_fields |= IPPF_NEXTHOP; 10892 else 10893 ipp->ipp_fields &= ~IPPF_NEXTHOP; 10894 } 10895 reterr = tcp_build_hdrs(q, tcp); 10896 if (reterr != 0) 10897 return (reterr); 10898 break; 10899 case IPV6_HOPOPTS: { 10900 ip6_hbh_t *hopts = (ip6_hbh_t *)invalp; 10901 10902 /* 10903 * Sanity checks - minimum size, size a multiple of 10904 * eight bytes, and matching size passed in. 10905 */ 10906 if (inlen != 0 && 10907 inlen != (8 * (hopts->ip6h_len + 1))) 10908 return (EINVAL); 10909 10910 if (checkonly) 10911 break; 10912 10913 reterr = optcom_pkt_set(invalp, inlen, B_TRUE, 10914 (uchar_t **)&ipp->ipp_hopopts, 10915 &ipp->ipp_hopoptslen, tcp->tcp_label_len); 10916 if (reterr != 0) 10917 return (reterr); 10918 if (ipp->ipp_hopoptslen == 0) 10919 ipp->ipp_fields &= ~IPPF_HOPOPTS; 10920 else 10921 ipp->ipp_fields |= IPPF_HOPOPTS; 10922 reterr = tcp_build_hdrs(q, tcp); 10923 if (reterr != 0) 10924 return (reterr); 10925 break; 10926 } 10927 case IPV6_RTHDRDSTOPTS: { 10928 ip6_dest_t *dopts = (ip6_dest_t *)invalp; 10929 10930 /* 10931 * Sanity checks - minimum size, size a multiple of 10932 * eight bytes, and matching size passed in. 10933 */ 10934 if (inlen != 0 && 10935 inlen != (8 * (dopts->ip6d_len + 1))) 10936 return (EINVAL); 10937 10938 if (checkonly) 10939 break; 10940 10941 reterr = optcom_pkt_set(invalp, inlen, B_TRUE, 10942 (uchar_t **)&ipp->ipp_rtdstopts, 10943 &ipp->ipp_rtdstoptslen, 0); 10944 if (reterr != 0) 10945 return (reterr); 10946 if (ipp->ipp_rtdstoptslen == 0) 10947 ipp->ipp_fields &= ~IPPF_RTDSTOPTS; 10948 else 10949 ipp->ipp_fields |= IPPF_RTDSTOPTS; 10950 reterr = tcp_build_hdrs(q, tcp); 10951 if (reterr != 0) 10952 return (reterr); 10953 break; 10954 } 10955 case IPV6_DSTOPTS: { 10956 ip6_dest_t *dopts = (ip6_dest_t *)invalp; 10957 10958 /* 10959 * Sanity checks - minimum size, size a multiple of 10960 * eight bytes, and matching size passed in. 10961 */ 10962 if (inlen != 0 && 10963 inlen != (8 * (dopts->ip6d_len + 1))) 10964 return (EINVAL); 10965 10966 if (checkonly) 10967 break; 10968 10969 reterr = optcom_pkt_set(invalp, inlen, B_TRUE, 10970 (uchar_t **)&ipp->ipp_dstopts, 10971 &ipp->ipp_dstoptslen, 0); 10972 if (reterr != 0) 10973 return (reterr); 10974 if (ipp->ipp_dstoptslen == 0) 10975 ipp->ipp_fields &= ~IPPF_DSTOPTS; 10976 else 10977 ipp->ipp_fields |= IPPF_DSTOPTS; 10978 reterr = tcp_build_hdrs(q, tcp); 10979 if (reterr != 0) 10980 return (reterr); 10981 break; 10982 } 10983 case IPV6_RTHDR: { 10984 ip6_rthdr_t *rt = (ip6_rthdr_t *)invalp; 10985 10986 /* 10987 * Sanity checks - minimum size, size a multiple of 10988 * eight bytes, and matching size passed in. 10989 */ 10990 if (inlen != 0 && 10991 inlen != (8 * (rt->ip6r_len + 1))) 10992 return (EINVAL); 10993 10994 if (checkonly) 10995 break; 10996 10997 reterr = optcom_pkt_set(invalp, inlen, B_TRUE, 10998 (uchar_t **)&ipp->ipp_rthdr, 10999 &ipp->ipp_rthdrlen, 0); 11000 if (reterr != 0) 11001 return (reterr); 11002 if (ipp->ipp_rthdrlen == 0) 11003 ipp->ipp_fields &= ~IPPF_RTHDR; 11004 else 11005 ipp->ipp_fields |= IPPF_RTHDR; 11006 reterr = tcp_build_hdrs(q, tcp); 11007 if (reterr != 0) 11008 return (reterr); 11009 break; 11010 } 11011 case IPV6_V6ONLY: 11012 if (!checkonly) 11013 tcp->tcp_connp->conn_ipv6_v6only = onoff; 11014 break; 11015 case IPV6_USE_MIN_MTU: 11016 if (inlen != sizeof (int)) 11017 return (EINVAL); 11018 11019 if (*i1 < -1 || *i1 > 1) 11020 return (EINVAL); 11021 11022 if (checkonly) 11023 break; 11024 11025 ipp->ipp_fields |= IPPF_USE_MIN_MTU; 11026 ipp->ipp_use_min_mtu = *i1; 11027 break; 11028 case IPV6_BOUND_PIF: 11029 /* Handled at the IP level */ 11030 return (-EINVAL); 11031 case IPV6_SEC_OPT: 11032 /* 11033 * We should not allow policy setting after 11034 * we start listening for connections. 11035 */ 11036 if (tcp->tcp_state == TCPS_LISTEN) { 11037 return (EINVAL); 11038 } else { 11039 /* Handled at the IP level */ 11040 return (-EINVAL); 11041 } 11042 case IPV6_SRC_PREFERENCES: 11043 if (inlen != sizeof (uint32_t)) 11044 return (EINVAL); 11045 reterr = ip6_set_src_preferences(tcp->tcp_connp, 11046 *(uint32_t *)invalp); 11047 if (reterr != 0) { 11048 *outlenp = 0; 11049 return (reterr); 11050 } 11051 break; 11052 default: 11053 *outlenp = 0; 11054 return (EINVAL); 11055 } 11056 break; 11057 } /* end IPPROTO_IPV6 */ 11058 default: 11059 *outlenp = 0; 11060 return (EINVAL); 11061 } 11062 /* 11063 * Common case of OK return with outval same as inval 11064 */ 11065 if (invalp != outvalp) { 11066 /* don't trust bcopy for identical src/dst */ 11067 (void) bcopy(invalp, outvalp, inlen); 11068 } 11069 *outlenp = inlen; 11070 return (0); 11071 } 11072 11073 /* 11074 * Update tcp_sticky_hdrs based on tcp_sticky_ipp. 11075 * The headers include ip6i_t (if needed), ip6_t, any sticky extension 11076 * headers, and the maximum size tcp header (to avoid reallocation 11077 * on the fly for additional tcp options). 11078 * Returns failure if can't allocate memory. 11079 */ 11080 static int 11081 tcp_build_hdrs(queue_t *q, tcp_t *tcp) 11082 { 11083 char *hdrs; 11084 uint_t hdrs_len; 11085 ip6i_t *ip6i; 11086 char buf[TCP_MAX_HDR_LENGTH]; 11087 ip6_pkt_t *ipp = &tcp->tcp_sticky_ipp; 11088 in6_addr_t src, dst; 11089 tcp_stack_t *tcps = tcp->tcp_tcps; 11090 11091 /* 11092 * save the existing tcp header and source/dest IP addresses 11093 */ 11094 bcopy(tcp->tcp_tcph, buf, tcp->tcp_tcp_hdr_len); 11095 src = tcp->tcp_ip6h->ip6_src; 11096 dst = tcp->tcp_ip6h->ip6_dst; 11097 hdrs_len = ip_total_hdrs_len_v6(ipp) + TCP_MAX_HDR_LENGTH; 11098 ASSERT(hdrs_len != 0); 11099 if (hdrs_len > tcp->tcp_iphc_len) { 11100 /* Need to reallocate */ 11101 hdrs = kmem_zalloc(hdrs_len, KM_NOSLEEP); 11102 if (hdrs == NULL) 11103 return (ENOMEM); 11104 if (tcp->tcp_iphc != NULL) { 11105 if (tcp->tcp_hdr_grown) { 11106 kmem_free(tcp->tcp_iphc, tcp->tcp_iphc_len); 11107 } else { 11108 bzero(tcp->tcp_iphc, tcp->tcp_iphc_len); 11109 kmem_cache_free(tcp_iphc_cache, tcp->tcp_iphc); 11110 } 11111 tcp->tcp_iphc_len = 0; 11112 } 11113 ASSERT(tcp->tcp_iphc_len == 0); 11114 tcp->tcp_iphc = hdrs; 11115 tcp->tcp_iphc_len = hdrs_len; 11116 tcp->tcp_hdr_grown = B_TRUE; 11117 } 11118 ip_build_hdrs_v6((uchar_t *)tcp->tcp_iphc, 11119 hdrs_len - TCP_MAX_HDR_LENGTH, ipp, IPPROTO_TCP); 11120 11121 /* Set header fields not in ipp */ 11122 if (ipp->ipp_fields & IPPF_HAS_IP6I) { 11123 ip6i = (ip6i_t *)tcp->tcp_iphc; 11124 tcp->tcp_ip6h = (ip6_t *)&ip6i[1]; 11125 } else { 11126 tcp->tcp_ip6h = (ip6_t *)tcp->tcp_iphc; 11127 } 11128 /* 11129 * tcp->tcp_ip_hdr_len will include ip6i_t if there is one. 11130 * 11131 * tcp->tcp_tcp_hdr_len doesn't change here. 11132 */ 11133 tcp->tcp_ip_hdr_len = hdrs_len - TCP_MAX_HDR_LENGTH; 11134 tcp->tcp_tcph = (tcph_t *)(tcp->tcp_iphc + tcp->tcp_ip_hdr_len); 11135 tcp->tcp_hdr_len = tcp->tcp_ip_hdr_len + tcp->tcp_tcp_hdr_len; 11136 11137 bcopy(buf, tcp->tcp_tcph, tcp->tcp_tcp_hdr_len); 11138 11139 tcp->tcp_ip6h->ip6_src = src; 11140 tcp->tcp_ip6h->ip6_dst = dst; 11141 11142 /* 11143 * If the hop limit was not set by ip_build_hdrs_v6(), set it to 11144 * the default value for TCP. 11145 */ 11146 if (!(ipp->ipp_fields & IPPF_UNICAST_HOPS)) 11147 tcp->tcp_ip6h->ip6_hops = tcps->tcps_ipv6_hoplimit; 11148 11149 /* 11150 * If we're setting extension headers after a connection 11151 * has been established, and if we have a routing header 11152 * among the extension headers, call ip_massage_options_v6 to 11153 * manipulate the routing header/ip6_dst set the checksum 11154 * difference in the tcp header template. 11155 * (This happens in tcp_connect_ipv6 if the routing header 11156 * is set prior to the connect.) 11157 * Set the tcp_sum to zero first in case we've cleared a 11158 * routing header or don't have one at all. 11159 */ 11160 tcp->tcp_sum = 0; 11161 if ((tcp->tcp_state >= TCPS_SYN_SENT) && 11162 (tcp->tcp_ipp_fields & IPPF_RTHDR)) { 11163 ip6_rthdr_t *rth = ip_find_rthdr_v6(tcp->tcp_ip6h, 11164 (uint8_t *)tcp->tcp_tcph); 11165 if (rth != NULL) { 11166 tcp->tcp_sum = ip_massage_options_v6(tcp->tcp_ip6h, 11167 rth, tcps->tcps_netstack); 11168 tcp->tcp_sum = ntohs((tcp->tcp_sum & 0xFFFF) + 11169 (tcp->tcp_sum >> 16)); 11170 } 11171 } 11172 11173 /* Try to get everything in a single mblk */ 11174 (void) mi_set_sth_wroff(RD(q), hdrs_len + tcps->tcps_wroff_xtra); 11175 return (0); 11176 } 11177 11178 /* 11179 * Transfer any source route option from ipha to buf/dst in reversed form. 11180 */ 11181 static int 11182 tcp_opt_rev_src_route(ipha_t *ipha, char *buf, uchar_t *dst) 11183 { 11184 ipoptp_t opts; 11185 uchar_t *opt; 11186 uint8_t optval; 11187 uint8_t optlen; 11188 uint32_t len = 0; 11189 11190 for (optval = ipoptp_first(&opts, ipha); 11191 optval != IPOPT_EOL; 11192 optval = ipoptp_next(&opts)) { 11193 opt = opts.ipoptp_cur; 11194 optlen = opts.ipoptp_len; 11195 switch (optval) { 11196 int off1, off2; 11197 case IPOPT_SSRR: 11198 case IPOPT_LSRR: 11199 11200 /* Reverse source route */ 11201 /* 11202 * First entry should be the next to last one in the 11203 * current source route (the last entry is our 11204 * address.) 11205 * The last entry should be the final destination. 11206 */ 11207 buf[IPOPT_OPTVAL] = (uint8_t)optval; 11208 buf[IPOPT_OLEN] = (uint8_t)optlen; 11209 off1 = IPOPT_MINOFF_SR - 1; 11210 off2 = opt[IPOPT_OFFSET] - IP_ADDR_LEN - 1; 11211 if (off2 < 0) { 11212 /* No entries in source route */ 11213 break; 11214 } 11215 bcopy(opt + off2, dst, IP_ADDR_LEN); 11216 /* 11217 * Note: use src since ipha has not had its src 11218 * and dst reversed (it is in the state it was 11219 * received. 11220 */ 11221 bcopy(&ipha->ipha_src, buf + off2, 11222 IP_ADDR_LEN); 11223 off2 -= IP_ADDR_LEN; 11224 11225 while (off2 > 0) { 11226 bcopy(opt + off2, buf + off1, 11227 IP_ADDR_LEN); 11228 off1 += IP_ADDR_LEN; 11229 off2 -= IP_ADDR_LEN; 11230 } 11231 buf[IPOPT_OFFSET] = IPOPT_MINOFF_SR; 11232 buf += optlen; 11233 len += optlen; 11234 break; 11235 } 11236 } 11237 done: 11238 /* Pad the resulting options */ 11239 while (len & 0x3) { 11240 *buf++ = IPOPT_EOL; 11241 len++; 11242 } 11243 return (len); 11244 } 11245 11246 11247 /* 11248 * Extract and revert a source route from ipha (if any) 11249 * and then update the relevant fields in both tcp_t and the standard header. 11250 */ 11251 static void 11252 tcp_opt_reverse(tcp_t *tcp, ipha_t *ipha) 11253 { 11254 char buf[TCP_MAX_HDR_LENGTH]; 11255 uint_t tcph_len; 11256 int len; 11257 11258 ASSERT(IPH_HDR_VERSION(ipha) == IPV4_VERSION); 11259 len = IPH_HDR_LENGTH(ipha); 11260 if (len == IP_SIMPLE_HDR_LENGTH) 11261 /* Nothing to do */ 11262 return; 11263 if (len > IP_SIMPLE_HDR_LENGTH + TCP_MAX_IP_OPTIONS_LENGTH || 11264 (len & 0x3)) 11265 return; 11266 11267 tcph_len = tcp->tcp_tcp_hdr_len; 11268 bcopy(tcp->tcp_tcph, buf, tcph_len); 11269 tcp->tcp_sum = (tcp->tcp_ipha->ipha_dst >> 16) + 11270 (tcp->tcp_ipha->ipha_dst & 0xffff); 11271 len = tcp_opt_rev_src_route(ipha, (char *)tcp->tcp_ipha + 11272 IP_SIMPLE_HDR_LENGTH, (uchar_t *)&tcp->tcp_ipha->ipha_dst); 11273 len += IP_SIMPLE_HDR_LENGTH; 11274 tcp->tcp_sum -= ((tcp->tcp_ipha->ipha_dst >> 16) + 11275 (tcp->tcp_ipha->ipha_dst & 0xffff)); 11276 if ((int)tcp->tcp_sum < 0) 11277 tcp->tcp_sum--; 11278 tcp->tcp_sum = (tcp->tcp_sum & 0xFFFF) + (tcp->tcp_sum >> 16); 11279 tcp->tcp_sum = ntohs((tcp->tcp_sum & 0xFFFF) + (tcp->tcp_sum >> 16)); 11280 tcp->tcp_tcph = (tcph_t *)((char *)tcp->tcp_ipha + len); 11281 bcopy(buf, tcp->tcp_tcph, tcph_len); 11282 tcp->tcp_ip_hdr_len = len; 11283 tcp->tcp_ipha->ipha_version_and_hdr_length = 11284 (IP_VERSION << 4) | (len >> 2); 11285 len += tcph_len; 11286 tcp->tcp_hdr_len = len; 11287 } 11288 11289 /* 11290 * Copy the standard header into its new location, 11291 * lay in the new options and then update the relevant 11292 * fields in both tcp_t and the standard header. 11293 */ 11294 static int 11295 tcp_opt_set_header(tcp_t *tcp, boolean_t checkonly, uchar_t *ptr, uint_t len) 11296 { 11297 uint_t tcph_len; 11298 uint8_t *ip_optp; 11299 tcph_t *new_tcph; 11300 tcp_stack_t *tcps = tcp->tcp_tcps; 11301 11302 if ((len > TCP_MAX_IP_OPTIONS_LENGTH) || (len & 0x3)) 11303 return (EINVAL); 11304 11305 if (len > IP_MAX_OPT_LENGTH - tcp->tcp_label_len) 11306 return (EINVAL); 11307 11308 if (checkonly) { 11309 /* 11310 * do not really set, just pretend to - T_CHECK 11311 */ 11312 return (0); 11313 } 11314 11315 ip_optp = (uint8_t *)tcp->tcp_ipha + IP_SIMPLE_HDR_LENGTH; 11316 if (tcp->tcp_label_len > 0) { 11317 int padlen; 11318 uint8_t opt; 11319 11320 /* convert list termination to no-ops */ 11321 padlen = tcp->tcp_label_len - ip_optp[IPOPT_OLEN]; 11322 ip_optp += ip_optp[IPOPT_OLEN]; 11323 opt = len > 0 ? IPOPT_NOP : IPOPT_EOL; 11324 while (--padlen >= 0) 11325 *ip_optp++ = opt; 11326 } 11327 tcph_len = tcp->tcp_tcp_hdr_len; 11328 new_tcph = (tcph_t *)(ip_optp + len); 11329 ovbcopy(tcp->tcp_tcph, new_tcph, tcph_len); 11330 tcp->tcp_tcph = new_tcph; 11331 bcopy(ptr, ip_optp, len); 11332 11333 len += IP_SIMPLE_HDR_LENGTH + tcp->tcp_label_len; 11334 11335 tcp->tcp_ip_hdr_len = len; 11336 tcp->tcp_ipha->ipha_version_and_hdr_length = 11337 (IP_VERSION << 4) | (len >> 2); 11338 tcp->tcp_hdr_len = len + tcph_len; 11339 if (!TCP_IS_DETACHED(tcp)) { 11340 /* Always allocate room for all options. */ 11341 (void) mi_set_sth_wroff(tcp->tcp_rq, 11342 TCP_MAX_COMBINED_HEADER_LENGTH + tcps->tcps_wroff_xtra); 11343 } 11344 return (0); 11345 } 11346 11347 /* Get callback routine passed to nd_load by tcp_param_register */ 11348 /* ARGSUSED */ 11349 static int 11350 tcp_param_get(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr) 11351 { 11352 tcpparam_t *tcppa = (tcpparam_t *)cp; 11353 11354 (void) mi_mpprintf(mp, "%u", tcppa->tcp_param_val); 11355 return (0); 11356 } 11357 11358 /* 11359 * Walk through the param array specified registering each element with the 11360 * named dispatch handler. 11361 */ 11362 static boolean_t 11363 tcp_param_register(IDP *ndp, tcpparam_t *tcppa, int cnt, tcp_stack_t *tcps) 11364 { 11365 for (; cnt-- > 0; tcppa++) { 11366 if (tcppa->tcp_param_name && tcppa->tcp_param_name[0]) { 11367 if (!nd_load(ndp, tcppa->tcp_param_name, 11368 tcp_param_get, tcp_param_set, 11369 (caddr_t)tcppa)) { 11370 nd_free(ndp); 11371 return (B_FALSE); 11372 } 11373 } 11374 } 11375 tcps->tcps_wroff_xtra_param = kmem_zalloc(sizeof (tcpparam_t), 11376 KM_SLEEP); 11377 bcopy(&lcl_tcp_wroff_xtra_param, tcps->tcps_wroff_xtra_param, 11378 sizeof (tcpparam_t)); 11379 if (!nd_load(ndp, tcps->tcps_wroff_xtra_param->tcp_param_name, 11380 tcp_param_get, tcp_param_set_aligned, 11381 (caddr_t)tcps->tcps_wroff_xtra_param)) { 11382 nd_free(ndp); 11383 return (B_FALSE); 11384 } 11385 tcps->tcps_mdt_head_param = kmem_zalloc(sizeof (tcpparam_t), 11386 KM_SLEEP); 11387 bcopy(&lcl_tcp_mdt_head_param, tcps->tcps_mdt_head_param, 11388 sizeof (tcpparam_t)); 11389 if (!nd_load(ndp, tcps->tcps_mdt_head_param->tcp_param_name, 11390 tcp_param_get, tcp_param_set_aligned, 11391 (caddr_t)tcps->tcps_mdt_head_param)) { 11392 nd_free(ndp); 11393 return (B_FALSE); 11394 } 11395 tcps->tcps_mdt_tail_param = kmem_zalloc(sizeof (tcpparam_t), 11396 KM_SLEEP); 11397 bcopy(&lcl_tcp_mdt_tail_param, tcps->tcps_mdt_tail_param, 11398 sizeof (tcpparam_t)); 11399 if (!nd_load(ndp, tcps->tcps_mdt_tail_param->tcp_param_name, 11400 tcp_param_get, tcp_param_set_aligned, 11401 (caddr_t)tcps->tcps_mdt_tail_param)) { 11402 nd_free(ndp); 11403 return (B_FALSE); 11404 } 11405 tcps->tcps_mdt_max_pbufs_param = kmem_zalloc(sizeof (tcpparam_t), 11406 KM_SLEEP); 11407 bcopy(&lcl_tcp_mdt_max_pbufs_param, tcps->tcps_mdt_max_pbufs_param, 11408 sizeof (tcpparam_t)); 11409 if (!nd_load(ndp, tcps->tcps_mdt_max_pbufs_param->tcp_param_name, 11410 tcp_param_get, tcp_param_set_aligned, 11411 (caddr_t)tcps->tcps_mdt_max_pbufs_param)) { 11412 nd_free(ndp); 11413 return (B_FALSE); 11414 } 11415 if (!nd_load(ndp, "tcp_extra_priv_ports", 11416 tcp_extra_priv_ports_get, NULL, NULL)) { 11417 nd_free(ndp); 11418 return (B_FALSE); 11419 } 11420 if (!nd_load(ndp, "tcp_extra_priv_ports_add", 11421 NULL, tcp_extra_priv_ports_add, NULL)) { 11422 nd_free(ndp); 11423 return (B_FALSE); 11424 } 11425 if (!nd_load(ndp, "tcp_extra_priv_ports_del", 11426 NULL, tcp_extra_priv_ports_del, NULL)) { 11427 nd_free(ndp); 11428 return (B_FALSE); 11429 } 11430 if (!nd_load(ndp, "tcp_status", tcp_status_report, NULL, 11431 NULL)) { 11432 nd_free(ndp); 11433 return (B_FALSE); 11434 } 11435 if (!nd_load(ndp, "tcp_bind_hash", tcp_bind_hash_report, 11436 NULL, NULL)) { 11437 nd_free(ndp); 11438 return (B_FALSE); 11439 } 11440 if (!nd_load(ndp, "tcp_listen_hash", 11441 tcp_listen_hash_report, NULL, NULL)) { 11442 nd_free(ndp); 11443 return (B_FALSE); 11444 } 11445 if (!nd_load(ndp, "tcp_conn_hash", tcp_conn_hash_report, 11446 NULL, NULL)) { 11447 nd_free(ndp); 11448 return (B_FALSE); 11449 } 11450 if (!nd_load(ndp, "tcp_acceptor_hash", 11451 tcp_acceptor_hash_report, NULL, NULL)) { 11452 nd_free(ndp); 11453 return (B_FALSE); 11454 } 11455 if (!nd_load(ndp, "tcp_1948_phrase", NULL, 11456 tcp_1948_phrase_set, NULL)) { 11457 nd_free(ndp); 11458 return (B_FALSE); 11459 } 11460 /* 11461 * Dummy ndd variables - only to convey obsolescence information 11462 * through printing of their name (no get or set routines) 11463 * XXX Remove in future releases ? 11464 */ 11465 if (!nd_load(ndp, 11466 "tcp_close_wait_interval(obsoleted - " 11467 "use tcp_time_wait_interval)", NULL, NULL, NULL)) { 11468 nd_free(ndp); 11469 return (B_FALSE); 11470 } 11471 return (B_TRUE); 11472 } 11473 11474 /* ndd set routine for tcp_wroff_xtra, tcp_mdt_hdr_{head,tail}_min. */ 11475 /* ARGSUSED */ 11476 static int 11477 tcp_param_set_aligned(queue_t *q, mblk_t *mp, char *value, caddr_t cp, 11478 cred_t *cr) 11479 { 11480 long new_value; 11481 tcpparam_t *tcppa = (tcpparam_t *)cp; 11482 11483 if (ddi_strtol(value, NULL, 10, &new_value) != 0 || 11484 new_value < tcppa->tcp_param_min || 11485 new_value > tcppa->tcp_param_max) { 11486 return (EINVAL); 11487 } 11488 /* 11489 * Need to make sure new_value is a multiple of 4. If it is not, 11490 * round it up. For future 64 bit requirement, we actually make it 11491 * a multiple of 8. 11492 */ 11493 if (new_value & 0x7) { 11494 new_value = (new_value & ~0x7) + 0x8; 11495 } 11496 tcppa->tcp_param_val = new_value; 11497 return (0); 11498 } 11499 11500 /* Set callback routine passed to nd_load by tcp_param_register */ 11501 /* ARGSUSED */ 11502 static int 11503 tcp_param_set(queue_t *q, mblk_t *mp, char *value, caddr_t cp, cred_t *cr) 11504 { 11505 long new_value; 11506 tcpparam_t *tcppa = (tcpparam_t *)cp; 11507 11508 if (ddi_strtol(value, NULL, 10, &new_value) != 0 || 11509 new_value < tcppa->tcp_param_min || 11510 new_value > tcppa->tcp_param_max) { 11511 return (EINVAL); 11512 } 11513 tcppa->tcp_param_val = new_value; 11514 return (0); 11515 } 11516 11517 /* 11518 * Add a new piece to the tcp reassembly queue. If the gap at the beginning 11519 * is filled, return as much as we can. The message passed in may be 11520 * multi-part, chained using b_cont. "start" is the starting sequence 11521 * number for this piece. 11522 */ 11523 static mblk_t * 11524 tcp_reass(tcp_t *tcp, mblk_t *mp, uint32_t start) 11525 { 11526 uint32_t end; 11527 mblk_t *mp1; 11528 mblk_t *mp2; 11529 mblk_t *next_mp; 11530 uint32_t u1; 11531 tcp_stack_t *tcps = tcp->tcp_tcps; 11532 11533 /* Walk through all the new pieces. */ 11534 do { 11535 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= 11536 (uintptr_t)INT_MAX); 11537 end = start + (int)(mp->b_wptr - mp->b_rptr); 11538 next_mp = mp->b_cont; 11539 if (start == end) { 11540 /* Empty. Blast it. */ 11541 freeb(mp); 11542 continue; 11543 } 11544 mp->b_cont = NULL; 11545 TCP_REASS_SET_SEQ(mp, start); 11546 TCP_REASS_SET_END(mp, end); 11547 mp1 = tcp->tcp_reass_tail; 11548 if (!mp1) { 11549 tcp->tcp_reass_tail = mp; 11550 tcp->tcp_reass_head = mp; 11551 BUMP_MIB(&tcps->tcps_mib, tcpInDataUnorderSegs); 11552 UPDATE_MIB(&tcps->tcps_mib, 11553 tcpInDataUnorderBytes, end - start); 11554 continue; 11555 } 11556 /* New stuff completely beyond tail? */ 11557 if (SEQ_GEQ(start, TCP_REASS_END(mp1))) { 11558 /* Link it on end. */ 11559 mp1->b_cont = mp; 11560 tcp->tcp_reass_tail = mp; 11561 BUMP_MIB(&tcps->tcps_mib, tcpInDataUnorderSegs); 11562 UPDATE_MIB(&tcps->tcps_mib, 11563 tcpInDataUnorderBytes, end - start); 11564 continue; 11565 } 11566 mp1 = tcp->tcp_reass_head; 11567 u1 = TCP_REASS_SEQ(mp1); 11568 /* New stuff at the front? */ 11569 if (SEQ_LT(start, u1)) { 11570 /* Yes... Check for overlap. */ 11571 mp->b_cont = mp1; 11572 tcp->tcp_reass_head = mp; 11573 tcp_reass_elim_overlap(tcp, mp); 11574 continue; 11575 } 11576 /* 11577 * The new piece fits somewhere between the head and tail. 11578 * We find our slot, where mp1 precedes us and mp2 trails. 11579 */ 11580 for (; (mp2 = mp1->b_cont) != NULL; mp1 = mp2) { 11581 u1 = TCP_REASS_SEQ(mp2); 11582 if (SEQ_LEQ(start, u1)) 11583 break; 11584 } 11585 /* Link ourselves in */ 11586 mp->b_cont = mp2; 11587 mp1->b_cont = mp; 11588 11589 /* Trim overlap with following mblk(s) first */ 11590 tcp_reass_elim_overlap(tcp, mp); 11591 11592 /* Trim overlap with preceding mblk */ 11593 tcp_reass_elim_overlap(tcp, mp1); 11594 11595 } while (start = end, mp = next_mp); 11596 mp1 = tcp->tcp_reass_head; 11597 /* Anything ready to go? */ 11598 if (TCP_REASS_SEQ(mp1) != tcp->tcp_rnxt) 11599 return (NULL); 11600 /* Eat what we can off the queue */ 11601 for (;;) { 11602 mp = mp1->b_cont; 11603 end = TCP_REASS_END(mp1); 11604 TCP_REASS_SET_SEQ(mp1, 0); 11605 TCP_REASS_SET_END(mp1, 0); 11606 if (!mp) { 11607 tcp->tcp_reass_tail = NULL; 11608 break; 11609 } 11610 if (end != TCP_REASS_SEQ(mp)) { 11611 mp1->b_cont = NULL; 11612 break; 11613 } 11614 mp1 = mp; 11615 } 11616 mp1 = tcp->tcp_reass_head; 11617 tcp->tcp_reass_head = mp; 11618 return (mp1); 11619 } 11620 11621 /* Eliminate any overlap that mp may have over later mblks */ 11622 static void 11623 tcp_reass_elim_overlap(tcp_t *tcp, mblk_t *mp) 11624 { 11625 uint32_t end; 11626 mblk_t *mp1; 11627 uint32_t u1; 11628 tcp_stack_t *tcps = tcp->tcp_tcps; 11629 11630 end = TCP_REASS_END(mp); 11631 while ((mp1 = mp->b_cont) != NULL) { 11632 u1 = TCP_REASS_SEQ(mp1); 11633 if (!SEQ_GT(end, u1)) 11634 break; 11635 if (!SEQ_GEQ(end, TCP_REASS_END(mp1))) { 11636 mp->b_wptr -= end - u1; 11637 TCP_REASS_SET_END(mp, u1); 11638 BUMP_MIB(&tcps->tcps_mib, tcpInDataPartDupSegs); 11639 UPDATE_MIB(&tcps->tcps_mib, 11640 tcpInDataPartDupBytes, end - u1); 11641 break; 11642 } 11643 mp->b_cont = mp1->b_cont; 11644 TCP_REASS_SET_SEQ(mp1, 0); 11645 TCP_REASS_SET_END(mp1, 0); 11646 freeb(mp1); 11647 BUMP_MIB(&tcps->tcps_mib, tcpInDataDupSegs); 11648 UPDATE_MIB(&tcps->tcps_mib, tcpInDataDupBytes, end - u1); 11649 } 11650 if (!mp1) 11651 tcp->tcp_reass_tail = mp; 11652 } 11653 11654 /* 11655 * Send up all messages queued on tcp_rcv_list. 11656 */ 11657 static uint_t 11658 tcp_rcv_drain(queue_t *q, tcp_t *tcp) 11659 { 11660 mblk_t *mp; 11661 uint_t ret = 0; 11662 uint_t thwin; 11663 #ifdef DEBUG 11664 uint_t cnt = 0; 11665 #endif 11666 tcp_stack_t *tcps = tcp->tcp_tcps; 11667 11668 /* Can't drain on an eager connection */ 11669 if (tcp->tcp_listener != NULL) 11670 return (ret); 11671 11672 /* Can't be sodirect enabled */ 11673 ASSERT(SOD_NOT_ENABLED(tcp)); 11674 11675 /* No need for the push timer now. */ 11676 if (tcp->tcp_push_tid != 0) { 11677 (void) TCP_TIMER_CANCEL(tcp, tcp->tcp_push_tid); 11678 tcp->tcp_push_tid = 0; 11679 } 11680 11681 /* 11682 * Handle two cases here: we are currently fused or we were 11683 * previously fused and have some urgent data to be delivered 11684 * upstream. The latter happens because we either ran out of 11685 * memory or were detached and therefore sending the SIGURG was 11686 * deferred until this point. In either case we pass control 11687 * over to tcp_fuse_rcv_drain() since it may need to complete 11688 * some work. 11689 */ 11690 if ((tcp->tcp_fused || tcp->tcp_fused_sigurg)) { 11691 ASSERT(tcp->tcp_fused_sigurg_mp != NULL); 11692 if (tcp_fuse_rcv_drain(q, tcp, tcp->tcp_fused ? NULL : 11693 &tcp->tcp_fused_sigurg_mp)) 11694 return (ret); 11695 } 11696 11697 while ((mp = tcp->tcp_rcv_list) != NULL) { 11698 tcp->tcp_rcv_list = mp->b_next; 11699 mp->b_next = NULL; 11700 #ifdef DEBUG 11701 cnt += msgdsize(mp); 11702 #endif 11703 /* Does this need SSL processing first? */ 11704 if ((tcp->tcp_kssl_ctx != NULL) && (DB_TYPE(mp) == M_DATA)) { 11705 DTRACE_PROBE1(kssl_mblk__ksslinput_rcvdrain, 11706 mblk_t *, mp); 11707 tcp_kssl_input(tcp, mp); 11708 continue; 11709 } 11710 putnext(q, mp); 11711 } 11712 ASSERT(cnt == tcp->tcp_rcv_cnt); 11713 tcp->tcp_rcv_last_head = NULL; 11714 tcp->tcp_rcv_last_tail = NULL; 11715 tcp->tcp_rcv_cnt = 0; 11716 11717 /* Learn the latest rwnd information that we sent to the other side. */ 11718 thwin = ((uint_t)BE16_TO_U16(tcp->tcp_tcph->th_win)) 11719 << tcp->tcp_rcv_ws; 11720 /* This is peer's calculated send window (our receive window). */ 11721 thwin -= tcp->tcp_rnxt - tcp->tcp_rack; 11722 /* 11723 * Increase the receive window to max. But we need to do receiver 11724 * SWS avoidance. This means that we need to check the increase of 11725 * of receive window is at least 1 MSS. 11726 */ 11727 if (canputnext(q) && (q->q_hiwat - thwin >= tcp->tcp_mss)) { 11728 /* 11729 * If the window that the other side knows is less than max 11730 * deferred acks segments, send an update immediately. 11731 */ 11732 if (thwin < tcp->tcp_rack_cur_max * tcp->tcp_mss) { 11733 BUMP_MIB(&tcps->tcps_mib, tcpOutWinUpdate); 11734 ret = TH_ACK_NEEDED; 11735 } 11736 tcp->tcp_rwnd = q->q_hiwat; 11737 } 11738 return (ret); 11739 } 11740 11741 /* 11742 * Queue data on tcp_rcv_list which is a b_next chain. 11743 * tcp_rcv_last_head/tail is the last element of this chain. 11744 * Each element of the chain is a b_cont chain. 11745 * 11746 * M_DATA messages are added to the current element. 11747 * Other messages are added as new (b_next) elements. 11748 */ 11749 void 11750 tcp_rcv_enqueue(tcp_t *tcp, mblk_t *mp, uint_t seg_len) 11751 { 11752 ASSERT(seg_len == msgdsize(mp)); 11753 ASSERT(tcp->tcp_rcv_list == NULL || tcp->tcp_rcv_last_head != NULL); 11754 11755 if (tcp->tcp_rcv_list == NULL) { 11756 ASSERT(tcp->tcp_rcv_last_head == NULL); 11757 tcp->tcp_rcv_list = mp; 11758 tcp->tcp_rcv_last_head = mp; 11759 } else if (DB_TYPE(mp) == DB_TYPE(tcp->tcp_rcv_last_head)) { 11760 tcp->tcp_rcv_last_tail->b_cont = mp; 11761 } else { 11762 tcp->tcp_rcv_last_head->b_next = mp; 11763 tcp->tcp_rcv_last_head = mp; 11764 } 11765 11766 while (mp->b_cont) 11767 mp = mp->b_cont; 11768 11769 tcp->tcp_rcv_last_tail = mp; 11770 tcp->tcp_rcv_cnt += seg_len; 11771 tcp->tcp_rwnd -= seg_len; 11772 } 11773 11774 /* 11775 * The tcp_rcv_sod_XXX() functions enqueue data directly to the socket 11776 * above, in addition when uioa is enabled schedule an asynchronous uio 11777 * prior to enqueuing. They implement the combinhed semantics of the 11778 * tcp_rcv_XXX() functions, tcp_rcv_list push logic, and STREAMS putnext() 11779 * canputnext(), i.e. flow-control with backenable. 11780 * 11781 * tcp_sod_wakeup() is called where tcp_rcv_drain() would be called in the 11782 * non sodirect connection but as there are no tcp_tcv_list mblk_t's we deal 11783 * with the rcv_wnd and push timer and call the sodirect wakeup function. 11784 * 11785 * Must be called with sodp->sod_lock held and will return with the lock 11786 * released. 11787 */ 11788 static uint_t 11789 tcp_rcv_sod_wakeup(tcp_t *tcp, sodirect_t *sodp) 11790 { 11791 queue_t *q = tcp->tcp_rq; 11792 uint_t thwin; 11793 tcp_stack_t *tcps = tcp->tcp_tcps; 11794 uint_t ret = 0; 11795 11796 /* Can't be an eager connection */ 11797 ASSERT(tcp->tcp_listener == NULL); 11798 11799 /* Caller must have lock held */ 11800 ASSERT(MUTEX_HELD(sodp->sod_lock)); 11801 11802 /* Sodirect mode so must not be a tcp_rcv_list */ 11803 ASSERT(tcp->tcp_rcv_list == NULL); 11804 11805 if (SOD_QFULL(sodp)) { 11806 /* Q is full, mark Q for need backenable */ 11807 SOD_QSETBE(sodp); 11808 } 11809 /* Last advertised rwnd, i.e. rwnd last sent in a packet */ 11810 thwin = ((uint_t)BE16_TO_U16(tcp->tcp_tcph->th_win)) 11811 << tcp->tcp_rcv_ws; 11812 /* This is peer's calculated send window (our available rwnd). */ 11813 thwin -= tcp->tcp_rnxt - tcp->tcp_rack; 11814 /* 11815 * Increase the receive window to max. But we need to do receiver 11816 * SWS avoidance. This means that we need to check the increase of 11817 * of receive window is at least 1 MSS. 11818 */ 11819 if (!SOD_QFULL(sodp) && (q->q_hiwat - thwin >= tcp->tcp_mss)) { 11820 /* 11821 * If the window that the other side knows is less than max 11822 * deferred acks segments, send an update immediately. 11823 */ 11824 if (thwin < tcp->tcp_rack_cur_max * tcp->tcp_mss) { 11825 BUMP_MIB(&tcps->tcps_mib, tcpOutWinUpdate); 11826 ret = TH_ACK_NEEDED; 11827 } 11828 tcp->tcp_rwnd = q->q_hiwat; 11829 } 11830 11831 if (!SOD_QEMPTY(sodp)) { 11832 /* Wakeup to socket */ 11833 sodp->sod_state &= SOD_WAKE_CLR; 11834 sodp->sod_state |= SOD_WAKE_DONE; 11835 (sodp->sod_wakeup)(sodp); 11836 /* wakeup() does the mutex_ext() */ 11837 } else { 11838 /* Q is empty, no need to wake */ 11839 sodp->sod_state &= SOD_WAKE_CLR; 11840 sodp->sod_state |= SOD_WAKE_NOT; 11841 mutex_exit(sodp->sod_lock); 11842 } 11843 11844 /* No need for the push timer now. */ 11845 if (tcp->tcp_push_tid != 0) { 11846 (void) TCP_TIMER_CANCEL(tcp, tcp->tcp_push_tid); 11847 tcp->tcp_push_tid = 0; 11848 } 11849 11850 return (ret); 11851 } 11852 11853 /* 11854 * Called where tcp_rcv_enqueue()/putnext(RD(q)) would be. For M_DATA 11855 * mblk_t's if uioa enabled then start a uioa asynchronous copy directly 11856 * to the user-land buffer and flag the mblk_t as such. 11857 * 11858 * Also, handle tcp_rwnd. 11859 */ 11860 uint_t 11861 tcp_rcv_sod_enqueue(tcp_t *tcp, sodirect_t *sodp, mblk_t *mp, uint_t seg_len) 11862 { 11863 uioa_t *uioap = &sodp->sod_uioa; 11864 boolean_t qfull; 11865 uint_t thwin; 11866 11867 /* Can't be an eager connection */ 11868 ASSERT(tcp->tcp_listener == NULL); 11869 11870 /* Caller must have lock held */ 11871 ASSERT(MUTEX_HELD(sodp->sod_lock)); 11872 11873 /* Sodirect mode so must not be a tcp_rcv_list */ 11874 ASSERT(tcp->tcp_rcv_list == NULL); 11875 11876 /* Passed in segment length must be equal to mblk_t chain data size */ 11877 ASSERT(seg_len == msgdsize(mp)); 11878 11879 if (DB_TYPE(mp) != M_DATA) { 11880 /* Only process M_DATA mblk_t's */ 11881 goto enq; 11882 } 11883 if (uioap->uioa_state & UIOA_ENABLED) { 11884 /* Uioa is enabled */ 11885 mblk_t *mp1 = mp; 11886 11887 if (seg_len > uioap->uio_resid) { 11888 /* 11889 * There isn't enough uio space for the mblk_t chain 11890 * so disable uioa such that this and any additional 11891 * mblk_t data is handled by the socket and schedule 11892 * the socket for wakeup to finish this uioa. 11893 */ 11894 uioap->uioa_state &= UIOA_CLR; 11895 uioap->uioa_state |= UIOA_FINI; 11896 if (sodp->sod_state & SOD_WAKE_NOT) { 11897 sodp->sod_state &= SOD_WAKE_CLR; 11898 sodp->sod_state |= SOD_WAKE_NEED; 11899 } 11900 goto enq; 11901 } 11902 do { 11903 uint32_t len = MBLKL(mp1); 11904 11905 if (!uioamove(mp1->b_rptr, len, UIO_READ, uioap)) { 11906 /* Scheduled, mark dblk_t as such */ 11907 DB_FLAGS(mp1) |= DBLK_UIOA; 11908 } else { 11909 /* Error, turn off async processing */ 11910 uioap->uioa_state &= UIOA_CLR; 11911 uioap->uioa_state |= UIOA_FINI; 11912 break; 11913 } 11914 } while ((mp1 = mp1->b_cont) != NULL); 11915 11916 if (mp1 != NULL || uioap->uio_resid == 0) { 11917 /* 11918 * Not all mblk_t(s) uioamoved (error) or all uio 11919 * space has been consumed so schedule the socket 11920 * for wakeup to finish this uio. 11921 */ 11922 sodp->sod_state &= SOD_WAKE_CLR; 11923 sodp->sod_state |= SOD_WAKE_NEED; 11924 } 11925 } else if (uioap->uioa_state & UIOA_FINI) { 11926 /* 11927 * Post UIO_ENABLED waiting for socket to finish processing 11928 * so just enqueue and update tcp_rwnd. 11929 */ 11930 if (SOD_QFULL(sodp)) 11931 tcp->tcp_rwnd -= seg_len; 11932 } else if (sodp->sod_want > 0) { 11933 /* 11934 * Uioa isn't enabled but sodirect has a pending read(). 11935 */ 11936 if (SOD_QCNT(sodp) + seg_len >= sodp->sod_want) { 11937 if (sodp->sod_state & SOD_WAKE_NOT) { 11938 /* Schedule socket for wakeup */ 11939 sodp->sod_state &= SOD_WAKE_CLR; 11940 sodp->sod_state |= SOD_WAKE_NEED; 11941 } 11942 tcp->tcp_rwnd -= seg_len; 11943 } 11944 } else if (SOD_QCNT(sodp) + seg_len >= tcp->tcp_rq->q_hiwat >> 3) { 11945 /* 11946 * No pending sodirect read() so used the default 11947 * TCP push logic to guess that a push is needed. 11948 */ 11949 if (sodp->sod_state & SOD_WAKE_NOT) { 11950 /* Schedule socket for wakeup */ 11951 sodp->sod_state &= SOD_WAKE_CLR; 11952 sodp->sod_state |= SOD_WAKE_NEED; 11953 } 11954 tcp->tcp_rwnd -= seg_len; 11955 } else { 11956 /* Just update tcp_rwnd */ 11957 tcp->tcp_rwnd -= seg_len; 11958 } 11959 enq: 11960 qfull = SOD_QFULL(sodp); 11961 11962 (sodp->sod_enqueue)(sodp, mp); 11963 11964 if (! qfull && SOD_QFULL(sodp)) { 11965 /* Wasn't QFULL, now QFULL, need back-enable */ 11966 SOD_QSETBE(sodp); 11967 } 11968 11969 /* 11970 * Check to see if remote avail swnd < mss due to delayed ACK, 11971 * first get advertised rwnd. 11972 */ 11973 thwin = ((uint_t)BE16_TO_U16(tcp->tcp_tcph->th_win)); 11974 /* Minus delayed ACK count */ 11975 thwin -= tcp->tcp_rnxt - tcp->tcp_rack; 11976 if (thwin < tcp->tcp_mss) { 11977 /* Remote avail swnd < mss, need ACK now */ 11978 return (TH_ACK_NEEDED); 11979 } 11980 11981 return (0); 11982 } 11983 11984 /* 11985 * DEFAULT TCP ENTRY POINT via squeue on READ side. 11986 * 11987 * This is the default entry function into TCP on the read side. TCP is 11988 * always entered via squeue i.e. using squeue's for mutual exclusion. 11989 * When classifier does a lookup to find the tcp, it also puts a reference 11990 * on the conn structure associated so the tcp is guaranteed to exist 11991 * when we come here. We still need to check the state because it might 11992 * as well has been closed. The squeue processing function i.e. squeue_enter, 11993 * squeue_enter_nodrain, or squeue_drain is responsible for doing the 11994 * CONN_DEC_REF. 11995 * 11996 * Apart from the default entry point, IP also sends packets directly to 11997 * tcp_rput_data for AF_INET fast path and tcp_conn_request for incoming 11998 * connections. 11999 */ 12000 void 12001 tcp_input(void *arg, mblk_t *mp, void *arg2) 12002 { 12003 conn_t *connp = (conn_t *)arg; 12004 tcp_t *tcp = (tcp_t *)connp->conn_tcp; 12005 12006 /* arg2 is the sqp */ 12007 ASSERT(arg2 != NULL); 12008 ASSERT(mp != NULL); 12009 12010 /* 12011 * Don't accept any input on a closed tcp as this TCP logically does 12012 * not exist on the system. Don't proceed further with this TCP. 12013 * For eg. this packet could trigger another close of this tcp 12014 * which would be disastrous for tcp_refcnt. tcp_close_detached / 12015 * tcp_clean_death / tcp_closei_local must be called at most once 12016 * on a TCP. In this case we need to refeed the packet into the 12017 * classifier and figure out where the packet should go. Need to 12018 * preserve the recv_ill somehow. Until we figure that out, for 12019 * now just drop the packet if we can't classify the packet. 12020 */ 12021 if (tcp->tcp_state == TCPS_CLOSED || 12022 tcp->tcp_state == TCPS_BOUND) { 12023 conn_t *new_connp; 12024 ip_stack_t *ipst = tcp->tcp_tcps->tcps_netstack->netstack_ip; 12025 12026 new_connp = ipcl_classify(mp, connp->conn_zoneid, ipst); 12027 if (new_connp != NULL) { 12028 tcp_reinput(new_connp, mp, arg2); 12029 return; 12030 } 12031 /* We failed to classify. For now just drop the packet */ 12032 freemsg(mp); 12033 return; 12034 } 12035 12036 if (DB_TYPE(mp) == M_DATA) 12037 tcp_rput_data(connp, mp, arg2); 12038 else 12039 tcp_rput_common(tcp, mp); 12040 } 12041 12042 /* 12043 * The read side put procedure. 12044 * The packets passed up by ip are assume to be aligned according to 12045 * OK_32PTR and the IP+TCP headers fitting in the first mblk. 12046 */ 12047 static void 12048 tcp_rput_common(tcp_t *tcp, mblk_t *mp) 12049 { 12050 /* 12051 * tcp_rput_data() does not expect M_CTL except for the case 12052 * where tcp_ipv6_recvancillary is set and we get a IN_PKTINFO 12053 * type. Need to make sure that any other M_CTLs don't make 12054 * it to tcp_rput_data since it is not expecting any and doesn't 12055 * check for it. 12056 */ 12057 if (DB_TYPE(mp) == M_CTL) { 12058 switch (*(uint32_t *)(mp->b_rptr)) { 12059 case TCP_IOC_ABORT_CONN: 12060 /* 12061 * Handle connection abort request. 12062 */ 12063 tcp_ioctl_abort_handler(tcp, mp); 12064 return; 12065 case IPSEC_IN: 12066 /* 12067 * Only secure icmp arrive in TCP and they 12068 * don't go through data path. 12069 */ 12070 tcp_icmp_error(tcp, mp); 12071 return; 12072 case IN_PKTINFO: 12073 /* 12074 * Handle IPV6_RECVPKTINFO socket option on AF_INET6 12075 * sockets that are receiving IPv4 traffic. tcp 12076 */ 12077 ASSERT(tcp->tcp_family == AF_INET6); 12078 ASSERT(tcp->tcp_ipv6_recvancillary & 12079 TCP_IPV6_RECVPKTINFO); 12080 tcp_rput_data(tcp->tcp_connp, mp, 12081 tcp->tcp_connp->conn_sqp); 12082 return; 12083 case MDT_IOC_INFO_UPDATE: 12084 /* 12085 * Handle Multidata information update; the 12086 * following routine will free the message. 12087 */ 12088 if (tcp->tcp_connp->conn_mdt_ok) { 12089 tcp_mdt_update(tcp, 12090 &((ip_mdt_info_t *)mp->b_rptr)->mdt_capab, 12091 B_FALSE); 12092 } 12093 freemsg(mp); 12094 return; 12095 case LSO_IOC_INFO_UPDATE: 12096 /* 12097 * Handle LSO information update; the following 12098 * routine will free the message. 12099 */ 12100 if (tcp->tcp_connp->conn_lso_ok) { 12101 tcp_lso_update(tcp, 12102 &((ip_lso_info_t *)mp->b_rptr)->lso_capab); 12103 } 12104 freemsg(mp); 12105 return; 12106 default: 12107 /* 12108 * tcp_icmp_err() will process the M_CTL packets. 12109 * Non-ICMP packets, if any, will be discarded in 12110 * tcp_icmp_err(). We will process the ICMP packet 12111 * even if we are TCP_IS_DETACHED_NONEAGER as the 12112 * incoming ICMP packet may result in changing 12113 * the tcp_mss, which we would need if we have 12114 * packets to retransmit. 12115 */ 12116 tcp_icmp_error(tcp, mp); 12117 return; 12118 } 12119 } 12120 12121 /* No point processing the message if tcp is already closed */ 12122 if (TCP_IS_DETACHED_NONEAGER(tcp)) { 12123 freemsg(mp); 12124 return; 12125 } 12126 12127 tcp_rput_other(tcp, mp); 12128 } 12129 12130 12131 /* The minimum of smoothed mean deviation in RTO calculation. */ 12132 #define TCP_SD_MIN 400 12133 12134 /* 12135 * Set RTO for this connection. The formula is from Jacobson and Karels' 12136 * "Congestion Avoidance and Control" in SIGCOMM '88. The variable names 12137 * are the same as those in Appendix A.2 of that paper. 12138 * 12139 * m = new measurement 12140 * sa = smoothed RTT average (8 * average estimates). 12141 * sv = smoothed mean deviation (mdev) of RTT (4 * deviation estimates). 12142 */ 12143 static void 12144 tcp_set_rto(tcp_t *tcp, clock_t rtt) 12145 { 12146 long m = TICK_TO_MSEC(rtt); 12147 clock_t sa = tcp->tcp_rtt_sa; 12148 clock_t sv = tcp->tcp_rtt_sd; 12149 clock_t rto; 12150 tcp_stack_t *tcps = tcp->tcp_tcps; 12151 12152 BUMP_MIB(&tcps->tcps_mib, tcpRttUpdate); 12153 tcp->tcp_rtt_update++; 12154 12155 /* tcp_rtt_sa is not 0 means this is a new sample. */ 12156 if (sa != 0) { 12157 /* 12158 * Update average estimator: 12159 * new rtt = 7/8 old rtt + 1/8 Error 12160 */ 12161 12162 /* m is now Error in estimate. */ 12163 m -= sa >> 3; 12164 if ((sa += m) <= 0) { 12165 /* 12166 * Don't allow the smoothed average to be negative. 12167 * We use 0 to denote reinitialization of the 12168 * variables. 12169 */ 12170 sa = 1; 12171 } 12172 12173 /* 12174 * Update deviation estimator: 12175 * new mdev = 3/4 old mdev + 1/4 (abs(Error) - old mdev) 12176 */ 12177 if (m < 0) 12178 m = -m; 12179 m -= sv >> 2; 12180 sv += m; 12181 } else { 12182 /* 12183 * This follows BSD's implementation. So the reinitialized 12184 * RTO is 3 * m. We cannot go less than 2 because if the 12185 * link is bandwidth dominated, doubling the window size 12186 * during slow start means doubling the RTT. We want to be 12187 * more conservative when we reinitialize our estimates. 3 12188 * is just a convenient number. 12189 */ 12190 sa = m << 3; 12191 sv = m << 1; 12192 } 12193 if (sv < TCP_SD_MIN) { 12194 /* 12195 * We do not know that if sa captures the delay ACK 12196 * effect as in a long train of segments, a receiver 12197 * does not delay its ACKs. So set the minimum of sv 12198 * to be TCP_SD_MIN, which is default to 400 ms, twice 12199 * of BSD DATO. That means the minimum of mean 12200 * deviation is 100 ms. 12201 * 12202 */ 12203 sv = TCP_SD_MIN; 12204 } 12205 tcp->tcp_rtt_sa = sa; 12206 tcp->tcp_rtt_sd = sv; 12207 /* 12208 * RTO = average estimates (sa / 8) + 4 * deviation estimates (sv) 12209 * 12210 * Add tcp_rexmit_interval extra in case of extreme environment 12211 * where the algorithm fails to work. The default value of 12212 * tcp_rexmit_interval_extra should be 0. 12213 * 12214 * As we use a finer grained clock than BSD and update 12215 * RTO for every ACKs, add in another .25 of RTT to the 12216 * deviation of RTO to accomodate burstiness of 1/4 of 12217 * window size. 12218 */ 12219 rto = (sa >> 3) + sv + tcps->tcps_rexmit_interval_extra + (sa >> 5); 12220 12221 if (rto > tcps->tcps_rexmit_interval_max) { 12222 tcp->tcp_rto = tcps->tcps_rexmit_interval_max; 12223 } else if (rto < tcps->tcps_rexmit_interval_min) { 12224 tcp->tcp_rto = tcps->tcps_rexmit_interval_min; 12225 } else { 12226 tcp->tcp_rto = rto; 12227 } 12228 12229 /* Now, we can reset tcp_timer_backoff to use the new RTO... */ 12230 tcp->tcp_timer_backoff = 0; 12231 } 12232 12233 /* 12234 * tcp_get_seg_mp() is called to get the pointer to a segment in the 12235 * send queue which starts at the given seq. no. 12236 * 12237 * Parameters: 12238 * tcp_t *tcp: the tcp instance pointer. 12239 * uint32_t seq: the starting seq. no of the requested segment. 12240 * int32_t *off: after the execution, *off will be the offset to 12241 * the returned mblk which points to the requested seq no. 12242 * It is the caller's responsibility to send in a non-null off. 12243 * 12244 * Return: 12245 * A mblk_t pointer pointing to the requested segment in send queue. 12246 */ 12247 static mblk_t * 12248 tcp_get_seg_mp(tcp_t *tcp, uint32_t seq, int32_t *off) 12249 { 12250 int32_t cnt; 12251 mblk_t *mp; 12252 12253 /* Defensive coding. Make sure we don't send incorrect data. */ 12254 if (SEQ_LT(seq, tcp->tcp_suna) || SEQ_GEQ(seq, tcp->tcp_snxt)) 12255 return (NULL); 12256 12257 cnt = seq - tcp->tcp_suna; 12258 mp = tcp->tcp_xmit_head; 12259 while (cnt > 0 && mp != NULL) { 12260 cnt -= mp->b_wptr - mp->b_rptr; 12261 if (cnt < 0) { 12262 cnt += mp->b_wptr - mp->b_rptr; 12263 break; 12264 } 12265 mp = mp->b_cont; 12266 } 12267 ASSERT(mp != NULL); 12268 *off = cnt; 12269 return (mp); 12270 } 12271 12272 /* 12273 * This function handles all retransmissions if SACK is enabled for this 12274 * connection. First it calculates how many segments can be retransmitted 12275 * based on tcp_pipe. Then it goes thru the notsack list to find eligible 12276 * segments. A segment is eligible if sack_cnt for that segment is greater 12277 * than or equal tcp_dupack_fast_retransmit. After it has retransmitted 12278 * all eligible segments, it checks to see if TCP can send some new segments 12279 * (fast recovery). If it can, set the appropriate flag for tcp_rput_data(). 12280 * 12281 * Parameters: 12282 * tcp_t *tcp: the tcp structure of the connection. 12283 * uint_t *flags: in return, appropriate value will be set for 12284 * tcp_rput_data(). 12285 */ 12286 static void 12287 tcp_sack_rxmit(tcp_t *tcp, uint_t *flags) 12288 { 12289 notsack_blk_t *notsack_blk; 12290 int32_t usable_swnd; 12291 int32_t mss; 12292 uint32_t seg_len; 12293 mblk_t *xmit_mp; 12294 tcp_stack_t *tcps = tcp->tcp_tcps; 12295 12296 ASSERT(tcp->tcp_sack_info != NULL); 12297 ASSERT(tcp->tcp_notsack_list != NULL); 12298 ASSERT(tcp->tcp_rexmit == B_FALSE); 12299 12300 /* Defensive coding in case there is a bug... */ 12301 if (tcp->tcp_notsack_list == NULL) { 12302 return; 12303 } 12304 notsack_blk = tcp->tcp_notsack_list; 12305 mss = tcp->tcp_mss; 12306 12307 /* 12308 * Limit the num of outstanding data in the network to be 12309 * tcp_cwnd_ssthresh, which is half of the original congestion wnd. 12310 */ 12311 usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe; 12312 12313 /* At least retransmit 1 MSS of data. */ 12314 if (usable_swnd <= 0) { 12315 usable_swnd = mss; 12316 } 12317 12318 /* Make sure no new RTT samples will be taken. */ 12319 tcp->tcp_csuna = tcp->tcp_snxt; 12320 12321 notsack_blk = tcp->tcp_notsack_list; 12322 while (usable_swnd > 0) { 12323 mblk_t *snxt_mp, *tmp_mp; 12324 tcp_seq begin = tcp->tcp_sack_snxt; 12325 tcp_seq end; 12326 int32_t off; 12327 12328 for (; notsack_blk != NULL; notsack_blk = notsack_blk->next) { 12329 if (SEQ_GT(notsack_blk->end, begin) && 12330 (notsack_blk->sack_cnt >= 12331 tcps->tcps_dupack_fast_retransmit)) { 12332 end = notsack_blk->end; 12333 if (SEQ_LT(begin, notsack_blk->begin)) { 12334 begin = notsack_blk->begin; 12335 } 12336 break; 12337 } 12338 } 12339 /* 12340 * All holes are filled. Manipulate tcp_cwnd to send more 12341 * if we can. Note that after the SACK recovery, tcp_cwnd is 12342 * set to tcp_cwnd_ssthresh. 12343 */ 12344 if (notsack_blk == NULL) { 12345 usable_swnd = tcp->tcp_cwnd_ssthresh - tcp->tcp_pipe; 12346 if (usable_swnd <= 0 || tcp->tcp_unsent == 0) { 12347 tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna; 12348 ASSERT(tcp->tcp_cwnd > 0); 12349 return; 12350 } else { 12351 usable_swnd = usable_swnd / mss; 12352 tcp->tcp_cwnd = tcp->tcp_snxt - tcp->tcp_suna + 12353 MAX(usable_swnd * mss, mss); 12354 *flags |= TH_XMIT_NEEDED; 12355 return; 12356 } 12357 } 12358 12359 /* 12360 * Note that we may send more than usable_swnd allows here 12361 * because of round off, but no more than 1 MSS of data. 12362 */ 12363 seg_len = end - begin; 12364 if (seg_len > mss) 12365 seg_len = mss; 12366 snxt_mp = tcp_get_seg_mp(tcp, begin, &off); 12367 ASSERT(snxt_mp != NULL); 12368 /* This should not happen. Defensive coding again... */ 12369 if (snxt_mp == NULL) { 12370 return; 12371 } 12372 12373 xmit_mp = tcp_xmit_mp(tcp, snxt_mp, seg_len, &off, 12374 &tmp_mp, begin, B_TRUE, &seg_len, B_TRUE); 12375 if (xmit_mp == NULL) 12376 return; 12377 12378 usable_swnd -= seg_len; 12379 tcp->tcp_pipe += seg_len; 12380 tcp->tcp_sack_snxt = begin + seg_len; 12381 12382 tcp_send_data(tcp, tcp->tcp_wq, xmit_mp); 12383 12384 /* 12385 * Update the send timestamp to avoid false retransmission. 12386 */ 12387 snxt_mp->b_prev = (mblk_t *)lbolt; 12388 12389 BUMP_MIB(&tcps->tcps_mib, tcpRetransSegs); 12390 UPDATE_MIB(&tcps->tcps_mib, tcpRetransBytes, seg_len); 12391 BUMP_MIB(&tcps->tcps_mib, tcpOutSackRetransSegs); 12392 /* 12393 * Update tcp_rexmit_max to extend this SACK recovery phase. 12394 * This happens when new data sent during fast recovery is 12395 * also lost. If TCP retransmits those new data, it needs 12396 * to extend SACK recover phase to avoid starting another 12397 * fast retransmit/recovery unnecessarily. 12398 */ 12399 if (SEQ_GT(tcp->tcp_sack_snxt, tcp->tcp_rexmit_max)) { 12400 tcp->tcp_rexmit_max = tcp->tcp_sack_snxt; 12401 } 12402 } 12403 } 12404 12405 /* 12406 * This function handles policy checking at TCP level for non-hard_bound/ 12407 * detached connections. 12408 */ 12409 static boolean_t 12410 tcp_check_policy(tcp_t *tcp, mblk_t *first_mp, ipha_t *ipha, ip6_t *ip6h, 12411 boolean_t secure, boolean_t mctl_present) 12412 { 12413 ipsec_latch_t *ipl = NULL; 12414 ipsec_action_t *act = NULL; 12415 mblk_t *data_mp; 12416 ipsec_in_t *ii; 12417 const char *reason; 12418 kstat_named_t *counter; 12419 tcp_stack_t *tcps = tcp->tcp_tcps; 12420 ipsec_stack_t *ipss; 12421 ip_stack_t *ipst; 12422 12423 ASSERT(mctl_present || !secure); 12424 12425 ASSERT((ipha == NULL && ip6h != NULL) || 12426 (ip6h == NULL && ipha != NULL)); 12427 12428 /* 12429 * We don't necessarily have an ipsec_in_act action to verify 12430 * policy because of assymetrical policy where we have only 12431 * outbound policy and no inbound policy (possible with global 12432 * policy). 12433 */ 12434 if (!secure) { 12435 if (act == NULL || act->ipa_act.ipa_type == IPSEC_ACT_BYPASS || 12436 act->ipa_act.ipa_type == IPSEC_ACT_CLEAR) 12437 return (B_TRUE); 12438 ipsec_log_policy_failure(IPSEC_POLICY_MISMATCH, 12439 "tcp_check_policy", ipha, ip6h, secure, 12440 tcps->tcps_netstack); 12441 ipss = tcps->tcps_netstack->netstack_ipsec; 12442 12443 ip_drop_packet(first_mp, B_TRUE, NULL, NULL, 12444 DROPPER(ipss, ipds_tcp_clear), 12445 &tcps->tcps_dropper); 12446 return (B_FALSE); 12447 } 12448 12449 /* 12450 * We have a secure packet. 12451 */ 12452 if (act == NULL) { 12453 ipsec_log_policy_failure(IPSEC_POLICY_NOT_NEEDED, 12454 "tcp_check_policy", ipha, ip6h, secure, 12455 tcps->tcps_netstack); 12456 ipss = tcps->tcps_netstack->netstack_ipsec; 12457 12458 ip_drop_packet(first_mp, B_TRUE, NULL, NULL, 12459 DROPPER(ipss, ipds_tcp_secure), 12460 &tcps->tcps_dropper); 12461 return (B_FALSE); 12462 } 12463 12464 /* 12465 * XXX This whole routine is currently incorrect. ipl should 12466 * be set to the latch pointer, but is currently not set, so 12467 * we initialize it to NULL to avoid picking up random garbage. 12468 */ 12469 if (ipl == NULL) 12470 return (B_TRUE); 12471 12472 data_mp = first_mp->b_cont; 12473 12474 ii = (ipsec_in_t *)first_mp->b_rptr; 12475 12476 ipst = tcps->tcps_netstack->netstack_ip; 12477 12478 if (ipsec_check_ipsecin_latch(ii, data_mp, ipl, ipha, ip6h, &reason, 12479 &counter, tcp->tcp_connp)) { 12480 BUMP_MIB(&ipst->ips_ip_mib, ipsecInSucceeded); 12481 return (B_TRUE); 12482 } 12483 (void) strlog(TCP_MOD_ID, 0, 0, SL_ERROR|SL_WARN|SL_CONSOLE, 12484 "tcp inbound policy mismatch: %s, packet dropped\n", 12485 reason); 12486 BUMP_MIB(&ipst->ips_ip_mib, ipsecInFailed); 12487 12488 ip_drop_packet(first_mp, B_TRUE, NULL, NULL, counter, 12489 &tcps->tcps_dropper); 12490 return (B_FALSE); 12491 } 12492 12493 /* 12494 * tcp_ss_rexmit() is called in tcp_rput_data() to do slow start 12495 * retransmission after a timeout. 12496 * 12497 * To limit the number of duplicate segments, we limit the number of segment 12498 * to be sent in one time to tcp_snd_burst, the burst variable. 12499 */ 12500 static void 12501 tcp_ss_rexmit(tcp_t *tcp) 12502 { 12503 uint32_t snxt; 12504 uint32_t smax; 12505 int32_t win; 12506 int32_t mss; 12507 int32_t off; 12508 int32_t burst = tcp->tcp_snd_burst; 12509 mblk_t *snxt_mp; 12510 tcp_stack_t *tcps = tcp->tcp_tcps; 12511 12512 /* 12513 * Note that tcp_rexmit can be set even though TCP has retransmitted 12514 * all unack'ed segments. 12515 */ 12516 if (SEQ_LT(tcp->tcp_rexmit_nxt, tcp->tcp_rexmit_max)) { 12517 smax = tcp->tcp_rexmit_max; 12518 snxt = tcp->tcp_rexmit_nxt; 12519 if (SEQ_LT(snxt, tcp->tcp_suna)) { 12520 snxt = tcp->tcp_suna; 12521 } 12522 win = MIN(tcp->tcp_cwnd, tcp->tcp_swnd); 12523 win -= snxt - tcp->tcp_suna; 12524 mss = tcp->tcp_mss; 12525 snxt_mp = tcp_get_seg_mp(tcp, snxt, &off); 12526 12527 while (SEQ_LT(snxt, smax) && (win > 0) && 12528 (burst > 0) && (snxt_mp != NULL)) { 12529 mblk_t *xmit_mp; 12530 mblk_t *old_snxt_mp = snxt_mp; 12531 uint32_t cnt = mss; 12532 12533 if (win < cnt) { 12534 cnt = win; 12535 } 12536 if (SEQ_GT(snxt + cnt, smax)) { 12537 cnt = smax - snxt; 12538 } 12539 xmit_mp = tcp_xmit_mp(tcp, snxt_mp, cnt, &off, 12540 &snxt_mp, snxt, B_TRUE, &cnt, B_TRUE); 12541 if (xmit_mp == NULL) 12542 return; 12543 12544 tcp_send_data(tcp, tcp->tcp_wq, xmit_mp); 12545 12546 snxt += cnt; 12547 win -= cnt; 12548 /* 12549 * Update the send timestamp to avoid false 12550 * retransmission. 12551 */ 12552 old_snxt_mp->b_prev = (mblk_t *)lbolt; 12553 BUMP_MIB(&tcps->tcps_mib, tcpRetransSegs); 12554 UPDATE_MIB(&tcps->tcps_mib, tcpRetransBytes, cnt); 12555 12556 tcp->tcp_rexmit_nxt = snxt; 12557 burst--; 12558 } 12559 /* 12560 * If we have transmitted all we have at the time 12561 * we started the retranmission, we can leave 12562 * the rest of the job to tcp_wput_data(). But we 12563 * need to check the send window first. If the 12564 * win is not 0, go on with tcp_wput_data(). 12565 */ 12566 if (SEQ_LT(snxt, smax) || win == 0) { 12567 return; 12568 } 12569 } 12570 /* Only call tcp_wput_data() if there is data to be sent. */ 12571 if (tcp->tcp_unsent) { 12572 tcp_wput_data(tcp, NULL, B_FALSE); 12573 } 12574 } 12575 12576 /* 12577 * Process all TCP option in SYN segment. Note that this function should 12578 * be called after tcp_adapt_ire() is called so that the necessary info 12579 * from IRE is already set in the tcp structure. 12580 * 12581 * This function sets up the correct tcp_mss value according to the 12582 * MSS option value and our header size. It also sets up the window scale 12583 * and timestamp values, and initialize SACK info blocks. But it does not 12584 * change receive window size after setting the tcp_mss value. The caller 12585 * should do the appropriate change. 12586 */ 12587 void 12588 tcp_process_options(tcp_t *tcp, tcph_t *tcph) 12589 { 12590 int options; 12591 tcp_opt_t tcpopt; 12592 uint32_t mss_max; 12593 char *tmp_tcph; 12594 tcp_stack_t *tcps = tcp->tcp_tcps; 12595 12596 tcpopt.tcp = NULL; 12597 options = tcp_parse_options(tcph, &tcpopt); 12598 12599 /* 12600 * Process MSS option. Note that MSS option value does not account 12601 * for IP or TCP options. This means that it is equal to MTU - minimum 12602 * IP+TCP header size, which is 40 bytes for IPv4 and 60 bytes for 12603 * IPv6. 12604 */ 12605 if (!(options & TCP_OPT_MSS_PRESENT)) { 12606 if (tcp->tcp_ipversion == IPV4_VERSION) 12607 tcpopt.tcp_opt_mss = tcps->tcps_mss_def_ipv4; 12608 else 12609 tcpopt.tcp_opt_mss = tcps->tcps_mss_def_ipv6; 12610 } else { 12611 if (tcp->tcp_ipversion == IPV4_VERSION) 12612 mss_max = tcps->tcps_mss_max_ipv4; 12613 else 12614 mss_max = tcps->tcps_mss_max_ipv6; 12615 if (tcpopt.tcp_opt_mss < tcps->tcps_mss_min) 12616 tcpopt.tcp_opt_mss = tcps->tcps_mss_min; 12617 else if (tcpopt.tcp_opt_mss > mss_max) 12618 tcpopt.tcp_opt_mss = mss_max; 12619 } 12620 12621 /* Process Window Scale option. */ 12622 if (options & TCP_OPT_WSCALE_PRESENT) { 12623 tcp->tcp_snd_ws = tcpopt.tcp_opt_wscale; 12624 tcp->tcp_snd_ws_ok = B_TRUE; 12625 } else { 12626 tcp->tcp_snd_ws = B_FALSE; 12627 tcp->tcp_snd_ws_ok = B_FALSE; 12628 tcp->tcp_rcv_ws = B_FALSE; 12629 } 12630 12631 /* Process Timestamp option. */ 12632 if ((options & TCP_OPT_TSTAMP_PRESENT) && 12633 (tcp->tcp_snd_ts_ok || TCP_IS_DETACHED(tcp))) { 12634 tmp_tcph = (char *)tcp->tcp_tcph; 12635 12636 tcp->tcp_snd_ts_ok = B_TRUE; 12637 tcp->tcp_ts_recent = tcpopt.tcp_opt_ts_val; 12638 tcp->tcp_last_rcv_lbolt = lbolt64; 12639 ASSERT(OK_32PTR(tmp_tcph)); 12640 ASSERT(tcp->tcp_tcp_hdr_len == TCP_MIN_HEADER_LENGTH); 12641 12642 /* Fill in our template header with basic timestamp option. */ 12643 tmp_tcph += tcp->tcp_tcp_hdr_len; 12644 tmp_tcph[0] = TCPOPT_NOP; 12645 tmp_tcph[1] = TCPOPT_NOP; 12646 tmp_tcph[2] = TCPOPT_TSTAMP; 12647 tmp_tcph[3] = TCPOPT_TSTAMP_LEN; 12648 tcp->tcp_hdr_len += TCPOPT_REAL_TS_LEN; 12649 tcp->tcp_tcp_hdr_len += TCPOPT_REAL_TS_LEN; 12650 tcp->tcp_tcph->th_offset_and_rsrvd[0] += (3 << 4); 12651 } else { 12652 tcp->tcp_snd_ts_ok = B_FALSE; 12653 } 12654 12655 /* 12656 * Process SACK options. If SACK is enabled for this connection, 12657 * then allocate the SACK info structure. Note the following ways 12658 * when tcp_snd_sack_ok is set to true. 12659 * 12660 * For active connection: in tcp_adapt_ire() called in 12661 * tcp_rput_other(), or in tcp_rput_other() when tcp_sack_permitted 12662 * is checked. 12663 * 12664 * For passive connection: in tcp_adapt_ire() called in 12665 * tcp_accept_comm(). 12666 * 12667 * That's the reason why the extra TCP_IS_DETACHED() check is there. 12668 * That check makes sure that if we did not send a SACK OK option, 12669 * we will not enable SACK for this connection even though the other 12670 * side sends us SACK OK option. For active connection, the SACK 12671 * info structure has already been allocated. So we need to free 12672 * it if SACK is disabled. 12673 */ 12674 if ((options & TCP_OPT_SACK_OK_PRESENT) && 12675 (tcp->tcp_snd_sack_ok || 12676 (tcps->tcps_sack_permitted != 0 && TCP_IS_DETACHED(tcp)))) { 12677 /* This should be true only in the passive case. */ 12678 if (tcp->tcp_sack_info == NULL) { 12679 ASSERT(TCP_IS_DETACHED(tcp)); 12680 tcp->tcp_sack_info = 12681 kmem_cache_alloc(tcp_sack_info_cache, KM_NOSLEEP); 12682 } 12683 if (tcp->tcp_sack_info == NULL) { 12684 tcp->tcp_snd_sack_ok = B_FALSE; 12685 } else { 12686 tcp->tcp_snd_sack_ok = B_TRUE; 12687 if (tcp->tcp_snd_ts_ok) { 12688 tcp->tcp_max_sack_blk = 3; 12689 } else { 12690 tcp->tcp_max_sack_blk = 4; 12691 } 12692 } 12693 } else { 12694 /* 12695 * Resetting tcp_snd_sack_ok to B_FALSE so that 12696 * no SACK info will be used for this 12697 * connection. This assumes that SACK usage 12698 * permission is negotiated. This may need 12699 * to be changed once this is clarified. 12700 */ 12701 if (tcp->tcp_sack_info != NULL) { 12702 ASSERT(tcp->tcp_notsack_list == NULL); 12703 kmem_cache_free(tcp_sack_info_cache, 12704 tcp->tcp_sack_info); 12705 tcp->tcp_sack_info = NULL; 12706 } 12707 tcp->tcp_snd_sack_ok = B_FALSE; 12708 } 12709 12710 /* 12711 * Now we know the exact TCP/IP header length, subtract 12712 * that from tcp_mss to get our side's MSS. 12713 */ 12714 tcp->tcp_mss -= tcp->tcp_hdr_len; 12715 /* 12716 * Here we assume that the other side's header size will be equal to 12717 * our header size. We calculate the real MSS accordingly. Need to 12718 * take into additional stuffs IPsec puts in. 12719 * 12720 * Real MSS = Opt.MSS - (our TCP/IP header - min TCP/IP header) 12721 */ 12722 tcpopt.tcp_opt_mss -= tcp->tcp_hdr_len + tcp->tcp_ipsec_overhead - 12723 ((tcp->tcp_ipversion == IPV4_VERSION ? 12724 IP_SIMPLE_HDR_LENGTH : IPV6_HDR_LEN) + TCP_MIN_HEADER_LENGTH); 12725 12726 /* 12727 * Set MSS to the smaller one of both ends of the connection. 12728 * We should not have called tcp_mss_set() before, but our 12729 * side of the MSS should have been set to a proper value 12730 * by tcp_adapt_ire(). tcp_mss_set() will also set up the 12731 * STREAM head parameters properly. 12732 * 12733 * If we have a larger-than-16-bit window but the other side 12734 * didn't want to do window scale, tcp_rwnd_set() will take 12735 * care of that. 12736 */ 12737 tcp_mss_set(tcp, MIN(tcpopt.tcp_opt_mss, tcp->tcp_mss), B_TRUE); 12738 } 12739 12740 /* 12741 * Sends the T_CONN_IND to the listener. The caller calls this 12742 * functions via squeue to get inside the listener's perimeter 12743 * once the 3 way hand shake is done a T_CONN_IND needs to be 12744 * sent. As an optimization, the caller can call this directly 12745 * if listener's perimeter is same as eager's. 12746 */ 12747 /* ARGSUSED */ 12748 void 12749 tcp_send_conn_ind(void *arg, mblk_t *mp, void *arg2) 12750 { 12751 conn_t *lconnp = (conn_t *)arg; 12752 tcp_t *listener = lconnp->conn_tcp; 12753 tcp_t *tcp; 12754 struct T_conn_ind *conn_ind; 12755 ipaddr_t *addr_cache; 12756 boolean_t need_send_conn_ind = B_FALSE; 12757 tcp_stack_t *tcps = listener->tcp_tcps; 12758 12759 /* retrieve the eager */ 12760 conn_ind = (struct T_conn_ind *)mp->b_rptr; 12761 ASSERT(conn_ind->OPT_offset != 0 && 12762 conn_ind->OPT_length == sizeof (intptr_t)); 12763 bcopy(mp->b_rptr + conn_ind->OPT_offset, &tcp, 12764 conn_ind->OPT_length); 12765 12766 /* 12767 * TLI/XTI applications will get confused by 12768 * sending eager as an option since it violates 12769 * the option semantics. So remove the eager as 12770 * option since TLI/XTI app doesn't need it anyway. 12771 */ 12772 if (!TCP_IS_SOCKET(listener)) { 12773 conn_ind->OPT_length = 0; 12774 conn_ind->OPT_offset = 0; 12775 } 12776 if (listener->tcp_state == TCPS_CLOSED || 12777 TCP_IS_DETACHED(listener)) { 12778 /* 12779 * If listener has closed, it would have caused a 12780 * a cleanup/blowoff to happen for the eager. We 12781 * just need to return. 12782 */ 12783 freemsg(mp); 12784 return; 12785 } 12786 12787 12788 /* 12789 * if the conn_req_q is full defer passing up the 12790 * T_CONN_IND until space is availabe after t_accept() 12791 * processing 12792 */ 12793 mutex_enter(&listener->tcp_eager_lock); 12794 12795 /* 12796 * Take the eager out, if it is in the list of droppable eagers 12797 * as we are here because the 3W handshake is over. 12798 */ 12799 MAKE_UNDROPPABLE(tcp); 12800 12801 if (listener->tcp_conn_req_cnt_q < listener->tcp_conn_req_max) { 12802 tcp_t *tail; 12803 12804 /* 12805 * The eager already has an extra ref put in tcp_rput_data 12806 * so that it stays till accept comes back even though it 12807 * might get into TCPS_CLOSED as a result of a TH_RST etc. 12808 */ 12809 ASSERT(listener->tcp_conn_req_cnt_q0 > 0); 12810 listener->tcp_conn_req_cnt_q0--; 12811 listener->tcp_conn_req_cnt_q++; 12812 12813 /* Move from SYN_RCVD to ESTABLISHED list */ 12814 tcp->tcp_eager_next_q0->tcp_eager_prev_q0 = 12815 tcp->tcp_eager_prev_q0; 12816 tcp->tcp_eager_prev_q0->tcp_eager_next_q0 = 12817 tcp->tcp_eager_next_q0; 12818 tcp->tcp_eager_prev_q0 = NULL; 12819 tcp->tcp_eager_next_q0 = NULL; 12820 12821 /* 12822 * Insert at end of the queue because sockfs 12823 * sends down T_CONN_RES in chronological 12824 * order. Leaving the older conn indications 12825 * at front of the queue helps reducing search 12826 * time. 12827 */ 12828 tail = listener->tcp_eager_last_q; 12829 if (tail != NULL) 12830 tail->tcp_eager_next_q = tcp; 12831 else 12832 listener->tcp_eager_next_q = tcp; 12833 listener->tcp_eager_last_q = tcp; 12834 tcp->tcp_eager_next_q = NULL; 12835 /* 12836 * Delay sending up the T_conn_ind until we are 12837 * done with the eager. Once we have have sent up 12838 * the T_conn_ind, the accept can potentially complete 12839 * any time and release the refhold we have on the eager. 12840 */ 12841 need_send_conn_ind = B_TRUE; 12842 } else { 12843 /* 12844 * Defer connection on q0 and set deferred 12845 * connection bit true 12846 */ 12847 tcp->tcp_conn_def_q0 = B_TRUE; 12848 12849 /* take tcp out of q0 ... */ 12850 tcp->tcp_eager_prev_q0->tcp_eager_next_q0 = 12851 tcp->tcp_eager_next_q0; 12852 tcp->tcp_eager_next_q0->tcp_eager_prev_q0 = 12853 tcp->tcp_eager_prev_q0; 12854 12855 /* ... and place it at the end of q0 */ 12856 tcp->tcp_eager_prev_q0 = listener->tcp_eager_prev_q0; 12857 tcp->tcp_eager_next_q0 = listener; 12858 listener->tcp_eager_prev_q0->tcp_eager_next_q0 = tcp; 12859 listener->tcp_eager_prev_q0 = tcp; 12860 tcp->tcp_conn.tcp_eager_conn_ind = mp; 12861 } 12862 12863 /* we have timed out before */ 12864 if (tcp->tcp_syn_rcvd_timeout != 0) { 12865 tcp->tcp_syn_rcvd_timeout = 0; 12866 listener->tcp_syn_rcvd_timeout--; 12867 if (listener->tcp_syn_defense && 12868 listener->tcp_syn_rcvd_timeout <= 12869 (tcps->tcps_conn_req_max_q0 >> 5) && 12870 10*MINUTES < TICK_TO_MSEC(lbolt64 - 12871 listener->tcp_last_rcv_lbolt)) { 12872 /* 12873 * Turn off the defense mode if we 12874 * believe the SYN attack is over. 12875 */ 12876 listener->tcp_syn_defense = B_FALSE; 12877 if (listener->tcp_ip_addr_cache) { 12878 kmem_free((void *)listener->tcp_ip_addr_cache, 12879 IP_ADDR_CACHE_SIZE * sizeof (ipaddr_t)); 12880 listener->tcp_ip_addr_cache = NULL; 12881 } 12882 } 12883 } 12884 addr_cache = (ipaddr_t *)(listener->tcp_ip_addr_cache); 12885 if (addr_cache != NULL) { 12886 /* 12887 * We have finished a 3-way handshake with this 12888 * remote host. This proves the IP addr is good. 12889 * Cache it! 12890 */ 12891 addr_cache[IP_ADDR_CACHE_HASH( 12892 tcp->tcp_remote)] = tcp->tcp_remote; 12893 } 12894 mutex_exit(&listener->tcp_eager_lock); 12895 if (need_send_conn_ind) 12896 putnext(listener->tcp_rq, mp); 12897 } 12898 12899 mblk_t * 12900 tcp_find_pktinfo(tcp_t *tcp, mblk_t *mp, uint_t *ipversp, uint_t *ip_hdr_lenp, 12901 uint_t *ifindexp, ip6_pkt_t *ippp) 12902 { 12903 ip_pktinfo_t *pinfo; 12904 ip6_t *ip6h; 12905 uchar_t *rptr; 12906 mblk_t *first_mp = mp; 12907 boolean_t mctl_present = B_FALSE; 12908 uint_t ifindex = 0; 12909 ip6_pkt_t ipp; 12910 uint_t ipvers; 12911 uint_t ip_hdr_len; 12912 tcp_stack_t *tcps = tcp->tcp_tcps; 12913 12914 rptr = mp->b_rptr; 12915 ASSERT(OK_32PTR(rptr)); 12916 ASSERT(tcp != NULL); 12917 ipp.ipp_fields = 0; 12918 12919 switch DB_TYPE(mp) { 12920 case M_CTL: 12921 mp = mp->b_cont; 12922 if (mp == NULL) { 12923 freemsg(first_mp); 12924 return (NULL); 12925 } 12926 if (DB_TYPE(mp) != M_DATA) { 12927 freemsg(first_mp); 12928 return (NULL); 12929 } 12930 mctl_present = B_TRUE; 12931 break; 12932 case M_DATA: 12933 break; 12934 default: 12935 cmn_err(CE_NOTE, "tcp_find_pktinfo: unknown db_type"); 12936 freemsg(mp); 12937 return (NULL); 12938 } 12939 ipvers = IPH_HDR_VERSION(rptr); 12940 if (ipvers == IPV4_VERSION) { 12941 if (tcp == NULL) { 12942 ip_hdr_len = IPH_HDR_LENGTH(rptr); 12943 goto done; 12944 } 12945 12946 ipp.ipp_fields |= IPPF_HOPLIMIT; 12947 ipp.ipp_hoplimit = ((ipha_t *)rptr)->ipha_ttl; 12948 12949 /* 12950 * If we have IN_PKTINFO in an M_CTL and tcp_ipv6_recvancillary 12951 * has TCP_IPV6_RECVPKTINFO set, pass I/F index along in ipp. 12952 */ 12953 if ((tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVPKTINFO) && 12954 mctl_present) { 12955 pinfo = (ip_pktinfo_t *)first_mp->b_rptr; 12956 if ((MBLKL(first_mp) == sizeof (ip_pktinfo_t)) && 12957 (pinfo->ip_pkt_ulp_type == IN_PKTINFO) && 12958 (pinfo->ip_pkt_flags & IPF_RECVIF)) { 12959 ipp.ipp_fields |= IPPF_IFINDEX; 12960 ipp.ipp_ifindex = pinfo->ip_pkt_ifindex; 12961 ifindex = pinfo->ip_pkt_ifindex; 12962 } 12963 freeb(first_mp); 12964 mctl_present = B_FALSE; 12965 } 12966 ip_hdr_len = IPH_HDR_LENGTH(rptr); 12967 } else { 12968 ip6h = (ip6_t *)rptr; 12969 12970 ASSERT(ipvers == IPV6_VERSION); 12971 ipp.ipp_fields = IPPF_HOPLIMIT | IPPF_TCLASS; 12972 ipp.ipp_tclass = (ip6h->ip6_flow & 0x0FF00000) >> 20; 12973 ipp.ipp_hoplimit = ip6h->ip6_hops; 12974 12975 if (ip6h->ip6_nxt != IPPROTO_TCP) { 12976 uint8_t nexthdrp; 12977 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 12978 12979 /* Look for ifindex information */ 12980 if (ip6h->ip6_nxt == IPPROTO_RAW) { 12981 ip6i_t *ip6i = (ip6i_t *)ip6h; 12982 if ((uchar_t *)&ip6i[1] > mp->b_wptr) { 12983 BUMP_MIB(&ipst->ips_ip_mib, tcpInErrs); 12984 freemsg(first_mp); 12985 return (NULL); 12986 } 12987 12988 if (ip6i->ip6i_flags & IP6I_IFINDEX) { 12989 ASSERT(ip6i->ip6i_ifindex != 0); 12990 ipp.ipp_fields |= IPPF_IFINDEX; 12991 ipp.ipp_ifindex = ip6i->ip6i_ifindex; 12992 ifindex = ip6i->ip6i_ifindex; 12993 } 12994 rptr = (uchar_t *)&ip6i[1]; 12995 mp->b_rptr = rptr; 12996 if (rptr == mp->b_wptr) { 12997 mblk_t *mp1; 12998 mp1 = mp->b_cont; 12999 freeb(mp); 13000 mp = mp1; 13001 rptr = mp->b_rptr; 13002 } 13003 if (MBLKL(mp) < IPV6_HDR_LEN + 13004 sizeof (tcph_t)) { 13005 BUMP_MIB(&ipst->ips_ip_mib, tcpInErrs); 13006 freemsg(first_mp); 13007 return (NULL); 13008 } 13009 ip6h = (ip6_t *)rptr; 13010 } 13011 13012 /* 13013 * Find any potentially interesting extension headers 13014 * as well as the length of the IPv6 + extension 13015 * headers. 13016 */ 13017 ip_hdr_len = ip_find_hdr_v6(mp, ip6h, &ipp, &nexthdrp); 13018 /* Verify if this is a TCP packet */ 13019 if (nexthdrp != IPPROTO_TCP) { 13020 BUMP_MIB(&ipst->ips_ip_mib, tcpInErrs); 13021 freemsg(first_mp); 13022 return (NULL); 13023 } 13024 } else { 13025 ip_hdr_len = IPV6_HDR_LEN; 13026 } 13027 } 13028 13029 done: 13030 if (ipversp != NULL) 13031 *ipversp = ipvers; 13032 if (ip_hdr_lenp != NULL) 13033 *ip_hdr_lenp = ip_hdr_len; 13034 if (ippp != NULL) 13035 *ippp = ipp; 13036 if (ifindexp != NULL) 13037 *ifindexp = ifindex; 13038 if (mctl_present) { 13039 freeb(first_mp); 13040 } 13041 return (mp); 13042 } 13043 13044 /* 13045 * Handle M_DATA messages from IP. Its called directly from IP via 13046 * squeue for AF_INET type sockets fast path. No M_CTL are expected 13047 * in this path. 13048 * 13049 * For everything else (including AF_INET6 sockets with 'tcp_ipversion' 13050 * v4 and v6), we are called through tcp_input() and a M_CTL can 13051 * be present for options but tcp_find_pktinfo() deals with it. We 13052 * only expect M_DATA packets after tcp_find_pktinfo() is done. 13053 * 13054 * The first argument is always the connp/tcp to which the mp belongs. 13055 * There are no exceptions to this rule. The caller has already put 13056 * a reference on this connp/tcp and once tcp_rput_data() returns, 13057 * the squeue will do the refrele. 13058 * 13059 * The TH_SYN for the listener directly go to tcp_conn_request via 13060 * squeue. 13061 * 13062 * sqp: NULL = recursive, sqp != NULL means called from squeue 13063 */ 13064 void 13065 tcp_rput_data(void *arg, mblk_t *mp, void *arg2) 13066 { 13067 int32_t bytes_acked; 13068 int32_t gap; 13069 mblk_t *mp1; 13070 uint_t flags; 13071 uint32_t new_swnd = 0; 13072 uchar_t *iphdr; 13073 uchar_t *rptr; 13074 int32_t rgap; 13075 uint32_t seg_ack; 13076 int seg_len; 13077 uint_t ip_hdr_len; 13078 uint32_t seg_seq; 13079 tcph_t *tcph; 13080 int urp; 13081 tcp_opt_t tcpopt; 13082 uint_t ipvers; 13083 ip6_pkt_t ipp; 13084 boolean_t ofo_seg = B_FALSE; /* Out of order segment */ 13085 uint32_t cwnd; 13086 uint32_t add; 13087 int npkt; 13088 int mss; 13089 conn_t *connp = (conn_t *)arg; 13090 squeue_t *sqp = (squeue_t *)arg2; 13091 tcp_t *tcp = connp->conn_tcp; 13092 tcp_stack_t *tcps = tcp->tcp_tcps; 13093 13094 /* 13095 * RST from fused tcp loopback peer should trigger an unfuse. 13096 */ 13097 if (tcp->tcp_fused) { 13098 TCP_STAT(tcps, tcp_fusion_aborted); 13099 tcp_unfuse(tcp); 13100 } 13101 13102 iphdr = mp->b_rptr; 13103 rptr = mp->b_rptr; 13104 ASSERT(OK_32PTR(rptr)); 13105 13106 /* 13107 * An AF_INET socket is not capable of receiving any pktinfo. Do inline 13108 * processing here. For rest call tcp_find_pktinfo to fill up the 13109 * necessary information. 13110 */ 13111 if (IPCL_IS_TCP4(connp)) { 13112 ipvers = IPV4_VERSION; 13113 ip_hdr_len = IPH_HDR_LENGTH(rptr); 13114 } else { 13115 mp = tcp_find_pktinfo(tcp, mp, &ipvers, &ip_hdr_len, 13116 NULL, &ipp); 13117 if (mp == NULL) { 13118 TCP_STAT(tcps, tcp_rput_v6_error); 13119 return; 13120 } 13121 iphdr = mp->b_rptr; 13122 rptr = mp->b_rptr; 13123 } 13124 ASSERT(DB_TYPE(mp) == M_DATA); 13125 13126 tcph = (tcph_t *)&rptr[ip_hdr_len]; 13127 seg_seq = ABE32_TO_U32(tcph->th_seq); 13128 seg_ack = ABE32_TO_U32(tcph->th_ack); 13129 ASSERT((uintptr_t)(mp->b_wptr - rptr) <= (uintptr_t)INT_MAX); 13130 seg_len = (int)(mp->b_wptr - rptr) - 13131 (ip_hdr_len + TCP_HDR_LENGTH(tcph)); 13132 if ((mp1 = mp->b_cont) != NULL && mp1->b_datap->db_type == M_DATA) { 13133 do { 13134 ASSERT((uintptr_t)(mp1->b_wptr - mp1->b_rptr) <= 13135 (uintptr_t)INT_MAX); 13136 seg_len += (int)(mp1->b_wptr - mp1->b_rptr); 13137 } while ((mp1 = mp1->b_cont) != NULL && 13138 mp1->b_datap->db_type == M_DATA); 13139 } 13140 13141 if (tcp->tcp_state == TCPS_TIME_WAIT) { 13142 tcp_time_wait_processing(tcp, mp, seg_seq, seg_ack, 13143 seg_len, tcph); 13144 return; 13145 } 13146 13147 if (sqp != NULL) { 13148 /* 13149 * This is the correct place to update tcp_last_recv_time. Note 13150 * that it is also updated for tcp structure that belongs to 13151 * global and listener queues which do not really need updating. 13152 * But that should not cause any harm. And it is updated for 13153 * all kinds of incoming segments, not only for data segments. 13154 */ 13155 tcp->tcp_last_recv_time = lbolt; 13156 } 13157 13158 flags = (unsigned int)tcph->th_flags[0] & 0xFF; 13159 13160 BUMP_LOCAL(tcp->tcp_ibsegs); 13161 DTRACE_PROBE2(tcp__trace__recv, mblk_t *, mp, tcp_t *, tcp); 13162 13163 if ((flags & TH_URG) && sqp != NULL) { 13164 /* 13165 * TCP can't handle urgent pointers that arrive before 13166 * the connection has been accept()ed since it can't 13167 * buffer OOB data. Discard segment if this happens. 13168 * 13169 * We can't just rely on a non-null tcp_listener to indicate 13170 * that the accept() has completed since unlinking of the 13171 * eager and completion of the accept are not atomic. 13172 * tcp_detached, when it is not set (B_FALSE) indicates 13173 * that the accept() has completed. 13174 * 13175 * Nor can it reassemble urgent pointers, so discard 13176 * if it's not the next segment expected. 13177 * 13178 * Otherwise, collapse chain into one mblk (discard if 13179 * that fails). This makes sure the headers, retransmitted 13180 * data, and new data all are in the same mblk. 13181 */ 13182 ASSERT(mp != NULL); 13183 if (tcp->tcp_detached || !pullupmsg(mp, -1)) { 13184 freemsg(mp); 13185 return; 13186 } 13187 /* Update pointers into message */ 13188 iphdr = rptr = mp->b_rptr; 13189 tcph = (tcph_t *)&rptr[ip_hdr_len]; 13190 if (SEQ_GT(seg_seq, tcp->tcp_rnxt)) { 13191 /* 13192 * Since we can't handle any data with this urgent 13193 * pointer that is out of sequence, we expunge 13194 * the data. This allows us to still register 13195 * the urgent mark and generate the M_PCSIG, 13196 * which we can do. 13197 */ 13198 mp->b_wptr = (uchar_t *)tcph + TCP_HDR_LENGTH(tcph); 13199 seg_len = 0; 13200 } 13201 } 13202 13203 switch (tcp->tcp_state) { 13204 case TCPS_SYN_SENT: 13205 if (flags & TH_ACK) { 13206 /* 13207 * Note that our stack cannot send data before a 13208 * connection is established, therefore the 13209 * following check is valid. Otherwise, it has 13210 * to be changed. 13211 */ 13212 if (SEQ_LEQ(seg_ack, tcp->tcp_iss) || 13213 SEQ_GT(seg_ack, tcp->tcp_snxt)) { 13214 freemsg(mp); 13215 if (flags & TH_RST) 13216 return; 13217 tcp_xmit_ctl("TCPS_SYN_SENT-Bad_seq", 13218 tcp, seg_ack, 0, TH_RST); 13219 return; 13220 } 13221 ASSERT(tcp->tcp_suna + 1 == seg_ack); 13222 } 13223 if (flags & TH_RST) { 13224 freemsg(mp); 13225 if (flags & TH_ACK) 13226 (void) tcp_clean_death(tcp, 13227 ECONNREFUSED, 13); 13228 return; 13229 } 13230 if (!(flags & TH_SYN)) { 13231 freemsg(mp); 13232 return; 13233 } 13234 13235 /* Process all TCP options. */ 13236 tcp_process_options(tcp, tcph); 13237 /* 13238 * The following changes our rwnd to be a multiple of the 13239 * MIN(peer MSS, our MSS) for performance reason. 13240 */ 13241 (void) tcp_rwnd_set(tcp, MSS_ROUNDUP(tcp->tcp_rq->q_hiwat, 13242 tcp->tcp_mss)); 13243 13244 /* Is the other end ECN capable? */ 13245 if (tcp->tcp_ecn_ok) { 13246 if ((flags & (TH_ECE|TH_CWR)) != TH_ECE) { 13247 tcp->tcp_ecn_ok = B_FALSE; 13248 } 13249 } 13250 /* 13251 * Clear ECN flags because it may interfere with later 13252 * processing. 13253 */ 13254 flags &= ~(TH_ECE|TH_CWR); 13255 13256 tcp->tcp_irs = seg_seq; 13257 tcp->tcp_rack = seg_seq; 13258 tcp->tcp_rnxt = seg_seq + 1; 13259 U32_TO_ABE32(tcp->tcp_rnxt, tcp->tcp_tcph->th_ack); 13260 if (!TCP_IS_DETACHED(tcp)) { 13261 /* Allocate room for SACK options if needed. */ 13262 if (tcp->tcp_snd_sack_ok) { 13263 (void) mi_set_sth_wroff(tcp->tcp_rq, 13264 tcp->tcp_hdr_len + TCPOPT_MAX_SACK_LEN + 13265 (tcp->tcp_loopback ? 0 : 13266 tcps->tcps_wroff_xtra)); 13267 } else { 13268 (void) mi_set_sth_wroff(tcp->tcp_rq, 13269 tcp->tcp_hdr_len + 13270 (tcp->tcp_loopback ? 0 : 13271 tcps->tcps_wroff_xtra)); 13272 } 13273 } 13274 if (flags & TH_ACK) { 13275 /* 13276 * If we can't get the confirmation upstream, pretend 13277 * we didn't even see this one. 13278 * 13279 * XXX: how can we pretend we didn't see it if we 13280 * have updated rnxt et. al. 13281 * 13282 * For loopback we defer sending up the T_CONN_CON 13283 * until after some checks below. 13284 */ 13285 mp1 = NULL; 13286 if (!tcp_conn_con(tcp, iphdr, tcph, mp, 13287 tcp->tcp_loopback ? &mp1 : NULL)) { 13288 freemsg(mp); 13289 return; 13290 } 13291 /* SYN was acked - making progress */ 13292 if (tcp->tcp_ipversion == IPV6_VERSION) 13293 tcp->tcp_ip_forward_progress = B_TRUE; 13294 13295 /* One for the SYN */ 13296 tcp->tcp_suna = tcp->tcp_iss + 1; 13297 tcp->tcp_valid_bits &= ~TCP_ISS_VALID; 13298 tcp->tcp_state = TCPS_ESTABLISHED; 13299 13300 /* 13301 * If SYN was retransmitted, need to reset all 13302 * retransmission info. This is because this 13303 * segment will be treated as a dup ACK. 13304 */ 13305 if (tcp->tcp_rexmit) { 13306 tcp->tcp_rexmit = B_FALSE; 13307 tcp->tcp_rexmit_nxt = tcp->tcp_snxt; 13308 tcp->tcp_rexmit_max = tcp->tcp_snxt; 13309 tcp->tcp_snd_burst = tcp->tcp_localnet ? 13310 TCP_CWND_INFINITE : TCP_CWND_NORMAL; 13311 tcp->tcp_ms_we_have_waited = 0; 13312 13313 /* 13314 * Set tcp_cwnd back to 1 MSS, per 13315 * recommendation from 13316 * draft-floyd-incr-init-win-01.txt, 13317 * Increasing TCP's Initial Window. 13318 */ 13319 tcp->tcp_cwnd = tcp->tcp_mss; 13320 } 13321 13322 tcp->tcp_swl1 = seg_seq; 13323 tcp->tcp_swl2 = seg_ack; 13324 13325 new_swnd = BE16_TO_U16(tcph->th_win); 13326 tcp->tcp_swnd = new_swnd; 13327 if (new_swnd > tcp->tcp_max_swnd) 13328 tcp->tcp_max_swnd = new_swnd; 13329 13330 /* 13331 * Always send the three-way handshake ack immediately 13332 * in order to make the connection complete as soon as 13333 * possible on the accepting host. 13334 */ 13335 flags |= TH_ACK_NEEDED; 13336 13337 /* 13338 * Special case for loopback. At this point we have 13339 * received SYN-ACK from the remote endpoint. In 13340 * order to ensure that both endpoints reach the 13341 * fused state prior to any data exchange, the final 13342 * ACK needs to be sent before we indicate T_CONN_CON 13343 * to the module upstream. 13344 */ 13345 if (tcp->tcp_loopback) { 13346 mblk_t *ack_mp; 13347 13348 ASSERT(!tcp->tcp_unfusable); 13349 ASSERT(mp1 != NULL); 13350 /* 13351 * For loopback, we always get a pure SYN-ACK 13352 * and only need to send back the final ACK 13353 * with no data (this is because the other 13354 * tcp is ours and we don't do T/TCP). This 13355 * final ACK triggers the passive side to 13356 * perform fusion in ESTABLISHED state. 13357 */ 13358 if ((ack_mp = tcp_ack_mp(tcp)) != NULL) { 13359 if (tcp->tcp_ack_tid != 0) { 13360 (void) TCP_TIMER_CANCEL(tcp, 13361 tcp->tcp_ack_tid); 13362 tcp->tcp_ack_tid = 0; 13363 } 13364 tcp_send_data(tcp, tcp->tcp_wq, ack_mp); 13365 BUMP_LOCAL(tcp->tcp_obsegs); 13366 BUMP_MIB(&tcps->tcps_mib, tcpOutAck); 13367 13368 /* Send up T_CONN_CON */ 13369 putnext(tcp->tcp_rq, mp1); 13370 13371 freemsg(mp); 13372 return; 13373 } 13374 /* 13375 * Forget fusion; we need to handle more 13376 * complex cases below. Send the deferred 13377 * T_CONN_CON message upstream and proceed 13378 * as usual. Mark this tcp as not capable 13379 * of fusion. 13380 */ 13381 TCP_STAT(tcps, tcp_fusion_unfusable); 13382 tcp->tcp_unfusable = B_TRUE; 13383 putnext(tcp->tcp_rq, mp1); 13384 } 13385 13386 /* 13387 * Check to see if there is data to be sent. If 13388 * yes, set the transmit flag. Then check to see 13389 * if received data processing needs to be done. 13390 * If not, go straight to xmit_check. This short 13391 * cut is OK as we don't support T/TCP. 13392 */ 13393 if (tcp->tcp_unsent) 13394 flags |= TH_XMIT_NEEDED; 13395 13396 if (seg_len == 0 && !(flags & TH_URG)) { 13397 freemsg(mp); 13398 goto xmit_check; 13399 } 13400 13401 flags &= ~TH_SYN; 13402 seg_seq++; 13403 break; 13404 } 13405 tcp->tcp_state = TCPS_SYN_RCVD; 13406 mp1 = tcp_xmit_mp(tcp, tcp->tcp_xmit_head, tcp->tcp_mss, 13407 NULL, NULL, tcp->tcp_iss, B_FALSE, NULL, B_FALSE); 13408 if (mp1) { 13409 DB_CPID(mp1) = tcp->tcp_cpid; 13410 tcp_send_data(tcp, tcp->tcp_wq, mp1); 13411 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 13412 } 13413 freemsg(mp); 13414 return; 13415 case TCPS_SYN_RCVD: 13416 if (flags & TH_ACK) { 13417 /* 13418 * In this state, a SYN|ACK packet is either bogus 13419 * because the other side must be ACKing our SYN which 13420 * indicates it has seen the ACK for their SYN and 13421 * shouldn't retransmit it or we're crossing SYNs 13422 * on active open. 13423 */ 13424 if ((flags & TH_SYN) && !tcp->tcp_active_open) { 13425 freemsg(mp); 13426 tcp_xmit_ctl("TCPS_SYN_RCVD-bad_syn", 13427 tcp, seg_ack, 0, TH_RST); 13428 return; 13429 } 13430 /* 13431 * NOTE: RFC 793 pg. 72 says this should be 13432 * tcp->tcp_suna <= seg_ack <= tcp->tcp_snxt 13433 * but that would mean we have an ack that ignored 13434 * our SYN. 13435 */ 13436 if (SEQ_LEQ(seg_ack, tcp->tcp_suna) || 13437 SEQ_GT(seg_ack, tcp->tcp_snxt)) { 13438 freemsg(mp); 13439 tcp_xmit_ctl("TCPS_SYN_RCVD-bad_ack", 13440 tcp, seg_ack, 0, TH_RST); 13441 return; 13442 } 13443 } 13444 break; 13445 case TCPS_LISTEN: 13446 /* 13447 * Only a TLI listener can come through this path when a 13448 * acceptor is going back to be a listener and a packet 13449 * for the acceptor hits the classifier. For a socket 13450 * listener, this can never happen because a listener 13451 * can never accept connection on itself and hence a 13452 * socket acceptor can not go back to being a listener. 13453 */ 13454 ASSERT(!TCP_IS_SOCKET(tcp)); 13455 /*FALLTHRU*/ 13456 case TCPS_CLOSED: 13457 case TCPS_BOUND: { 13458 conn_t *new_connp; 13459 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 13460 13461 new_connp = ipcl_classify(mp, connp->conn_zoneid, ipst); 13462 if (new_connp != NULL) { 13463 tcp_reinput(new_connp, mp, connp->conn_sqp); 13464 return; 13465 } 13466 /* We failed to classify. For now just drop the packet */ 13467 freemsg(mp); 13468 return; 13469 } 13470 case TCPS_IDLE: 13471 /* 13472 * Handle the case where the tcp_clean_death() has happened 13473 * on a connection (application hasn't closed yet) but a packet 13474 * was already queued on squeue before tcp_clean_death() 13475 * was processed. Calling tcp_clean_death() twice on same 13476 * connection can result in weird behaviour. 13477 */ 13478 freemsg(mp); 13479 return; 13480 default: 13481 break; 13482 } 13483 13484 /* 13485 * Already on the correct queue/perimeter. 13486 * If this is a detached connection and not an eager 13487 * connection hanging off a listener then new data 13488 * (past the FIN) will cause a reset. 13489 * We do a special check here where it 13490 * is out of the main line, rather than check 13491 * if we are detached every time we see new 13492 * data down below. 13493 */ 13494 if (TCP_IS_DETACHED_NONEAGER(tcp) && 13495 (seg_len > 0 && SEQ_GT(seg_seq + seg_len, tcp->tcp_rnxt))) { 13496 BUMP_MIB(&tcps->tcps_mib, tcpInClosed); 13497 DTRACE_PROBE2(tcp__trace__recv, mblk_t *, mp, tcp_t *, tcp); 13498 13499 freemsg(mp); 13500 /* 13501 * This could be an SSL closure alert. We're detached so just 13502 * acknowledge it this last time. 13503 */ 13504 if (tcp->tcp_kssl_ctx != NULL) { 13505 kssl_release_ctx(tcp->tcp_kssl_ctx); 13506 tcp->tcp_kssl_ctx = NULL; 13507 13508 tcp->tcp_rnxt += seg_len; 13509 U32_TO_ABE32(tcp->tcp_rnxt, tcp->tcp_tcph->th_ack); 13510 flags |= TH_ACK_NEEDED; 13511 goto ack_check; 13512 } 13513 13514 tcp_xmit_ctl("new data when detached", tcp, 13515 tcp->tcp_snxt, 0, TH_RST); 13516 (void) tcp_clean_death(tcp, EPROTO, 12); 13517 return; 13518 } 13519 13520 mp->b_rptr = (uchar_t *)tcph + TCP_HDR_LENGTH(tcph); 13521 urp = BE16_TO_U16(tcph->th_urp) - TCP_OLD_URP_INTERPRETATION; 13522 new_swnd = BE16_TO_U16(tcph->th_win) << 13523 ((tcph->th_flags[0] & TH_SYN) ? 0 : tcp->tcp_snd_ws); 13524 13525 if (tcp->tcp_snd_ts_ok) { 13526 if (!tcp_paws_check(tcp, tcph, &tcpopt)) { 13527 /* 13528 * This segment is not acceptable. 13529 * Drop it and send back an ACK. 13530 */ 13531 freemsg(mp); 13532 flags |= TH_ACK_NEEDED; 13533 goto ack_check; 13534 } 13535 } else if (tcp->tcp_snd_sack_ok) { 13536 ASSERT(tcp->tcp_sack_info != NULL); 13537 tcpopt.tcp = tcp; 13538 /* 13539 * SACK info in already updated in tcp_parse_options. Ignore 13540 * all other TCP options... 13541 */ 13542 (void) tcp_parse_options(tcph, &tcpopt); 13543 } 13544 try_again:; 13545 mss = tcp->tcp_mss; 13546 gap = seg_seq - tcp->tcp_rnxt; 13547 rgap = tcp->tcp_rwnd - (gap + seg_len); 13548 /* 13549 * gap is the amount of sequence space between what we expect to see 13550 * and what we got for seg_seq. A positive value for gap means 13551 * something got lost. A negative value means we got some old stuff. 13552 */ 13553 if (gap < 0) { 13554 /* Old stuff present. Is the SYN in there? */ 13555 if (seg_seq == tcp->tcp_irs && (flags & TH_SYN) && 13556 (seg_len != 0)) { 13557 flags &= ~TH_SYN; 13558 seg_seq++; 13559 urp--; 13560 /* Recompute the gaps after noting the SYN. */ 13561 goto try_again; 13562 } 13563 BUMP_MIB(&tcps->tcps_mib, tcpInDataDupSegs); 13564 UPDATE_MIB(&tcps->tcps_mib, tcpInDataDupBytes, 13565 (seg_len > -gap ? -gap : seg_len)); 13566 /* Remove the old stuff from seg_len. */ 13567 seg_len += gap; 13568 /* 13569 * Anything left? 13570 * Make sure to check for unack'd FIN when rest of data 13571 * has been previously ack'd. 13572 */ 13573 if (seg_len < 0 || (seg_len == 0 && !(flags & TH_FIN))) { 13574 /* 13575 * Resets are only valid if they lie within our offered 13576 * window. If the RST bit is set, we just ignore this 13577 * segment. 13578 */ 13579 if (flags & TH_RST) { 13580 freemsg(mp); 13581 return; 13582 } 13583 13584 /* 13585 * The arriving of dup data packets indicate that we 13586 * may have postponed an ack for too long, or the other 13587 * side's RTT estimate is out of shape. Start acking 13588 * more often. 13589 */ 13590 if (SEQ_GEQ(seg_seq + seg_len - gap, tcp->tcp_rack) && 13591 tcp->tcp_rack_cnt >= 1 && 13592 tcp->tcp_rack_abs_max > 2) { 13593 tcp->tcp_rack_abs_max--; 13594 } 13595 tcp->tcp_rack_cur_max = 1; 13596 13597 /* 13598 * This segment is "unacceptable". None of its 13599 * sequence space lies within our advertized window. 13600 * 13601 * Adjust seg_len to the original value for tracing. 13602 */ 13603 seg_len -= gap; 13604 if (tcp->tcp_debug) { 13605 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE, 13606 "tcp_rput: unacceptable, gap %d, rgap %d, " 13607 "flags 0x%x, seg_seq %u, seg_ack %u, " 13608 "seg_len %d, rnxt %u, snxt %u, %s", 13609 gap, rgap, flags, seg_seq, seg_ack, 13610 seg_len, tcp->tcp_rnxt, tcp->tcp_snxt, 13611 tcp_display(tcp, NULL, 13612 DISP_ADDR_AND_PORT)); 13613 } 13614 13615 /* 13616 * Arrange to send an ACK in response to the 13617 * unacceptable segment per RFC 793 page 69. There 13618 * is only one small difference between ours and the 13619 * acceptability test in the RFC - we accept ACK-only 13620 * packet with SEG.SEQ = RCV.NXT+RCV.WND and no ACK 13621 * will be generated. 13622 * 13623 * Note that we have to ACK an ACK-only packet at least 13624 * for stacks that send 0-length keep-alives with 13625 * SEG.SEQ = SND.NXT-1 as recommended by RFC1122, 13626 * section 4.2.3.6. As long as we don't ever generate 13627 * an unacceptable packet in response to an incoming 13628 * packet that is unacceptable, it should not cause 13629 * "ACK wars". 13630 */ 13631 flags |= TH_ACK_NEEDED; 13632 13633 /* 13634 * Continue processing this segment in order to use the 13635 * ACK information it contains, but skip all other 13636 * sequence-number processing. Processing the ACK 13637 * information is necessary in order to 13638 * re-synchronize connections that may have lost 13639 * synchronization. 13640 * 13641 * We clear seg_len and flag fields related to 13642 * sequence number processing as they are not 13643 * to be trusted for an unacceptable segment. 13644 */ 13645 seg_len = 0; 13646 flags &= ~(TH_SYN | TH_FIN | TH_URG); 13647 goto process_ack; 13648 } 13649 13650 /* Fix seg_seq, and chew the gap off the front. */ 13651 seg_seq = tcp->tcp_rnxt; 13652 urp += gap; 13653 do { 13654 mblk_t *mp2; 13655 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= 13656 (uintptr_t)UINT_MAX); 13657 gap += (uint_t)(mp->b_wptr - mp->b_rptr); 13658 if (gap > 0) { 13659 mp->b_rptr = mp->b_wptr - gap; 13660 break; 13661 } 13662 mp2 = mp; 13663 mp = mp->b_cont; 13664 freeb(mp2); 13665 } while (gap < 0); 13666 /* 13667 * If the urgent data has already been acknowledged, we 13668 * should ignore TH_URG below 13669 */ 13670 if (urp < 0) 13671 flags &= ~TH_URG; 13672 } 13673 /* 13674 * rgap is the amount of stuff received out of window. A negative 13675 * value is the amount out of window. 13676 */ 13677 if (rgap < 0) { 13678 mblk_t *mp2; 13679 13680 if (tcp->tcp_rwnd == 0) { 13681 BUMP_MIB(&tcps->tcps_mib, tcpInWinProbe); 13682 } else { 13683 BUMP_MIB(&tcps->tcps_mib, tcpInDataPastWinSegs); 13684 UPDATE_MIB(&tcps->tcps_mib, 13685 tcpInDataPastWinBytes, -rgap); 13686 } 13687 13688 /* 13689 * seg_len does not include the FIN, so if more than 13690 * just the FIN is out of window, we act like we don't 13691 * see it. (If just the FIN is out of window, rgap 13692 * will be zero and we will go ahead and acknowledge 13693 * the FIN.) 13694 */ 13695 flags &= ~TH_FIN; 13696 13697 /* Fix seg_len and make sure there is something left. */ 13698 seg_len += rgap; 13699 if (seg_len <= 0) { 13700 /* 13701 * Resets are only valid if they lie within our offered 13702 * window. If the RST bit is set, we just ignore this 13703 * segment. 13704 */ 13705 if (flags & TH_RST) { 13706 freemsg(mp); 13707 return; 13708 } 13709 13710 /* Per RFC 793, we need to send back an ACK. */ 13711 flags |= TH_ACK_NEEDED; 13712 13713 /* 13714 * Send SIGURG as soon as possible i.e. even 13715 * if the TH_URG was delivered in a window probe 13716 * packet (which will be unacceptable). 13717 * 13718 * We generate a signal if none has been generated 13719 * for this connection or if this is a new urgent 13720 * byte. Also send a zero-length "unmarked" message 13721 * to inform SIOCATMARK that this is not the mark. 13722 * 13723 * tcp_urp_last_valid is cleared when the T_exdata_ind 13724 * is sent up. This plus the check for old data 13725 * (gap >= 0) handles the wraparound of the sequence 13726 * number space without having to always track the 13727 * correct MAX(tcp_urp_last, tcp_rnxt). (BSD tracks 13728 * this max in its rcv_up variable). 13729 * 13730 * This prevents duplicate SIGURGS due to a "late" 13731 * zero-window probe when the T_EXDATA_IND has already 13732 * been sent up. 13733 */ 13734 if ((flags & TH_URG) && 13735 (!tcp->tcp_urp_last_valid || SEQ_GT(urp + seg_seq, 13736 tcp->tcp_urp_last))) { 13737 mp1 = allocb(0, BPRI_MED); 13738 if (mp1 == NULL) { 13739 freemsg(mp); 13740 return; 13741 } 13742 if (!TCP_IS_DETACHED(tcp) && 13743 !putnextctl1(tcp->tcp_rq, M_PCSIG, 13744 SIGURG)) { 13745 /* Try again on the rexmit. */ 13746 freemsg(mp1); 13747 freemsg(mp); 13748 return; 13749 } 13750 /* 13751 * If the next byte would be the mark 13752 * then mark with MARKNEXT else mark 13753 * with NOTMARKNEXT. 13754 */ 13755 if (gap == 0 && urp == 0) 13756 mp1->b_flag |= MSGMARKNEXT; 13757 else 13758 mp1->b_flag |= MSGNOTMARKNEXT; 13759 freemsg(tcp->tcp_urp_mark_mp); 13760 tcp->tcp_urp_mark_mp = mp1; 13761 flags |= TH_SEND_URP_MARK; 13762 tcp->tcp_urp_last_valid = B_TRUE; 13763 tcp->tcp_urp_last = urp + seg_seq; 13764 } 13765 /* 13766 * If this is a zero window probe, continue to 13767 * process the ACK part. But we need to set seg_len 13768 * to 0 to avoid data processing. Otherwise just 13769 * drop the segment and send back an ACK. 13770 */ 13771 if (tcp->tcp_rwnd == 0 && seg_seq == tcp->tcp_rnxt) { 13772 flags &= ~(TH_SYN | TH_URG); 13773 seg_len = 0; 13774 goto process_ack; 13775 } else { 13776 freemsg(mp); 13777 goto ack_check; 13778 } 13779 } 13780 /* Pitch out of window stuff off the end. */ 13781 rgap = seg_len; 13782 mp2 = mp; 13783 do { 13784 ASSERT((uintptr_t)(mp2->b_wptr - mp2->b_rptr) <= 13785 (uintptr_t)INT_MAX); 13786 rgap -= (int)(mp2->b_wptr - mp2->b_rptr); 13787 if (rgap < 0) { 13788 mp2->b_wptr += rgap; 13789 if ((mp1 = mp2->b_cont) != NULL) { 13790 mp2->b_cont = NULL; 13791 freemsg(mp1); 13792 } 13793 break; 13794 } 13795 } while ((mp2 = mp2->b_cont) != NULL); 13796 } 13797 ok:; 13798 /* 13799 * TCP should check ECN info for segments inside the window only. 13800 * Therefore the check should be done here. 13801 */ 13802 if (tcp->tcp_ecn_ok) { 13803 if (flags & TH_CWR) { 13804 tcp->tcp_ecn_echo_on = B_FALSE; 13805 } 13806 /* 13807 * Note that both ECN_CE and CWR can be set in the 13808 * same segment. In this case, we once again turn 13809 * on ECN_ECHO. 13810 */ 13811 if (tcp->tcp_ipversion == IPV4_VERSION) { 13812 uchar_t tos = ((ipha_t *)rptr)->ipha_type_of_service; 13813 13814 if ((tos & IPH_ECN_CE) == IPH_ECN_CE) { 13815 tcp->tcp_ecn_echo_on = B_TRUE; 13816 } 13817 } else { 13818 uint32_t vcf = ((ip6_t *)rptr)->ip6_vcf; 13819 13820 if ((vcf & htonl(IPH_ECN_CE << 20)) == 13821 htonl(IPH_ECN_CE << 20)) { 13822 tcp->tcp_ecn_echo_on = B_TRUE; 13823 } 13824 } 13825 } 13826 13827 /* 13828 * Check whether we can update tcp_ts_recent. This test is 13829 * NOT the one in RFC 1323 3.4. It is from Braden, 1993, "TCP 13830 * Extensions for High Performance: An Update", Internet Draft. 13831 */ 13832 if (tcp->tcp_snd_ts_ok && 13833 TSTMP_GEQ(tcpopt.tcp_opt_ts_val, tcp->tcp_ts_recent) && 13834 SEQ_LEQ(seg_seq, tcp->tcp_rack)) { 13835 tcp->tcp_ts_recent = tcpopt.tcp_opt_ts_val; 13836 tcp->tcp_last_rcv_lbolt = lbolt64; 13837 } 13838 13839 if (seg_seq != tcp->tcp_rnxt || tcp->tcp_reass_head) { 13840 /* 13841 * FIN in an out of order segment. We record this in 13842 * tcp_valid_bits and the seq num of FIN in tcp_ofo_fin_seq. 13843 * Clear the FIN so that any check on FIN flag will fail. 13844 * Remember that FIN also counts in the sequence number 13845 * space. So we need to ack out of order FIN only segments. 13846 */ 13847 if (flags & TH_FIN) { 13848 tcp->tcp_valid_bits |= TCP_OFO_FIN_VALID; 13849 tcp->tcp_ofo_fin_seq = seg_seq + seg_len; 13850 flags &= ~TH_FIN; 13851 flags |= TH_ACK_NEEDED; 13852 } 13853 if (seg_len > 0) { 13854 /* Fill in the SACK blk list. */ 13855 if (tcp->tcp_snd_sack_ok) { 13856 ASSERT(tcp->tcp_sack_info != NULL); 13857 tcp_sack_insert(tcp->tcp_sack_list, 13858 seg_seq, seg_seq + seg_len, 13859 &(tcp->tcp_num_sack_blk)); 13860 } 13861 13862 /* 13863 * Attempt reassembly and see if we have something 13864 * ready to go. 13865 */ 13866 mp = tcp_reass(tcp, mp, seg_seq); 13867 /* Always ack out of order packets */ 13868 flags |= TH_ACK_NEEDED | TH_PUSH; 13869 if (mp) { 13870 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= 13871 (uintptr_t)INT_MAX); 13872 seg_len = mp->b_cont ? msgdsize(mp) : 13873 (int)(mp->b_wptr - mp->b_rptr); 13874 seg_seq = tcp->tcp_rnxt; 13875 /* 13876 * A gap is filled and the seq num and len 13877 * of the gap match that of a previously 13878 * received FIN, put the FIN flag back in. 13879 */ 13880 if ((tcp->tcp_valid_bits & TCP_OFO_FIN_VALID) && 13881 seg_seq + seg_len == tcp->tcp_ofo_fin_seq) { 13882 flags |= TH_FIN; 13883 tcp->tcp_valid_bits &= 13884 ~TCP_OFO_FIN_VALID; 13885 } 13886 } else { 13887 /* 13888 * Keep going even with NULL mp. 13889 * There may be a useful ACK or something else 13890 * we don't want to miss. 13891 * 13892 * But TCP should not perform fast retransmit 13893 * because of the ack number. TCP uses 13894 * seg_len == 0 to determine if it is a pure 13895 * ACK. And this is not a pure ACK. 13896 */ 13897 seg_len = 0; 13898 ofo_seg = B_TRUE; 13899 } 13900 } 13901 } else if (seg_len > 0) { 13902 BUMP_MIB(&tcps->tcps_mib, tcpInDataInorderSegs); 13903 UPDATE_MIB(&tcps->tcps_mib, tcpInDataInorderBytes, seg_len); 13904 /* 13905 * If an out of order FIN was received before, and the seq 13906 * num and len of the new segment match that of the FIN, 13907 * put the FIN flag back in. 13908 */ 13909 if ((tcp->tcp_valid_bits & TCP_OFO_FIN_VALID) && 13910 seg_seq + seg_len == tcp->tcp_ofo_fin_seq) { 13911 flags |= TH_FIN; 13912 tcp->tcp_valid_bits &= ~TCP_OFO_FIN_VALID; 13913 } 13914 } 13915 if ((flags & (TH_RST | TH_SYN | TH_URG | TH_ACK)) != TH_ACK) { 13916 if (flags & TH_RST) { 13917 freemsg(mp); 13918 switch (tcp->tcp_state) { 13919 case TCPS_SYN_RCVD: 13920 (void) tcp_clean_death(tcp, ECONNREFUSED, 14); 13921 break; 13922 case TCPS_ESTABLISHED: 13923 case TCPS_FIN_WAIT_1: 13924 case TCPS_FIN_WAIT_2: 13925 case TCPS_CLOSE_WAIT: 13926 (void) tcp_clean_death(tcp, ECONNRESET, 15); 13927 break; 13928 case TCPS_CLOSING: 13929 case TCPS_LAST_ACK: 13930 (void) tcp_clean_death(tcp, 0, 16); 13931 break; 13932 default: 13933 ASSERT(tcp->tcp_state != TCPS_TIME_WAIT); 13934 (void) tcp_clean_death(tcp, ENXIO, 17); 13935 break; 13936 } 13937 return; 13938 } 13939 if (flags & TH_SYN) { 13940 /* 13941 * See RFC 793, Page 71 13942 * 13943 * The seq number must be in the window as it should 13944 * be "fixed" above. If it is outside window, it should 13945 * be already rejected. Note that we allow seg_seq to be 13946 * rnxt + rwnd because we want to accept 0 window probe. 13947 */ 13948 ASSERT(SEQ_GEQ(seg_seq, tcp->tcp_rnxt) && 13949 SEQ_LEQ(seg_seq, tcp->tcp_rnxt + tcp->tcp_rwnd)); 13950 freemsg(mp); 13951 /* 13952 * If the ACK flag is not set, just use our snxt as the 13953 * seq number of the RST segment. 13954 */ 13955 if (!(flags & TH_ACK)) { 13956 seg_ack = tcp->tcp_snxt; 13957 } 13958 tcp_xmit_ctl("TH_SYN", tcp, seg_ack, seg_seq + 1, 13959 TH_RST|TH_ACK); 13960 ASSERT(tcp->tcp_state != TCPS_TIME_WAIT); 13961 (void) tcp_clean_death(tcp, ECONNRESET, 18); 13962 return; 13963 } 13964 /* 13965 * urp could be -1 when the urp field in the packet is 0 13966 * and TCP_OLD_URP_INTERPRETATION is set. This implies that the urgent 13967 * byte was at seg_seq - 1, in which case we ignore the urgent flag. 13968 */ 13969 if (flags & TH_URG && urp >= 0) { 13970 if (!tcp->tcp_urp_last_valid || 13971 SEQ_GT(urp + seg_seq, tcp->tcp_urp_last)) { 13972 /* 13973 * If we haven't generated the signal yet for this 13974 * urgent pointer value, do it now. Also, send up a 13975 * zero-length M_DATA indicating whether or not this is 13976 * the mark. The latter is not needed when a 13977 * T_EXDATA_IND is sent up. However, if there are 13978 * allocation failures this code relies on the sender 13979 * retransmitting and the socket code for determining 13980 * the mark should not block waiting for the peer to 13981 * transmit. Thus, for simplicity we always send up the 13982 * mark indication. 13983 */ 13984 mp1 = allocb(0, BPRI_MED); 13985 if (mp1 == NULL) { 13986 freemsg(mp); 13987 return; 13988 } 13989 if (!TCP_IS_DETACHED(tcp) && 13990 !putnextctl1(tcp->tcp_rq, M_PCSIG, SIGURG)) { 13991 /* Try again on the rexmit. */ 13992 freemsg(mp1); 13993 freemsg(mp); 13994 return; 13995 } 13996 /* 13997 * Mark with NOTMARKNEXT for now. 13998 * The code below will change this to MARKNEXT 13999 * if we are at the mark. 14000 * 14001 * If there are allocation failures (e.g. in dupmsg 14002 * below) the next time tcp_rput_data sees the urgent 14003 * segment it will send up the MSG*MARKNEXT message. 14004 */ 14005 mp1->b_flag |= MSGNOTMARKNEXT; 14006 freemsg(tcp->tcp_urp_mark_mp); 14007 tcp->tcp_urp_mark_mp = mp1; 14008 flags |= TH_SEND_URP_MARK; 14009 #ifdef DEBUG 14010 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE, 14011 "tcp_rput: sent M_PCSIG 2 seq %x urp %x " 14012 "last %x, %s", 14013 seg_seq, urp, tcp->tcp_urp_last, 14014 tcp_display(tcp, NULL, DISP_PORT_ONLY)); 14015 #endif /* DEBUG */ 14016 tcp->tcp_urp_last_valid = B_TRUE; 14017 tcp->tcp_urp_last = urp + seg_seq; 14018 } else if (tcp->tcp_urp_mark_mp != NULL) { 14019 /* 14020 * An allocation failure prevented the previous 14021 * tcp_rput_data from sending up the allocated 14022 * MSG*MARKNEXT message - send it up this time 14023 * around. 14024 */ 14025 flags |= TH_SEND_URP_MARK; 14026 } 14027 14028 /* 14029 * If the urgent byte is in this segment, make sure that it is 14030 * all by itself. This makes it much easier to deal with the 14031 * possibility of an allocation failure on the T_exdata_ind. 14032 * Note that seg_len is the number of bytes in the segment, and 14033 * urp is the offset into the segment of the urgent byte. 14034 * urp < seg_len means that the urgent byte is in this segment. 14035 */ 14036 if (urp < seg_len) { 14037 if (seg_len != 1) { 14038 uint32_t tmp_rnxt; 14039 /* 14040 * Break it up and feed it back in. 14041 * Re-attach the IP header. 14042 */ 14043 mp->b_rptr = iphdr; 14044 if (urp > 0) { 14045 /* 14046 * There is stuff before the urgent 14047 * byte. 14048 */ 14049 mp1 = dupmsg(mp); 14050 if (!mp1) { 14051 /* 14052 * Trim from urgent byte on. 14053 * The rest will come back. 14054 */ 14055 (void) adjmsg(mp, 14056 urp - seg_len); 14057 tcp_rput_data(connp, 14058 mp, NULL); 14059 return; 14060 } 14061 (void) adjmsg(mp1, urp - seg_len); 14062 /* Feed this piece back in. */ 14063 tmp_rnxt = tcp->tcp_rnxt; 14064 tcp_rput_data(connp, mp1, NULL); 14065 /* 14066 * If the data passed back in was not 14067 * processed (ie: bad ACK) sending 14068 * the remainder back in will cause a 14069 * loop. In this case, drop the 14070 * packet and let the sender try 14071 * sending a good packet. 14072 */ 14073 if (tmp_rnxt == tcp->tcp_rnxt) { 14074 freemsg(mp); 14075 return; 14076 } 14077 } 14078 if (urp != seg_len - 1) { 14079 uint32_t tmp_rnxt; 14080 /* 14081 * There is stuff after the urgent 14082 * byte. 14083 */ 14084 mp1 = dupmsg(mp); 14085 if (!mp1) { 14086 /* 14087 * Trim everything beyond the 14088 * urgent byte. The rest will 14089 * come back. 14090 */ 14091 (void) adjmsg(mp, 14092 urp + 1 - seg_len); 14093 tcp_rput_data(connp, 14094 mp, NULL); 14095 return; 14096 } 14097 (void) adjmsg(mp1, urp + 1 - seg_len); 14098 tmp_rnxt = tcp->tcp_rnxt; 14099 tcp_rput_data(connp, mp1, NULL); 14100 /* 14101 * If the data passed back in was not 14102 * processed (ie: bad ACK) sending 14103 * the remainder back in will cause a 14104 * loop. In this case, drop the 14105 * packet and let the sender try 14106 * sending a good packet. 14107 */ 14108 if (tmp_rnxt == tcp->tcp_rnxt) { 14109 freemsg(mp); 14110 return; 14111 } 14112 } 14113 tcp_rput_data(connp, mp, NULL); 14114 return; 14115 } 14116 /* 14117 * This segment contains only the urgent byte. We 14118 * have to allocate the T_exdata_ind, if we can. 14119 */ 14120 if (!tcp->tcp_urp_mp) { 14121 struct T_exdata_ind *tei; 14122 mp1 = allocb(sizeof (struct T_exdata_ind), 14123 BPRI_MED); 14124 if (!mp1) { 14125 /* 14126 * Sigh... It'll be back. 14127 * Generate any MSG*MARK message now. 14128 */ 14129 freemsg(mp); 14130 seg_len = 0; 14131 if (flags & TH_SEND_URP_MARK) { 14132 14133 14134 ASSERT(tcp->tcp_urp_mark_mp); 14135 tcp->tcp_urp_mark_mp->b_flag &= 14136 ~MSGNOTMARKNEXT; 14137 tcp->tcp_urp_mark_mp->b_flag |= 14138 MSGMARKNEXT; 14139 } 14140 goto ack_check; 14141 } 14142 mp1->b_datap->db_type = M_PROTO; 14143 tei = (struct T_exdata_ind *)mp1->b_rptr; 14144 tei->PRIM_type = T_EXDATA_IND; 14145 tei->MORE_flag = 0; 14146 mp1->b_wptr = (uchar_t *)&tei[1]; 14147 tcp->tcp_urp_mp = mp1; 14148 #ifdef DEBUG 14149 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE, 14150 "tcp_rput: allocated exdata_ind %s", 14151 tcp_display(tcp, NULL, 14152 DISP_PORT_ONLY)); 14153 #endif /* DEBUG */ 14154 /* 14155 * There is no need to send a separate MSG*MARK 14156 * message since the T_EXDATA_IND will be sent 14157 * now. 14158 */ 14159 flags &= ~TH_SEND_URP_MARK; 14160 freemsg(tcp->tcp_urp_mark_mp); 14161 tcp->tcp_urp_mark_mp = NULL; 14162 } 14163 /* 14164 * Now we are all set. On the next putnext upstream, 14165 * tcp_urp_mp will be non-NULL and will get prepended 14166 * to what has to be this piece containing the urgent 14167 * byte. If for any reason we abort this segment below, 14168 * if it comes back, we will have this ready, or it 14169 * will get blown off in close. 14170 */ 14171 } else if (urp == seg_len) { 14172 /* 14173 * The urgent byte is the next byte after this sequence 14174 * number. If there is data it is marked with 14175 * MSGMARKNEXT and any tcp_urp_mark_mp is discarded 14176 * since it is not needed. Otherwise, if the code 14177 * above just allocated a zero-length tcp_urp_mark_mp 14178 * message, that message is tagged with MSGMARKNEXT. 14179 * Sending up these MSGMARKNEXT messages makes 14180 * SIOCATMARK work correctly even though 14181 * the T_EXDATA_IND will not be sent up until the 14182 * urgent byte arrives. 14183 */ 14184 if (seg_len != 0) { 14185 flags |= TH_MARKNEXT_NEEDED; 14186 freemsg(tcp->tcp_urp_mark_mp); 14187 tcp->tcp_urp_mark_mp = NULL; 14188 flags &= ~TH_SEND_URP_MARK; 14189 } else if (tcp->tcp_urp_mark_mp != NULL) { 14190 flags |= TH_SEND_URP_MARK; 14191 tcp->tcp_urp_mark_mp->b_flag &= 14192 ~MSGNOTMARKNEXT; 14193 tcp->tcp_urp_mark_mp->b_flag |= MSGMARKNEXT; 14194 } 14195 #ifdef DEBUG 14196 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE, 14197 "tcp_rput: AT MARK, len %d, flags 0x%x, %s", 14198 seg_len, flags, 14199 tcp_display(tcp, NULL, DISP_PORT_ONLY)); 14200 #endif /* DEBUG */ 14201 } else { 14202 /* Data left until we hit mark */ 14203 #ifdef DEBUG 14204 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE, 14205 "tcp_rput: URP %d bytes left, %s", 14206 urp - seg_len, tcp_display(tcp, NULL, 14207 DISP_PORT_ONLY)); 14208 #endif /* DEBUG */ 14209 } 14210 } 14211 14212 process_ack: 14213 if (!(flags & TH_ACK)) { 14214 freemsg(mp); 14215 goto xmit_check; 14216 } 14217 } 14218 bytes_acked = (int)(seg_ack - tcp->tcp_suna); 14219 14220 if (tcp->tcp_ipversion == IPV6_VERSION && bytes_acked > 0) 14221 tcp->tcp_ip_forward_progress = B_TRUE; 14222 if (tcp->tcp_state == TCPS_SYN_RCVD) { 14223 if ((tcp->tcp_conn.tcp_eager_conn_ind != NULL) && 14224 ((tcp->tcp_kssl_ent == NULL) || !tcp->tcp_kssl_pending)) { 14225 /* 3-way handshake complete - pass up the T_CONN_IND */ 14226 tcp_t *listener = tcp->tcp_listener; 14227 mblk_t *mp = tcp->tcp_conn.tcp_eager_conn_ind; 14228 14229 tcp->tcp_tconnind_started = B_TRUE; 14230 tcp->tcp_conn.tcp_eager_conn_ind = NULL; 14231 /* 14232 * We are here means eager is fine but it can 14233 * get a TH_RST at any point between now and till 14234 * accept completes and disappear. We need to 14235 * ensure that reference to eager is valid after 14236 * we get out of eager's perimeter. So we do 14237 * an extra refhold. 14238 */ 14239 CONN_INC_REF(connp); 14240 14241 /* 14242 * The listener also exists because of the refhold 14243 * done in tcp_conn_request. Its possible that it 14244 * might have closed. We will check that once we 14245 * get inside listeners context. 14246 */ 14247 CONN_INC_REF(listener->tcp_connp); 14248 if (listener->tcp_connp->conn_sqp == 14249 connp->conn_sqp) { 14250 tcp_send_conn_ind(listener->tcp_connp, mp, 14251 listener->tcp_connp->conn_sqp); 14252 CONN_DEC_REF(listener->tcp_connp); 14253 } else if (!tcp->tcp_loopback) { 14254 squeue_fill(listener->tcp_connp->conn_sqp, mp, 14255 tcp_send_conn_ind, 14256 listener->tcp_connp, SQTAG_TCP_CONN_IND); 14257 } else { 14258 squeue_enter(listener->tcp_connp->conn_sqp, mp, 14259 tcp_send_conn_ind, listener->tcp_connp, 14260 SQTAG_TCP_CONN_IND); 14261 } 14262 } 14263 14264 if (tcp->tcp_active_open) { 14265 /* 14266 * We are seeing the final ack in the three way 14267 * hand shake of a active open'ed connection 14268 * so we must send up a T_CONN_CON 14269 */ 14270 if (!tcp_conn_con(tcp, iphdr, tcph, mp, NULL)) { 14271 freemsg(mp); 14272 return; 14273 } 14274 /* 14275 * Don't fuse the loopback endpoints for 14276 * simultaneous active opens. 14277 */ 14278 if (tcp->tcp_loopback) { 14279 TCP_STAT(tcps, tcp_fusion_unfusable); 14280 tcp->tcp_unfusable = B_TRUE; 14281 } 14282 } 14283 14284 tcp->tcp_suna = tcp->tcp_iss + 1; /* One for the SYN */ 14285 bytes_acked--; 14286 /* SYN was acked - making progress */ 14287 if (tcp->tcp_ipversion == IPV6_VERSION) 14288 tcp->tcp_ip_forward_progress = B_TRUE; 14289 14290 /* 14291 * If SYN was retransmitted, need to reset all 14292 * retransmission info as this segment will be 14293 * treated as a dup ACK. 14294 */ 14295 if (tcp->tcp_rexmit) { 14296 tcp->tcp_rexmit = B_FALSE; 14297 tcp->tcp_rexmit_nxt = tcp->tcp_snxt; 14298 tcp->tcp_rexmit_max = tcp->tcp_snxt; 14299 tcp->tcp_snd_burst = tcp->tcp_localnet ? 14300 TCP_CWND_INFINITE : TCP_CWND_NORMAL; 14301 tcp->tcp_ms_we_have_waited = 0; 14302 tcp->tcp_cwnd = mss; 14303 } 14304 14305 /* 14306 * We set the send window to zero here. 14307 * This is needed if there is data to be 14308 * processed already on the queue. 14309 * Later (at swnd_update label), the 14310 * "new_swnd > tcp_swnd" condition is satisfied 14311 * the XMIT_NEEDED flag is set in the current 14312 * (SYN_RCVD) state. This ensures tcp_wput_data() is 14313 * called if there is already data on queue in 14314 * this state. 14315 */ 14316 tcp->tcp_swnd = 0; 14317 14318 if (new_swnd > tcp->tcp_max_swnd) 14319 tcp->tcp_max_swnd = new_swnd; 14320 tcp->tcp_swl1 = seg_seq; 14321 tcp->tcp_swl2 = seg_ack; 14322 tcp->tcp_state = TCPS_ESTABLISHED; 14323 tcp->tcp_valid_bits &= ~TCP_ISS_VALID; 14324 14325 /* Fuse when both sides are in ESTABLISHED state */ 14326 if (tcp->tcp_loopback && do_tcp_fusion) 14327 tcp_fuse(tcp, iphdr, tcph); 14328 14329 } 14330 /* This code follows 4.4BSD-Lite2 mostly. */ 14331 if (bytes_acked < 0) 14332 goto est; 14333 14334 /* 14335 * If TCP is ECN capable and the congestion experience bit is 14336 * set, reduce tcp_cwnd and tcp_ssthresh. But this should only be 14337 * done once per window (or more loosely, per RTT). 14338 */ 14339 if (tcp->tcp_cwr && SEQ_GT(seg_ack, tcp->tcp_cwr_snd_max)) 14340 tcp->tcp_cwr = B_FALSE; 14341 if (tcp->tcp_ecn_ok && (flags & TH_ECE)) { 14342 if (!tcp->tcp_cwr) { 14343 npkt = ((tcp->tcp_snxt - tcp->tcp_suna) >> 1) / mss; 14344 tcp->tcp_cwnd_ssthresh = MAX(npkt, 2) * mss; 14345 tcp->tcp_cwnd = npkt * mss; 14346 /* 14347 * If the cwnd is 0, use the timer to clock out 14348 * new segments. This is required by the ECN spec. 14349 */ 14350 if (npkt == 0) { 14351 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 14352 /* 14353 * This makes sure that when the ACK comes 14354 * back, we will increase tcp_cwnd by 1 MSS. 14355 */ 14356 tcp->tcp_cwnd_cnt = 0; 14357 } 14358 tcp->tcp_cwr = B_TRUE; 14359 /* 14360 * This marks the end of the current window of in 14361 * flight data. That is why we don't use 14362 * tcp_suna + tcp_swnd. Only data in flight can 14363 * provide ECN info. 14364 */ 14365 tcp->tcp_cwr_snd_max = tcp->tcp_snxt; 14366 tcp->tcp_ecn_cwr_sent = B_FALSE; 14367 } 14368 } 14369 14370 mp1 = tcp->tcp_xmit_head; 14371 if (bytes_acked == 0) { 14372 if (!ofo_seg && seg_len == 0 && new_swnd == tcp->tcp_swnd) { 14373 int dupack_cnt; 14374 14375 BUMP_MIB(&tcps->tcps_mib, tcpInDupAck); 14376 /* 14377 * Fast retransmit. When we have seen exactly three 14378 * identical ACKs while we have unacked data 14379 * outstanding we take it as a hint that our peer 14380 * dropped something. 14381 * 14382 * If TCP is retransmitting, don't do fast retransmit. 14383 */ 14384 if (mp1 && tcp->tcp_suna != tcp->tcp_snxt && 14385 ! tcp->tcp_rexmit) { 14386 /* Do Limited Transmit */ 14387 if ((dupack_cnt = ++tcp->tcp_dupack_cnt) < 14388 tcps->tcps_dupack_fast_retransmit) { 14389 /* 14390 * RFC 3042 14391 * 14392 * What we need to do is temporarily 14393 * increase tcp_cwnd so that new 14394 * data can be sent if it is allowed 14395 * by the receive window (tcp_rwnd). 14396 * tcp_wput_data() will take care of 14397 * the rest. 14398 * 14399 * If the connection is SACK capable, 14400 * only do limited xmit when there 14401 * is SACK info. 14402 * 14403 * Note how tcp_cwnd is incremented. 14404 * The first dup ACK will increase 14405 * it by 1 MSS. The second dup ACK 14406 * will increase it by 2 MSS. This 14407 * means that only 1 new segment will 14408 * be sent for each dup ACK. 14409 */ 14410 if (tcp->tcp_unsent > 0 && 14411 (!tcp->tcp_snd_sack_ok || 14412 (tcp->tcp_snd_sack_ok && 14413 tcp->tcp_notsack_list != NULL))) { 14414 tcp->tcp_cwnd += mss << 14415 (tcp->tcp_dupack_cnt - 1); 14416 flags |= TH_LIMIT_XMIT; 14417 } 14418 } else if (dupack_cnt == 14419 tcps->tcps_dupack_fast_retransmit) { 14420 14421 /* 14422 * If we have reduced tcp_ssthresh 14423 * because of ECN, do not reduce it again 14424 * unless it is already one window of data 14425 * away. After one window of data, tcp_cwr 14426 * should then be cleared. Note that 14427 * for non ECN capable connection, tcp_cwr 14428 * should always be false. 14429 * 14430 * Adjust cwnd since the duplicate 14431 * ack indicates that a packet was 14432 * dropped (due to congestion.) 14433 */ 14434 if (!tcp->tcp_cwr) { 14435 npkt = ((tcp->tcp_snxt - 14436 tcp->tcp_suna) >> 1) / mss; 14437 tcp->tcp_cwnd_ssthresh = MAX(npkt, 2) * 14438 mss; 14439 tcp->tcp_cwnd = (npkt + 14440 tcp->tcp_dupack_cnt) * mss; 14441 } 14442 if (tcp->tcp_ecn_ok) { 14443 tcp->tcp_cwr = B_TRUE; 14444 tcp->tcp_cwr_snd_max = tcp->tcp_snxt; 14445 tcp->tcp_ecn_cwr_sent = B_FALSE; 14446 } 14447 14448 /* 14449 * We do Hoe's algorithm. Refer to her 14450 * paper "Improving the Start-up Behavior 14451 * of a Congestion Control Scheme for TCP," 14452 * appeared in SIGCOMM'96. 14453 * 14454 * Save highest seq no we have sent so far. 14455 * Be careful about the invisible FIN byte. 14456 */ 14457 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && 14458 (tcp->tcp_unsent == 0)) { 14459 tcp->tcp_rexmit_max = tcp->tcp_fss; 14460 } else { 14461 tcp->tcp_rexmit_max = tcp->tcp_snxt; 14462 } 14463 14464 /* 14465 * Do not allow bursty traffic during. 14466 * fast recovery. Refer to Fall and Floyd's 14467 * paper "Simulation-based Comparisons of 14468 * Tahoe, Reno and SACK TCP" (in CCR?) 14469 * This is a best current practise. 14470 */ 14471 tcp->tcp_snd_burst = TCP_CWND_SS; 14472 14473 /* 14474 * For SACK: 14475 * Calculate tcp_pipe, which is the 14476 * estimated number of bytes in 14477 * network. 14478 * 14479 * tcp_fack is the highest sack'ed seq num 14480 * TCP has received. 14481 * 14482 * tcp_pipe is explained in the above quoted 14483 * Fall and Floyd's paper. tcp_fack is 14484 * explained in Mathis and Mahdavi's 14485 * "Forward Acknowledgment: Refining TCP 14486 * Congestion Control" in SIGCOMM '96. 14487 */ 14488 if (tcp->tcp_snd_sack_ok) { 14489 ASSERT(tcp->tcp_sack_info != NULL); 14490 if (tcp->tcp_notsack_list != NULL) { 14491 tcp->tcp_pipe = tcp->tcp_snxt - 14492 tcp->tcp_fack; 14493 tcp->tcp_sack_snxt = seg_ack; 14494 flags |= TH_NEED_SACK_REXMIT; 14495 } else { 14496 /* 14497 * Always initialize tcp_pipe 14498 * even though we don't have 14499 * any SACK info. If later 14500 * we get SACK info and 14501 * tcp_pipe is not initialized, 14502 * funny things will happen. 14503 */ 14504 tcp->tcp_pipe = 14505 tcp->tcp_cwnd_ssthresh; 14506 } 14507 } else { 14508 flags |= TH_REXMIT_NEEDED; 14509 } /* tcp_snd_sack_ok */ 14510 14511 } else { 14512 /* 14513 * Here we perform congestion 14514 * avoidance, but NOT slow start. 14515 * This is known as the Fast 14516 * Recovery Algorithm. 14517 */ 14518 if (tcp->tcp_snd_sack_ok && 14519 tcp->tcp_notsack_list != NULL) { 14520 flags |= TH_NEED_SACK_REXMIT; 14521 tcp->tcp_pipe -= mss; 14522 if (tcp->tcp_pipe < 0) 14523 tcp->tcp_pipe = 0; 14524 } else { 14525 /* 14526 * We know that one more packet has 14527 * left the pipe thus we can update 14528 * cwnd. 14529 */ 14530 cwnd = tcp->tcp_cwnd + mss; 14531 if (cwnd > tcp->tcp_cwnd_max) 14532 cwnd = tcp->tcp_cwnd_max; 14533 tcp->tcp_cwnd = cwnd; 14534 if (tcp->tcp_unsent > 0) 14535 flags |= TH_XMIT_NEEDED; 14536 } 14537 } 14538 } 14539 } else if (tcp->tcp_zero_win_probe) { 14540 /* 14541 * If the window has opened, need to arrange 14542 * to send additional data. 14543 */ 14544 if (new_swnd != 0) { 14545 /* tcp_suna != tcp_snxt */ 14546 /* Packet contains a window update */ 14547 BUMP_MIB(&tcps->tcps_mib, tcpInWinUpdate); 14548 tcp->tcp_zero_win_probe = 0; 14549 tcp->tcp_timer_backoff = 0; 14550 tcp->tcp_ms_we_have_waited = 0; 14551 14552 /* 14553 * Transmit starting with tcp_suna since 14554 * the one byte probe is not ack'ed. 14555 * If TCP has sent more than one identical 14556 * probe, tcp_rexmit will be set. That means 14557 * tcp_ss_rexmit() will send out the one 14558 * byte along with new data. Otherwise, 14559 * fake the retransmission. 14560 */ 14561 flags |= TH_XMIT_NEEDED; 14562 if (!tcp->tcp_rexmit) { 14563 tcp->tcp_rexmit = B_TRUE; 14564 tcp->tcp_dupack_cnt = 0; 14565 tcp->tcp_rexmit_nxt = tcp->tcp_suna; 14566 tcp->tcp_rexmit_max = tcp->tcp_suna + 1; 14567 } 14568 } 14569 } 14570 goto swnd_update; 14571 } 14572 14573 /* 14574 * Check for "acceptability" of ACK value per RFC 793, pages 72 - 73. 14575 * If the ACK value acks something that we have not yet sent, it might 14576 * be an old duplicate segment. Send an ACK to re-synchronize the 14577 * other side. 14578 * Note: reset in response to unacceptable ACK in SYN_RECEIVE 14579 * state is handled above, so we can always just drop the segment and 14580 * send an ACK here. 14581 * 14582 * Should we send ACKs in response to ACK only segments? 14583 */ 14584 if (SEQ_GT(seg_ack, tcp->tcp_snxt)) { 14585 BUMP_MIB(&tcps->tcps_mib, tcpInAckUnsent); 14586 /* drop the received segment */ 14587 freemsg(mp); 14588 14589 /* 14590 * Send back an ACK. If tcp_drop_ack_unsent_cnt is 14591 * greater than 0, check if the number of such 14592 * bogus ACks is greater than that count. If yes, 14593 * don't send back any ACK. This prevents TCP from 14594 * getting into an ACK storm if somehow an attacker 14595 * successfully spoofs an acceptable segment to our 14596 * peer. 14597 */ 14598 if (tcp_drop_ack_unsent_cnt > 0 && 14599 ++tcp->tcp_in_ack_unsent > tcp_drop_ack_unsent_cnt) { 14600 TCP_STAT(tcps, tcp_in_ack_unsent_drop); 14601 return; 14602 } 14603 mp = tcp_ack_mp(tcp); 14604 if (mp != NULL) { 14605 BUMP_LOCAL(tcp->tcp_obsegs); 14606 BUMP_MIB(&tcps->tcps_mib, tcpOutAck); 14607 tcp_send_data(tcp, tcp->tcp_wq, mp); 14608 } 14609 return; 14610 } 14611 14612 /* 14613 * TCP gets a new ACK, update the notsack'ed list to delete those 14614 * blocks that are covered by this ACK. 14615 */ 14616 if (tcp->tcp_snd_sack_ok && tcp->tcp_notsack_list != NULL) { 14617 tcp_notsack_remove(&(tcp->tcp_notsack_list), seg_ack, 14618 &(tcp->tcp_num_notsack_blk), &(tcp->tcp_cnt_notsack_list)); 14619 } 14620 14621 /* 14622 * If we got an ACK after fast retransmit, check to see 14623 * if it is a partial ACK. If it is not and the congestion 14624 * window was inflated to account for the other side's 14625 * cached packets, retract it. If it is, do Hoe's algorithm. 14626 */ 14627 if (tcp->tcp_dupack_cnt >= tcps->tcps_dupack_fast_retransmit) { 14628 ASSERT(tcp->tcp_rexmit == B_FALSE); 14629 if (SEQ_GEQ(seg_ack, tcp->tcp_rexmit_max)) { 14630 tcp->tcp_dupack_cnt = 0; 14631 /* 14632 * Restore the orig tcp_cwnd_ssthresh after 14633 * fast retransmit phase. 14634 */ 14635 if (tcp->tcp_cwnd > tcp->tcp_cwnd_ssthresh) { 14636 tcp->tcp_cwnd = tcp->tcp_cwnd_ssthresh; 14637 } 14638 tcp->tcp_rexmit_max = seg_ack; 14639 tcp->tcp_cwnd_cnt = 0; 14640 tcp->tcp_snd_burst = tcp->tcp_localnet ? 14641 TCP_CWND_INFINITE : TCP_CWND_NORMAL; 14642 14643 /* 14644 * Remove all notsack info to avoid confusion with 14645 * the next fast retrasnmit/recovery phase. 14646 */ 14647 if (tcp->tcp_snd_sack_ok && 14648 tcp->tcp_notsack_list != NULL) { 14649 TCP_NOTSACK_REMOVE_ALL(tcp->tcp_notsack_list); 14650 } 14651 } else { 14652 if (tcp->tcp_snd_sack_ok && 14653 tcp->tcp_notsack_list != NULL) { 14654 flags |= TH_NEED_SACK_REXMIT; 14655 tcp->tcp_pipe -= mss; 14656 if (tcp->tcp_pipe < 0) 14657 tcp->tcp_pipe = 0; 14658 } else { 14659 /* 14660 * Hoe's algorithm: 14661 * 14662 * Retransmit the unack'ed segment and 14663 * restart fast recovery. Note that we 14664 * need to scale back tcp_cwnd to the 14665 * original value when we started fast 14666 * recovery. This is to prevent overly 14667 * aggressive behaviour in sending new 14668 * segments. 14669 */ 14670 tcp->tcp_cwnd = tcp->tcp_cwnd_ssthresh + 14671 tcps->tcps_dupack_fast_retransmit * mss; 14672 tcp->tcp_cwnd_cnt = tcp->tcp_cwnd; 14673 flags |= TH_REXMIT_NEEDED; 14674 } 14675 } 14676 } else { 14677 tcp->tcp_dupack_cnt = 0; 14678 if (tcp->tcp_rexmit) { 14679 /* 14680 * TCP is retranmitting. If the ACK ack's all 14681 * outstanding data, update tcp_rexmit_max and 14682 * tcp_rexmit_nxt. Otherwise, update tcp_rexmit_nxt 14683 * to the correct value. 14684 * 14685 * Note that SEQ_LEQ() is used. This is to avoid 14686 * unnecessary fast retransmit caused by dup ACKs 14687 * received when TCP does slow start retransmission 14688 * after a time out. During this phase, TCP may 14689 * send out segments which are already received. 14690 * This causes dup ACKs to be sent back. 14691 */ 14692 if (SEQ_LEQ(seg_ack, tcp->tcp_rexmit_max)) { 14693 if (SEQ_GT(seg_ack, tcp->tcp_rexmit_nxt)) { 14694 tcp->tcp_rexmit_nxt = seg_ack; 14695 } 14696 if (seg_ack != tcp->tcp_rexmit_max) { 14697 flags |= TH_XMIT_NEEDED; 14698 } 14699 } else { 14700 tcp->tcp_rexmit = B_FALSE; 14701 tcp->tcp_xmit_zc_clean = B_FALSE; 14702 tcp->tcp_rexmit_nxt = tcp->tcp_snxt; 14703 tcp->tcp_snd_burst = tcp->tcp_localnet ? 14704 TCP_CWND_INFINITE : TCP_CWND_NORMAL; 14705 } 14706 tcp->tcp_ms_we_have_waited = 0; 14707 } 14708 } 14709 14710 BUMP_MIB(&tcps->tcps_mib, tcpInAckSegs); 14711 UPDATE_MIB(&tcps->tcps_mib, tcpInAckBytes, bytes_acked); 14712 tcp->tcp_suna = seg_ack; 14713 if (tcp->tcp_zero_win_probe != 0) { 14714 tcp->tcp_zero_win_probe = 0; 14715 tcp->tcp_timer_backoff = 0; 14716 } 14717 14718 /* 14719 * If tcp_xmit_head is NULL, then it must be the FIN being ack'ed. 14720 * Note that it cannot be the SYN being ack'ed. The code flow 14721 * will not reach here. 14722 */ 14723 if (mp1 == NULL) { 14724 goto fin_acked; 14725 } 14726 14727 /* 14728 * Update the congestion window. 14729 * 14730 * If TCP is not ECN capable or TCP is ECN capable but the 14731 * congestion experience bit is not set, increase the tcp_cwnd as 14732 * usual. 14733 */ 14734 if (!tcp->tcp_ecn_ok || !(flags & TH_ECE)) { 14735 cwnd = tcp->tcp_cwnd; 14736 add = mss; 14737 14738 if (cwnd >= tcp->tcp_cwnd_ssthresh) { 14739 /* 14740 * This is to prevent an increase of less than 1 MSS of 14741 * tcp_cwnd. With partial increase, tcp_wput_data() 14742 * may send out tinygrams in order to preserve mblk 14743 * boundaries. 14744 * 14745 * By initializing tcp_cwnd_cnt to new tcp_cwnd and 14746 * decrementing it by 1 MSS for every ACKs, tcp_cwnd is 14747 * increased by 1 MSS for every RTTs. 14748 */ 14749 if (tcp->tcp_cwnd_cnt <= 0) { 14750 tcp->tcp_cwnd_cnt = cwnd + add; 14751 } else { 14752 tcp->tcp_cwnd_cnt -= add; 14753 add = 0; 14754 } 14755 } 14756 tcp->tcp_cwnd = MIN(cwnd + add, tcp->tcp_cwnd_max); 14757 } 14758 14759 /* See if the latest urgent data has been acknowledged */ 14760 if ((tcp->tcp_valid_bits & TCP_URG_VALID) && 14761 SEQ_GT(seg_ack, tcp->tcp_urg)) 14762 tcp->tcp_valid_bits &= ~TCP_URG_VALID; 14763 14764 /* Can we update the RTT estimates? */ 14765 if (tcp->tcp_snd_ts_ok) { 14766 /* Ignore zero timestamp echo-reply. */ 14767 if (tcpopt.tcp_opt_ts_ecr != 0) { 14768 tcp_set_rto(tcp, (int32_t)lbolt - 14769 (int32_t)tcpopt.tcp_opt_ts_ecr); 14770 } 14771 14772 /* If needed, restart the timer. */ 14773 if (tcp->tcp_set_timer == 1) { 14774 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 14775 tcp->tcp_set_timer = 0; 14776 } 14777 /* 14778 * Update tcp_csuna in case the other side stops sending 14779 * us timestamps. 14780 */ 14781 tcp->tcp_csuna = tcp->tcp_snxt; 14782 } else if (SEQ_GT(seg_ack, tcp->tcp_csuna)) { 14783 /* 14784 * An ACK sequence we haven't seen before, so get the RTT 14785 * and update the RTO. But first check if the timestamp is 14786 * valid to use. 14787 */ 14788 if ((mp1->b_next != NULL) && 14789 SEQ_GT(seg_ack, (uint32_t)(uintptr_t)(mp1->b_next))) 14790 tcp_set_rto(tcp, (int32_t)lbolt - 14791 (int32_t)(intptr_t)mp1->b_prev); 14792 else 14793 BUMP_MIB(&tcps->tcps_mib, tcpRttNoUpdate); 14794 14795 /* Remeber the last sequence to be ACKed */ 14796 tcp->tcp_csuna = seg_ack; 14797 if (tcp->tcp_set_timer == 1) { 14798 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 14799 tcp->tcp_set_timer = 0; 14800 } 14801 } else { 14802 BUMP_MIB(&tcps->tcps_mib, tcpRttNoUpdate); 14803 } 14804 14805 /* Eat acknowledged bytes off the xmit queue. */ 14806 for (;;) { 14807 mblk_t *mp2; 14808 uchar_t *wptr; 14809 14810 wptr = mp1->b_wptr; 14811 ASSERT((uintptr_t)(wptr - mp1->b_rptr) <= (uintptr_t)INT_MAX); 14812 bytes_acked -= (int)(wptr - mp1->b_rptr); 14813 if (bytes_acked < 0) { 14814 mp1->b_rptr = wptr + bytes_acked; 14815 /* 14816 * Set a new timestamp if all the bytes timed by the 14817 * old timestamp have been ack'ed. 14818 */ 14819 if (SEQ_GT(seg_ack, 14820 (uint32_t)(uintptr_t)(mp1->b_next))) { 14821 mp1->b_prev = (mblk_t *)(uintptr_t)lbolt; 14822 mp1->b_next = NULL; 14823 } 14824 break; 14825 } 14826 mp1->b_next = NULL; 14827 mp1->b_prev = NULL; 14828 mp2 = mp1; 14829 mp1 = mp1->b_cont; 14830 14831 /* 14832 * This notification is required for some zero-copy 14833 * clients to maintain a copy semantic. After the data 14834 * is ack'ed, client is safe to modify or reuse the buffer. 14835 */ 14836 if (tcp->tcp_snd_zcopy_aware && 14837 (mp2->b_datap->db_struioflag & STRUIO_ZCNOTIFY)) 14838 tcp_zcopy_notify(tcp); 14839 freeb(mp2); 14840 if (bytes_acked == 0) { 14841 if (mp1 == NULL) { 14842 /* Everything is ack'ed, clear the tail. */ 14843 tcp->tcp_xmit_tail = NULL; 14844 /* 14845 * Cancel the timer unless we are still 14846 * waiting for an ACK for the FIN packet. 14847 */ 14848 if (tcp->tcp_timer_tid != 0 && 14849 tcp->tcp_snxt == tcp->tcp_suna) { 14850 (void) TCP_TIMER_CANCEL(tcp, 14851 tcp->tcp_timer_tid); 14852 tcp->tcp_timer_tid = 0; 14853 } 14854 goto pre_swnd_update; 14855 } 14856 if (mp2 != tcp->tcp_xmit_tail) 14857 break; 14858 tcp->tcp_xmit_tail = mp1; 14859 ASSERT((uintptr_t)(mp1->b_wptr - mp1->b_rptr) <= 14860 (uintptr_t)INT_MAX); 14861 tcp->tcp_xmit_tail_unsent = (int)(mp1->b_wptr - 14862 mp1->b_rptr); 14863 break; 14864 } 14865 if (mp1 == NULL) { 14866 /* 14867 * More was acked but there is nothing more 14868 * outstanding. This means that the FIN was 14869 * just acked or that we're talking to a clown. 14870 */ 14871 fin_acked: 14872 ASSERT(tcp->tcp_fin_sent); 14873 tcp->tcp_xmit_tail = NULL; 14874 if (tcp->tcp_fin_sent) { 14875 /* FIN was acked - making progress */ 14876 if (tcp->tcp_ipversion == IPV6_VERSION && 14877 !tcp->tcp_fin_acked) 14878 tcp->tcp_ip_forward_progress = B_TRUE; 14879 tcp->tcp_fin_acked = B_TRUE; 14880 if (tcp->tcp_linger_tid != 0 && 14881 TCP_TIMER_CANCEL(tcp, 14882 tcp->tcp_linger_tid) >= 0) { 14883 tcp_stop_lingering(tcp); 14884 freemsg(mp); 14885 mp = NULL; 14886 } 14887 } else { 14888 /* 14889 * We should never get here because 14890 * we have already checked that the 14891 * number of bytes ack'ed should be 14892 * smaller than or equal to what we 14893 * have sent so far (it is the 14894 * acceptability check of the ACK). 14895 * We can only get here if the send 14896 * queue is corrupted. 14897 * 14898 * Terminate the connection and 14899 * panic the system. It is better 14900 * for us to panic instead of 14901 * continuing to avoid other disaster. 14902 */ 14903 tcp_xmit_ctl(NULL, tcp, tcp->tcp_snxt, 14904 tcp->tcp_rnxt, TH_RST|TH_ACK); 14905 panic("Memory corruption " 14906 "detected for connection %s.", 14907 tcp_display(tcp, NULL, 14908 DISP_ADDR_AND_PORT)); 14909 /*NOTREACHED*/ 14910 } 14911 goto pre_swnd_update; 14912 } 14913 ASSERT(mp2 != tcp->tcp_xmit_tail); 14914 } 14915 if (tcp->tcp_unsent) { 14916 flags |= TH_XMIT_NEEDED; 14917 } 14918 pre_swnd_update: 14919 tcp->tcp_xmit_head = mp1; 14920 swnd_update: 14921 /* 14922 * The following check is different from most other implementations. 14923 * For bi-directional transfer, when segments are dropped, the 14924 * "normal" check will not accept a window update in those 14925 * retransmitted segemnts. Failing to do that, TCP may send out 14926 * segments which are outside receiver's window. As TCP accepts 14927 * the ack in those retransmitted segments, if the window update in 14928 * the same segment is not accepted, TCP will incorrectly calculates 14929 * that it can send more segments. This can create a deadlock 14930 * with the receiver if its window becomes zero. 14931 */ 14932 if (SEQ_LT(tcp->tcp_swl2, seg_ack) || 14933 SEQ_LT(tcp->tcp_swl1, seg_seq) || 14934 (tcp->tcp_swl1 == seg_seq && new_swnd > tcp->tcp_swnd)) { 14935 /* 14936 * The criteria for update is: 14937 * 14938 * 1. the segment acknowledges some data. Or 14939 * 2. the segment is new, i.e. it has a higher seq num. Or 14940 * 3. the segment is not old and the advertised window is 14941 * larger than the previous advertised window. 14942 */ 14943 if (tcp->tcp_unsent && new_swnd > tcp->tcp_swnd) 14944 flags |= TH_XMIT_NEEDED; 14945 tcp->tcp_swnd = new_swnd; 14946 if (new_swnd > tcp->tcp_max_swnd) 14947 tcp->tcp_max_swnd = new_swnd; 14948 tcp->tcp_swl1 = seg_seq; 14949 tcp->tcp_swl2 = seg_ack; 14950 } 14951 est: 14952 if (tcp->tcp_state > TCPS_ESTABLISHED) { 14953 14954 switch (tcp->tcp_state) { 14955 case TCPS_FIN_WAIT_1: 14956 if (tcp->tcp_fin_acked) { 14957 tcp->tcp_state = TCPS_FIN_WAIT_2; 14958 /* 14959 * We implement the non-standard BSD/SunOS 14960 * FIN_WAIT_2 flushing algorithm. 14961 * If there is no user attached to this 14962 * TCP endpoint, then this TCP struct 14963 * could hang around forever in FIN_WAIT_2 14964 * state if the peer forgets to send us 14965 * a FIN. To prevent this, we wait only 14966 * 2*MSL (a convenient time value) for 14967 * the FIN to arrive. If it doesn't show up, 14968 * we flush the TCP endpoint. This algorithm, 14969 * though a violation of RFC-793, has worked 14970 * for over 10 years in BSD systems. 14971 * Note: SunOS 4.x waits 675 seconds before 14972 * flushing the FIN_WAIT_2 connection. 14973 */ 14974 TCP_TIMER_RESTART(tcp, 14975 tcps->tcps_fin_wait_2_flush_interval); 14976 } 14977 break; 14978 case TCPS_FIN_WAIT_2: 14979 break; /* Shutdown hook? */ 14980 case TCPS_LAST_ACK: 14981 freemsg(mp); 14982 if (tcp->tcp_fin_acked) { 14983 (void) tcp_clean_death(tcp, 0, 19); 14984 return; 14985 } 14986 goto xmit_check; 14987 case TCPS_CLOSING: 14988 if (tcp->tcp_fin_acked) { 14989 tcp->tcp_state = TCPS_TIME_WAIT; 14990 /* 14991 * Unconditionally clear the exclusive binding 14992 * bit so this TIME-WAIT connection won't 14993 * interfere with new ones. 14994 */ 14995 tcp->tcp_exclbind = 0; 14996 if (!TCP_IS_DETACHED(tcp)) { 14997 TCP_TIMER_RESTART(tcp, 14998 tcps->tcps_time_wait_interval); 14999 } else { 15000 tcp_time_wait_append(tcp); 15001 TCP_DBGSTAT(tcps, tcp_rput_time_wait); 15002 } 15003 } 15004 /*FALLTHRU*/ 15005 case TCPS_CLOSE_WAIT: 15006 freemsg(mp); 15007 goto xmit_check; 15008 default: 15009 ASSERT(tcp->tcp_state != TCPS_TIME_WAIT); 15010 break; 15011 } 15012 } 15013 if (flags & TH_FIN) { 15014 /* Make sure we ack the fin */ 15015 flags |= TH_ACK_NEEDED; 15016 if (!tcp->tcp_fin_rcvd) { 15017 tcp->tcp_fin_rcvd = B_TRUE; 15018 tcp->tcp_rnxt++; 15019 tcph = tcp->tcp_tcph; 15020 U32_TO_ABE32(tcp->tcp_rnxt, tcph->th_ack); 15021 15022 /* 15023 * Generate the ordrel_ind at the end unless we 15024 * are an eager guy. 15025 * In the eager case tcp_rsrv will do this when run 15026 * after tcp_accept is done. 15027 */ 15028 if (tcp->tcp_listener == NULL && 15029 !TCP_IS_DETACHED(tcp) && (!tcp->tcp_hard_binding)) 15030 flags |= TH_ORDREL_NEEDED; 15031 switch (tcp->tcp_state) { 15032 case TCPS_SYN_RCVD: 15033 case TCPS_ESTABLISHED: 15034 tcp->tcp_state = TCPS_CLOSE_WAIT; 15035 /* Keepalive? */ 15036 break; 15037 case TCPS_FIN_WAIT_1: 15038 if (!tcp->tcp_fin_acked) { 15039 tcp->tcp_state = TCPS_CLOSING; 15040 break; 15041 } 15042 /* FALLTHRU */ 15043 case TCPS_FIN_WAIT_2: 15044 tcp->tcp_state = TCPS_TIME_WAIT; 15045 /* 15046 * Unconditionally clear the exclusive binding 15047 * bit so this TIME-WAIT connection won't 15048 * interfere with new ones. 15049 */ 15050 tcp->tcp_exclbind = 0; 15051 if (!TCP_IS_DETACHED(tcp)) { 15052 TCP_TIMER_RESTART(tcp, 15053 tcps->tcps_time_wait_interval); 15054 } else { 15055 tcp_time_wait_append(tcp); 15056 TCP_DBGSTAT(tcps, tcp_rput_time_wait); 15057 } 15058 if (seg_len) { 15059 /* 15060 * implies data piggybacked on FIN. 15061 * break to handle data. 15062 */ 15063 break; 15064 } 15065 freemsg(mp); 15066 goto ack_check; 15067 } 15068 } 15069 } 15070 if (mp == NULL) 15071 goto xmit_check; 15072 if (seg_len == 0) { 15073 freemsg(mp); 15074 goto xmit_check; 15075 } 15076 if (mp->b_rptr == mp->b_wptr) { 15077 /* 15078 * The header has been consumed, so we remove the 15079 * zero-length mblk here. 15080 */ 15081 mp1 = mp; 15082 mp = mp->b_cont; 15083 freeb(mp1); 15084 } 15085 tcph = tcp->tcp_tcph; 15086 tcp->tcp_rack_cnt++; 15087 { 15088 uint32_t cur_max; 15089 15090 cur_max = tcp->tcp_rack_cur_max; 15091 if (tcp->tcp_rack_cnt >= cur_max) { 15092 /* 15093 * We have more unacked data than we should - send 15094 * an ACK now. 15095 */ 15096 flags |= TH_ACK_NEEDED; 15097 cur_max++; 15098 if (cur_max > tcp->tcp_rack_abs_max) 15099 tcp->tcp_rack_cur_max = tcp->tcp_rack_abs_max; 15100 else 15101 tcp->tcp_rack_cur_max = cur_max; 15102 } else if (TCP_IS_DETACHED(tcp)) { 15103 /* We don't have an ACK timer for detached TCP. */ 15104 flags |= TH_ACK_NEEDED; 15105 } else if (seg_len < mss) { 15106 /* 15107 * If we get a segment that is less than an mss, and we 15108 * already have unacknowledged data, and the amount 15109 * unacknowledged is not a multiple of mss, then we 15110 * better generate an ACK now. Otherwise, this may be 15111 * the tail piece of a transaction, and we would rather 15112 * wait for the response. 15113 */ 15114 uint32_t udif; 15115 ASSERT((uintptr_t)(tcp->tcp_rnxt - tcp->tcp_rack) <= 15116 (uintptr_t)INT_MAX); 15117 udif = (int)(tcp->tcp_rnxt - tcp->tcp_rack); 15118 if (udif && (udif % mss)) 15119 flags |= TH_ACK_NEEDED; 15120 else 15121 flags |= TH_ACK_TIMER_NEEDED; 15122 } else { 15123 /* Start delayed ack timer */ 15124 flags |= TH_ACK_TIMER_NEEDED; 15125 } 15126 } 15127 tcp->tcp_rnxt += seg_len; 15128 U32_TO_ABE32(tcp->tcp_rnxt, tcph->th_ack); 15129 15130 /* Update SACK list */ 15131 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) { 15132 tcp_sack_remove(tcp->tcp_sack_list, tcp->tcp_rnxt, 15133 &(tcp->tcp_num_sack_blk)); 15134 } 15135 15136 if (tcp->tcp_urp_mp) { 15137 tcp->tcp_urp_mp->b_cont = mp; 15138 mp = tcp->tcp_urp_mp; 15139 tcp->tcp_urp_mp = NULL; 15140 /* Ready for a new signal. */ 15141 tcp->tcp_urp_last_valid = B_FALSE; 15142 #ifdef DEBUG 15143 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE, 15144 "tcp_rput: sending exdata_ind %s", 15145 tcp_display(tcp, NULL, DISP_PORT_ONLY)); 15146 #endif /* DEBUG */ 15147 } 15148 15149 /* 15150 * Check for ancillary data changes compared to last segment. 15151 */ 15152 if (tcp->tcp_ipv6_recvancillary != 0) { 15153 mp = tcp_rput_add_ancillary(tcp, mp, &ipp); 15154 if (mp == NULL) 15155 return; 15156 } 15157 15158 if (tcp->tcp_listener || tcp->tcp_hard_binding) { 15159 /* 15160 * Side queue inbound data until the accept happens. 15161 * tcp_accept/tcp_rput drains this when the accept happens. 15162 * M_DATA is queued on b_cont. Otherwise (T_OPTDATA_IND or 15163 * T_EXDATA_IND) it is queued on b_next. 15164 * XXX Make urgent data use this. Requires: 15165 * Removing tcp_listener check for TH_URG 15166 * Making M_PCPROTO and MARK messages skip the eager case 15167 */ 15168 15169 if (tcp->tcp_kssl_pending) { 15170 DTRACE_PROBE1(kssl_mblk__ksslinput_pending, 15171 mblk_t *, mp); 15172 tcp_kssl_input(tcp, mp); 15173 } else { 15174 tcp_rcv_enqueue(tcp, mp, seg_len); 15175 } 15176 } else { 15177 sodirect_t *sodp = tcp->tcp_sodirect; 15178 15179 /* 15180 * If an sodirect connection and an enabled sodirect_t then 15181 * sodp will be set to point to the tcp_t/sonode_t shared 15182 * sodirect_t and the sodirect_t's lock will be held. 15183 */ 15184 if (sodp != NULL) { 15185 mutex_enter(sodp->sod_lock); 15186 if (!(sodp->sod_state & SOD_ENABLED)) { 15187 mutex_exit(sodp->sod_lock); 15188 sodp = NULL; 15189 } else if (tcp->tcp_kssl_ctx != NULL && 15190 DB_TYPE(mp) == M_DATA) { 15191 mutex_exit(sodp->sod_lock); 15192 sodp = NULL; 15193 } 15194 } 15195 if (mp->b_datap->db_type != M_DATA || 15196 (flags & TH_MARKNEXT_NEEDED)) { 15197 if (sodp != NULL) { 15198 if (!SOD_QEMPTY(sodp) && 15199 (sodp->sod_state & SOD_WAKE_NOT)) { 15200 flags |= tcp_rcv_sod_wakeup(tcp, sodp); 15201 /* sod_wakeup() did the mutex_exit() */ 15202 mutex_enter(sodp->sod_lock); 15203 } 15204 } else if (tcp->tcp_rcv_list != NULL) { 15205 flags |= tcp_rcv_drain(tcp->tcp_rq, tcp); 15206 } 15207 ASSERT(tcp->tcp_rcv_list == NULL || 15208 tcp->tcp_fused_sigurg); 15209 15210 if (flags & TH_MARKNEXT_NEEDED) { 15211 #ifdef DEBUG 15212 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE, 15213 "tcp_rput: sending MSGMARKNEXT %s", 15214 tcp_display(tcp, NULL, 15215 DISP_PORT_ONLY)); 15216 #endif /* DEBUG */ 15217 mp->b_flag |= MSGMARKNEXT; 15218 flags &= ~TH_MARKNEXT_NEEDED; 15219 } 15220 15221 /* Does this need SSL processing first? */ 15222 if ((tcp->tcp_kssl_ctx != NULL) && 15223 (DB_TYPE(mp) == M_DATA)) { 15224 DTRACE_PROBE1(kssl_mblk__ksslinput_data1, 15225 mblk_t *, mp); 15226 tcp_kssl_input(tcp, mp); 15227 } else { 15228 if (sodp) { 15229 /* 15230 * Done with sodirect, use putnext 15231 * to push this non M_DATA headed 15232 * mblk_t chain. 15233 */ 15234 mutex_exit(sodp->sod_lock); 15235 } 15236 putnext(tcp->tcp_rq, mp); 15237 if (!canputnext(tcp->tcp_rq)) 15238 tcp->tcp_rwnd -= seg_len; 15239 } 15240 } else if ((tcp->tcp_kssl_ctx != NULL) && 15241 (DB_TYPE(mp) == M_DATA)) { 15242 /* Do SSL processing first */ 15243 DTRACE_PROBE1(kssl_mblk__ksslinput_data2, 15244 mblk_t *, mp); 15245 tcp_kssl_input(tcp, mp); 15246 } else if (sodp != NULL) { 15247 /* 15248 * Sodirect so all mblk_t's are queued on the 15249 * socket directly, check for wakeup of blocked 15250 * reader (if any), and last if flow-controled. 15251 */ 15252 flags |= tcp_rcv_sod_enqueue(tcp, sodp, mp, seg_len); 15253 if ((sodp->sod_state & SOD_WAKE_NEED) || 15254 (flags & (TH_PUSH|TH_FIN))) { 15255 flags |= tcp_rcv_sod_wakeup(tcp, sodp); 15256 /* sod_wakeup() did the mutex_exit() */ 15257 } else { 15258 if (SOD_QFULL(sodp)) { 15259 /* Q is full, need backenable */ 15260 SOD_QSETBE(sodp); 15261 } 15262 mutex_exit(sodp->sod_lock); 15263 } 15264 } else if ((flags & (TH_PUSH|TH_FIN)) || 15265 tcp->tcp_rcv_cnt + seg_len >= tcp->tcp_rq->q_hiwat >> 3) { 15266 if (tcp->tcp_rcv_list != NULL) { 15267 /* 15268 * Enqueue the new segment first and then 15269 * call tcp_rcv_drain() to send all data 15270 * up. The other way to do this is to 15271 * send all queued data up and then call 15272 * putnext() to send the new segment up. 15273 * This way can remove the else part later 15274 * on. 15275 * 15276 * We don't this to avoid one more call to 15277 * canputnext() as tcp_rcv_drain() needs to 15278 * call canputnext(). 15279 */ 15280 tcp_rcv_enqueue(tcp, mp, seg_len); 15281 flags |= tcp_rcv_drain(tcp->tcp_rq, tcp); 15282 } else { 15283 putnext(tcp->tcp_rq, mp); 15284 if (!canputnext(tcp->tcp_rq)) 15285 tcp->tcp_rwnd -= seg_len; 15286 } 15287 } else { 15288 /* 15289 * Enqueue all packets when processing an mblk 15290 * from the co queue and also enqueue normal packets. 15291 */ 15292 tcp_rcv_enqueue(tcp, mp, seg_len); 15293 } 15294 /* 15295 * Make sure the timer is running if we have data waiting 15296 * for a push bit. This provides resiliency against 15297 * implementations that do not correctly generate push bits. 15298 * 15299 * Note, for sodirect if Q isn't empty and there's not a 15300 * pending wakeup then we need a timer. Also note that sodp 15301 * is assumed to be still valid after exit()ing the sod_lock 15302 * above and while the SOD state can change it can only change 15303 * such that the Q is empty now even though data was added 15304 * above. 15305 */ 15306 if (((sodp != NULL && !SOD_QEMPTY(sodp) && 15307 (sodp->sod_state & SOD_WAKE_NOT)) || 15308 (sodp == NULL && tcp->tcp_rcv_list != NULL)) && 15309 tcp->tcp_push_tid == 0) { 15310 /* 15311 * The connection may be closed at this point, so don't 15312 * do anything for a detached tcp. 15313 */ 15314 if (!TCP_IS_DETACHED(tcp)) 15315 tcp->tcp_push_tid = TCP_TIMER(tcp, 15316 tcp_push_timer, 15317 MSEC_TO_TICK( 15318 tcps->tcps_push_timer_interval)); 15319 } 15320 } 15321 15322 xmit_check: 15323 /* Is there anything left to do? */ 15324 ASSERT(!(flags & TH_MARKNEXT_NEEDED)); 15325 if ((flags & (TH_REXMIT_NEEDED|TH_XMIT_NEEDED|TH_ACK_NEEDED| 15326 TH_NEED_SACK_REXMIT|TH_LIMIT_XMIT|TH_ACK_TIMER_NEEDED| 15327 TH_ORDREL_NEEDED|TH_SEND_URP_MARK)) == 0) 15328 goto done; 15329 15330 /* Any transmit work to do and a non-zero window? */ 15331 if ((flags & (TH_REXMIT_NEEDED|TH_XMIT_NEEDED|TH_NEED_SACK_REXMIT| 15332 TH_LIMIT_XMIT)) && tcp->tcp_swnd != 0) { 15333 if (flags & TH_REXMIT_NEEDED) { 15334 uint32_t snd_size = tcp->tcp_snxt - tcp->tcp_suna; 15335 15336 BUMP_MIB(&tcps->tcps_mib, tcpOutFastRetrans); 15337 if (snd_size > mss) 15338 snd_size = mss; 15339 if (snd_size > tcp->tcp_swnd) 15340 snd_size = tcp->tcp_swnd; 15341 mp1 = tcp_xmit_mp(tcp, tcp->tcp_xmit_head, snd_size, 15342 NULL, NULL, tcp->tcp_suna, B_TRUE, &snd_size, 15343 B_TRUE); 15344 15345 if (mp1 != NULL) { 15346 tcp->tcp_xmit_head->b_prev = (mblk_t *)lbolt; 15347 tcp->tcp_csuna = tcp->tcp_snxt; 15348 BUMP_MIB(&tcps->tcps_mib, tcpRetransSegs); 15349 UPDATE_MIB(&tcps->tcps_mib, 15350 tcpRetransBytes, snd_size); 15351 tcp_send_data(tcp, tcp->tcp_wq, mp1); 15352 } 15353 } 15354 if (flags & TH_NEED_SACK_REXMIT) { 15355 tcp_sack_rxmit(tcp, &flags); 15356 } 15357 /* 15358 * For TH_LIMIT_XMIT, tcp_wput_data() is called to send 15359 * out new segment. Note that tcp_rexmit should not be 15360 * set, otherwise TH_LIMIT_XMIT should not be set. 15361 */ 15362 if (flags & (TH_XMIT_NEEDED|TH_LIMIT_XMIT)) { 15363 if (!tcp->tcp_rexmit) { 15364 tcp_wput_data(tcp, NULL, B_FALSE); 15365 } else { 15366 tcp_ss_rexmit(tcp); 15367 } 15368 } 15369 /* 15370 * Adjust tcp_cwnd back to normal value after sending 15371 * new data segments. 15372 */ 15373 if (flags & TH_LIMIT_XMIT) { 15374 tcp->tcp_cwnd -= mss << (tcp->tcp_dupack_cnt - 1); 15375 /* 15376 * This will restart the timer. Restarting the 15377 * timer is used to avoid a timeout before the 15378 * limited transmitted segment's ACK gets back. 15379 */ 15380 if (tcp->tcp_xmit_head != NULL) 15381 tcp->tcp_xmit_head->b_prev = (mblk_t *)lbolt; 15382 } 15383 15384 /* Anything more to do? */ 15385 if ((flags & (TH_ACK_NEEDED|TH_ACK_TIMER_NEEDED| 15386 TH_ORDREL_NEEDED|TH_SEND_URP_MARK)) == 0) 15387 goto done; 15388 } 15389 ack_check: 15390 if (flags & TH_SEND_URP_MARK) { 15391 ASSERT(tcp->tcp_urp_mark_mp); 15392 /* 15393 * Send up any queued data and then send the mark message 15394 */ 15395 sodirect_t *sodp; 15396 15397 SOD_PTR_ENTER(tcp, sodp); 15398 15399 mp1 = tcp->tcp_urp_mark_mp; 15400 tcp->tcp_urp_mark_mp = NULL; 15401 if (sodp != NULL) { 15402 15403 ASSERT(tcp->tcp_rcv_list == NULL); 15404 15405 flags |= tcp_rcv_sod_wakeup(tcp, sodp); 15406 /* sod_wakeup() does the mutex_exit() */ 15407 } else if (tcp->tcp_rcv_list != NULL) { 15408 flags |= tcp_rcv_drain(tcp->tcp_rq, tcp); 15409 15410 ASSERT(tcp->tcp_rcv_list == NULL || 15411 tcp->tcp_fused_sigurg); 15412 15413 } 15414 putnext(tcp->tcp_rq, mp1); 15415 #ifdef DEBUG 15416 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE, 15417 "tcp_rput: sending zero-length %s %s", 15418 ((mp1->b_flag & MSGMARKNEXT) ? "MSGMARKNEXT" : 15419 "MSGNOTMARKNEXT"), 15420 tcp_display(tcp, NULL, DISP_PORT_ONLY)); 15421 #endif /* DEBUG */ 15422 flags &= ~TH_SEND_URP_MARK; 15423 } 15424 if (flags & TH_ACK_NEEDED) { 15425 /* 15426 * Time to send an ack for some reason. 15427 */ 15428 mp1 = tcp_ack_mp(tcp); 15429 15430 if (mp1 != NULL) { 15431 tcp_send_data(tcp, tcp->tcp_wq, mp1); 15432 BUMP_LOCAL(tcp->tcp_obsegs); 15433 BUMP_MIB(&tcps->tcps_mib, tcpOutAck); 15434 } 15435 if (tcp->tcp_ack_tid != 0) { 15436 (void) TCP_TIMER_CANCEL(tcp, tcp->tcp_ack_tid); 15437 tcp->tcp_ack_tid = 0; 15438 } 15439 } 15440 if (flags & TH_ACK_TIMER_NEEDED) { 15441 /* 15442 * Arrange for deferred ACK or push wait timeout. 15443 * Start timer if it is not already running. 15444 */ 15445 if (tcp->tcp_ack_tid == 0) { 15446 tcp->tcp_ack_tid = TCP_TIMER(tcp, tcp_ack_timer, 15447 MSEC_TO_TICK(tcp->tcp_localnet ? 15448 (clock_t)tcps->tcps_local_dack_interval : 15449 (clock_t)tcps->tcps_deferred_ack_interval)); 15450 } 15451 } 15452 if (flags & TH_ORDREL_NEEDED) { 15453 /* 15454 * Send up the ordrel_ind unless we are an eager guy. 15455 * In the eager case tcp_rsrv will do this when run 15456 * after tcp_accept is done. 15457 */ 15458 sodirect_t *sodp; 15459 15460 ASSERT(tcp->tcp_listener == NULL); 15461 15462 SOD_PTR_ENTER(tcp, sodp); 15463 if (sodp != NULL) { 15464 /* No more sodirect */ 15465 tcp->tcp_sodirect = NULL; 15466 if (!SOD_QEMPTY(sodp)) { 15467 /* Mblk(s) to process, notify */ 15468 flags |= tcp_rcv_sod_wakeup(tcp, sodp); 15469 /* sod_wakeup() does the mutex_exit() */ 15470 } else { 15471 /* Nothing to process */ 15472 mutex_exit(sodp->sod_lock); 15473 } 15474 } else if (tcp->tcp_rcv_list != NULL) { 15475 /* 15476 * Push any mblk(s) enqueued from co processing. 15477 */ 15478 flags |= tcp_rcv_drain(tcp->tcp_rq, tcp); 15479 15480 ASSERT(tcp->tcp_rcv_list == NULL || 15481 tcp->tcp_fused_sigurg); 15482 } 15483 15484 if ((mp1 = mi_tpi_ordrel_ind()) != NULL) { 15485 tcp->tcp_ordrel_done = B_TRUE; 15486 putnext(tcp->tcp_rq, mp1); 15487 if (tcp->tcp_deferred_clean_death) { 15488 /* 15489 * tcp_clean_death was deferred 15490 * for T_ORDREL_IND - do it now 15491 */ 15492 (void) tcp_clean_death(tcp, 15493 tcp->tcp_client_errno, 20); 15494 tcp->tcp_deferred_clean_death = B_FALSE; 15495 } 15496 } else { 15497 /* 15498 * Run the orderly release in the 15499 * service routine. 15500 */ 15501 qenable(tcp->tcp_rq); 15502 /* 15503 * Caveat(XXX): The machine may be so 15504 * overloaded that tcp_rsrv() is not scheduled 15505 * until after the endpoint has transitioned 15506 * to TCPS_TIME_WAIT 15507 * and tcp_time_wait_interval expires. Then 15508 * tcp_timer() will blow away state in tcp_t 15509 * and T_ORDREL_IND will never be delivered 15510 * upstream. Unlikely but potentially 15511 * a problem. 15512 */ 15513 } 15514 } 15515 done: 15516 ASSERT(!(flags & TH_MARKNEXT_NEEDED)); 15517 } 15518 15519 /* 15520 * This function does PAWS protection check. Returns B_TRUE if the 15521 * segment passes the PAWS test, else returns B_FALSE. 15522 */ 15523 boolean_t 15524 tcp_paws_check(tcp_t *tcp, tcph_t *tcph, tcp_opt_t *tcpoptp) 15525 { 15526 uint8_t flags; 15527 int options; 15528 uint8_t *up; 15529 15530 flags = (unsigned int)tcph->th_flags[0] & 0xFF; 15531 /* 15532 * If timestamp option is aligned nicely, get values inline, 15533 * otherwise call general routine to parse. Only do that 15534 * if timestamp is the only option. 15535 */ 15536 if (TCP_HDR_LENGTH(tcph) == (uint32_t)TCP_MIN_HEADER_LENGTH + 15537 TCPOPT_REAL_TS_LEN && 15538 OK_32PTR((up = ((uint8_t *)tcph) + 15539 TCP_MIN_HEADER_LENGTH)) && 15540 *(uint32_t *)up == TCPOPT_NOP_NOP_TSTAMP) { 15541 tcpoptp->tcp_opt_ts_val = ABE32_TO_U32((up+4)); 15542 tcpoptp->tcp_opt_ts_ecr = ABE32_TO_U32((up+8)); 15543 15544 options = TCP_OPT_TSTAMP_PRESENT; 15545 } else { 15546 if (tcp->tcp_snd_sack_ok) { 15547 tcpoptp->tcp = tcp; 15548 } else { 15549 tcpoptp->tcp = NULL; 15550 } 15551 options = tcp_parse_options(tcph, tcpoptp); 15552 } 15553 15554 if (options & TCP_OPT_TSTAMP_PRESENT) { 15555 /* 15556 * Do PAWS per RFC 1323 section 4.2. Accept RST 15557 * regardless of the timestamp, page 18 RFC 1323.bis. 15558 */ 15559 if ((flags & TH_RST) == 0 && 15560 TSTMP_LT(tcpoptp->tcp_opt_ts_val, 15561 tcp->tcp_ts_recent)) { 15562 if (TSTMP_LT(lbolt64, tcp->tcp_last_rcv_lbolt + 15563 PAWS_TIMEOUT)) { 15564 /* This segment is not acceptable. */ 15565 return (B_FALSE); 15566 } else { 15567 /* 15568 * Connection has been idle for 15569 * too long. Reset the timestamp 15570 * and assume the segment is valid. 15571 */ 15572 tcp->tcp_ts_recent = 15573 tcpoptp->tcp_opt_ts_val; 15574 } 15575 } 15576 } else { 15577 /* 15578 * If we don't get a timestamp on every packet, we 15579 * figure we can't really trust 'em, so we stop sending 15580 * and parsing them. 15581 */ 15582 tcp->tcp_snd_ts_ok = B_FALSE; 15583 15584 tcp->tcp_hdr_len -= TCPOPT_REAL_TS_LEN; 15585 tcp->tcp_tcp_hdr_len -= TCPOPT_REAL_TS_LEN; 15586 tcp->tcp_tcph->th_offset_and_rsrvd[0] -= (3 << 4); 15587 /* 15588 * Adjust the tcp_mss accordingly. We also need to 15589 * adjust tcp_cwnd here in accordance with the new mss. 15590 * But we avoid doing a slow start here so as to not 15591 * to lose on the transfer rate built up so far. 15592 */ 15593 tcp_mss_set(tcp, tcp->tcp_mss + TCPOPT_REAL_TS_LEN, B_FALSE); 15594 if (tcp->tcp_snd_sack_ok) { 15595 ASSERT(tcp->tcp_sack_info != NULL); 15596 tcp->tcp_max_sack_blk = 4; 15597 } 15598 } 15599 return (B_TRUE); 15600 } 15601 15602 /* 15603 * Attach ancillary data to a received TCP segments for the 15604 * ancillary pieces requested by the application that are 15605 * different than they were in the previous data segment. 15606 * 15607 * Save the "current" values once memory allocation is ok so that 15608 * when memory allocation fails we can just wait for the next data segment. 15609 */ 15610 static mblk_t * 15611 tcp_rput_add_ancillary(tcp_t *tcp, mblk_t *mp, ip6_pkt_t *ipp) 15612 { 15613 struct T_optdata_ind *todi; 15614 int optlen; 15615 uchar_t *optptr; 15616 struct T_opthdr *toh; 15617 uint_t addflag; /* Which pieces to add */ 15618 mblk_t *mp1; 15619 15620 optlen = 0; 15621 addflag = 0; 15622 /* If app asked for pktinfo and the index has changed ... */ 15623 if ((ipp->ipp_fields & IPPF_IFINDEX) && 15624 ipp->ipp_ifindex != tcp->tcp_recvifindex && 15625 (tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVPKTINFO)) { 15626 optlen += sizeof (struct T_opthdr) + 15627 sizeof (struct in6_pktinfo); 15628 addflag |= TCP_IPV6_RECVPKTINFO; 15629 } 15630 /* If app asked for hoplimit and it has changed ... */ 15631 if ((ipp->ipp_fields & IPPF_HOPLIMIT) && 15632 ipp->ipp_hoplimit != tcp->tcp_recvhops && 15633 (tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVHOPLIMIT)) { 15634 optlen += sizeof (struct T_opthdr) + sizeof (uint_t); 15635 addflag |= TCP_IPV6_RECVHOPLIMIT; 15636 } 15637 /* If app asked for tclass and it has changed ... */ 15638 if ((ipp->ipp_fields & IPPF_TCLASS) && 15639 ipp->ipp_tclass != tcp->tcp_recvtclass && 15640 (tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVTCLASS)) { 15641 optlen += sizeof (struct T_opthdr) + sizeof (uint_t); 15642 addflag |= TCP_IPV6_RECVTCLASS; 15643 } 15644 /* 15645 * If app asked for hopbyhop headers and it has changed ... 15646 * For security labels, note that (1) security labels can't change on 15647 * a connected socket at all, (2) we're connected to at most one peer, 15648 * (3) if anything changes, then it must be some other extra option. 15649 */ 15650 if ((tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVHOPOPTS) && 15651 ip_cmpbuf(tcp->tcp_hopopts, tcp->tcp_hopoptslen, 15652 (ipp->ipp_fields & IPPF_HOPOPTS), 15653 ipp->ipp_hopopts, ipp->ipp_hopoptslen)) { 15654 optlen += sizeof (struct T_opthdr) + ipp->ipp_hopoptslen - 15655 tcp->tcp_label_len; 15656 addflag |= TCP_IPV6_RECVHOPOPTS; 15657 if (!ip_allocbuf((void **)&tcp->tcp_hopopts, 15658 &tcp->tcp_hopoptslen, (ipp->ipp_fields & IPPF_HOPOPTS), 15659 ipp->ipp_hopopts, ipp->ipp_hopoptslen)) 15660 return (mp); 15661 } 15662 /* If app asked for dst headers before routing headers ... */ 15663 if ((tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVRTDSTOPTS) && 15664 ip_cmpbuf(tcp->tcp_rtdstopts, tcp->tcp_rtdstoptslen, 15665 (ipp->ipp_fields & IPPF_RTDSTOPTS), 15666 ipp->ipp_rtdstopts, ipp->ipp_rtdstoptslen)) { 15667 optlen += sizeof (struct T_opthdr) + 15668 ipp->ipp_rtdstoptslen; 15669 addflag |= TCP_IPV6_RECVRTDSTOPTS; 15670 if (!ip_allocbuf((void **)&tcp->tcp_rtdstopts, 15671 &tcp->tcp_rtdstoptslen, (ipp->ipp_fields & IPPF_RTDSTOPTS), 15672 ipp->ipp_rtdstopts, ipp->ipp_rtdstoptslen)) 15673 return (mp); 15674 } 15675 /* If app asked for routing headers and it has changed ... */ 15676 if ((tcp->tcp_ipv6_recvancillary & TCP_IPV6_RECVRTHDR) && 15677 ip_cmpbuf(tcp->tcp_rthdr, tcp->tcp_rthdrlen, 15678 (ipp->ipp_fields & IPPF_RTHDR), 15679 ipp->ipp_rthdr, ipp->ipp_rthdrlen)) { 15680 optlen += sizeof (struct T_opthdr) + ipp->ipp_rthdrlen; 15681 addflag |= TCP_IPV6_RECVRTHDR; 15682 if (!ip_allocbuf((void **)&tcp->tcp_rthdr, 15683 &tcp->tcp_rthdrlen, (ipp->ipp_fields & IPPF_RTHDR), 15684 ipp->ipp_rthdr, ipp->ipp_rthdrlen)) 15685 return (mp); 15686 } 15687 /* If app asked for dest headers and it has changed ... */ 15688 if ((tcp->tcp_ipv6_recvancillary & 15689 (TCP_IPV6_RECVDSTOPTS | TCP_OLD_IPV6_RECVDSTOPTS)) && 15690 ip_cmpbuf(tcp->tcp_dstopts, tcp->tcp_dstoptslen, 15691 (ipp->ipp_fields & IPPF_DSTOPTS), 15692 ipp->ipp_dstopts, ipp->ipp_dstoptslen)) { 15693 optlen += sizeof (struct T_opthdr) + ipp->ipp_dstoptslen; 15694 addflag |= TCP_IPV6_RECVDSTOPTS; 15695 if (!ip_allocbuf((void **)&tcp->tcp_dstopts, 15696 &tcp->tcp_dstoptslen, (ipp->ipp_fields & IPPF_DSTOPTS), 15697 ipp->ipp_dstopts, ipp->ipp_dstoptslen)) 15698 return (mp); 15699 } 15700 15701 if (optlen == 0) { 15702 /* Nothing to add */ 15703 return (mp); 15704 } 15705 mp1 = allocb(sizeof (struct T_optdata_ind) + optlen, BPRI_MED); 15706 if (mp1 == NULL) { 15707 /* 15708 * Defer sending ancillary data until the next TCP segment 15709 * arrives. 15710 */ 15711 return (mp); 15712 } 15713 mp1->b_cont = mp; 15714 mp = mp1; 15715 mp->b_wptr += sizeof (*todi) + optlen; 15716 mp->b_datap->db_type = M_PROTO; 15717 todi = (struct T_optdata_ind *)mp->b_rptr; 15718 todi->PRIM_type = T_OPTDATA_IND; 15719 todi->DATA_flag = 1; /* MORE data */ 15720 todi->OPT_length = optlen; 15721 todi->OPT_offset = sizeof (*todi); 15722 optptr = (uchar_t *)&todi[1]; 15723 /* 15724 * If app asked for pktinfo and the index has changed ... 15725 * Note that the local address never changes for the connection. 15726 */ 15727 if (addflag & TCP_IPV6_RECVPKTINFO) { 15728 struct in6_pktinfo *pkti; 15729 15730 toh = (struct T_opthdr *)optptr; 15731 toh->level = IPPROTO_IPV6; 15732 toh->name = IPV6_PKTINFO; 15733 toh->len = sizeof (*toh) + sizeof (*pkti); 15734 toh->status = 0; 15735 optptr += sizeof (*toh); 15736 pkti = (struct in6_pktinfo *)optptr; 15737 if (tcp->tcp_ipversion == IPV6_VERSION) 15738 pkti->ipi6_addr = tcp->tcp_ip6h->ip6_src; 15739 else 15740 IN6_IPADDR_TO_V4MAPPED(tcp->tcp_ipha->ipha_src, 15741 &pkti->ipi6_addr); 15742 pkti->ipi6_ifindex = ipp->ipp_ifindex; 15743 optptr += sizeof (*pkti); 15744 ASSERT(OK_32PTR(optptr)); 15745 /* Save as "last" value */ 15746 tcp->tcp_recvifindex = ipp->ipp_ifindex; 15747 } 15748 /* If app asked for hoplimit and it has changed ... */ 15749 if (addflag & TCP_IPV6_RECVHOPLIMIT) { 15750 toh = (struct T_opthdr *)optptr; 15751 toh->level = IPPROTO_IPV6; 15752 toh->name = IPV6_HOPLIMIT; 15753 toh->len = sizeof (*toh) + sizeof (uint_t); 15754 toh->status = 0; 15755 optptr += sizeof (*toh); 15756 *(uint_t *)optptr = ipp->ipp_hoplimit; 15757 optptr += sizeof (uint_t); 15758 ASSERT(OK_32PTR(optptr)); 15759 /* Save as "last" value */ 15760 tcp->tcp_recvhops = ipp->ipp_hoplimit; 15761 } 15762 /* If app asked for tclass and it has changed ... */ 15763 if (addflag & TCP_IPV6_RECVTCLASS) { 15764 toh = (struct T_opthdr *)optptr; 15765 toh->level = IPPROTO_IPV6; 15766 toh->name = IPV6_TCLASS; 15767 toh->len = sizeof (*toh) + sizeof (uint_t); 15768 toh->status = 0; 15769 optptr += sizeof (*toh); 15770 *(uint_t *)optptr = ipp->ipp_tclass; 15771 optptr += sizeof (uint_t); 15772 ASSERT(OK_32PTR(optptr)); 15773 /* Save as "last" value */ 15774 tcp->tcp_recvtclass = ipp->ipp_tclass; 15775 } 15776 if (addflag & TCP_IPV6_RECVHOPOPTS) { 15777 toh = (struct T_opthdr *)optptr; 15778 toh->level = IPPROTO_IPV6; 15779 toh->name = IPV6_HOPOPTS; 15780 toh->len = sizeof (*toh) + ipp->ipp_hopoptslen - 15781 tcp->tcp_label_len; 15782 toh->status = 0; 15783 optptr += sizeof (*toh); 15784 bcopy((uchar_t *)ipp->ipp_hopopts + tcp->tcp_label_len, optptr, 15785 ipp->ipp_hopoptslen - tcp->tcp_label_len); 15786 optptr += ipp->ipp_hopoptslen - tcp->tcp_label_len; 15787 ASSERT(OK_32PTR(optptr)); 15788 /* Save as last value */ 15789 ip_savebuf((void **)&tcp->tcp_hopopts, &tcp->tcp_hopoptslen, 15790 (ipp->ipp_fields & IPPF_HOPOPTS), 15791 ipp->ipp_hopopts, ipp->ipp_hopoptslen); 15792 } 15793 if (addflag & TCP_IPV6_RECVRTDSTOPTS) { 15794 toh = (struct T_opthdr *)optptr; 15795 toh->level = IPPROTO_IPV6; 15796 toh->name = IPV6_RTHDRDSTOPTS; 15797 toh->len = sizeof (*toh) + ipp->ipp_rtdstoptslen; 15798 toh->status = 0; 15799 optptr += sizeof (*toh); 15800 bcopy(ipp->ipp_rtdstopts, optptr, ipp->ipp_rtdstoptslen); 15801 optptr += ipp->ipp_rtdstoptslen; 15802 ASSERT(OK_32PTR(optptr)); 15803 /* Save as last value */ 15804 ip_savebuf((void **)&tcp->tcp_rtdstopts, 15805 &tcp->tcp_rtdstoptslen, 15806 (ipp->ipp_fields & IPPF_RTDSTOPTS), 15807 ipp->ipp_rtdstopts, ipp->ipp_rtdstoptslen); 15808 } 15809 if (addflag & TCP_IPV6_RECVRTHDR) { 15810 toh = (struct T_opthdr *)optptr; 15811 toh->level = IPPROTO_IPV6; 15812 toh->name = IPV6_RTHDR; 15813 toh->len = sizeof (*toh) + ipp->ipp_rthdrlen; 15814 toh->status = 0; 15815 optptr += sizeof (*toh); 15816 bcopy(ipp->ipp_rthdr, optptr, ipp->ipp_rthdrlen); 15817 optptr += ipp->ipp_rthdrlen; 15818 ASSERT(OK_32PTR(optptr)); 15819 /* Save as last value */ 15820 ip_savebuf((void **)&tcp->tcp_rthdr, &tcp->tcp_rthdrlen, 15821 (ipp->ipp_fields & IPPF_RTHDR), 15822 ipp->ipp_rthdr, ipp->ipp_rthdrlen); 15823 } 15824 if (addflag & (TCP_IPV6_RECVDSTOPTS | TCP_OLD_IPV6_RECVDSTOPTS)) { 15825 toh = (struct T_opthdr *)optptr; 15826 toh->level = IPPROTO_IPV6; 15827 toh->name = IPV6_DSTOPTS; 15828 toh->len = sizeof (*toh) + ipp->ipp_dstoptslen; 15829 toh->status = 0; 15830 optptr += sizeof (*toh); 15831 bcopy(ipp->ipp_dstopts, optptr, ipp->ipp_dstoptslen); 15832 optptr += ipp->ipp_dstoptslen; 15833 ASSERT(OK_32PTR(optptr)); 15834 /* Save as last value */ 15835 ip_savebuf((void **)&tcp->tcp_dstopts, &tcp->tcp_dstoptslen, 15836 (ipp->ipp_fields & IPPF_DSTOPTS), 15837 ipp->ipp_dstopts, ipp->ipp_dstoptslen); 15838 } 15839 ASSERT(optptr == mp->b_wptr); 15840 return (mp); 15841 } 15842 15843 15844 /* 15845 * Handle a *T_BIND_REQ that has failed either due to a T_ERROR_ACK 15846 * or a "bad" IRE detected by tcp_adapt_ire. 15847 * We can't tell if the failure was due to the laddr or the faddr 15848 * thus we clear out all addresses and ports. 15849 */ 15850 static void 15851 tcp_bind_failed(tcp_t *tcp, mblk_t *mp, int error) 15852 { 15853 queue_t *q = tcp->tcp_rq; 15854 tcph_t *tcph; 15855 struct T_error_ack *tea; 15856 conn_t *connp = tcp->tcp_connp; 15857 15858 15859 ASSERT(mp->b_datap->db_type == M_PCPROTO); 15860 15861 if (mp->b_cont) { 15862 freemsg(mp->b_cont); 15863 mp->b_cont = NULL; 15864 } 15865 tea = (struct T_error_ack *)mp->b_rptr; 15866 switch (tea->PRIM_type) { 15867 case T_BIND_ACK: 15868 /* 15869 * Need to unbind with classifier since we were just told that 15870 * our bind succeeded. 15871 */ 15872 tcp->tcp_hard_bound = B_FALSE; 15873 tcp->tcp_hard_binding = B_FALSE; 15874 15875 ipcl_hash_remove(connp); 15876 /* Reuse the mblk if possible */ 15877 ASSERT(mp->b_datap->db_lim - mp->b_datap->db_base >= 15878 sizeof (*tea)); 15879 mp->b_rptr = mp->b_datap->db_base; 15880 mp->b_wptr = mp->b_rptr + sizeof (*tea); 15881 tea = (struct T_error_ack *)mp->b_rptr; 15882 tea->PRIM_type = T_ERROR_ACK; 15883 tea->TLI_error = TSYSERR; 15884 tea->UNIX_error = error; 15885 if (tcp->tcp_state >= TCPS_SYN_SENT) { 15886 tea->ERROR_prim = T_CONN_REQ; 15887 } else { 15888 tea->ERROR_prim = O_T_BIND_REQ; 15889 } 15890 break; 15891 15892 case T_ERROR_ACK: 15893 if (tcp->tcp_state >= TCPS_SYN_SENT) 15894 tea->ERROR_prim = T_CONN_REQ; 15895 break; 15896 default: 15897 panic("tcp_bind_failed: unexpected TPI type"); 15898 /*NOTREACHED*/ 15899 } 15900 15901 tcp->tcp_state = TCPS_IDLE; 15902 if (tcp->tcp_ipversion == IPV4_VERSION) 15903 tcp->tcp_ipha->ipha_src = 0; 15904 else 15905 V6_SET_ZERO(tcp->tcp_ip6h->ip6_src); 15906 /* 15907 * Copy of the src addr. in tcp_t is needed since 15908 * the lookup funcs. can only look at tcp_t 15909 */ 15910 V6_SET_ZERO(tcp->tcp_ip_src_v6); 15911 15912 tcph = tcp->tcp_tcph; 15913 tcph->th_lport[0] = 0; 15914 tcph->th_lport[1] = 0; 15915 tcp_bind_hash_remove(tcp); 15916 bzero(&connp->u_port, sizeof (connp->u_port)); 15917 /* blow away saved option results if any */ 15918 if (tcp->tcp_conn.tcp_opts_conn_req != NULL) 15919 tcp_close_mpp(&tcp->tcp_conn.tcp_opts_conn_req); 15920 15921 conn_delete_ire(tcp->tcp_connp, NULL); 15922 putnext(q, mp); 15923 } 15924 15925 /* 15926 * tcp_rput_other is called by tcp_rput to handle everything other than M_DATA 15927 * messages. 15928 */ 15929 void 15930 tcp_rput_other(tcp_t *tcp, mblk_t *mp) 15931 { 15932 mblk_t *mp1; 15933 uchar_t *rptr = mp->b_rptr; 15934 queue_t *q = tcp->tcp_rq; 15935 struct T_error_ack *tea; 15936 uint32_t mss; 15937 mblk_t *syn_mp; 15938 mblk_t *mdti; 15939 mblk_t *lsoi; 15940 int retval; 15941 mblk_t *ire_mp; 15942 tcp_stack_t *tcps = tcp->tcp_tcps; 15943 15944 switch (mp->b_datap->db_type) { 15945 case M_PROTO: 15946 case M_PCPROTO: 15947 ASSERT((uintptr_t)(mp->b_wptr - rptr) <= (uintptr_t)INT_MAX); 15948 if ((mp->b_wptr - rptr) < sizeof (t_scalar_t)) 15949 break; 15950 tea = (struct T_error_ack *)rptr; 15951 switch (tea->PRIM_type) { 15952 case T_BIND_ACK: 15953 /* 15954 * Adapt Multidata information, if any. The 15955 * following tcp_mdt_update routine will free 15956 * the message. 15957 */ 15958 if ((mdti = tcp_mdt_info_mp(mp)) != NULL) { 15959 tcp_mdt_update(tcp, &((ip_mdt_info_t *)mdti-> 15960 b_rptr)->mdt_capab, B_TRUE); 15961 freemsg(mdti); 15962 } 15963 15964 /* 15965 * Check to update LSO information with tcp, and 15966 * tcp_lso_update routine will free the message. 15967 */ 15968 if ((lsoi = tcp_lso_info_mp(mp)) != NULL) { 15969 tcp_lso_update(tcp, &((ip_lso_info_t *)lsoi-> 15970 b_rptr)->lso_capab); 15971 freemsg(lsoi); 15972 } 15973 15974 /* Get the IRE, if we had requested for it */ 15975 ire_mp = tcp_ire_mp(mp); 15976 15977 if (tcp->tcp_hard_binding) { 15978 tcp->tcp_hard_binding = B_FALSE; 15979 tcp->tcp_hard_bound = B_TRUE; 15980 CL_INET_CONNECT(tcp); 15981 } else { 15982 if (ire_mp != NULL) 15983 freeb(ire_mp); 15984 goto after_syn_sent; 15985 } 15986 15987 retval = tcp_adapt_ire(tcp, ire_mp); 15988 if (ire_mp != NULL) 15989 freeb(ire_mp); 15990 if (retval == 0) { 15991 tcp_bind_failed(tcp, mp, 15992 (int)((tcp->tcp_state >= TCPS_SYN_SENT) ? 15993 ENETUNREACH : EADDRNOTAVAIL)); 15994 return; 15995 } 15996 /* 15997 * Don't let an endpoint connect to itself. 15998 * Also checked in tcp_connect() but that 15999 * check can't handle the case when the 16000 * local IP address is INADDR_ANY. 16001 */ 16002 if (tcp->tcp_ipversion == IPV4_VERSION) { 16003 if ((tcp->tcp_ipha->ipha_dst == 16004 tcp->tcp_ipha->ipha_src) && 16005 (BE16_EQL(tcp->tcp_tcph->th_lport, 16006 tcp->tcp_tcph->th_fport))) { 16007 tcp_bind_failed(tcp, mp, EADDRNOTAVAIL); 16008 return; 16009 } 16010 } else { 16011 if (IN6_ARE_ADDR_EQUAL( 16012 &tcp->tcp_ip6h->ip6_dst, 16013 &tcp->tcp_ip6h->ip6_src) && 16014 (BE16_EQL(tcp->tcp_tcph->th_lport, 16015 tcp->tcp_tcph->th_fport))) { 16016 tcp_bind_failed(tcp, mp, EADDRNOTAVAIL); 16017 return; 16018 } 16019 } 16020 ASSERT(tcp->tcp_state == TCPS_SYN_SENT); 16021 /* 16022 * This should not be possible! Just for 16023 * defensive coding... 16024 */ 16025 if (tcp->tcp_state != TCPS_SYN_SENT) 16026 goto after_syn_sent; 16027 16028 if (is_system_labeled() && 16029 !tcp_update_label(tcp, CONN_CRED(tcp->tcp_connp))) { 16030 tcp_bind_failed(tcp, mp, EHOSTUNREACH); 16031 return; 16032 } 16033 16034 ASSERT(q == tcp->tcp_rq); 16035 /* 16036 * tcp_adapt_ire() does not adjust 16037 * for TCP/IP header length. 16038 */ 16039 mss = tcp->tcp_mss - tcp->tcp_hdr_len; 16040 16041 /* 16042 * Just make sure our rwnd is at 16043 * least tcp_recv_hiwat_mss * MSS 16044 * large, and round up to the nearest 16045 * MSS. 16046 * 16047 * We do the round up here because 16048 * we need to get the interface 16049 * MTU first before we can do the 16050 * round up. 16051 */ 16052 tcp->tcp_rwnd = MAX(MSS_ROUNDUP(tcp->tcp_rwnd, mss), 16053 tcps->tcps_recv_hiwat_minmss * mss); 16054 q->q_hiwat = tcp->tcp_rwnd; 16055 tcp_set_ws_value(tcp); 16056 U32_TO_ABE16((tcp->tcp_rwnd >> tcp->tcp_rcv_ws), 16057 tcp->tcp_tcph->th_win); 16058 if (tcp->tcp_rcv_ws > 0 || tcps->tcps_wscale_always) 16059 tcp->tcp_snd_ws_ok = B_TRUE; 16060 16061 /* 16062 * Set tcp_snd_ts_ok to true 16063 * so that tcp_xmit_mp will 16064 * include the timestamp 16065 * option in the SYN segment. 16066 */ 16067 if (tcps->tcps_tstamp_always || 16068 (tcp->tcp_rcv_ws && tcps->tcps_tstamp_if_wscale)) { 16069 tcp->tcp_snd_ts_ok = B_TRUE; 16070 } 16071 16072 /* 16073 * tcp_snd_sack_ok can be set in 16074 * tcp_adapt_ire() if the sack metric 16075 * is set. So check it here also. 16076 */ 16077 if (tcps->tcps_sack_permitted == 2 || 16078 tcp->tcp_snd_sack_ok) { 16079 if (tcp->tcp_sack_info == NULL) { 16080 tcp->tcp_sack_info = 16081 kmem_cache_alloc( 16082 tcp_sack_info_cache, 16083 KM_SLEEP); 16084 } 16085 tcp->tcp_snd_sack_ok = B_TRUE; 16086 } 16087 16088 /* 16089 * Should we use ECN? Note that the current 16090 * default value (SunOS 5.9) of tcp_ecn_permitted 16091 * is 1. The reason for doing this is that there 16092 * are equipments out there that will drop ECN 16093 * enabled IP packets. Setting it to 1 avoids 16094 * compatibility problems. 16095 */ 16096 if (tcps->tcps_ecn_permitted == 2) 16097 tcp->tcp_ecn_ok = B_TRUE; 16098 16099 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 16100 syn_mp = tcp_xmit_mp(tcp, NULL, 0, NULL, NULL, 16101 tcp->tcp_iss, B_FALSE, NULL, B_FALSE); 16102 if (syn_mp) { 16103 cred_t *cr; 16104 pid_t pid; 16105 16106 /* 16107 * Obtain the credential from the 16108 * thread calling connect(); the credential 16109 * lives on in the second mblk which 16110 * originated from T_CONN_REQ and is echoed 16111 * with the T_BIND_ACK from ip. If none 16112 * can be found, default to the creator 16113 * of the socket. 16114 */ 16115 if (mp->b_cont == NULL || 16116 (cr = DB_CRED(mp->b_cont)) == NULL) { 16117 cr = tcp->tcp_cred; 16118 pid = tcp->tcp_cpid; 16119 } else { 16120 pid = DB_CPID(mp->b_cont); 16121 } 16122 mblk_setcred(syn_mp, cr); 16123 DB_CPID(syn_mp) = pid; 16124 tcp_send_data(tcp, tcp->tcp_wq, syn_mp); 16125 } 16126 after_syn_sent: 16127 /* 16128 * A trailer mblk indicates a waiting client upstream. 16129 * We complete here the processing begun in 16130 * either tcp_bind() or tcp_connect() by passing 16131 * upstream the reply message they supplied. 16132 */ 16133 mp1 = mp; 16134 mp = mp->b_cont; 16135 freeb(mp1); 16136 if (mp) 16137 break; 16138 return; 16139 case T_ERROR_ACK: 16140 if (tcp->tcp_debug) { 16141 (void) strlog(TCP_MOD_ID, 0, 1, 16142 SL_TRACE|SL_ERROR, 16143 "tcp_rput_other: case T_ERROR_ACK, " 16144 "ERROR_prim == %d", 16145 tea->ERROR_prim); 16146 } 16147 switch (tea->ERROR_prim) { 16148 case O_T_BIND_REQ: 16149 case T_BIND_REQ: 16150 tcp_bind_failed(tcp, mp, 16151 (int)((tcp->tcp_state >= TCPS_SYN_SENT) ? 16152 ENETUNREACH : EADDRNOTAVAIL)); 16153 return; 16154 case T_UNBIND_REQ: 16155 tcp->tcp_hard_binding = B_FALSE; 16156 tcp->tcp_hard_bound = B_FALSE; 16157 if (mp->b_cont) { 16158 freemsg(mp->b_cont); 16159 mp->b_cont = NULL; 16160 } 16161 if (tcp->tcp_unbind_pending) 16162 tcp->tcp_unbind_pending = 0; 16163 else { 16164 /* From tcp_ip_unbind() - free */ 16165 freemsg(mp); 16166 return; 16167 } 16168 break; 16169 case T_SVR4_OPTMGMT_REQ: 16170 if (tcp->tcp_drop_opt_ack_cnt > 0) { 16171 /* T_OPTMGMT_REQ generated by TCP */ 16172 printf("T_SVR4_OPTMGMT_REQ failed " 16173 "%d/%d - dropped (cnt %d)\n", 16174 tea->TLI_error, tea->UNIX_error, 16175 tcp->tcp_drop_opt_ack_cnt); 16176 freemsg(mp); 16177 tcp->tcp_drop_opt_ack_cnt--; 16178 return; 16179 } 16180 break; 16181 } 16182 if (tea->ERROR_prim == T_SVR4_OPTMGMT_REQ && 16183 tcp->tcp_drop_opt_ack_cnt > 0) { 16184 printf("T_SVR4_OPTMGMT_REQ failed %d/%d " 16185 "- dropped (cnt %d)\n", 16186 tea->TLI_error, tea->UNIX_error, 16187 tcp->tcp_drop_opt_ack_cnt); 16188 freemsg(mp); 16189 tcp->tcp_drop_opt_ack_cnt--; 16190 return; 16191 } 16192 break; 16193 case T_OPTMGMT_ACK: 16194 if (tcp->tcp_drop_opt_ack_cnt > 0) { 16195 /* T_OPTMGMT_REQ generated by TCP */ 16196 freemsg(mp); 16197 tcp->tcp_drop_opt_ack_cnt--; 16198 return; 16199 } 16200 break; 16201 default: 16202 break; 16203 } 16204 break; 16205 case M_FLUSH: 16206 if (*rptr & FLUSHR) 16207 flushq(q, FLUSHDATA); 16208 break; 16209 default: 16210 /* M_CTL will be directly sent to tcp_icmp_error() */ 16211 ASSERT(DB_TYPE(mp) != M_CTL); 16212 break; 16213 } 16214 /* 16215 * Make sure we set this bit before sending the ACK for 16216 * bind. Otherwise accept could possibly run and free 16217 * this tcp struct. 16218 */ 16219 putnext(q, mp); 16220 } 16221 16222 /* 16223 * Called as the result of a qbufcall or a qtimeout to remedy a failure 16224 * to allocate a T_ordrel_ind in tcp_rsrv(). qenable(q) will make 16225 * tcp_rsrv() try again. 16226 */ 16227 static void 16228 tcp_ordrel_kick(void *arg) 16229 { 16230 conn_t *connp = (conn_t *)arg; 16231 tcp_t *tcp = connp->conn_tcp; 16232 16233 tcp->tcp_ordrelid = 0; 16234 tcp->tcp_timeout = B_FALSE; 16235 if (!TCP_IS_DETACHED(tcp) && tcp->tcp_rq != NULL && 16236 tcp->tcp_fin_rcvd && !tcp->tcp_ordrel_done) { 16237 qenable(tcp->tcp_rq); 16238 } 16239 } 16240 16241 /* ARGSUSED */ 16242 static void 16243 tcp_rsrv_input(void *arg, mblk_t *mp, void *arg2) 16244 { 16245 conn_t *connp = (conn_t *)arg; 16246 tcp_t *tcp = connp->conn_tcp; 16247 queue_t *q = tcp->tcp_rq; 16248 uint_t thwin; 16249 tcp_stack_t *tcps = tcp->tcp_tcps; 16250 sodirect_t *sodp; 16251 boolean_t fc; 16252 16253 freeb(mp); 16254 16255 TCP_STAT(tcps, tcp_rsrv_calls); 16256 16257 if (TCP_IS_DETACHED(tcp) || q == NULL) { 16258 return; 16259 } 16260 16261 if (tcp->tcp_fused) { 16262 tcp_t *peer_tcp = tcp->tcp_loopback_peer; 16263 16264 ASSERT(tcp->tcp_fused); 16265 ASSERT(peer_tcp != NULL && peer_tcp->tcp_fused); 16266 ASSERT(peer_tcp->tcp_loopback_peer == tcp); 16267 ASSERT(!TCP_IS_DETACHED(tcp)); 16268 ASSERT(tcp->tcp_connp->conn_sqp == 16269 peer_tcp->tcp_connp->conn_sqp); 16270 16271 /* 16272 * Normally we would not get backenabled in synchronous 16273 * streams mode, but in case this happens, we need to plug 16274 * synchronous streams during our drain to prevent a race 16275 * with tcp_fuse_rrw() or tcp_fuse_rinfop(). 16276 */ 16277 TCP_FUSE_SYNCSTR_PLUG_DRAIN(tcp); 16278 if (tcp->tcp_rcv_list != NULL) 16279 (void) tcp_rcv_drain(tcp->tcp_rq, tcp); 16280 16281 if (peer_tcp > tcp) { 16282 mutex_enter(&peer_tcp->tcp_non_sq_lock); 16283 mutex_enter(&tcp->tcp_non_sq_lock); 16284 } else { 16285 mutex_enter(&tcp->tcp_non_sq_lock); 16286 mutex_enter(&peer_tcp->tcp_non_sq_lock); 16287 } 16288 16289 if (peer_tcp->tcp_flow_stopped && 16290 (TCP_UNSENT_BYTES(peer_tcp) <= 16291 peer_tcp->tcp_xmit_lowater)) { 16292 tcp_clrqfull(peer_tcp); 16293 } 16294 mutex_exit(&peer_tcp->tcp_non_sq_lock); 16295 mutex_exit(&tcp->tcp_non_sq_lock); 16296 16297 TCP_FUSE_SYNCSTR_UNPLUG_DRAIN(tcp); 16298 TCP_STAT(tcps, tcp_fusion_backenabled); 16299 return; 16300 } 16301 16302 SOD_PTR_ENTER(tcp, sodp); 16303 if (sodp != NULL) { 16304 /* An sodirect connection */ 16305 if (SOD_QFULL(sodp)) { 16306 /* Flow-controlled, need another back-enable */ 16307 fc = B_TRUE; 16308 SOD_QSETBE(sodp); 16309 } else { 16310 /* Not flow-controlled */ 16311 fc = B_FALSE; 16312 } 16313 mutex_exit(sodp->sod_lock); 16314 } else if (canputnext(q)) { 16315 /* STREAMS, not flow-controlled */ 16316 fc = B_FALSE; 16317 } else { 16318 /* STREAMS, flow-controlled */ 16319 fc = B_TRUE; 16320 } 16321 if (!fc) { 16322 /* Not flow-controlled, open rwnd */ 16323 tcp->tcp_rwnd = q->q_hiwat; 16324 thwin = ((uint_t)BE16_TO_U16(tcp->tcp_tcph->th_win)) 16325 << tcp->tcp_rcv_ws; 16326 thwin -= tcp->tcp_rnxt - tcp->tcp_rack; 16327 /* 16328 * Send back a window update immediately if TCP is above 16329 * ESTABLISHED state and the increase of the rcv window 16330 * that the other side knows is at least 1 MSS after flow 16331 * control is lifted. 16332 */ 16333 if (tcp->tcp_state >= TCPS_ESTABLISHED && 16334 (q->q_hiwat - thwin >= tcp->tcp_mss)) { 16335 tcp_xmit_ctl(NULL, tcp, 16336 (tcp->tcp_swnd == 0) ? tcp->tcp_suna : 16337 tcp->tcp_snxt, tcp->tcp_rnxt, TH_ACK); 16338 BUMP_MIB(&tcps->tcps_mib, tcpOutWinUpdate); 16339 } 16340 } 16341 16342 /* Handle a failure to allocate a T_ORDREL_IND here */ 16343 if (tcp->tcp_fin_rcvd && !tcp->tcp_ordrel_done) { 16344 ASSERT(tcp->tcp_listener == NULL); 16345 16346 SOD_PTR_ENTER(tcp, sodp); 16347 if (sodp != NULL) { 16348 /* No more sodirect */ 16349 tcp->tcp_sodirect = NULL; 16350 if (!SOD_QEMPTY(sodp)) { 16351 /* Notify mblk(s) to process */ 16352 (void) tcp_rcv_sod_wakeup(tcp, sodp); 16353 /* sod_wakeup() does the mutex_exit() */ 16354 } else { 16355 /* Nothing to process */ 16356 mutex_exit(sodp->sod_lock); 16357 } 16358 } else if (tcp->tcp_rcv_list != NULL) { 16359 /* 16360 * Push any mblk(s) enqueued from co processing. 16361 */ 16362 (void) tcp_rcv_drain(tcp->tcp_rq, tcp); 16363 ASSERT(tcp->tcp_rcv_list == NULL || 16364 tcp->tcp_fused_sigurg); 16365 } 16366 16367 mp = mi_tpi_ordrel_ind(); 16368 if (mp) { 16369 tcp->tcp_ordrel_done = B_TRUE; 16370 putnext(q, mp); 16371 if (tcp->tcp_deferred_clean_death) { 16372 /* 16373 * tcp_clean_death was deferred for 16374 * T_ORDREL_IND - do it now 16375 */ 16376 tcp->tcp_deferred_clean_death = B_FALSE; 16377 (void) tcp_clean_death(tcp, 16378 tcp->tcp_client_errno, 22); 16379 } 16380 } else if (!tcp->tcp_timeout && tcp->tcp_ordrelid == 0) { 16381 /* 16382 * If there isn't already a timer running 16383 * start one. Use a 4 second 16384 * timer as a fallback since it can't fail. 16385 */ 16386 tcp->tcp_timeout = B_TRUE; 16387 tcp->tcp_ordrelid = TCP_TIMER(tcp, tcp_ordrel_kick, 16388 MSEC_TO_TICK(4000)); 16389 } 16390 } 16391 } 16392 16393 /* 16394 * The read side service routine is called mostly when we get back-enabled as a 16395 * result of flow control relief. Since we don't actually queue anything in 16396 * TCP, we have no data to send out of here. What we do is clear the receive 16397 * window, and send out a window update. 16398 * This routine is also called to drive an orderly release message upstream 16399 * if the attempt in tcp_rput failed. 16400 */ 16401 static void 16402 tcp_rsrv(queue_t *q) 16403 { 16404 conn_t *connp = Q_TO_CONN(q); 16405 tcp_t *tcp = connp->conn_tcp; 16406 mblk_t *mp; 16407 tcp_stack_t *tcps = tcp->tcp_tcps; 16408 16409 /* No code does a putq on the read side */ 16410 ASSERT(q->q_first == NULL); 16411 16412 /* Nothing to do for the default queue */ 16413 if (q == tcps->tcps_g_q) { 16414 return; 16415 } 16416 16417 mp = allocb(0, BPRI_HI); 16418 if (mp == NULL) { 16419 /* 16420 * We are under memory pressure. Return for now and we 16421 * we will be called again later. 16422 */ 16423 if (!tcp->tcp_timeout && tcp->tcp_ordrelid == 0) { 16424 /* 16425 * If there isn't already a timer running 16426 * start one. Use a 4 second 16427 * timer as a fallback since it can't fail. 16428 */ 16429 tcp->tcp_timeout = B_TRUE; 16430 tcp->tcp_ordrelid = TCP_TIMER(tcp, tcp_ordrel_kick, 16431 MSEC_TO_TICK(4000)); 16432 } 16433 return; 16434 } 16435 CONN_INC_REF(connp); 16436 squeue_enter(connp->conn_sqp, mp, tcp_rsrv_input, connp, 16437 SQTAG_TCP_RSRV); 16438 } 16439 16440 /* 16441 * tcp_rwnd_set() is called to adjust the receive window to a desired value. 16442 * We do not allow the receive window to shrink. After setting rwnd, 16443 * set the flow control hiwat of the stream. 16444 * 16445 * This function is called in 2 cases: 16446 * 16447 * 1) Before data transfer begins, in tcp_accept_comm() for accepting a 16448 * connection (passive open) and in tcp_rput_data() for active connect. 16449 * This is called after tcp_mss_set() when the desired MSS value is known. 16450 * This makes sure that our window size is a mutiple of the other side's 16451 * MSS. 16452 * 2) Handling SO_RCVBUF option. 16453 * 16454 * It is ASSUMED that the requested size is a multiple of the current MSS. 16455 * 16456 * XXX - Should allow a lower rwnd than tcp_recv_hiwat_minmss * mss if the 16457 * user requests so. 16458 */ 16459 static int 16460 tcp_rwnd_set(tcp_t *tcp, uint32_t rwnd) 16461 { 16462 uint32_t mss = tcp->tcp_mss; 16463 uint32_t old_max_rwnd; 16464 uint32_t max_transmittable_rwnd; 16465 boolean_t tcp_detached = TCP_IS_DETACHED(tcp); 16466 tcp_stack_t *tcps = tcp->tcp_tcps; 16467 16468 if (tcp->tcp_fused) { 16469 size_t sth_hiwat; 16470 tcp_t *peer_tcp = tcp->tcp_loopback_peer; 16471 16472 ASSERT(peer_tcp != NULL); 16473 /* 16474 * Record the stream head's high water mark for 16475 * this endpoint; this is used for flow-control 16476 * purposes in tcp_fuse_output(). 16477 */ 16478 sth_hiwat = tcp_fuse_set_rcv_hiwat(tcp, rwnd); 16479 if (!tcp_detached) 16480 (void) mi_set_sth_hiwat(tcp->tcp_rq, sth_hiwat); 16481 16482 /* 16483 * In the fusion case, the maxpsz stream head value of 16484 * our peer is set according to its send buffer size 16485 * and our receive buffer size; since the latter may 16486 * have changed we need to update the peer's maxpsz. 16487 */ 16488 (void) tcp_maxpsz_set(peer_tcp, B_TRUE); 16489 return (rwnd); 16490 } 16491 16492 if (tcp_detached) 16493 old_max_rwnd = tcp->tcp_rwnd; 16494 else 16495 old_max_rwnd = tcp->tcp_rq->q_hiwat; 16496 16497 /* 16498 * Insist on a receive window that is at least 16499 * tcp_recv_hiwat_minmss * MSS (default 4 * MSS) to avoid 16500 * funny TCP interactions of Nagle algorithm, SWS avoidance 16501 * and delayed acknowledgement. 16502 */ 16503 rwnd = MAX(rwnd, tcps->tcps_recv_hiwat_minmss * mss); 16504 16505 /* 16506 * If window size info has already been exchanged, TCP should not 16507 * shrink the window. Shrinking window is doable if done carefully. 16508 * We may add that support later. But so far there is not a real 16509 * need to do that. 16510 */ 16511 if (rwnd < old_max_rwnd && tcp->tcp_state > TCPS_SYN_SENT) { 16512 /* MSS may have changed, do a round up again. */ 16513 rwnd = MSS_ROUNDUP(old_max_rwnd, mss); 16514 } 16515 16516 /* 16517 * tcp_rcv_ws starts with TCP_MAX_WINSHIFT so the following check 16518 * can be applied even before the window scale option is decided. 16519 */ 16520 max_transmittable_rwnd = TCP_MAXWIN << tcp->tcp_rcv_ws; 16521 if (rwnd > max_transmittable_rwnd) { 16522 rwnd = max_transmittable_rwnd - 16523 (max_transmittable_rwnd % mss); 16524 if (rwnd < mss) 16525 rwnd = max_transmittable_rwnd; 16526 /* 16527 * If we're over the limit we may have to back down tcp_rwnd. 16528 * The increment below won't work for us. So we set all three 16529 * here and the increment below will have no effect. 16530 */ 16531 tcp->tcp_rwnd = old_max_rwnd = rwnd; 16532 } 16533 if (tcp->tcp_localnet) { 16534 tcp->tcp_rack_abs_max = 16535 MIN(tcps->tcps_local_dacks_max, rwnd / mss / 2); 16536 } else { 16537 /* 16538 * For a remote host on a different subnet (through a router), 16539 * we ack every other packet to be conforming to RFC1122. 16540 * tcp_deferred_acks_max is default to 2. 16541 */ 16542 tcp->tcp_rack_abs_max = 16543 MIN(tcps->tcps_deferred_acks_max, rwnd / mss / 2); 16544 } 16545 if (tcp->tcp_rack_cur_max > tcp->tcp_rack_abs_max) 16546 tcp->tcp_rack_cur_max = tcp->tcp_rack_abs_max; 16547 else 16548 tcp->tcp_rack_cur_max = 0; 16549 /* 16550 * Increment the current rwnd by the amount the maximum grew (we 16551 * can not overwrite it since we might be in the middle of a 16552 * connection.) 16553 */ 16554 tcp->tcp_rwnd += rwnd - old_max_rwnd; 16555 U32_TO_ABE16(tcp->tcp_rwnd >> tcp->tcp_rcv_ws, tcp->tcp_tcph->th_win); 16556 if ((tcp->tcp_rcv_ws > 0) && rwnd > tcp->tcp_cwnd_max) 16557 tcp->tcp_cwnd_max = rwnd; 16558 16559 if (tcp_detached) 16560 return (rwnd); 16561 /* 16562 * We set the maximum receive window into rq->q_hiwat. 16563 * This is not actually used for flow control. 16564 */ 16565 tcp->tcp_rq->q_hiwat = rwnd; 16566 /* 16567 * Set the Stream head high water mark. This doesn't have to be 16568 * here, since we are simply using default values, but we would 16569 * prefer to choose these values algorithmically, with a likely 16570 * relationship to rwnd. 16571 */ 16572 (void) mi_set_sth_hiwat(tcp->tcp_rq, 16573 MAX(rwnd, tcps->tcps_sth_rcv_hiwat)); 16574 return (rwnd); 16575 } 16576 16577 /* 16578 * Return SNMP stuff in buffer in mpdata. 16579 */ 16580 mblk_t * 16581 tcp_snmp_get(queue_t *q, mblk_t *mpctl) 16582 { 16583 mblk_t *mpdata; 16584 mblk_t *mp_conn_ctl = NULL; 16585 mblk_t *mp_conn_tail; 16586 mblk_t *mp_attr_ctl = NULL; 16587 mblk_t *mp_attr_tail; 16588 mblk_t *mp6_conn_ctl = NULL; 16589 mblk_t *mp6_conn_tail; 16590 mblk_t *mp6_attr_ctl = NULL; 16591 mblk_t *mp6_attr_tail; 16592 struct opthdr *optp; 16593 mib2_tcpConnEntry_t tce; 16594 mib2_tcp6ConnEntry_t tce6; 16595 mib2_transportMLPEntry_t mlp; 16596 connf_t *connfp; 16597 int i; 16598 boolean_t ispriv; 16599 zoneid_t zoneid; 16600 int v4_conn_idx; 16601 int v6_conn_idx; 16602 conn_t *connp = Q_TO_CONN(q); 16603 tcp_stack_t *tcps; 16604 ip_stack_t *ipst; 16605 mblk_t *mp2ctl; 16606 16607 /* 16608 * make a copy of the original message 16609 */ 16610 mp2ctl = copymsg(mpctl); 16611 16612 if (mpctl == NULL || 16613 (mpdata = mpctl->b_cont) == NULL || 16614 (mp_conn_ctl = copymsg(mpctl)) == NULL || 16615 (mp_attr_ctl = copymsg(mpctl)) == NULL || 16616 (mp6_conn_ctl = copymsg(mpctl)) == NULL || 16617 (mp6_attr_ctl = copymsg(mpctl)) == NULL) { 16618 freemsg(mp_conn_ctl); 16619 freemsg(mp_attr_ctl); 16620 freemsg(mp6_conn_ctl); 16621 freemsg(mp6_attr_ctl); 16622 freemsg(mpctl); 16623 freemsg(mp2ctl); 16624 return (NULL); 16625 } 16626 16627 ipst = connp->conn_netstack->netstack_ip; 16628 tcps = connp->conn_netstack->netstack_tcp; 16629 16630 /* build table of connections -- need count in fixed part */ 16631 SET_MIB(tcps->tcps_mib.tcpRtoAlgorithm, 4); /* vanj */ 16632 SET_MIB(tcps->tcps_mib.tcpRtoMin, tcps->tcps_rexmit_interval_min); 16633 SET_MIB(tcps->tcps_mib.tcpRtoMax, tcps->tcps_rexmit_interval_max); 16634 SET_MIB(tcps->tcps_mib.tcpMaxConn, -1); 16635 SET_MIB(tcps->tcps_mib.tcpCurrEstab, 0); 16636 16637 ispriv = 16638 secpolicy_ip_config((Q_TO_CONN(q))->conn_cred, B_TRUE) == 0; 16639 zoneid = Q_TO_CONN(q)->conn_zoneid; 16640 16641 v4_conn_idx = v6_conn_idx = 0; 16642 mp_conn_tail = mp_attr_tail = mp6_conn_tail = mp6_attr_tail = NULL; 16643 16644 for (i = 0; i < CONN_G_HASH_SIZE; i++) { 16645 ipst = tcps->tcps_netstack->netstack_ip; 16646 16647 connfp = &ipst->ips_ipcl_globalhash_fanout[i]; 16648 16649 connp = NULL; 16650 16651 while ((connp = 16652 ipcl_get_next_conn(connfp, connp, IPCL_TCP)) != NULL) { 16653 tcp_t *tcp; 16654 boolean_t needattr; 16655 16656 if (connp->conn_zoneid != zoneid) 16657 continue; /* not in this zone */ 16658 16659 tcp = connp->conn_tcp; 16660 UPDATE_MIB(&tcps->tcps_mib, 16661 tcpHCInSegs, tcp->tcp_ibsegs); 16662 tcp->tcp_ibsegs = 0; 16663 UPDATE_MIB(&tcps->tcps_mib, 16664 tcpHCOutSegs, tcp->tcp_obsegs); 16665 tcp->tcp_obsegs = 0; 16666 16667 tce6.tcp6ConnState = tce.tcpConnState = 16668 tcp_snmp_state(tcp); 16669 if (tce.tcpConnState == MIB2_TCP_established || 16670 tce.tcpConnState == MIB2_TCP_closeWait) 16671 BUMP_MIB(&tcps->tcps_mib, tcpCurrEstab); 16672 16673 needattr = B_FALSE; 16674 bzero(&mlp, sizeof (mlp)); 16675 if (connp->conn_mlp_type != mlptSingle) { 16676 if (connp->conn_mlp_type == mlptShared || 16677 connp->conn_mlp_type == mlptBoth) 16678 mlp.tme_flags |= MIB2_TMEF_SHARED; 16679 if (connp->conn_mlp_type == mlptPrivate || 16680 connp->conn_mlp_type == mlptBoth) 16681 mlp.tme_flags |= MIB2_TMEF_PRIVATE; 16682 needattr = B_TRUE; 16683 } 16684 if (connp->conn_peercred != NULL) { 16685 ts_label_t *tsl; 16686 16687 tsl = crgetlabel(connp->conn_peercred); 16688 mlp.tme_doi = label2doi(tsl); 16689 mlp.tme_label = *label2bslabel(tsl); 16690 needattr = B_TRUE; 16691 } 16692 16693 /* Create a message to report on IPv6 entries */ 16694 if (tcp->tcp_ipversion == IPV6_VERSION) { 16695 tce6.tcp6ConnLocalAddress = tcp->tcp_ip_src_v6; 16696 tce6.tcp6ConnRemAddress = tcp->tcp_remote_v6; 16697 tce6.tcp6ConnLocalPort = ntohs(tcp->tcp_lport); 16698 tce6.tcp6ConnRemPort = ntohs(tcp->tcp_fport); 16699 tce6.tcp6ConnIfIndex = tcp->tcp_bound_if; 16700 /* Don't want just anybody seeing these... */ 16701 if (ispriv) { 16702 tce6.tcp6ConnEntryInfo.ce_snxt = 16703 tcp->tcp_snxt; 16704 tce6.tcp6ConnEntryInfo.ce_suna = 16705 tcp->tcp_suna; 16706 tce6.tcp6ConnEntryInfo.ce_rnxt = 16707 tcp->tcp_rnxt; 16708 tce6.tcp6ConnEntryInfo.ce_rack = 16709 tcp->tcp_rack; 16710 } else { 16711 /* 16712 * Netstat, unfortunately, uses this to 16713 * get send/receive queue sizes. How to fix? 16714 * Why not compute the difference only? 16715 */ 16716 tce6.tcp6ConnEntryInfo.ce_snxt = 16717 tcp->tcp_snxt - tcp->tcp_suna; 16718 tce6.tcp6ConnEntryInfo.ce_suna = 0; 16719 tce6.tcp6ConnEntryInfo.ce_rnxt = 16720 tcp->tcp_rnxt - tcp->tcp_rack; 16721 tce6.tcp6ConnEntryInfo.ce_rack = 0; 16722 } 16723 16724 tce6.tcp6ConnEntryInfo.ce_swnd = tcp->tcp_swnd; 16725 tce6.tcp6ConnEntryInfo.ce_rwnd = tcp->tcp_rwnd; 16726 tce6.tcp6ConnEntryInfo.ce_rto = tcp->tcp_rto; 16727 tce6.tcp6ConnEntryInfo.ce_mss = tcp->tcp_mss; 16728 tce6.tcp6ConnEntryInfo.ce_state = tcp->tcp_state; 16729 16730 tce6.tcp6ConnCreationProcess = 16731 (tcp->tcp_cpid < 0) ? MIB2_UNKNOWN_PROCESS : 16732 tcp->tcp_cpid; 16733 tce6.tcp6ConnCreationTime = tcp->tcp_open_time; 16734 16735 (void) snmp_append_data2(mp6_conn_ctl->b_cont, 16736 &mp6_conn_tail, (char *)&tce6, sizeof (tce6)); 16737 16738 mlp.tme_connidx = v6_conn_idx++; 16739 if (needattr) 16740 (void) snmp_append_data2(mp6_attr_ctl->b_cont, 16741 &mp6_attr_tail, (char *)&mlp, sizeof (mlp)); 16742 } 16743 /* 16744 * Create an IPv4 table entry for IPv4 entries and also 16745 * for IPv6 entries which are bound to in6addr_any 16746 * but don't have IPV6_V6ONLY set. 16747 * (i.e. anything an IPv4 peer could connect to) 16748 */ 16749 if (tcp->tcp_ipversion == IPV4_VERSION || 16750 (tcp->tcp_state <= TCPS_LISTEN && 16751 !tcp->tcp_connp->conn_ipv6_v6only && 16752 IN6_IS_ADDR_UNSPECIFIED(&tcp->tcp_ip_src_v6))) { 16753 if (tcp->tcp_ipversion == IPV6_VERSION) { 16754 tce.tcpConnRemAddress = INADDR_ANY; 16755 tce.tcpConnLocalAddress = INADDR_ANY; 16756 } else { 16757 tce.tcpConnRemAddress = 16758 tcp->tcp_remote; 16759 tce.tcpConnLocalAddress = 16760 tcp->tcp_ip_src; 16761 } 16762 tce.tcpConnLocalPort = ntohs(tcp->tcp_lport); 16763 tce.tcpConnRemPort = ntohs(tcp->tcp_fport); 16764 /* Don't want just anybody seeing these... */ 16765 if (ispriv) { 16766 tce.tcpConnEntryInfo.ce_snxt = 16767 tcp->tcp_snxt; 16768 tce.tcpConnEntryInfo.ce_suna = 16769 tcp->tcp_suna; 16770 tce.tcpConnEntryInfo.ce_rnxt = 16771 tcp->tcp_rnxt; 16772 tce.tcpConnEntryInfo.ce_rack = 16773 tcp->tcp_rack; 16774 } else { 16775 /* 16776 * Netstat, unfortunately, uses this to 16777 * get send/receive queue sizes. How 16778 * to fix? 16779 * Why not compute the difference only? 16780 */ 16781 tce.tcpConnEntryInfo.ce_snxt = 16782 tcp->tcp_snxt - tcp->tcp_suna; 16783 tce.tcpConnEntryInfo.ce_suna = 0; 16784 tce.tcpConnEntryInfo.ce_rnxt = 16785 tcp->tcp_rnxt - tcp->tcp_rack; 16786 tce.tcpConnEntryInfo.ce_rack = 0; 16787 } 16788 16789 tce.tcpConnEntryInfo.ce_swnd = tcp->tcp_swnd; 16790 tce.tcpConnEntryInfo.ce_rwnd = tcp->tcp_rwnd; 16791 tce.tcpConnEntryInfo.ce_rto = tcp->tcp_rto; 16792 tce.tcpConnEntryInfo.ce_mss = tcp->tcp_mss; 16793 tce.tcpConnEntryInfo.ce_state = 16794 tcp->tcp_state; 16795 16796 tce.tcpConnCreationProcess = 16797 (tcp->tcp_cpid < 0) ? MIB2_UNKNOWN_PROCESS : 16798 tcp->tcp_cpid; 16799 tce.tcpConnCreationTime = tcp->tcp_open_time; 16800 16801 (void) snmp_append_data2(mp_conn_ctl->b_cont, 16802 &mp_conn_tail, (char *)&tce, sizeof (tce)); 16803 16804 mlp.tme_connidx = v4_conn_idx++; 16805 if (needattr) 16806 (void) snmp_append_data2( 16807 mp_attr_ctl->b_cont, 16808 &mp_attr_tail, (char *)&mlp, 16809 sizeof (mlp)); 16810 } 16811 } 16812 } 16813 16814 /* fixed length structure for IPv4 and IPv6 counters */ 16815 SET_MIB(tcps->tcps_mib.tcpConnTableSize, sizeof (mib2_tcpConnEntry_t)); 16816 SET_MIB(tcps->tcps_mib.tcp6ConnTableSize, 16817 sizeof (mib2_tcp6ConnEntry_t)); 16818 /* synchronize 32- and 64-bit counters */ 16819 SYNC32_MIB(&tcps->tcps_mib, tcpInSegs, tcpHCInSegs); 16820 SYNC32_MIB(&tcps->tcps_mib, tcpOutSegs, tcpHCOutSegs); 16821 optp = (struct opthdr *)&mpctl->b_rptr[sizeof (struct T_optmgmt_ack)]; 16822 optp->level = MIB2_TCP; 16823 optp->name = 0; 16824 (void) snmp_append_data(mpdata, (char *)&tcps->tcps_mib, 16825 sizeof (tcps->tcps_mib)); 16826 optp->len = msgdsize(mpdata); 16827 qreply(q, mpctl); 16828 16829 /* table of connections... */ 16830 optp = (struct opthdr *)&mp_conn_ctl->b_rptr[ 16831 sizeof (struct T_optmgmt_ack)]; 16832 optp->level = MIB2_TCP; 16833 optp->name = MIB2_TCP_CONN; 16834 optp->len = msgdsize(mp_conn_ctl->b_cont); 16835 qreply(q, mp_conn_ctl); 16836 16837 /* table of MLP attributes... */ 16838 optp = (struct opthdr *)&mp_attr_ctl->b_rptr[ 16839 sizeof (struct T_optmgmt_ack)]; 16840 optp->level = MIB2_TCP; 16841 optp->name = EXPER_XPORT_MLP; 16842 optp->len = msgdsize(mp_attr_ctl->b_cont); 16843 if (optp->len == 0) 16844 freemsg(mp_attr_ctl); 16845 else 16846 qreply(q, mp_attr_ctl); 16847 16848 /* table of IPv6 connections... */ 16849 optp = (struct opthdr *)&mp6_conn_ctl->b_rptr[ 16850 sizeof (struct T_optmgmt_ack)]; 16851 optp->level = MIB2_TCP6; 16852 optp->name = MIB2_TCP6_CONN; 16853 optp->len = msgdsize(mp6_conn_ctl->b_cont); 16854 qreply(q, mp6_conn_ctl); 16855 16856 /* table of IPv6 MLP attributes... */ 16857 optp = (struct opthdr *)&mp6_attr_ctl->b_rptr[ 16858 sizeof (struct T_optmgmt_ack)]; 16859 optp->level = MIB2_TCP6; 16860 optp->name = EXPER_XPORT_MLP; 16861 optp->len = msgdsize(mp6_attr_ctl->b_cont); 16862 if (optp->len == 0) 16863 freemsg(mp6_attr_ctl); 16864 else 16865 qreply(q, mp6_attr_ctl); 16866 return (mp2ctl); 16867 } 16868 16869 /* Return 0 if invalid set request, 1 otherwise, including non-tcp requests */ 16870 /* ARGSUSED */ 16871 int 16872 tcp_snmp_set(queue_t *q, int level, int name, uchar_t *ptr, int len) 16873 { 16874 mib2_tcpConnEntry_t *tce = (mib2_tcpConnEntry_t *)ptr; 16875 16876 switch (level) { 16877 case MIB2_TCP: 16878 switch (name) { 16879 case 13: 16880 if (tce->tcpConnState != MIB2_TCP_deleteTCB) 16881 return (0); 16882 /* TODO: delete entry defined by tce */ 16883 return (1); 16884 default: 16885 return (0); 16886 } 16887 default: 16888 return (1); 16889 } 16890 } 16891 16892 /* Translate TCP state to MIB2 TCP state. */ 16893 static int 16894 tcp_snmp_state(tcp_t *tcp) 16895 { 16896 if (tcp == NULL) 16897 return (0); 16898 16899 switch (tcp->tcp_state) { 16900 case TCPS_CLOSED: 16901 case TCPS_IDLE: /* RFC1213 doesn't have analogue for IDLE & BOUND */ 16902 case TCPS_BOUND: 16903 return (MIB2_TCP_closed); 16904 case TCPS_LISTEN: 16905 return (MIB2_TCP_listen); 16906 case TCPS_SYN_SENT: 16907 return (MIB2_TCP_synSent); 16908 case TCPS_SYN_RCVD: 16909 return (MIB2_TCP_synReceived); 16910 case TCPS_ESTABLISHED: 16911 return (MIB2_TCP_established); 16912 case TCPS_CLOSE_WAIT: 16913 return (MIB2_TCP_closeWait); 16914 case TCPS_FIN_WAIT_1: 16915 return (MIB2_TCP_finWait1); 16916 case TCPS_CLOSING: 16917 return (MIB2_TCP_closing); 16918 case TCPS_LAST_ACK: 16919 return (MIB2_TCP_lastAck); 16920 case TCPS_FIN_WAIT_2: 16921 return (MIB2_TCP_finWait2); 16922 case TCPS_TIME_WAIT: 16923 return (MIB2_TCP_timeWait); 16924 default: 16925 return (0); 16926 } 16927 } 16928 16929 static char tcp_report_header[] = 16930 "TCP " MI_COL_HDRPAD_STR 16931 "zone dest snxt suna " 16932 "swnd rnxt rack rwnd rto mss w sw rw t " 16933 "recent [lport,fport] state"; 16934 16935 /* 16936 * TCP status report triggered via the Named Dispatch mechanism. 16937 */ 16938 /* ARGSUSED */ 16939 static void 16940 tcp_report_item(mblk_t *mp, tcp_t *tcp, int hashval, tcp_t *thisstream, 16941 cred_t *cr) 16942 { 16943 char hash[10], addrbuf[INET6_ADDRSTRLEN]; 16944 boolean_t ispriv = secpolicy_ip_config(cr, B_TRUE) == 0; 16945 char cflag; 16946 in6_addr_t v6dst; 16947 char buf[80]; 16948 uint_t print_len, buf_len; 16949 16950 buf_len = mp->b_datap->db_lim - mp->b_wptr; 16951 if (buf_len <= 0) 16952 return; 16953 16954 if (hashval >= 0) 16955 (void) sprintf(hash, "%03d ", hashval); 16956 else 16957 hash[0] = '\0'; 16958 16959 /* 16960 * Note that we use the remote address in the tcp_b structure. 16961 * This means that it will print out the real destination address, 16962 * not the next hop's address if source routing is used. This 16963 * avoid the confusion on the output because user may not 16964 * know that source routing is used for a connection. 16965 */ 16966 if (tcp->tcp_ipversion == IPV4_VERSION) { 16967 IN6_IPADDR_TO_V4MAPPED(tcp->tcp_remote, &v6dst); 16968 } else { 16969 v6dst = tcp->tcp_remote_v6; 16970 } 16971 (void) inet_ntop(AF_INET6, &v6dst, addrbuf, sizeof (addrbuf)); 16972 /* 16973 * the ispriv checks are so that normal users cannot determine 16974 * sequence number information using NDD. 16975 */ 16976 16977 if (TCP_IS_DETACHED(tcp)) 16978 cflag = '*'; 16979 else 16980 cflag = ' '; 16981 print_len = snprintf((char *)mp->b_wptr, buf_len, 16982 "%s " MI_COL_PTRFMT_STR "%d %s %08x %08x %010d %08x %08x " 16983 "%010d %05ld %05d %1d %02d %02d %1d %08x %s%c\n", 16984 hash, 16985 (void *)tcp, 16986 tcp->tcp_connp->conn_zoneid, 16987 addrbuf, 16988 (ispriv) ? tcp->tcp_snxt : 0, 16989 (ispriv) ? tcp->tcp_suna : 0, 16990 tcp->tcp_swnd, 16991 (ispriv) ? tcp->tcp_rnxt : 0, 16992 (ispriv) ? tcp->tcp_rack : 0, 16993 tcp->tcp_rwnd, 16994 tcp->tcp_rto, 16995 tcp->tcp_mss, 16996 tcp->tcp_snd_ws_ok, 16997 tcp->tcp_snd_ws, 16998 tcp->tcp_rcv_ws, 16999 tcp->tcp_snd_ts_ok, 17000 tcp->tcp_ts_recent, 17001 tcp_display(tcp, buf, DISP_PORT_ONLY), cflag); 17002 if (print_len < buf_len) { 17003 ((mblk_t *)mp)->b_wptr += print_len; 17004 } else { 17005 ((mblk_t *)mp)->b_wptr += buf_len; 17006 } 17007 } 17008 17009 /* 17010 * TCP status report (for listeners only) triggered via the Named Dispatch 17011 * mechanism. 17012 */ 17013 /* ARGSUSED */ 17014 static void 17015 tcp_report_listener(mblk_t *mp, tcp_t *tcp, int hashval) 17016 { 17017 char addrbuf[INET6_ADDRSTRLEN]; 17018 in6_addr_t v6dst; 17019 uint_t print_len, buf_len; 17020 17021 buf_len = mp->b_datap->db_lim - mp->b_wptr; 17022 if (buf_len <= 0) 17023 return; 17024 17025 if (tcp->tcp_ipversion == IPV4_VERSION) { 17026 IN6_IPADDR_TO_V4MAPPED(tcp->tcp_ipha->ipha_src, &v6dst); 17027 (void) inet_ntop(AF_INET6, &v6dst, addrbuf, sizeof (addrbuf)); 17028 } else { 17029 (void) inet_ntop(AF_INET6, &tcp->tcp_ip6h->ip6_src, 17030 addrbuf, sizeof (addrbuf)); 17031 } 17032 print_len = snprintf((char *)mp->b_wptr, buf_len, 17033 "%03d " 17034 MI_COL_PTRFMT_STR 17035 "%d %s %05u %08u %d/%d/%d%c\n", 17036 hashval, (void *)tcp, 17037 tcp->tcp_connp->conn_zoneid, 17038 addrbuf, 17039 (uint_t)BE16_TO_U16(tcp->tcp_tcph->th_lport), 17040 tcp->tcp_conn_req_seqnum, 17041 tcp->tcp_conn_req_cnt_q0, tcp->tcp_conn_req_cnt_q, 17042 tcp->tcp_conn_req_max, 17043 tcp->tcp_syn_defense ? '*' : ' '); 17044 if (print_len < buf_len) { 17045 ((mblk_t *)mp)->b_wptr += print_len; 17046 } else { 17047 ((mblk_t *)mp)->b_wptr += buf_len; 17048 } 17049 } 17050 17051 /* TCP status report triggered via the Named Dispatch mechanism. */ 17052 /* ARGSUSED */ 17053 static int 17054 tcp_status_report(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr) 17055 { 17056 tcp_t *tcp; 17057 int i; 17058 conn_t *connp; 17059 connf_t *connfp; 17060 zoneid_t zoneid; 17061 tcp_stack_t *tcps; 17062 ip_stack_t *ipst; 17063 17064 zoneid = Q_TO_CONN(q)->conn_zoneid; 17065 tcps = Q_TO_TCP(q)->tcp_tcps; 17066 17067 /* 17068 * Because of the ndd constraint, at most we can have 64K buffer 17069 * to put in all TCP info. So to be more efficient, just 17070 * allocate a 64K buffer here, assuming we need that large buffer. 17071 * This may be a problem as any user can read tcp_status. Therefore 17072 * we limit the rate of doing this using tcp_ndd_get_info_interval. 17073 * This should be OK as normal users should not do this too often. 17074 */ 17075 if (cr == NULL || secpolicy_ip_config(cr, B_TRUE) != 0) { 17076 if (ddi_get_lbolt() - tcps->tcps_last_ndd_get_info_time < 17077 drv_usectohz(tcps->tcps_ndd_get_info_interval * 1000)) { 17078 (void) mi_mpprintf(mp, NDD_TOO_QUICK_MSG); 17079 return (0); 17080 } 17081 } 17082 if ((mp->b_cont = allocb(ND_MAX_BUF_LEN, BPRI_HI)) == NULL) { 17083 /* The following may work even if we cannot get a large buf. */ 17084 (void) mi_mpprintf(mp, NDD_OUT_OF_BUF_MSG); 17085 return (0); 17086 } 17087 17088 (void) mi_mpprintf(mp, "%s", tcp_report_header); 17089 17090 for (i = 0; i < CONN_G_HASH_SIZE; i++) { 17091 17092 ipst = tcps->tcps_netstack->netstack_ip; 17093 connfp = &ipst->ips_ipcl_globalhash_fanout[i]; 17094 17095 connp = NULL; 17096 17097 while ((connp = 17098 ipcl_get_next_conn(connfp, connp, IPCL_TCP)) != NULL) { 17099 tcp = connp->conn_tcp; 17100 if (zoneid != GLOBAL_ZONEID && 17101 zoneid != connp->conn_zoneid) 17102 continue; 17103 tcp_report_item(mp->b_cont, tcp, -1, tcp, 17104 cr); 17105 } 17106 17107 } 17108 17109 tcps->tcps_last_ndd_get_info_time = ddi_get_lbolt(); 17110 return (0); 17111 } 17112 17113 /* TCP status report triggered via the Named Dispatch mechanism. */ 17114 /* ARGSUSED */ 17115 static int 17116 tcp_bind_hash_report(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr) 17117 { 17118 tf_t *tbf; 17119 tcp_t *tcp; 17120 int i; 17121 zoneid_t zoneid; 17122 tcp_stack_t *tcps = Q_TO_TCP(q)->tcp_tcps; 17123 17124 zoneid = Q_TO_CONN(q)->conn_zoneid; 17125 17126 /* Refer to comments in tcp_status_report(). */ 17127 if (cr == NULL || secpolicy_ip_config(cr, B_TRUE) != 0) { 17128 if (ddi_get_lbolt() - tcps->tcps_last_ndd_get_info_time < 17129 drv_usectohz(tcps->tcps_ndd_get_info_interval * 1000)) { 17130 (void) mi_mpprintf(mp, NDD_TOO_QUICK_MSG); 17131 return (0); 17132 } 17133 } 17134 if ((mp->b_cont = allocb(ND_MAX_BUF_LEN, BPRI_HI)) == NULL) { 17135 /* The following may work even if we cannot get a large buf. */ 17136 (void) mi_mpprintf(mp, NDD_OUT_OF_BUF_MSG); 17137 return (0); 17138 } 17139 17140 (void) mi_mpprintf(mp, " %s", tcp_report_header); 17141 17142 for (i = 0; i < TCP_BIND_FANOUT_SIZE; i++) { 17143 tbf = &tcps->tcps_bind_fanout[i]; 17144 mutex_enter(&tbf->tf_lock); 17145 for (tcp = tbf->tf_tcp; tcp != NULL; 17146 tcp = tcp->tcp_bind_hash) { 17147 if (zoneid != GLOBAL_ZONEID && 17148 zoneid != tcp->tcp_connp->conn_zoneid) 17149 continue; 17150 CONN_INC_REF(tcp->tcp_connp); 17151 tcp_report_item(mp->b_cont, tcp, i, 17152 Q_TO_TCP(q), cr); 17153 CONN_DEC_REF(tcp->tcp_connp); 17154 } 17155 mutex_exit(&tbf->tf_lock); 17156 } 17157 tcps->tcps_last_ndd_get_info_time = ddi_get_lbolt(); 17158 return (0); 17159 } 17160 17161 /* TCP status report triggered via the Named Dispatch mechanism. */ 17162 /* ARGSUSED */ 17163 static int 17164 tcp_listen_hash_report(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr) 17165 { 17166 connf_t *connfp; 17167 conn_t *connp; 17168 tcp_t *tcp; 17169 int i; 17170 zoneid_t zoneid; 17171 tcp_stack_t *tcps; 17172 ip_stack_t *ipst; 17173 17174 zoneid = Q_TO_CONN(q)->conn_zoneid; 17175 tcps = Q_TO_TCP(q)->tcp_tcps; 17176 17177 /* Refer to comments in tcp_status_report(). */ 17178 if (cr == NULL || secpolicy_ip_config(cr, B_TRUE) != 0) { 17179 if (ddi_get_lbolt() - tcps->tcps_last_ndd_get_info_time < 17180 drv_usectohz(tcps->tcps_ndd_get_info_interval * 1000)) { 17181 (void) mi_mpprintf(mp, NDD_TOO_QUICK_MSG); 17182 return (0); 17183 } 17184 } 17185 if ((mp->b_cont = allocb(ND_MAX_BUF_LEN, BPRI_HI)) == NULL) { 17186 /* The following may work even if we cannot get a large buf. */ 17187 (void) mi_mpprintf(mp, NDD_OUT_OF_BUF_MSG); 17188 return (0); 17189 } 17190 17191 (void) mi_mpprintf(mp, 17192 " TCP " MI_COL_HDRPAD_STR 17193 "zone IP addr port seqnum backlog (q0/q/max)"); 17194 17195 ipst = tcps->tcps_netstack->netstack_ip; 17196 17197 for (i = 0; i < ipst->ips_ipcl_bind_fanout_size; i++) { 17198 connfp = &ipst->ips_ipcl_bind_fanout[i]; 17199 connp = NULL; 17200 while ((connp = 17201 ipcl_get_next_conn(connfp, connp, IPCL_TCP)) != NULL) { 17202 tcp = connp->conn_tcp; 17203 if (zoneid != GLOBAL_ZONEID && 17204 zoneid != connp->conn_zoneid) 17205 continue; 17206 tcp_report_listener(mp->b_cont, tcp, i); 17207 } 17208 } 17209 17210 tcps->tcps_last_ndd_get_info_time = ddi_get_lbolt(); 17211 return (0); 17212 } 17213 17214 /* TCP status report triggered via the Named Dispatch mechanism. */ 17215 /* ARGSUSED */ 17216 static int 17217 tcp_conn_hash_report(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr) 17218 { 17219 connf_t *connfp; 17220 conn_t *connp; 17221 tcp_t *tcp; 17222 int i; 17223 zoneid_t zoneid; 17224 tcp_stack_t *tcps; 17225 ip_stack_t *ipst; 17226 17227 zoneid = Q_TO_CONN(q)->conn_zoneid; 17228 tcps = Q_TO_TCP(q)->tcp_tcps; 17229 ipst = tcps->tcps_netstack->netstack_ip; 17230 17231 /* Refer to comments in tcp_status_report(). */ 17232 if (cr == NULL || secpolicy_ip_config(cr, B_TRUE) != 0) { 17233 if (ddi_get_lbolt() - tcps->tcps_last_ndd_get_info_time < 17234 drv_usectohz(tcps->tcps_ndd_get_info_interval * 1000)) { 17235 (void) mi_mpprintf(mp, NDD_TOO_QUICK_MSG); 17236 return (0); 17237 } 17238 } 17239 if ((mp->b_cont = allocb(ND_MAX_BUF_LEN, BPRI_HI)) == NULL) { 17240 /* The following may work even if we cannot get a large buf. */ 17241 (void) mi_mpprintf(mp, NDD_OUT_OF_BUF_MSG); 17242 return (0); 17243 } 17244 17245 (void) mi_mpprintf(mp, "tcp_conn_hash_size = %d", 17246 ipst->ips_ipcl_conn_fanout_size); 17247 (void) mi_mpprintf(mp, " %s", tcp_report_header); 17248 17249 for (i = 0; i < ipst->ips_ipcl_conn_fanout_size; i++) { 17250 connfp = &ipst->ips_ipcl_conn_fanout[i]; 17251 connp = NULL; 17252 while ((connp = 17253 ipcl_get_next_conn(connfp, connp, IPCL_TCP)) != NULL) { 17254 tcp = connp->conn_tcp; 17255 if (zoneid != GLOBAL_ZONEID && 17256 zoneid != connp->conn_zoneid) 17257 continue; 17258 tcp_report_item(mp->b_cont, tcp, i, 17259 Q_TO_TCP(q), cr); 17260 } 17261 } 17262 17263 tcps->tcps_last_ndd_get_info_time = ddi_get_lbolt(); 17264 return (0); 17265 } 17266 17267 /* TCP status report triggered via the Named Dispatch mechanism. */ 17268 /* ARGSUSED */ 17269 static int 17270 tcp_acceptor_hash_report(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr) 17271 { 17272 tf_t *tf; 17273 tcp_t *tcp; 17274 int i; 17275 zoneid_t zoneid; 17276 tcp_stack_t *tcps; 17277 17278 zoneid = Q_TO_CONN(q)->conn_zoneid; 17279 tcps = Q_TO_TCP(q)->tcp_tcps; 17280 17281 /* Refer to comments in tcp_status_report(). */ 17282 if (cr == NULL || secpolicy_ip_config(cr, B_TRUE) != 0) { 17283 if (ddi_get_lbolt() - tcps->tcps_last_ndd_get_info_time < 17284 drv_usectohz(tcps->tcps_ndd_get_info_interval * 1000)) { 17285 (void) mi_mpprintf(mp, NDD_TOO_QUICK_MSG); 17286 return (0); 17287 } 17288 } 17289 if ((mp->b_cont = allocb(ND_MAX_BUF_LEN, BPRI_HI)) == NULL) { 17290 /* The following may work even if we cannot get a large buf. */ 17291 (void) mi_mpprintf(mp, NDD_OUT_OF_BUF_MSG); 17292 return (0); 17293 } 17294 17295 (void) mi_mpprintf(mp, " %s", tcp_report_header); 17296 17297 for (i = 0; i < TCP_FANOUT_SIZE; i++) { 17298 tf = &tcps->tcps_acceptor_fanout[i]; 17299 mutex_enter(&tf->tf_lock); 17300 for (tcp = tf->tf_tcp; tcp != NULL; 17301 tcp = tcp->tcp_acceptor_hash) { 17302 if (zoneid != GLOBAL_ZONEID && 17303 zoneid != tcp->tcp_connp->conn_zoneid) 17304 continue; 17305 tcp_report_item(mp->b_cont, tcp, i, 17306 Q_TO_TCP(q), cr); 17307 } 17308 mutex_exit(&tf->tf_lock); 17309 } 17310 tcps->tcps_last_ndd_get_info_time = ddi_get_lbolt(); 17311 return (0); 17312 } 17313 17314 /* 17315 * tcp_timer is the timer service routine. It handles the retransmission, 17316 * FIN_WAIT_2 flush, and zero window probe timeout events. It figures out 17317 * from the state of the tcp instance what kind of action needs to be done 17318 * at the time it is called. 17319 */ 17320 static void 17321 tcp_timer(void *arg) 17322 { 17323 mblk_t *mp; 17324 clock_t first_threshold; 17325 clock_t second_threshold; 17326 clock_t ms; 17327 uint32_t mss; 17328 conn_t *connp = (conn_t *)arg; 17329 tcp_t *tcp = connp->conn_tcp; 17330 tcp_stack_t *tcps = tcp->tcp_tcps; 17331 17332 tcp->tcp_timer_tid = 0; 17333 17334 if (tcp->tcp_fused) 17335 return; 17336 17337 first_threshold = tcp->tcp_first_timer_threshold; 17338 second_threshold = tcp->tcp_second_timer_threshold; 17339 switch (tcp->tcp_state) { 17340 case TCPS_IDLE: 17341 case TCPS_BOUND: 17342 case TCPS_LISTEN: 17343 return; 17344 case TCPS_SYN_RCVD: { 17345 tcp_t *listener = tcp->tcp_listener; 17346 17347 if (tcp->tcp_syn_rcvd_timeout == 0 && (listener != NULL)) { 17348 ASSERT(tcp->tcp_rq == listener->tcp_rq); 17349 /* it's our first timeout */ 17350 tcp->tcp_syn_rcvd_timeout = 1; 17351 mutex_enter(&listener->tcp_eager_lock); 17352 listener->tcp_syn_rcvd_timeout++; 17353 if (!tcp->tcp_dontdrop && !tcp->tcp_closemp_used) { 17354 /* 17355 * Make this eager available for drop if we 17356 * need to drop one to accomodate a new 17357 * incoming SYN request. 17358 */ 17359 MAKE_DROPPABLE(listener, tcp); 17360 } 17361 if (!listener->tcp_syn_defense && 17362 (listener->tcp_syn_rcvd_timeout > 17363 (tcps->tcps_conn_req_max_q0 >> 2)) && 17364 (tcps->tcps_conn_req_max_q0 > 200)) { 17365 /* We may be under attack. Put on a defense. */ 17366 listener->tcp_syn_defense = B_TRUE; 17367 cmn_err(CE_WARN, "High TCP connect timeout " 17368 "rate! System (port %d) may be under a " 17369 "SYN flood attack!", 17370 BE16_TO_U16(listener->tcp_tcph->th_lport)); 17371 17372 listener->tcp_ip_addr_cache = kmem_zalloc( 17373 IP_ADDR_CACHE_SIZE * sizeof (ipaddr_t), 17374 KM_NOSLEEP); 17375 } 17376 mutex_exit(&listener->tcp_eager_lock); 17377 } else if (listener != NULL) { 17378 mutex_enter(&listener->tcp_eager_lock); 17379 tcp->tcp_syn_rcvd_timeout++; 17380 if (tcp->tcp_syn_rcvd_timeout > 1 && 17381 !tcp->tcp_closemp_used) { 17382 /* 17383 * This is our second timeout. Put the tcp in 17384 * the list of droppable eagers to allow it to 17385 * be dropped, if needed. We don't check 17386 * whether tcp_dontdrop is set or not to 17387 * protect ourselve from a SYN attack where a 17388 * remote host can spoof itself as one of the 17389 * good IP source and continue to hold 17390 * resources too long. 17391 */ 17392 MAKE_DROPPABLE(listener, tcp); 17393 } 17394 mutex_exit(&listener->tcp_eager_lock); 17395 } 17396 } 17397 /* FALLTHRU */ 17398 case TCPS_SYN_SENT: 17399 first_threshold = tcp->tcp_first_ctimer_threshold; 17400 second_threshold = tcp->tcp_second_ctimer_threshold; 17401 break; 17402 case TCPS_ESTABLISHED: 17403 case TCPS_FIN_WAIT_1: 17404 case TCPS_CLOSING: 17405 case TCPS_CLOSE_WAIT: 17406 case TCPS_LAST_ACK: 17407 /* If we have data to rexmit */ 17408 if (tcp->tcp_suna != tcp->tcp_snxt) { 17409 clock_t time_to_wait; 17410 17411 BUMP_MIB(&tcps->tcps_mib, tcpTimRetrans); 17412 if (!tcp->tcp_xmit_head) 17413 break; 17414 time_to_wait = lbolt - 17415 (clock_t)tcp->tcp_xmit_head->b_prev; 17416 time_to_wait = tcp->tcp_rto - 17417 TICK_TO_MSEC(time_to_wait); 17418 /* 17419 * If the timer fires too early, 1 clock tick earlier, 17420 * restart the timer. 17421 */ 17422 if (time_to_wait > msec_per_tick) { 17423 TCP_STAT(tcps, tcp_timer_fire_early); 17424 TCP_TIMER_RESTART(tcp, time_to_wait); 17425 return; 17426 } 17427 /* 17428 * When we probe zero windows, we force the swnd open. 17429 * If our peer acks with a closed window swnd will be 17430 * set to zero by tcp_rput(). As long as we are 17431 * receiving acks tcp_rput will 17432 * reset 'tcp_ms_we_have_waited' so as not to trip the 17433 * first and second interval actions. NOTE: the timer 17434 * interval is allowed to continue its exponential 17435 * backoff. 17436 */ 17437 if (tcp->tcp_swnd == 0 || tcp->tcp_zero_win_probe) { 17438 if (tcp->tcp_debug) { 17439 (void) strlog(TCP_MOD_ID, 0, 1, 17440 SL_TRACE, "tcp_timer: zero win"); 17441 } 17442 } else { 17443 /* 17444 * After retransmission, we need to do 17445 * slow start. Set the ssthresh to one 17446 * half of current effective window and 17447 * cwnd to one MSS. Also reset 17448 * tcp_cwnd_cnt. 17449 * 17450 * Note that if tcp_ssthresh is reduced because 17451 * of ECN, do not reduce it again unless it is 17452 * already one window of data away (tcp_cwr 17453 * should then be cleared) or this is a 17454 * timeout for a retransmitted segment. 17455 */ 17456 uint32_t npkt; 17457 17458 if (!tcp->tcp_cwr || tcp->tcp_rexmit) { 17459 npkt = ((tcp->tcp_timer_backoff ? 17460 tcp->tcp_cwnd_ssthresh : 17461 tcp->tcp_snxt - 17462 tcp->tcp_suna) >> 1) / tcp->tcp_mss; 17463 tcp->tcp_cwnd_ssthresh = MAX(npkt, 2) * 17464 tcp->tcp_mss; 17465 } 17466 tcp->tcp_cwnd = tcp->tcp_mss; 17467 tcp->tcp_cwnd_cnt = 0; 17468 if (tcp->tcp_ecn_ok) { 17469 tcp->tcp_cwr = B_TRUE; 17470 tcp->tcp_cwr_snd_max = tcp->tcp_snxt; 17471 tcp->tcp_ecn_cwr_sent = B_FALSE; 17472 } 17473 } 17474 break; 17475 } 17476 /* 17477 * We have something to send yet we cannot send. The 17478 * reason can be: 17479 * 17480 * 1. Zero send window: we need to do zero window probe. 17481 * 2. Zero cwnd: because of ECN, we need to "clock out 17482 * segments. 17483 * 3. SWS avoidance: receiver may have shrunk window, 17484 * reset our knowledge. 17485 * 17486 * Note that condition 2 can happen with either 1 or 17487 * 3. But 1 and 3 are exclusive. 17488 */ 17489 if (tcp->tcp_unsent != 0) { 17490 if (tcp->tcp_cwnd == 0) { 17491 /* 17492 * Set tcp_cwnd to 1 MSS so that a 17493 * new segment can be sent out. We 17494 * are "clocking out" new data when 17495 * the network is really congested. 17496 */ 17497 ASSERT(tcp->tcp_ecn_ok); 17498 tcp->tcp_cwnd = tcp->tcp_mss; 17499 } 17500 if (tcp->tcp_swnd == 0) { 17501 /* Extend window for zero window probe */ 17502 tcp->tcp_swnd++; 17503 tcp->tcp_zero_win_probe = B_TRUE; 17504 BUMP_MIB(&tcps->tcps_mib, tcpOutWinProbe); 17505 } else { 17506 /* 17507 * Handle timeout from sender SWS avoidance. 17508 * Reset our knowledge of the max send window 17509 * since the receiver might have reduced its 17510 * receive buffer. Avoid setting tcp_max_swnd 17511 * to one since that will essentially disable 17512 * the SWS checks. 17513 * 17514 * Note that since we don't have a SWS 17515 * state variable, if the timeout is set 17516 * for ECN but not for SWS, this 17517 * code will also be executed. This is 17518 * fine as tcp_max_swnd is updated 17519 * constantly and it will not affect 17520 * anything. 17521 */ 17522 tcp->tcp_max_swnd = MAX(tcp->tcp_swnd, 2); 17523 } 17524 tcp_wput_data(tcp, NULL, B_FALSE); 17525 return; 17526 } 17527 /* Is there a FIN that needs to be to re retransmitted? */ 17528 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && 17529 !tcp->tcp_fin_acked) 17530 break; 17531 /* Nothing to do, return without restarting timer. */ 17532 TCP_STAT(tcps, tcp_timer_fire_miss); 17533 return; 17534 case TCPS_FIN_WAIT_2: 17535 /* 17536 * User closed the TCP endpoint and peer ACK'ed our FIN. 17537 * We waited some time for for peer's FIN, but it hasn't 17538 * arrived. We flush the connection now to avoid 17539 * case where the peer has rebooted. 17540 */ 17541 if (TCP_IS_DETACHED(tcp)) { 17542 (void) tcp_clean_death(tcp, 0, 23); 17543 } else { 17544 TCP_TIMER_RESTART(tcp, 17545 tcps->tcps_fin_wait_2_flush_interval); 17546 } 17547 return; 17548 case TCPS_TIME_WAIT: 17549 (void) tcp_clean_death(tcp, 0, 24); 17550 return; 17551 default: 17552 if (tcp->tcp_debug) { 17553 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE|SL_ERROR, 17554 "tcp_timer: strange state (%d) %s", 17555 tcp->tcp_state, tcp_display(tcp, NULL, 17556 DISP_PORT_ONLY)); 17557 } 17558 return; 17559 } 17560 if ((ms = tcp->tcp_ms_we_have_waited) > second_threshold) { 17561 /* 17562 * For zero window probe, we need to send indefinitely, 17563 * unless we have not heard from the other side for some 17564 * time... 17565 */ 17566 if ((tcp->tcp_zero_win_probe == 0) || 17567 (TICK_TO_MSEC(lbolt - tcp->tcp_last_recv_time) > 17568 second_threshold)) { 17569 BUMP_MIB(&tcps->tcps_mib, tcpTimRetransDrop); 17570 /* 17571 * If TCP is in SYN_RCVD state, send back a 17572 * RST|ACK as BSD does. Note that tcp_zero_win_probe 17573 * should be zero in TCPS_SYN_RCVD state. 17574 */ 17575 if (tcp->tcp_state == TCPS_SYN_RCVD) { 17576 tcp_xmit_ctl("tcp_timer: RST sent on timeout " 17577 "in SYN_RCVD", 17578 tcp, tcp->tcp_snxt, 17579 tcp->tcp_rnxt, TH_RST | TH_ACK); 17580 } 17581 (void) tcp_clean_death(tcp, 17582 tcp->tcp_client_errno ? 17583 tcp->tcp_client_errno : ETIMEDOUT, 25); 17584 return; 17585 } else { 17586 /* 17587 * Set tcp_ms_we_have_waited to second_threshold 17588 * so that in next timeout, we will do the above 17589 * check (lbolt - tcp_last_recv_time). This is 17590 * also to avoid overflow. 17591 * 17592 * We don't need to decrement tcp_timer_backoff 17593 * to avoid overflow because it will be decremented 17594 * later if new timeout value is greater than 17595 * tcp_rexmit_interval_max. In the case when 17596 * tcp_rexmit_interval_max is greater than 17597 * second_threshold, it means that we will wait 17598 * longer than second_threshold to send the next 17599 * window probe. 17600 */ 17601 tcp->tcp_ms_we_have_waited = second_threshold; 17602 } 17603 } else if (ms > first_threshold) { 17604 if (tcp->tcp_snd_zcopy_aware && (!tcp->tcp_xmit_zc_clean) && 17605 tcp->tcp_xmit_head != NULL) { 17606 tcp->tcp_xmit_head = 17607 tcp_zcopy_backoff(tcp, tcp->tcp_xmit_head, 1); 17608 } 17609 /* 17610 * We have been retransmitting for too long... The RTT 17611 * we calculated is probably incorrect. Reinitialize it. 17612 * Need to compensate for 0 tcp_rtt_sa. Reset 17613 * tcp_rtt_update so that we won't accidentally cache a 17614 * bad value. But only do this if this is not a zero 17615 * window probe. 17616 */ 17617 if (tcp->tcp_rtt_sa != 0 && tcp->tcp_zero_win_probe == 0) { 17618 tcp->tcp_rtt_sd += (tcp->tcp_rtt_sa >> 3) + 17619 (tcp->tcp_rtt_sa >> 5); 17620 tcp->tcp_rtt_sa = 0; 17621 tcp_ip_notify(tcp); 17622 tcp->tcp_rtt_update = 0; 17623 } 17624 } 17625 tcp->tcp_timer_backoff++; 17626 if ((ms = (tcp->tcp_rtt_sa >> 3) + tcp->tcp_rtt_sd + 17627 tcps->tcps_rexmit_interval_extra + (tcp->tcp_rtt_sa >> 5)) < 17628 tcps->tcps_rexmit_interval_min) { 17629 /* 17630 * This means the original RTO is tcp_rexmit_interval_min. 17631 * So we will use tcp_rexmit_interval_min as the RTO value 17632 * and do the backoff. 17633 */ 17634 ms = tcps->tcps_rexmit_interval_min << tcp->tcp_timer_backoff; 17635 } else { 17636 ms <<= tcp->tcp_timer_backoff; 17637 } 17638 if (ms > tcps->tcps_rexmit_interval_max) { 17639 ms = tcps->tcps_rexmit_interval_max; 17640 /* 17641 * ms is at max, decrement tcp_timer_backoff to avoid 17642 * overflow. 17643 */ 17644 tcp->tcp_timer_backoff--; 17645 } 17646 tcp->tcp_ms_we_have_waited += ms; 17647 if (tcp->tcp_zero_win_probe == 0) { 17648 tcp->tcp_rto = ms; 17649 } 17650 TCP_TIMER_RESTART(tcp, ms); 17651 /* 17652 * This is after a timeout and tcp_rto is backed off. Set 17653 * tcp_set_timer to 1 so that next time RTO is updated, we will 17654 * restart the timer with a correct value. 17655 */ 17656 tcp->tcp_set_timer = 1; 17657 mss = tcp->tcp_snxt - tcp->tcp_suna; 17658 if (mss > tcp->tcp_mss) 17659 mss = tcp->tcp_mss; 17660 if (mss > tcp->tcp_swnd && tcp->tcp_swnd != 0) 17661 mss = tcp->tcp_swnd; 17662 17663 if ((mp = tcp->tcp_xmit_head) != NULL) 17664 mp->b_prev = (mblk_t *)lbolt; 17665 mp = tcp_xmit_mp(tcp, mp, mss, NULL, NULL, tcp->tcp_suna, B_TRUE, &mss, 17666 B_TRUE); 17667 17668 /* 17669 * When slow start after retransmission begins, start with 17670 * this seq no. tcp_rexmit_max marks the end of special slow 17671 * start phase. tcp_snd_burst controls how many segments 17672 * can be sent because of an ack. 17673 */ 17674 tcp->tcp_rexmit_nxt = tcp->tcp_suna; 17675 tcp->tcp_snd_burst = TCP_CWND_SS; 17676 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && 17677 (tcp->tcp_unsent == 0)) { 17678 tcp->tcp_rexmit_max = tcp->tcp_fss; 17679 } else { 17680 tcp->tcp_rexmit_max = tcp->tcp_snxt; 17681 } 17682 tcp->tcp_rexmit = B_TRUE; 17683 tcp->tcp_dupack_cnt = 0; 17684 17685 /* 17686 * Remove all rexmit SACK blk to start from fresh. 17687 */ 17688 if (tcp->tcp_snd_sack_ok && tcp->tcp_notsack_list != NULL) { 17689 TCP_NOTSACK_REMOVE_ALL(tcp->tcp_notsack_list); 17690 tcp->tcp_num_notsack_blk = 0; 17691 tcp->tcp_cnt_notsack_list = 0; 17692 } 17693 if (mp == NULL) { 17694 return; 17695 } 17696 /* Attach credentials to retransmitted initial SYNs. */ 17697 if (tcp->tcp_state == TCPS_SYN_SENT) { 17698 mblk_setcred(mp, tcp->tcp_cred); 17699 DB_CPID(mp) = tcp->tcp_cpid; 17700 } 17701 17702 tcp->tcp_csuna = tcp->tcp_snxt; 17703 BUMP_MIB(&tcps->tcps_mib, tcpRetransSegs); 17704 UPDATE_MIB(&tcps->tcps_mib, tcpRetransBytes, mss); 17705 tcp_send_data(tcp, tcp->tcp_wq, mp); 17706 17707 } 17708 17709 /* tcp_unbind is called by tcp_wput_proto to handle T_UNBIND_REQ messages. */ 17710 static void 17711 tcp_unbind(tcp_t *tcp, mblk_t *mp) 17712 { 17713 conn_t *connp; 17714 17715 switch (tcp->tcp_state) { 17716 case TCPS_BOUND: 17717 case TCPS_LISTEN: 17718 break; 17719 default: 17720 tcp_err_ack(tcp, mp, TOUTSTATE, 0); 17721 return; 17722 } 17723 17724 /* 17725 * Need to clean up all the eagers since after the unbind, segments 17726 * will no longer be delivered to this listener stream. 17727 */ 17728 mutex_enter(&tcp->tcp_eager_lock); 17729 if (tcp->tcp_conn_req_cnt_q0 != 0 || tcp->tcp_conn_req_cnt_q != 0) { 17730 tcp_eager_cleanup(tcp, 0); 17731 } 17732 mutex_exit(&tcp->tcp_eager_lock); 17733 17734 if (tcp->tcp_ipversion == IPV4_VERSION) { 17735 tcp->tcp_ipha->ipha_src = 0; 17736 } else { 17737 V6_SET_ZERO(tcp->tcp_ip6h->ip6_src); 17738 } 17739 V6_SET_ZERO(tcp->tcp_ip_src_v6); 17740 bzero(tcp->tcp_tcph->th_lport, sizeof (tcp->tcp_tcph->th_lport)); 17741 tcp_bind_hash_remove(tcp); 17742 tcp->tcp_state = TCPS_IDLE; 17743 tcp->tcp_mdt = B_FALSE; 17744 /* Send M_FLUSH according to TPI */ 17745 (void) putnextctl1(tcp->tcp_rq, M_FLUSH, FLUSHRW); 17746 connp = tcp->tcp_connp; 17747 connp->conn_mdt_ok = B_FALSE; 17748 ipcl_hash_remove(connp); 17749 bzero(&connp->conn_ports, sizeof (connp->conn_ports)); 17750 mp = mi_tpi_ok_ack_alloc(mp); 17751 putnext(tcp->tcp_rq, mp); 17752 } 17753 17754 /* 17755 * Don't let port fall into the privileged range. 17756 * Since the extra privileged ports can be arbitrary we also 17757 * ensure that we exclude those from consideration. 17758 * tcp_g_epriv_ports is not sorted thus we loop over it until 17759 * there are no changes. 17760 * 17761 * Note: No locks are held when inspecting tcp_g_*epriv_ports 17762 * but instead the code relies on: 17763 * - the fact that the address of the array and its size never changes 17764 * - the atomic assignment of the elements of the array 17765 * 17766 * Returns 0 if there are no more ports available. 17767 * 17768 * TS note: skip multilevel ports. 17769 */ 17770 static in_port_t 17771 tcp_update_next_port(in_port_t port, const tcp_t *tcp, boolean_t random) 17772 { 17773 int i; 17774 boolean_t restart = B_FALSE; 17775 tcp_stack_t *tcps = tcp->tcp_tcps; 17776 17777 if (random && tcp_random_anon_port != 0) { 17778 (void) random_get_pseudo_bytes((uint8_t *)&port, 17779 sizeof (in_port_t)); 17780 /* 17781 * Unless changed by a sys admin, the smallest anon port 17782 * is 32768 and the largest anon port is 65535. It is 17783 * very likely (50%) for the random port to be smaller 17784 * than the smallest anon port. When that happens, 17785 * add port % (anon port range) to the smallest anon 17786 * port to get the random port. It should fall into the 17787 * valid anon port range. 17788 */ 17789 if (port < tcps->tcps_smallest_anon_port) { 17790 port = tcps->tcps_smallest_anon_port + 17791 port % (tcps->tcps_largest_anon_port - 17792 tcps->tcps_smallest_anon_port); 17793 } 17794 } 17795 17796 retry: 17797 if (port < tcps->tcps_smallest_anon_port) 17798 port = (in_port_t)tcps->tcps_smallest_anon_port; 17799 17800 if (port > tcps->tcps_largest_anon_port) { 17801 if (restart) 17802 return (0); 17803 restart = B_TRUE; 17804 port = (in_port_t)tcps->tcps_smallest_anon_port; 17805 } 17806 17807 if (port < tcps->tcps_smallest_nonpriv_port) 17808 port = (in_port_t)tcps->tcps_smallest_nonpriv_port; 17809 17810 for (i = 0; i < tcps->tcps_g_num_epriv_ports; i++) { 17811 if (port == tcps->tcps_g_epriv_ports[i]) { 17812 port++; 17813 /* 17814 * Make sure whether the port is in the 17815 * valid range. 17816 */ 17817 goto retry; 17818 } 17819 } 17820 if (is_system_labeled() && 17821 (i = tsol_next_port(crgetzone(tcp->tcp_cred), port, 17822 IPPROTO_TCP, B_TRUE)) != 0) { 17823 port = i; 17824 goto retry; 17825 } 17826 return (port); 17827 } 17828 17829 /* 17830 * Return the next anonymous port in the privileged port range for 17831 * bind checking. It starts at IPPORT_RESERVED - 1 and goes 17832 * downwards. This is the same behavior as documented in the userland 17833 * library call rresvport(3N). 17834 * 17835 * TS note: skip multilevel ports. 17836 */ 17837 static in_port_t 17838 tcp_get_next_priv_port(const tcp_t *tcp) 17839 { 17840 static in_port_t next_priv_port = IPPORT_RESERVED - 1; 17841 in_port_t nextport; 17842 boolean_t restart = B_FALSE; 17843 tcp_stack_t *tcps = tcp->tcp_tcps; 17844 retry: 17845 if (next_priv_port < tcps->tcps_min_anonpriv_port || 17846 next_priv_port >= IPPORT_RESERVED) { 17847 next_priv_port = IPPORT_RESERVED - 1; 17848 if (restart) 17849 return (0); 17850 restart = B_TRUE; 17851 } 17852 if (is_system_labeled() && 17853 (nextport = tsol_next_port(crgetzone(tcp->tcp_cred), 17854 next_priv_port, IPPROTO_TCP, B_FALSE)) != 0) { 17855 next_priv_port = nextport; 17856 goto retry; 17857 } 17858 return (next_priv_port--); 17859 } 17860 17861 /* The write side r/w procedure. */ 17862 17863 #if CCS_STATS 17864 struct { 17865 struct { 17866 int64_t count, bytes; 17867 } tot, hit; 17868 } wrw_stats; 17869 #endif 17870 17871 /* 17872 * Call by tcp_wput() to handle all non data, except M_PROTO and M_PCPROTO, 17873 * messages. 17874 */ 17875 /* ARGSUSED */ 17876 static void 17877 tcp_wput_nondata(void *arg, mblk_t *mp, void *arg2) 17878 { 17879 conn_t *connp = (conn_t *)arg; 17880 tcp_t *tcp = connp->conn_tcp; 17881 queue_t *q = tcp->tcp_wq; 17882 17883 ASSERT(DB_TYPE(mp) != M_IOCTL); 17884 /* 17885 * TCP is D_MP and qprocsoff() is done towards the end of the tcp_close. 17886 * Once the close starts, streamhead and sockfs will not let any data 17887 * packets come down (close ensures that there are no threads using the 17888 * queue and no new threads will come down) but since qprocsoff() 17889 * hasn't happened yet, a M_FLUSH or some non data message might 17890 * get reflected back (in response to our own FLUSHRW) and get 17891 * processed after tcp_close() is done. The conn would still be valid 17892 * because a ref would have added but we need to check the state 17893 * before actually processing the packet. 17894 */ 17895 if (TCP_IS_DETACHED(tcp) || (tcp->tcp_state == TCPS_CLOSED)) { 17896 freemsg(mp); 17897 return; 17898 } 17899 17900 switch (DB_TYPE(mp)) { 17901 case M_IOCDATA: 17902 tcp_wput_iocdata(tcp, mp); 17903 break; 17904 case M_FLUSH: 17905 tcp_wput_flush(tcp, mp); 17906 break; 17907 default: 17908 CALL_IP_WPUT(connp, q, mp); 17909 break; 17910 } 17911 } 17912 17913 /* 17914 * The TCP fast path write put procedure. 17915 * NOTE: the logic of the fast path is duplicated from tcp_wput_data() 17916 */ 17917 /* ARGSUSED */ 17918 void 17919 tcp_output(void *arg, mblk_t *mp, void *arg2) 17920 { 17921 int len; 17922 int hdrlen; 17923 int plen; 17924 mblk_t *mp1; 17925 uchar_t *rptr; 17926 uint32_t snxt; 17927 tcph_t *tcph; 17928 struct datab *db; 17929 uint32_t suna; 17930 uint32_t mss; 17931 ipaddr_t *dst; 17932 ipaddr_t *src; 17933 uint32_t sum; 17934 int usable; 17935 conn_t *connp = (conn_t *)arg; 17936 tcp_t *tcp = connp->conn_tcp; 17937 uint32_t msize; 17938 tcp_stack_t *tcps = tcp->tcp_tcps; 17939 17940 /* 17941 * Try and ASSERT the minimum possible references on the 17942 * conn early enough. Since we are executing on write side, 17943 * the connection is obviously not detached and that means 17944 * there is a ref each for TCP and IP. Since we are behind 17945 * the squeue, the minimum references needed are 3. If the 17946 * conn is in classifier hash list, there should be an 17947 * extra ref for that (we check both the possibilities). 17948 */ 17949 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) || 17950 (connp->conn_fanout == NULL && connp->conn_ref >= 3)); 17951 17952 ASSERT(DB_TYPE(mp) == M_DATA); 17953 msize = (mp->b_cont == NULL) ? MBLKL(mp) : msgdsize(mp); 17954 17955 mutex_enter(&tcp->tcp_non_sq_lock); 17956 tcp->tcp_squeue_bytes -= msize; 17957 mutex_exit(&tcp->tcp_non_sq_lock); 17958 17959 /* Bypass tcp protocol for fused tcp loopback */ 17960 if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize)) 17961 return; 17962 17963 mss = tcp->tcp_mss; 17964 if (tcp->tcp_xmit_zc_clean) 17965 mp = tcp_zcopy_backoff(tcp, mp, 0); 17966 17967 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= (uintptr_t)INT_MAX); 17968 len = (int)(mp->b_wptr - mp->b_rptr); 17969 17970 /* 17971 * Criteria for fast path: 17972 * 17973 * 1. no unsent data 17974 * 2. single mblk in request 17975 * 3. connection established 17976 * 4. data in mblk 17977 * 5. len <= mss 17978 * 6. no tcp_valid bits 17979 */ 17980 if ((tcp->tcp_unsent != 0) || 17981 (tcp->tcp_cork) || 17982 (mp->b_cont != NULL) || 17983 (tcp->tcp_state != TCPS_ESTABLISHED) || 17984 (len == 0) || 17985 (len > mss) || 17986 (tcp->tcp_valid_bits != 0)) { 17987 tcp_wput_data(tcp, mp, B_FALSE); 17988 return; 17989 } 17990 17991 ASSERT(tcp->tcp_xmit_tail_unsent == 0); 17992 ASSERT(tcp->tcp_fin_sent == 0); 17993 17994 /* queue new packet onto retransmission queue */ 17995 if (tcp->tcp_xmit_head == NULL) { 17996 tcp->tcp_xmit_head = mp; 17997 } else { 17998 tcp->tcp_xmit_last->b_cont = mp; 17999 } 18000 tcp->tcp_xmit_last = mp; 18001 tcp->tcp_xmit_tail = mp; 18002 18003 /* find out how much we can send */ 18004 /* BEGIN CSTYLED */ 18005 /* 18006 * un-acked usable 18007 * |--------------|-----------------| 18008 * tcp_suna tcp_snxt tcp_suna+tcp_swnd 18009 */ 18010 /* END CSTYLED */ 18011 18012 /* start sending from tcp_snxt */ 18013 snxt = tcp->tcp_snxt; 18014 18015 /* 18016 * Check to see if this connection has been idled for some 18017 * time and no ACK is expected. If it is, we need to slow 18018 * start again to get back the connection's "self-clock" as 18019 * described in VJ's paper. 18020 * 18021 * Refer to the comment in tcp_mss_set() for the calculation 18022 * of tcp_cwnd after idle. 18023 */ 18024 if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet && 18025 (TICK_TO_MSEC(lbolt - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) { 18026 SET_TCP_INIT_CWND(tcp, mss, tcps->tcps_slow_start_after_idle); 18027 } 18028 18029 usable = tcp->tcp_swnd; /* tcp window size */ 18030 if (usable > tcp->tcp_cwnd) 18031 usable = tcp->tcp_cwnd; /* congestion window smaller */ 18032 usable -= snxt; /* subtract stuff already sent */ 18033 suna = tcp->tcp_suna; 18034 usable += suna; 18035 /* usable can be < 0 if the congestion window is smaller */ 18036 if (len > usable) { 18037 /* Can't send complete M_DATA in one shot */ 18038 goto slow; 18039 } 18040 18041 mutex_enter(&tcp->tcp_non_sq_lock); 18042 if (tcp->tcp_flow_stopped && 18043 TCP_UNSENT_BYTES(tcp) <= tcp->tcp_xmit_lowater) { 18044 tcp_clrqfull(tcp); 18045 } 18046 mutex_exit(&tcp->tcp_non_sq_lock); 18047 18048 /* 18049 * determine if anything to send (Nagle). 18050 * 18051 * 1. len < tcp_mss (i.e. small) 18052 * 2. unacknowledged data present 18053 * 3. len < nagle limit 18054 * 4. last packet sent < nagle limit (previous packet sent) 18055 */ 18056 if ((len < mss) && (snxt != suna) && 18057 (len < (int)tcp->tcp_naglim) && 18058 (tcp->tcp_last_sent_len < tcp->tcp_naglim)) { 18059 /* 18060 * This was the first unsent packet and normally 18061 * mss < xmit_hiwater so there is no need to worry 18062 * about flow control. The next packet will go 18063 * through the flow control check in tcp_wput_data(). 18064 */ 18065 /* leftover work from above */ 18066 tcp->tcp_unsent = len; 18067 tcp->tcp_xmit_tail_unsent = len; 18068 18069 return; 18070 } 18071 18072 /* len <= tcp->tcp_mss && len == unsent so no silly window */ 18073 18074 if (snxt == suna) { 18075 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 18076 } 18077 18078 /* we have always sent something */ 18079 tcp->tcp_rack_cnt = 0; 18080 18081 tcp->tcp_snxt = snxt + len; 18082 tcp->tcp_rack = tcp->tcp_rnxt; 18083 18084 if ((mp1 = dupb(mp)) == 0) 18085 goto no_memory; 18086 mp->b_prev = (mblk_t *)(uintptr_t)lbolt; 18087 mp->b_next = (mblk_t *)(uintptr_t)snxt; 18088 18089 /* adjust tcp header information */ 18090 tcph = tcp->tcp_tcph; 18091 tcph->th_flags[0] = (TH_ACK|TH_PUSH); 18092 18093 sum = len + tcp->tcp_tcp_hdr_len + tcp->tcp_sum; 18094 sum = (sum >> 16) + (sum & 0xFFFF); 18095 U16_TO_ABE16(sum, tcph->th_sum); 18096 18097 U32_TO_ABE32(snxt, tcph->th_seq); 18098 18099 BUMP_MIB(&tcps->tcps_mib, tcpOutDataSegs); 18100 UPDATE_MIB(&tcps->tcps_mib, tcpOutDataBytes, len); 18101 BUMP_LOCAL(tcp->tcp_obsegs); 18102 18103 /* Update the latest receive window size in TCP header. */ 18104 U32_TO_ABE16(tcp->tcp_rwnd >> tcp->tcp_rcv_ws, 18105 tcph->th_win); 18106 18107 tcp->tcp_last_sent_len = (ushort_t)len; 18108 18109 plen = len + tcp->tcp_hdr_len; 18110 18111 if (tcp->tcp_ipversion == IPV4_VERSION) { 18112 tcp->tcp_ipha->ipha_length = htons(plen); 18113 } else { 18114 tcp->tcp_ip6h->ip6_plen = htons(plen - 18115 ((char *)&tcp->tcp_ip6h[1] - tcp->tcp_iphc)); 18116 } 18117 18118 /* see if we need to allocate a mblk for the headers */ 18119 hdrlen = tcp->tcp_hdr_len; 18120 rptr = mp1->b_rptr - hdrlen; 18121 db = mp1->b_datap; 18122 if ((db->db_ref != 2) || rptr < db->db_base || 18123 (!OK_32PTR(rptr))) { 18124 /* NOTE: we assume allocb returns an OK_32PTR */ 18125 mp = allocb(tcp->tcp_ip_hdr_len + TCP_MAX_HDR_LENGTH + 18126 tcps->tcps_wroff_xtra, BPRI_MED); 18127 if (!mp) { 18128 freemsg(mp1); 18129 goto no_memory; 18130 } 18131 mp->b_cont = mp1; 18132 mp1 = mp; 18133 /* Leave room for Link Level header */ 18134 /* hdrlen = tcp->tcp_hdr_len; */ 18135 rptr = &mp1->b_rptr[tcps->tcps_wroff_xtra]; 18136 mp1->b_wptr = &rptr[hdrlen]; 18137 } 18138 mp1->b_rptr = rptr; 18139 18140 /* Fill in the timestamp option. */ 18141 if (tcp->tcp_snd_ts_ok) { 18142 U32_TO_BE32((uint32_t)lbolt, 18143 (char *)tcph+TCP_MIN_HEADER_LENGTH+4); 18144 U32_TO_BE32(tcp->tcp_ts_recent, 18145 (char *)tcph+TCP_MIN_HEADER_LENGTH+8); 18146 } else { 18147 ASSERT(tcp->tcp_tcp_hdr_len == TCP_MIN_HEADER_LENGTH); 18148 } 18149 18150 /* copy header into outgoing packet */ 18151 dst = (ipaddr_t *)rptr; 18152 src = (ipaddr_t *)tcp->tcp_iphc; 18153 dst[0] = src[0]; 18154 dst[1] = src[1]; 18155 dst[2] = src[2]; 18156 dst[3] = src[3]; 18157 dst[4] = src[4]; 18158 dst[5] = src[5]; 18159 dst[6] = src[6]; 18160 dst[7] = src[7]; 18161 dst[8] = src[8]; 18162 dst[9] = src[9]; 18163 if (hdrlen -= 40) { 18164 hdrlen >>= 2; 18165 dst += 10; 18166 src += 10; 18167 do { 18168 *dst++ = *src++; 18169 } while (--hdrlen); 18170 } 18171 18172 /* 18173 * Set the ECN info in the TCP header. Note that this 18174 * is not the template header. 18175 */ 18176 if (tcp->tcp_ecn_ok) { 18177 SET_ECT(tcp, rptr); 18178 18179 tcph = (tcph_t *)(rptr + tcp->tcp_ip_hdr_len); 18180 if (tcp->tcp_ecn_echo_on) 18181 tcph->th_flags[0] |= TH_ECE; 18182 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) { 18183 tcph->th_flags[0] |= TH_CWR; 18184 tcp->tcp_ecn_cwr_sent = B_TRUE; 18185 } 18186 } 18187 18188 if (tcp->tcp_ip_forward_progress) { 18189 ASSERT(tcp->tcp_ipversion == IPV6_VERSION); 18190 *(uint32_t *)mp1->b_rptr |= IP_FORWARD_PROG; 18191 tcp->tcp_ip_forward_progress = B_FALSE; 18192 } 18193 tcp_send_data(tcp, tcp->tcp_wq, mp1); 18194 return; 18195 18196 /* 18197 * If we ran out of memory, we pretend to have sent the packet 18198 * and that it was lost on the wire. 18199 */ 18200 no_memory: 18201 return; 18202 18203 slow: 18204 /* leftover work from above */ 18205 tcp->tcp_unsent = len; 18206 tcp->tcp_xmit_tail_unsent = len; 18207 tcp_wput_data(tcp, NULL, B_FALSE); 18208 } 18209 18210 /* 18211 * The function called through squeue to get behind eager's perimeter to 18212 * finish the accept processing. 18213 */ 18214 /* ARGSUSED */ 18215 void 18216 tcp_accept_finish(void *arg, mblk_t *mp, void *arg2) 18217 { 18218 conn_t *connp = (conn_t *)arg; 18219 tcp_t *tcp = connp->conn_tcp; 18220 queue_t *q = tcp->tcp_rq; 18221 mblk_t *mp1; 18222 mblk_t *stropt_mp = mp; 18223 struct stroptions *stropt; 18224 uint_t thwin; 18225 tcp_stack_t *tcps = tcp->tcp_tcps; 18226 18227 /* 18228 * Drop the eager's ref on the listener, that was placed when 18229 * this eager began life in tcp_conn_request. 18230 */ 18231 CONN_DEC_REF(tcp->tcp_saved_listener->tcp_connp); 18232 18233 tcp->tcp_detached = B_FALSE; 18234 18235 if (tcp->tcp_state <= TCPS_BOUND || tcp->tcp_accept_error) { 18236 /* 18237 * Someone blewoff the eager before we could finish 18238 * the accept. 18239 * 18240 * The only reason eager exists it because we put in 18241 * a ref on it when conn ind went up. We need to send 18242 * a disconnect indication up while the last reference 18243 * on the eager will be dropped by the squeue when we 18244 * return. 18245 */ 18246 ASSERT(tcp->tcp_listener == NULL); 18247 if (tcp->tcp_issocket || tcp->tcp_send_discon_ind) { 18248 struct T_discon_ind *tdi; 18249 18250 (void) putnextctl1(q, M_FLUSH, FLUSHRW); 18251 /* 18252 * Let us reuse the incoming mblk to avoid memory 18253 * allocation failure problems. We know that the 18254 * size of the incoming mblk i.e. stroptions is greater 18255 * than sizeof T_discon_ind. So the reallocb below 18256 * can't fail. 18257 */ 18258 freemsg(mp->b_cont); 18259 mp->b_cont = NULL; 18260 ASSERT(DB_REF(mp) == 1); 18261 mp = reallocb(mp, sizeof (struct T_discon_ind), 18262 B_FALSE); 18263 ASSERT(mp != NULL); 18264 DB_TYPE(mp) = M_PROTO; 18265 ((union T_primitives *)mp->b_rptr)->type = T_DISCON_IND; 18266 tdi = (struct T_discon_ind *)mp->b_rptr; 18267 if (tcp->tcp_issocket) { 18268 tdi->DISCON_reason = ECONNREFUSED; 18269 tdi->SEQ_number = 0; 18270 } else { 18271 tdi->DISCON_reason = ENOPROTOOPT; 18272 tdi->SEQ_number = 18273 tcp->tcp_conn_req_seqnum; 18274 } 18275 mp->b_wptr = mp->b_rptr + sizeof (struct T_discon_ind); 18276 putnext(q, mp); 18277 } else { 18278 freemsg(mp); 18279 } 18280 if (tcp->tcp_hard_binding) { 18281 tcp->tcp_hard_binding = B_FALSE; 18282 tcp->tcp_hard_bound = B_TRUE; 18283 } 18284 return; 18285 } 18286 18287 mp1 = stropt_mp->b_cont; 18288 stropt_mp->b_cont = NULL; 18289 ASSERT(DB_TYPE(stropt_mp) == M_SETOPTS); 18290 stropt = (struct stroptions *)stropt_mp->b_rptr; 18291 18292 while (mp1 != NULL) { 18293 mp = mp1; 18294 mp1 = mp1->b_cont; 18295 mp->b_cont = NULL; 18296 tcp->tcp_drop_opt_ack_cnt++; 18297 CALL_IP_WPUT(connp, tcp->tcp_wq, mp); 18298 } 18299 mp = NULL; 18300 18301 /* 18302 * For a loopback connection with tcp_direct_sockfs on, note that 18303 * we don't have to protect tcp_rcv_list yet because synchronous 18304 * streams has not yet been enabled and tcp_fuse_rrw() cannot 18305 * possibly race with us. 18306 */ 18307 18308 /* 18309 * Set the max window size (tcp_rq->q_hiwat) of the acceptor 18310 * properly. This is the first time we know of the acceptor' 18311 * queue. So we do it here. 18312 */ 18313 if (tcp->tcp_rcv_list == NULL) { 18314 /* 18315 * Recv queue is empty, tcp_rwnd should not have changed. 18316 * That means it should be equal to the listener's tcp_rwnd. 18317 */ 18318 tcp->tcp_rq->q_hiwat = tcp->tcp_rwnd; 18319 } else { 18320 #ifdef DEBUG 18321 uint_t cnt = 0; 18322 18323 mp1 = tcp->tcp_rcv_list; 18324 while ((mp = mp1) != NULL) { 18325 mp1 = mp->b_next; 18326 cnt += msgdsize(mp); 18327 } 18328 ASSERT(cnt != 0 && tcp->tcp_rcv_cnt == cnt); 18329 #endif 18330 /* There is some data, add them back to get the max. */ 18331 tcp->tcp_rq->q_hiwat = tcp->tcp_rwnd + tcp->tcp_rcv_cnt; 18332 } 18333 /* 18334 * This is the first time we run on the correct 18335 * queue after tcp_accept. So fix all the q parameters 18336 * here. 18337 */ 18338 stropt->so_flags = SO_HIWAT | SO_MAXBLK | SO_WROFF; 18339 stropt->so_maxblk = tcp_maxpsz_set(tcp, B_FALSE); 18340 18341 /* 18342 * Record the stream head's high water mark for this endpoint; 18343 * this is used for flow-control purposes. 18344 */ 18345 stropt->so_hiwat = tcp->tcp_fused ? 18346 tcp_fuse_set_rcv_hiwat(tcp, q->q_hiwat) : 18347 MAX(q->q_hiwat, tcps->tcps_sth_rcv_hiwat); 18348 18349 /* 18350 * Determine what write offset value to use depending on SACK and 18351 * whether the endpoint is fused or not. 18352 */ 18353 if (tcp->tcp_fused) { 18354 ASSERT(tcp->tcp_loopback); 18355 ASSERT(tcp->tcp_loopback_peer != NULL); 18356 /* 18357 * For fused tcp loopback, set the stream head's write 18358 * offset value to zero since we won't be needing any room 18359 * for TCP/IP headers. This would also improve performance 18360 * since it would reduce the amount of work done by kmem. 18361 * Non-fused tcp loopback case is handled separately below. 18362 */ 18363 stropt->so_wroff = 0; 18364 /* 18365 * Update the peer's transmit parameters according to 18366 * our recently calculated high water mark value. 18367 */ 18368 (void) tcp_maxpsz_set(tcp->tcp_loopback_peer, B_TRUE); 18369 } else if (tcp->tcp_snd_sack_ok) { 18370 stropt->so_wroff = tcp->tcp_hdr_len + TCPOPT_MAX_SACK_LEN + 18371 (tcp->tcp_loopback ? 0 : tcps->tcps_wroff_xtra); 18372 } else { 18373 stropt->so_wroff = tcp->tcp_hdr_len + (tcp->tcp_loopback ? 0 : 18374 tcps->tcps_wroff_xtra); 18375 } 18376 18377 /* 18378 * If this is endpoint is handling SSL, then reserve extra 18379 * offset and space at the end. 18380 * Also have the stream head allocate SSL3_MAX_RECORD_LEN packets, 18381 * overriding the previous setting. The extra cost of signing and 18382 * encrypting multiple MSS-size records (12 of them with Ethernet), 18383 * instead of a single contiguous one by the stream head 18384 * largely outweighs the statistical reduction of ACKs, when 18385 * applicable. The peer will also save on decryption and verification 18386 * costs. 18387 */ 18388 if (tcp->tcp_kssl_ctx != NULL) { 18389 stropt->so_wroff += SSL3_WROFFSET; 18390 18391 stropt->so_flags |= SO_TAIL; 18392 stropt->so_tail = SSL3_MAX_TAIL_LEN; 18393 18394 stropt->so_flags |= SO_COPYOPT; 18395 stropt->so_copyopt = ZCVMUNSAFE; 18396 18397 stropt->so_maxblk = SSL3_MAX_RECORD_LEN; 18398 } 18399 18400 /* Send the options up */ 18401 putnext(q, stropt_mp); 18402 18403 /* 18404 * Pass up any data and/or a fin that has been received. 18405 * 18406 * Adjust receive window in case it had decreased 18407 * (because there is data <=> tcp_rcv_list != NULL) 18408 * while the connection was detached. Note that 18409 * in case the eager was flow-controlled, w/o this 18410 * code, the rwnd may never open up again! 18411 */ 18412 if (tcp->tcp_rcv_list != NULL) { 18413 /* We drain directly in case of fused tcp loopback */ 18414 sodirect_t *sodp; 18415 18416 if (!tcp->tcp_fused && canputnext(q)) { 18417 tcp->tcp_rwnd = q->q_hiwat; 18418 thwin = ((uint_t)BE16_TO_U16(tcp->tcp_tcph->th_win)) 18419 << tcp->tcp_rcv_ws; 18420 thwin -= tcp->tcp_rnxt - tcp->tcp_rack; 18421 if (tcp->tcp_state >= TCPS_ESTABLISHED && 18422 (q->q_hiwat - thwin >= tcp->tcp_mss)) { 18423 tcp_xmit_ctl(NULL, 18424 tcp, (tcp->tcp_swnd == 0) ? 18425 tcp->tcp_suna : tcp->tcp_snxt, 18426 tcp->tcp_rnxt, TH_ACK); 18427 BUMP_MIB(&tcps->tcps_mib, tcpOutWinUpdate); 18428 } 18429 18430 } 18431 18432 SOD_PTR_ENTER(tcp, sodp); 18433 if (sodp != NULL) { 18434 /* Sodirect, move from rcv_list */ 18435 ASSERT(!tcp->tcp_fused); 18436 while ((mp = tcp->tcp_rcv_list) != NULL) { 18437 tcp->tcp_rcv_list = mp->b_next; 18438 mp->b_next = NULL; 18439 (void) tcp_rcv_sod_enqueue(tcp, sodp, mp, 18440 msgdsize(mp)); 18441 } 18442 tcp->tcp_rcv_last_head = NULL; 18443 tcp->tcp_rcv_last_tail = NULL; 18444 tcp->tcp_rcv_cnt = 0; 18445 (void) tcp_rcv_sod_wakeup(tcp, sodp); 18446 /* sod_wakeup() did the mutex_exit() */ 18447 } else { 18448 /* Not sodirect, drain */ 18449 (void) tcp_rcv_drain(q, tcp); 18450 } 18451 18452 /* 18453 * For fused tcp loopback, back-enable peer endpoint 18454 * if it's currently flow-controlled. 18455 */ 18456 if (tcp->tcp_fused) { 18457 tcp_t *peer_tcp = tcp->tcp_loopback_peer; 18458 18459 ASSERT(peer_tcp != NULL); 18460 ASSERT(peer_tcp->tcp_fused); 18461 /* 18462 * In order to change the peer's tcp_flow_stopped, 18463 * we need to take locks for both end points. The 18464 * highest address is taken first. 18465 */ 18466 if (peer_tcp > tcp) { 18467 mutex_enter(&peer_tcp->tcp_non_sq_lock); 18468 mutex_enter(&tcp->tcp_non_sq_lock); 18469 } else { 18470 mutex_enter(&tcp->tcp_non_sq_lock); 18471 mutex_enter(&peer_tcp->tcp_non_sq_lock); 18472 } 18473 if (peer_tcp->tcp_flow_stopped) { 18474 tcp_clrqfull(peer_tcp); 18475 TCP_STAT(tcps, tcp_fusion_backenabled); 18476 } 18477 mutex_exit(&peer_tcp->tcp_non_sq_lock); 18478 mutex_exit(&tcp->tcp_non_sq_lock); 18479 } 18480 } 18481 ASSERT(tcp->tcp_rcv_list == NULL || tcp->tcp_fused_sigurg); 18482 if (tcp->tcp_fin_rcvd && !tcp->tcp_ordrel_done) { 18483 mp = mi_tpi_ordrel_ind(); 18484 if (mp) { 18485 tcp->tcp_ordrel_done = B_TRUE; 18486 putnext(q, mp); 18487 if (tcp->tcp_deferred_clean_death) { 18488 /* 18489 * tcp_clean_death was deferred 18490 * for T_ORDREL_IND - do it now 18491 */ 18492 (void) tcp_clean_death(tcp, 18493 tcp->tcp_client_errno, 21); 18494 tcp->tcp_deferred_clean_death = B_FALSE; 18495 } 18496 } else { 18497 /* 18498 * Run the orderly release in the 18499 * service routine. 18500 */ 18501 qenable(q); 18502 } 18503 } 18504 if (tcp->tcp_hard_binding) { 18505 tcp->tcp_hard_binding = B_FALSE; 18506 tcp->tcp_hard_bound = B_TRUE; 18507 } 18508 18509 /* We can enable synchronous streams now */ 18510 if (tcp->tcp_fused) { 18511 tcp_fuse_syncstr_enable_pair(tcp); 18512 } 18513 18514 if (tcp->tcp_ka_enabled) { 18515 tcp->tcp_ka_last_intrvl = 0; 18516 tcp->tcp_ka_tid = TCP_TIMER(tcp, tcp_keepalive_killer, 18517 MSEC_TO_TICK(tcp->tcp_ka_interval)); 18518 } 18519 18520 /* 18521 * At this point, eager is fully established and will 18522 * have the following references - 18523 * 18524 * 2 references for connection to exist (1 for TCP and 1 for IP). 18525 * 1 reference for the squeue which will be dropped by the squeue as 18526 * soon as this function returns. 18527 * There will be 1 additonal reference for being in classifier 18528 * hash list provided something bad hasn't happened. 18529 */ 18530 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) || 18531 (connp->conn_fanout == NULL && connp->conn_ref >= 3)); 18532 } 18533 18534 /* 18535 * The function called through squeue to get behind listener's perimeter to 18536 * send a deffered conn_ind. 18537 */ 18538 /* ARGSUSED */ 18539 void 18540 tcp_send_pending(void *arg, mblk_t *mp, void *arg2) 18541 { 18542 conn_t *connp = (conn_t *)arg; 18543 tcp_t *listener = connp->conn_tcp; 18544 18545 if (listener->tcp_state == TCPS_CLOSED || 18546 TCP_IS_DETACHED(listener)) { 18547 /* 18548 * If listener has closed, it would have caused a 18549 * a cleanup/blowoff to happen for the eager. 18550 */ 18551 tcp_t *tcp; 18552 struct T_conn_ind *conn_ind; 18553 18554 conn_ind = (struct T_conn_ind *)mp->b_rptr; 18555 bcopy(mp->b_rptr + conn_ind->OPT_offset, &tcp, 18556 conn_ind->OPT_length); 18557 /* 18558 * We need to drop the ref on eager that was put 18559 * tcp_rput_data() before trying to send the conn_ind 18560 * to listener. The conn_ind was deferred in tcp_send_conn_ind 18561 * and tcp_wput_accept() is sending this deferred conn_ind but 18562 * listener is closed so we drop the ref. 18563 */ 18564 CONN_DEC_REF(tcp->tcp_connp); 18565 freemsg(mp); 18566 return; 18567 } 18568 putnext(listener->tcp_rq, mp); 18569 } 18570 18571 18572 /* 18573 * This is the STREAMS entry point for T_CONN_RES coming down on 18574 * Acceptor STREAM when sockfs listener does accept processing. 18575 * Read the block comment on top of tcp_conn_request(). 18576 */ 18577 void 18578 tcp_wput_accept(queue_t *q, mblk_t *mp) 18579 { 18580 queue_t *rq = RD(q); 18581 struct T_conn_res *conn_res; 18582 tcp_t *eager; 18583 tcp_t *listener; 18584 struct T_ok_ack *ok; 18585 t_scalar_t PRIM_type; 18586 mblk_t *opt_mp; 18587 conn_t *econnp; 18588 18589 ASSERT(DB_TYPE(mp) == M_PROTO); 18590 18591 conn_res = (struct T_conn_res *)mp->b_rptr; 18592 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= (uintptr_t)INT_MAX); 18593 if ((mp->b_wptr - mp->b_rptr) < sizeof (struct T_conn_res)) { 18594 mp = mi_tpi_err_ack_alloc(mp, TPROTO, 0); 18595 if (mp != NULL) 18596 putnext(rq, mp); 18597 return; 18598 } 18599 switch (conn_res->PRIM_type) { 18600 case O_T_CONN_RES: 18601 case T_CONN_RES: 18602 /* 18603 * We pass up an err ack if allocb fails. This will 18604 * cause sockfs to issue a T_DISCON_REQ which will cause 18605 * tcp_eager_blowoff to be called. sockfs will then call 18606 * rq->q_qinfo->qi_qclose to cleanup the acceptor stream. 18607 * we need to do the allocb up here because we have to 18608 * make sure rq->q_qinfo->qi_qclose still points to the 18609 * correct function (tcpclose_accept) in case allocb 18610 * fails. 18611 */ 18612 opt_mp = allocb(sizeof (struct stroptions), BPRI_HI); 18613 if (opt_mp == NULL) { 18614 mp = mi_tpi_err_ack_alloc(mp, TPROTO, 0); 18615 if (mp != NULL) 18616 putnext(rq, mp); 18617 return; 18618 } 18619 18620 bcopy(mp->b_rptr + conn_res->OPT_offset, 18621 &eager, conn_res->OPT_length); 18622 PRIM_type = conn_res->PRIM_type; 18623 mp->b_datap->db_type = M_PCPROTO; 18624 mp->b_wptr = mp->b_rptr + sizeof (struct T_ok_ack); 18625 ok = (struct T_ok_ack *)mp->b_rptr; 18626 ok->PRIM_type = T_OK_ACK; 18627 ok->CORRECT_prim = PRIM_type; 18628 econnp = eager->tcp_connp; 18629 econnp->conn_dev = (dev_t)RD(q)->q_ptr; 18630 econnp->conn_minor_arena = (vmem_t *)(WR(q)->q_ptr); 18631 eager->tcp_rq = rq; 18632 eager->tcp_wq = q; 18633 rq->q_ptr = econnp; 18634 rq->q_qinfo = &tcp_rinitv4; /* No open - same as rinitv6 */ 18635 q->q_ptr = econnp; 18636 q->q_qinfo = &tcp_winit; 18637 listener = eager->tcp_listener; 18638 eager->tcp_issocket = B_TRUE; 18639 18640 /* 18641 * TCP is _D_SODIRECT and sockfs is directly above so 18642 * save shared sodirect_t pointer (if any). 18643 * 18644 * If tcp_fused and sodirect enabled disable it. 18645 */ 18646 eager->tcp_sodirect = SOD_QTOSODP(eager->tcp_rq); 18647 if (eager->tcp_fused && eager->tcp_sodirect != NULL) { 18648 /* Fused, disable sodirect */ 18649 mutex_enter(eager->tcp_sodirect->sod_lock); 18650 SOD_DISABLE(eager->tcp_sodirect); 18651 mutex_exit(eager->tcp_sodirect->sod_lock); 18652 eager->tcp_sodirect = NULL; 18653 } 18654 18655 econnp->conn_zoneid = listener->tcp_connp->conn_zoneid; 18656 econnp->conn_allzones = listener->tcp_connp->conn_allzones; 18657 ASSERT(econnp->conn_netstack == 18658 listener->tcp_connp->conn_netstack); 18659 ASSERT(eager->tcp_tcps == listener->tcp_tcps); 18660 18661 /* Put the ref for IP */ 18662 CONN_INC_REF(econnp); 18663 18664 /* 18665 * We should have minimum of 3 references on the conn 18666 * at this point. One each for TCP and IP and one for 18667 * the T_conn_ind that was sent up when the 3-way handshake 18668 * completed. In the normal case we would also have another 18669 * reference (making a total of 4) for the conn being in the 18670 * classifier hash list. However the eager could have received 18671 * an RST subsequently and tcp_closei_local could have removed 18672 * the eager from the classifier hash list, hence we can't 18673 * assert that reference. 18674 */ 18675 ASSERT(econnp->conn_ref >= 3); 18676 18677 /* 18678 * Send the new local address also up to sockfs. There 18679 * should already be enough space in the mp that came 18680 * down from soaccept(). 18681 */ 18682 if (eager->tcp_family == AF_INET) { 18683 sin_t *sin; 18684 18685 ASSERT((mp->b_datap->db_lim - mp->b_datap->db_base) >= 18686 (sizeof (struct T_ok_ack) + sizeof (sin_t))); 18687 sin = (sin_t *)mp->b_wptr; 18688 mp->b_wptr += sizeof (sin_t); 18689 sin->sin_family = AF_INET; 18690 sin->sin_port = eager->tcp_lport; 18691 sin->sin_addr.s_addr = eager->tcp_ipha->ipha_src; 18692 } else { 18693 sin6_t *sin6; 18694 18695 ASSERT((mp->b_datap->db_lim - mp->b_datap->db_base) >= 18696 sizeof (struct T_ok_ack) + sizeof (sin6_t)); 18697 sin6 = (sin6_t *)mp->b_wptr; 18698 mp->b_wptr += sizeof (sin6_t); 18699 sin6->sin6_family = AF_INET6; 18700 sin6->sin6_port = eager->tcp_lport; 18701 if (eager->tcp_ipversion == IPV4_VERSION) { 18702 sin6->sin6_flowinfo = 0; 18703 IN6_IPADDR_TO_V4MAPPED( 18704 eager->tcp_ipha->ipha_src, 18705 &sin6->sin6_addr); 18706 } else { 18707 ASSERT(eager->tcp_ip6h != NULL); 18708 sin6->sin6_flowinfo = 18709 eager->tcp_ip6h->ip6_vcf & 18710 ~IPV6_VERS_AND_FLOW_MASK; 18711 sin6->sin6_addr = eager->tcp_ip6h->ip6_src; 18712 } 18713 sin6->sin6_scope_id = 0; 18714 sin6->__sin6_src_id = 0; 18715 } 18716 18717 putnext(rq, mp); 18718 18719 opt_mp->b_datap->db_type = M_SETOPTS; 18720 opt_mp->b_wptr += sizeof (struct stroptions); 18721 18722 /* 18723 * Prepare for inheriting IPV6_BOUND_IF and IPV6_RECVPKTINFO 18724 * from listener to acceptor. The message is chained on the 18725 * bind_mp which tcp_rput_other will send down to IP. 18726 */ 18727 if (listener->tcp_bound_if != 0) { 18728 /* allocate optmgmt req */ 18729 mp = tcp_setsockopt_mp(IPPROTO_IPV6, 18730 IPV6_BOUND_IF, (char *)&listener->tcp_bound_if, 18731 sizeof (int)); 18732 if (mp != NULL) 18733 linkb(opt_mp, mp); 18734 } 18735 if (listener->tcp_ipv6_recvancillary & TCP_IPV6_RECVPKTINFO) { 18736 uint_t on = 1; 18737 18738 /* allocate optmgmt req */ 18739 mp = tcp_setsockopt_mp(IPPROTO_IPV6, 18740 IPV6_RECVPKTINFO, (char *)&on, sizeof (on)); 18741 if (mp != NULL) 18742 linkb(opt_mp, mp); 18743 } 18744 18745 18746 mutex_enter(&listener->tcp_eager_lock); 18747 18748 if (listener->tcp_eager_prev_q0->tcp_conn_def_q0) { 18749 18750 tcp_t *tail; 18751 tcp_t *tcp; 18752 mblk_t *mp1; 18753 18754 tcp = listener->tcp_eager_prev_q0; 18755 /* 18756 * listener->tcp_eager_prev_q0 points to the TAIL of the 18757 * deferred T_conn_ind queue. We need to get to the head 18758 * of the queue in order to send up T_conn_ind the same 18759 * order as how the 3WHS is completed. 18760 */ 18761 while (tcp != listener) { 18762 if (!tcp->tcp_eager_prev_q0->tcp_conn_def_q0 && 18763 !tcp->tcp_kssl_pending) 18764 break; 18765 else 18766 tcp = tcp->tcp_eager_prev_q0; 18767 } 18768 /* None of the pending eagers can be sent up now */ 18769 if (tcp == listener) 18770 goto no_more_eagers; 18771 18772 mp1 = tcp->tcp_conn.tcp_eager_conn_ind; 18773 tcp->tcp_conn.tcp_eager_conn_ind = NULL; 18774 /* Move from q0 to q */ 18775 ASSERT(listener->tcp_conn_req_cnt_q0 > 0); 18776 listener->tcp_conn_req_cnt_q0--; 18777 listener->tcp_conn_req_cnt_q++; 18778 tcp->tcp_eager_next_q0->tcp_eager_prev_q0 = 18779 tcp->tcp_eager_prev_q0; 18780 tcp->tcp_eager_prev_q0->tcp_eager_next_q0 = 18781 tcp->tcp_eager_next_q0; 18782 tcp->tcp_eager_prev_q0 = NULL; 18783 tcp->tcp_eager_next_q0 = NULL; 18784 tcp->tcp_conn_def_q0 = B_FALSE; 18785 18786 /* Make sure the tcp isn't in the list of droppables */ 18787 ASSERT(tcp->tcp_eager_next_drop_q0 == NULL && 18788 tcp->tcp_eager_prev_drop_q0 == NULL); 18789 18790 /* 18791 * Insert at end of the queue because sockfs sends 18792 * down T_CONN_RES in chronological order. Leaving 18793 * the older conn indications at front of the queue 18794 * helps reducing search time. 18795 */ 18796 tail = listener->tcp_eager_last_q; 18797 if (tail != NULL) { 18798 tail->tcp_eager_next_q = tcp; 18799 } else { 18800 listener->tcp_eager_next_q = tcp; 18801 } 18802 listener->tcp_eager_last_q = tcp; 18803 tcp->tcp_eager_next_q = NULL; 18804 18805 /* Need to get inside the listener perimeter */ 18806 CONN_INC_REF(listener->tcp_connp); 18807 squeue_fill(listener->tcp_connp->conn_sqp, mp1, 18808 tcp_send_pending, listener->tcp_connp, 18809 SQTAG_TCP_SEND_PENDING); 18810 } 18811 no_more_eagers: 18812 tcp_eager_unlink(eager); 18813 mutex_exit(&listener->tcp_eager_lock); 18814 18815 /* 18816 * At this point, the eager is detached from the listener 18817 * but we still have an extra refs on eager (apart from the 18818 * usual tcp references). The ref was placed in tcp_rput_data 18819 * before sending the conn_ind in tcp_send_conn_ind. 18820 * The ref will be dropped in tcp_accept_finish(). 18821 */ 18822 squeue_enter_nodrain(econnp->conn_sqp, opt_mp, 18823 tcp_accept_finish, econnp, SQTAG_TCP_ACCEPT_FINISH_Q0); 18824 return; 18825 default: 18826 mp = mi_tpi_err_ack_alloc(mp, TNOTSUPPORT, 0); 18827 if (mp != NULL) 18828 putnext(rq, mp); 18829 return; 18830 } 18831 } 18832 18833 static int 18834 tcp_getmyname(tcp_t *tcp, struct sockaddr *sa, uint_t *salenp) 18835 { 18836 sin_t *sin = (sin_t *)sa; 18837 sin6_t *sin6 = (sin6_t *)sa; 18838 18839 switch (tcp->tcp_family) { 18840 case AF_INET: 18841 ASSERT(tcp->tcp_ipversion == IPV4_VERSION); 18842 18843 if (*salenp < sizeof (sin_t)) 18844 return (EINVAL); 18845 18846 *sin = sin_null; 18847 sin->sin_family = AF_INET; 18848 sin->sin_port = tcp->tcp_lport; 18849 sin->sin_addr.s_addr = tcp->tcp_ipha->ipha_src; 18850 break; 18851 18852 case AF_INET6: 18853 if (*salenp < sizeof (sin6_t)) 18854 return (EINVAL); 18855 18856 *sin6 = sin6_null; 18857 sin6->sin6_family = AF_INET6; 18858 sin6->sin6_port = tcp->tcp_lport; 18859 if (tcp->tcp_ipversion == IPV4_VERSION) { 18860 IN6_IPADDR_TO_V4MAPPED(tcp->tcp_ipha->ipha_src, 18861 &sin6->sin6_addr); 18862 } else { 18863 sin6->sin6_addr = tcp->tcp_ip6h->ip6_src; 18864 } 18865 break; 18866 } 18867 18868 return (0); 18869 } 18870 18871 static int 18872 tcp_getpeername(tcp_t *tcp, struct sockaddr *sa, uint_t *salenp) 18873 { 18874 sin_t *sin = (sin_t *)sa; 18875 sin6_t *sin6 = (sin6_t *)sa; 18876 18877 if (tcp->tcp_state < TCPS_SYN_RCVD) 18878 return (ENOTCONN); 18879 18880 switch (tcp->tcp_family) { 18881 case AF_INET: 18882 ASSERT(tcp->tcp_ipversion == IPV4_VERSION); 18883 18884 if (*salenp < sizeof (sin_t)) 18885 return (EINVAL); 18886 18887 *sin = sin_null; 18888 sin->sin_family = AF_INET; 18889 sin->sin_port = tcp->tcp_fport; 18890 IN6_V4MAPPED_TO_IPADDR(&tcp->tcp_remote_v6, 18891 sin->sin_addr.s_addr); 18892 break; 18893 18894 case AF_INET6: 18895 if (*salenp < sizeof (sin6_t)) 18896 return (EINVAL); 18897 18898 *sin6 = sin6_null; 18899 sin6->sin6_family = AF_INET6; 18900 sin6->sin6_port = tcp->tcp_fport; 18901 sin6->sin6_addr = tcp->tcp_remote_v6; 18902 if (tcp->tcp_ipversion == IPV6_VERSION) { 18903 sin6->sin6_flowinfo = tcp->tcp_ip6h->ip6_vcf & 18904 ~IPV6_VERS_AND_FLOW_MASK; 18905 } 18906 break; 18907 } 18908 18909 return (0); 18910 } 18911 18912 /* 18913 * Handle special out-of-band ioctl requests (see PSARC/2008/265). 18914 */ 18915 static void 18916 tcp_wput_cmdblk(queue_t *q, mblk_t *mp) 18917 { 18918 void *data; 18919 mblk_t *datamp = mp->b_cont; 18920 tcp_t *tcp = Q_TO_TCP(q); 18921 cmdblk_t *cmdp = (cmdblk_t *)mp->b_rptr; 18922 18923 if (datamp == NULL || MBLKL(datamp) < cmdp->cb_len) { 18924 cmdp->cb_error = EPROTO; 18925 qreply(q, mp); 18926 return; 18927 } 18928 18929 data = datamp->b_rptr; 18930 18931 switch (cmdp->cb_cmd) { 18932 case TI_GETPEERNAME: 18933 cmdp->cb_error = tcp_getpeername(tcp, data, &cmdp->cb_len); 18934 break; 18935 case TI_GETMYNAME: 18936 cmdp->cb_error = tcp_getmyname(tcp, data, &cmdp->cb_len); 18937 break; 18938 default: 18939 cmdp->cb_error = EINVAL; 18940 break; 18941 } 18942 18943 qreply(q, mp); 18944 } 18945 18946 void 18947 tcp_wput(queue_t *q, mblk_t *mp) 18948 { 18949 conn_t *connp = Q_TO_CONN(q); 18950 tcp_t *tcp; 18951 void (*output_proc)(); 18952 t_scalar_t type; 18953 uchar_t *rptr; 18954 struct iocblk *iocp; 18955 uint32_t msize; 18956 tcp_stack_t *tcps = Q_TO_TCP(q)->tcp_tcps; 18957 18958 ASSERT(connp->conn_ref >= 2); 18959 18960 switch (DB_TYPE(mp)) { 18961 case M_DATA: 18962 tcp = connp->conn_tcp; 18963 ASSERT(tcp != NULL); 18964 18965 msize = msgdsize(mp); 18966 18967 mutex_enter(&tcp->tcp_non_sq_lock); 18968 tcp->tcp_squeue_bytes += msize; 18969 if (TCP_UNSENT_BYTES(tcp) > tcp->tcp_xmit_hiwater) { 18970 tcp_setqfull(tcp); 18971 } 18972 mutex_exit(&tcp->tcp_non_sq_lock); 18973 18974 CONN_INC_REF(connp); 18975 (*tcp_squeue_wput_proc)(connp->conn_sqp, mp, 18976 tcp_output, connp, SQTAG_TCP_OUTPUT); 18977 return; 18978 18979 case M_CMD: 18980 tcp_wput_cmdblk(q, mp); 18981 return; 18982 18983 case M_PROTO: 18984 case M_PCPROTO: 18985 /* 18986 * if it is a snmp message, don't get behind the squeue 18987 */ 18988 tcp = connp->conn_tcp; 18989 rptr = mp->b_rptr; 18990 if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) { 18991 type = ((union T_primitives *)rptr)->type; 18992 } else { 18993 if (tcp->tcp_debug) { 18994 (void) strlog(TCP_MOD_ID, 0, 1, 18995 SL_ERROR|SL_TRACE, 18996 "tcp_wput_proto, dropping one..."); 18997 } 18998 freemsg(mp); 18999 return; 19000 } 19001 if (type == T_SVR4_OPTMGMT_REQ) { 19002 cred_t *cr = DB_CREDDEF(mp, tcp->tcp_cred); 19003 if (snmpcom_req(q, mp, tcp_snmp_set, ip_snmp_get, 19004 cr)) { 19005 /* 19006 * This was a SNMP request 19007 */ 19008 return; 19009 } else { 19010 output_proc = tcp_wput_proto; 19011 } 19012 } else { 19013 output_proc = tcp_wput_proto; 19014 } 19015 break; 19016 case M_IOCTL: 19017 /* 19018 * Most ioctls can be processed right away without going via 19019 * squeues - process them right here. Those that do require 19020 * squeue (currently TCP_IOC_DEFAULT_Q and _SIOCSOCKFALLBACK) 19021 * are processed by tcp_wput_ioctl(). 19022 */ 19023 iocp = (struct iocblk *)mp->b_rptr; 19024 tcp = connp->conn_tcp; 19025 19026 switch (iocp->ioc_cmd) { 19027 case TCP_IOC_ABORT_CONN: 19028 tcp_ioctl_abort_conn(q, mp); 19029 return; 19030 case TI_GETPEERNAME: 19031 case TI_GETMYNAME: 19032 mi_copyin(q, mp, NULL, 19033 SIZEOF_STRUCT(strbuf, iocp->ioc_flag)); 19034 return; 19035 case ND_SET: 19036 /* nd_getset does the necessary checks */ 19037 case ND_GET: 19038 if (!nd_getset(q, tcps->tcps_g_nd, mp)) { 19039 CALL_IP_WPUT(connp, q, mp); 19040 return; 19041 } 19042 qreply(q, mp); 19043 return; 19044 case TCP_IOC_DEFAULT_Q: 19045 /* 19046 * Wants to be the default wq. Check the credentials 19047 * first, the rest is executed via squeue. 19048 */ 19049 if (secpolicy_ip_config(iocp->ioc_cr, B_FALSE) != 0) { 19050 iocp->ioc_error = EPERM; 19051 iocp->ioc_count = 0; 19052 mp->b_datap->db_type = M_IOCACK; 19053 qreply(q, mp); 19054 return; 19055 } 19056 output_proc = tcp_wput_ioctl; 19057 break; 19058 default: 19059 output_proc = tcp_wput_ioctl; 19060 break; 19061 } 19062 break; 19063 default: 19064 output_proc = tcp_wput_nondata; 19065 break; 19066 } 19067 19068 CONN_INC_REF(connp); 19069 (*tcp_squeue_wput_proc)(connp->conn_sqp, mp, 19070 output_proc, connp, SQTAG_TCP_WPUT_OTHER); 19071 } 19072 19073 /* 19074 * Initial STREAMS write side put() procedure for sockets. It tries to 19075 * handle the T_CAPABILITY_REQ which sockfs sends down while setting 19076 * up the socket without using the squeue. Non T_CAPABILITY_REQ messages 19077 * are handled by tcp_wput() as usual. 19078 * 19079 * All further messages will also be handled by tcp_wput() because we cannot 19080 * be sure that the above short cut is safe later. 19081 */ 19082 static void 19083 tcp_wput_sock(queue_t *wq, mblk_t *mp) 19084 { 19085 conn_t *connp = Q_TO_CONN(wq); 19086 tcp_t *tcp = connp->conn_tcp; 19087 struct T_capability_req *car = (struct T_capability_req *)mp->b_rptr; 19088 19089 ASSERT(wq->q_qinfo == &tcp_sock_winit); 19090 wq->q_qinfo = &tcp_winit; 19091 19092 ASSERT(IPCL_IS_TCP(connp)); 19093 ASSERT(TCP_IS_SOCKET(tcp)); 19094 19095 if (DB_TYPE(mp) == M_PCPROTO && 19096 MBLKL(mp) == sizeof (struct T_capability_req) && 19097 car->PRIM_type == T_CAPABILITY_REQ) { 19098 tcp_capability_req(tcp, mp); 19099 return; 19100 } 19101 19102 tcp_wput(wq, mp); 19103 } 19104 19105 static boolean_t 19106 tcp_zcopy_check(tcp_t *tcp) 19107 { 19108 conn_t *connp = tcp->tcp_connp; 19109 ire_t *ire; 19110 boolean_t zc_enabled = B_FALSE; 19111 tcp_stack_t *tcps = tcp->tcp_tcps; 19112 19113 if (do_tcpzcopy == 2) 19114 zc_enabled = B_TRUE; 19115 else if (tcp->tcp_ipversion == IPV4_VERSION && 19116 IPCL_IS_CONNECTED(connp) && 19117 (connp->conn_flags & IPCL_CHECK_POLICY) == 0 && 19118 connp->conn_dontroute == 0 && 19119 !connp->conn_nexthop_set && 19120 connp->conn_outgoing_ill == NULL && 19121 connp->conn_nofailover_ill == NULL && 19122 do_tcpzcopy == 1) { 19123 /* 19124 * the checks above closely resemble the fast path checks 19125 * in tcp_send_data(). 19126 */ 19127 mutex_enter(&connp->conn_lock); 19128 ire = connp->conn_ire_cache; 19129 ASSERT(!(connp->conn_state_flags & CONN_INCIPIENT)); 19130 if (ire != NULL && !(ire->ire_marks & IRE_MARK_CONDEMNED)) { 19131 IRE_REFHOLD(ire); 19132 if (ire->ire_stq != NULL) { 19133 ill_t *ill = (ill_t *)ire->ire_stq->q_ptr; 19134 19135 zc_enabled = ill && (ill->ill_capabilities & 19136 ILL_CAPAB_ZEROCOPY) && 19137 (ill->ill_zerocopy_capab-> 19138 ill_zerocopy_flags != 0); 19139 } 19140 IRE_REFRELE(ire); 19141 } 19142 mutex_exit(&connp->conn_lock); 19143 } 19144 tcp->tcp_snd_zcopy_on = zc_enabled; 19145 if (!TCP_IS_DETACHED(tcp)) { 19146 if (zc_enabled) { 19147 (void) mi_set_sth_copyopt(tcp->tcp_rq, ZCVMSAFE); 19148 TCP_STAT(tcps, tcp_zcopy_on); 19149 } else { 19150 (void) mi_set_sth_copyopt(tcp->tcp_rq, ZCVMUNSAFE); 19151 TCP_STAT(tcps, tcp_zcopy_off); 19152 } 19153 } 19154 return (zc_enabled); 19155 } 19156 19157 static mblk_t * 19158 tcp_zcopy_disable(tcp_t *tcp, mblk_t *bp) 19159 { 19160 tcp_stack_t *tcps = tcp->tcp_tcps; 19161 19162 if (do_tcpzcopy == 2) 19163 return (bp); 19164 else if (tcp->tcp_snd_zcopy_on) { 19165 tcp->tcp_snd_zcopy_on = B_FALSE; 19166 if (!TCP_IS_DETACHED(tcp)) { 19167 (void) mi_set_sth_copyopt(tcp->tcp_rq, ZCVMUNSAFE); 19168 TCP_STAT(tcps, tcp_zcopy_disable); 19169 } 19170 } 19171 return (tcp_zcopy_backoff(tcp, bp, 0)); 19172 } 19173 19174 /* 19175 * Backoff from a zero-copy mblk by copying data to a new mblk and freeing 19176 * the original desballoca'ed segmapped mblk. 19177 */ 19178 static mblk_t * 19179 tcp_zcopy_backoff(tcp_t *tcp, mblk_t *bp, int fix_xmitlist) 19180 { 19181 mblk_t *head, *tail, *nbp; 19182 tcp_stack_t *tcps = tcp->tcp_tcps; 19183 19184 if (IS_VMLOANED_MBLK(bp)) { 19185 TCP_STAT(tcps, tcp_zcopy_backoff); 19186 if ((head = copyb(bp)) == NULL) { 19187 /* fail to backoff; leave it for the next backoff */ 19188 tcp->tcp_xmit_zc_clean = B_FALSE; 19189 return (bp); 19190 } 19191 if (bp->b_datap->db_struioflag & STRUIO_ZCNOTIFY) { 19192 if (fix_xmitlist) 19193 tcp_zcopy_notify(tcp); 19194 else 19195 head->b_datap->db_struioflag |= STRUIO_ZCNOTIFY; 19196 } 19197 nbp = bp->b_cont; 19198 if (fix_xmitlist) { 19199 head->b_prev = bp->b_prev; 19200 head->b_next = bp->b_next; 19201 if (tcp->tcp_xmit_tail == bp) 19202 tcp->tcp_xmit_tail = head; 19203 } 19204 bp->b_next = NULL; 19205 bp->b_prev = NULL; 19206 freeb(bp); 19207 } else { 19208 head = bp; 19209 nbp = bp->b_cont; 19210 } 19211 tail = head; 19212 while (nbp) { 19213 if (IS_VMLOANED_MBLK(nbp)) { 19214 TCP_STAT(tcps, tcp_zcopy_backoff); 19215 if ((tail->b_cont = copyb(nbp)) == NULL) { 19216 tcp->tcp_xmit_zc_clean = B_FALSE; 19217 tail->b_cont = nbp; 19218 return (head); 19219 } 19220 tail = tail->b_cont; 19221 if (nbp->b_datap->db_struioflag & STRUIO_ZCNOTIFY) { 19222 if (fix_xmitlist) 19223 tcp_zcopy_notify(tcp); 19224 else 19225 tail->b_datap->db_struioflag |= 19226 STRUIO_ZCNOTIFY; 19227 } 19228 bp = nbp; 19229 nbp = nbp->b_cont; 19230 if (fix_xmitlist) { 19231 tail->b_prev = bp->b_prev; 19232 tail->b_next = bp->b_next; 19233 if (tcp->tcp_xmit_tail == bp) 19234 tcp->tcp_xmit_tail = tail; 19235 } 19236 bp->b_next = NULL; 19237 bp->b_prev = NULL; 19238 freeb(bp); 19239 } else { 19240 tail->b_cont = nbp; 19241 tail = nbp; 19242 nbp = nbp->b_cont; 19243 } 19244 } 19245 if (fix_xmitlist) { 19246 tcp->tcp_xmit_last = tail; 19247 tcp->tcp_xmit_zc_clean = B_TRUE; 19248 } 19249 return (head); 19250 } 19251 19252 static void 19253 tcp_zcopy_notify(tcp_t *tcp) 19254 { 19255 struct stdata *stp; 19256 19257 if (tcp->tcp_detached) 19258 return; 19259 stp = STREAM(tcp->tcp_rq); 19260 mutex_enter(&stp->sd_lock); 19261 stp->sd_flag |= STZCNOTIFY; 19262 cv_broadcast(&stp->sd_zcopy_wait); 19263 mutex_exit(&stp->sd_lock); 19264 } 19265 19266 static boolean_t 19267 tcp_send_find_ire(tcp_t *tcp, ipaddr_t *dst, ire_t **irep) 19268 { 19269 ire_t *ire; 19270 conn_t *connp = tcp->tcp_connp; 19271 tcp_stack_t *tcps = tcp->tcp_tcps; 19272 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 19273 19274 mutex_enter(&connp->conn_lock); 19275 ire = connp->conn_ire_cache; 19276 ASSERT(!(connp->conn_state_flags & CONN_INCIPIENT)); 19277 19278 if ((ire != NULL) && 19279 (((dst != NULL) && (ire->ire_addr == *dst)) || ((dst == NULL) && 19280 IN6_ARE_ADDR_EQUAL(&ire->ire_addr_v6, &tcp->tcp_ip6h->ip6_dst))) && 19281 !(ire->ire_marks & IRE_MARK_CONDEMNED)) { 19282 IRE_REFHOLD(ire); 19283 mutex_exit(&connp->conn_lock); 19284 } else { 19285 boolean_t cached = B_FALSE; 19286 ts_label_t *tsl; 19287 19288 /* force a recheck later on */ 19289 tcp->tcp_ire_ill_check_done = B_FALSE; 19290 19291 TCP_DBGSTAT(tcps, tcp_ire_null1); 19292 connp->conn_ire_cache = NULL; 19293 mutex_exit(&connp->conn_lock); 19294 19295 if (ire != NULL) 19296 IRE_REFRELE_NOTR(ire); 19297 19298 tsl = crgetlabel(CONN_CRED(connp)); 19299 ire = (dst ? 19300 ire_cache_lookup(*dst, connp->conn_zoneid, tsl, ipst) : 19301 ire_cache_lookup_v6(&tcp->tcp_ip6h->ip6_dst, 19302 connp->conn_zoneid, tsl, ipst)); 19303 19304 if (ire == NULL) { 19305 TCP_STAT(tcps, tcp_ire_null); 19306 return (B_FALSE); 19307 } 19308 19309 IRE_REFHOLD_NOTR(ire); 19310 19311 mutex_enter(&connp->conn_lock); 19312 if (CONN_CACHE_IRE(connp)) { 19313 rw_enter(&ire->ire_bucket->irb_lock, RW_READER); 19314 if (!(ire->ire_marks & IRE_MARK_CONDEMNED)) { 19315 TCP_CHECK_IREINFO(tcp, ire); 19316 connp->conn_ire_cache = ire; 19317 cached = B_TRUE; 19318 } 19319 rw_exit(&ire->ire_bucket->irb_lock); 19320 } 19321 mutex_exit(&connp->conn_lock); 19322 19323 /* 19324 * We can continue to use the ire but since it was 19325 * not cached, we should drop the extra reference. 19326 */ 19327 if (!cached) 19328 IRE_REFRELE_NOTR(ire); 19329 19330 /* 19331 * Rampart note: no need to select a new label here, since 19332 * labels are not allowed to change during the life of a TCP 19333 * connection. 19334 */ 19335 } 19336 19337 *irep = ire; 19338 19339 return (B_TRUE); 19340 } 19341 19342 /* 19343 * Called from tcp_send() or tcp_send_data() to find workable IRE. 19344 * 19345 * 0 = success; 19346 * 1 = failed to find ire and ill. 19347 */ 19348 static boolean_t 19349 tcp_send_find_ire_ill(tcp_t *tcp, mblk_t *mp, ire_t **irep, ill_t **illp) 19350 { 19351 ipha_t *ipha; 19352 ipaddr_t dst; 19353 ire_t *ire; 19354 ill_t *ill; 19355 conn_t *connp = tcp->tcp_connp; 19356 mblk_t *ire_fp_mp; 19357 tcp_stack_t *tcps = tcp->tcp_tcps; 19358 19359 if (mp != NULL) 19360 ipha = (ipha_t *)mp->b_rptr; 19361 else 19362 ipha = tcp->tcp_ipha; 19363 dst = ipha->ipha_dst; 19364 19365 if (!tcp_send_find_ire(tcp, &dst, &ire)) 19366 return (B_FALSE); 19367 19368 if ((ire->ire_flags & RTF_MULTIRT) || 19369 (ire->ire_stq == NULL) || 19370 (ire->ire_nce == NULL) || 19371 ((ire_fp_mp = ire->ire_nce->nce_fp_mp) == NULL) || 19372 ((mp != NULL) && (ire->ire_max_frag < ntohs(ipha->ipha_length) || 19373 MBLKL(ire_fp_mp) > MBLKHEAD(mp)))) { 19374 TCP_STAT(tcps, tcp_ip_ire_send); 19375 IRE_REFRELE(ire); 19376 return (B_FALSE); 19377 } 19378 19379 ill = ire_to_ill(ire); 19380 if (connp->conn_outgoing_ill != NULL) { 19381 ill_t *conn_outgoing_ill = NULL; 19382 /* 19383 * Choose a good ill in the group to send the packets on. 19384 */ 19385 ire = conn_set_outgoing_ill(connp, ire, &conn_outgoing_ill); 19386 ill = ire_to_ill(ire); 19387 } 19388 ASSERT(ill != NULL); 19389 19390 if (!tcp->tcp_ire_ill_check_done) { 19391 tcp_ire_ill_check(tcp, ire, ill, B_TRUE); 19392 tcp->tcp_ire_ill_check_done = B_TRUE; 19393 } 19394 19395 *irep = ire; 19396 *illp = ill; 19397 19398 return (B_TRUE); 19399 } 19400 19401 static void 19402 tcp_send_data(tcp_t *tcp, queue_t *q, mblk_t *mp) 19403 { 19404 ipha_t *ipha; 19405 ipaddr_t src; 19406 ipaddr_t dst; 19407 uint32_t cksum; 19408 ire_t *ire; 19409 uint16_t *up; 19410 ill_t *ill; 19411 conn_t *connp = tcp->tcp_connp; 19412 uint32_t hcksum_txflags = 0; 19413 mblk_t *ire_fp_mp; 19414 uint_t ire_fp_mp_len; 19415 tcp_stack_t *tcps = tcp->tcp_tcps; 19416 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 19417 19418 ASSERT(DB_TYPE(mp) == M_DATA); 19419 19420 if (DB_CRED(mp) == NULL) 19421 mblk_setcred(mp, CONN_CRED(connp)); 19422 19423 ipha = (ipha_t *)mp->b_rptr; 19424 src = ipha->ipha_src; 19425 dst = ipha->ipha_dst; 19426 19427 DTRACE_PROBE2(tcp__trace__send, mblk_t *, mp, tcp_t *, tcp); 19428 19429 /* 19430 * Drop off fast path for IPv6 and also if options are present or 19431 * we need to resolve a TS label. 19432 */ 19433 if (tcp->tcp_ipversion != IPV4_VERSION || 19434 !IPCL_IS_CONNECTED(connp) || 19435 !CONN_IS_LSO_MD_FASTPATH(connp) || 19436 (connp->conn_flags & IPCL_CHECK_POLICY) != 0 || 19437 !connp->conn_ulp_labeled || 19438 ipha->ipha_ident == IP_HDR_INCLUDED || 19439 ipha->ipha_version_and_hdr_length != IP_SIMPLE_HDR_VERSION || 19440 IPP_ENABLED(IPP_LOCAL_OUT, ipst)) { 19441 if (tcp->tcp_snd_zcopy_aware) 19442 mp = tcp_zcopy_disable(tcp, mp); 19443 TCP_STAT(tcps, tcp_ip_send); 19444 CALL_IP_WPUT(connp, q, mp); 19445 return; 19446 } 19447 19448 if (!tcp_send_find_ire_ill(tcp, mp, &ire, &ill)) { 19449 if (tcp->tcp_snd_zcopy_aware) 19450 mp = tcp_zcopy_backoff(tcp, mp, 0); 19451 CALL_IP_WPUT(connp, q, mp); 19452 return; 19453 } 19454 ire_fp_mp = ire->ire_nce->nce_fp_mp; 19455 ire_fp_mp_len = MBLKL(ire_fp_mp); 19456 19457 ASSERT(ipha->ipha_ident == 0 || ipha->ipha_ident == IP_HDR_INCLUDED); 19458 ipha->ipha_ident = (uint16_t)atomic_add_32_nv(&ire->ire_ident, 1); 19459 #ifndef _BIG_ENDIAN 19460 ipha->ipha_ident = (ipha->ipha_ident << 8) | (ipha->ipha_ident >> 8); 19461 #endif 19462 19463 /* 19464 * Check to see if we need to re-enable LSO/MDT for this connection 19465 * because it was previously disabled due to changes in the ill; 19466 * note that by doing it here, this re-enabling only applies when 19467 * the packet is not dispatched through CALL_IP_WPUT(). 19468 * 19469 * That means for IPv4, it is worth re-enabling LSO/MDT for the fastpath 19470 * case, since that's how we ended up here. For IPv6, we do the 19471 * re-enabling work in ip_xmit_v6(), albeit indirectly via squeue. 19472 */ 19473 if (connp->conn_lso_ok && !tcp->tcp_lso && ILL_LSO_TCP_USABLE(ill)) { 19474 /* 19475 * Restore LSO for this connection, so that next time around 19476 * it is eligible to go through tcp_lsosend() path again. 19477 */ 19478 TCP_STAT(tcps, tcp_lso_enabled); 19479 tcp->tcp_lso = B_TRUE; 19480 ip1dbg(("tcp_send_data: reenabling LSO for connp %p on " 19481 "interface %s\n", (void *)connp, ill->ill_name)); 19482 } else if (connp->conn_mdt_ok && !tcp->tcp_mdt && ILL_MDT_USABLE(ill)) { 19483 /* 19484 * Restore MDT for this connection, so that next time around 19485 * it is eligible to go through tcp_multisend() path again. 19486 */ 19487 TCP_STAT(tcps, tcp_mdt_conn_resumed1); 19488 tcp->tcp_mdt = B_TRUE; 19489 ip1dbg(("tcp_send_data: reenabling MDT for connp %p on " 19490 "interface %s\n", (void *)connp, ill->ill_name)); 19491 } 19492 19493 if (tcp->tcp_snd_zcopy_aware) { 19494 if ((ill->ill_capabilities & ILL_CAPAB_ZEROCOPY) == 0 || 19495 (ill->ill_zerocopy_capab->ill_zerocopy_flags == 0)) 19496 mp = tcp_zcopy_disable(tcp, mp); 19497 /* 19498 * we shouldn't need to reset ipha as the mp containing 19499 * ipha should never be a zero-copy mp. 19500 */ 19501 } 19502 19503 if (ILL_HCKSUM_CAPABLE(ill) && dohwcksum) { 19504 ASSERT(ill->ill_hcksum_capab != NULL); 19505 hcksum_txflags = ill->ill_hcksum_capab->ill_hcksum_txflags; 19506 } 19507 19508 /* pseudo-header checksum (do it in parts for IP header checksum) */ 19509 cksum = (dst >> 16) + (dst & 0xFFFF) + (src >> 16) + (src & 0xFFFF); 19510 19511 ASSERT(ipha->ipha_version_and_hdr_length == IP_SIMPLE_HDR_VERSION); 19512 up = IPH_TCPH_CHECKSUMP(ipha, IP_SIMPLE_HDR_LENGTH); 19513 19514 IP_CKSUM_XMIT_FAST(ire->ire_ipversion, hcksum_txflags, mp, ipha, up, 19515 IPPROTO_TCP, IP_SIMPLE_HDR_LENGTH, ntohs(ipha->ipha_length), cksum); 19516 19517 /* Software checksum? */ 19518 if (DB_CKSUMFLAGS(mp) == 0) { 19519 TCP_STAT(tcps, tcp_out_sw_cksum); 19520 TCP_STAT_UPDATE(tcps, tcp_out_sw_cksum_bytes, 19521 ntohs(ipha->ipha_length) - IP_SIMPLE_HDR_LENGTH); 19522 } 19523 19524 ipha->ipha_fragment_offset_and_flags |= 19525 (uint32_t)htons(ire->ire_frag_flag); 19526 19527 /* Calculate IP header checksum if hardware isn't capable */ 19528 if (!(DB_CKSUMFLAGS(mp) & HCK_IPV4_HDRCKSUM)) { 19529 IP_HDR_CKSUM(ipha, cksum, ((uint32_t *)ipha)[0], 19530 ((uint16_t *)ipha)[4]); 19531 } 19532 19533 ASSERT(DB_TYPE(ire_fp_mp) == M_DATA); 19534 mp->b_rptr = (uchar_t *)ipha - ire_fp_mp_len; 19535 bcopy(ire_fp_mp->b_rptr, mp->b_rptr, ire_fp_mp_len); 19536 19537 UPDATE_OB_PKT_COUNT(ire); 19538 ire->ire_last_used_time = lbolt; 19539 19540 BUMP_MIB(ill->ill_ip_mib, ipIfStatsHCOutRequests); 19541 BUMP_MIB(ill->ill_ip_mib, ipIfStatsHCOutTransmits); 19542 UPDATE_MIB(ill->ill_ip_mib, ipIfStatsHCOutOctets, 19543 ntohs(ipha->ipha_length)); 19544 19545 if (ILL_DLS_CAPABLE(ill)) { 19546 /* 19547 * Send the packet directly to DLD, where it may be queued 19548 * depending on the availability of transmit resources at 19549 * the media layer. 19550 */ 19551 IP_DLS_ILL_TX(ill, ipha, mp, ipst); 19552 } else { 19553 ill_t *out_ill = (ill_t *)ire->ire_stq->q_ptr; 19554 DTRACE_PROBE4(ip4__physical__out__start, 19555 ill_t *, NULL, ill_t *, out_ill, 19556 ipha_t *, ipha, mblk_t *, mp); 19557 FW_HOOKS(ipst->ips_ip4_physical_out_event, 19558 ipst->ips_ipv4firewall_physical_out, 19559 NULL, out_ill, ipha, mp, mp, 0, ipst); 19560 DTRACE_PROBE1(ip4__physical__out__end, mblk_t *, mp); 19561 19562 if (mp != NULL) { 19563 DTRACE_IP_FASTPATH(mp, ipha, out_ill, ipha, NULL); 19564 putnext(ire->ire_stq, mp); 19565 } 19566 } 19567 IRE_REFRELE(ire); 19568 } 19569 19570 /* 19571 * This handles the case when the receiver has shrunk its win. Per RFC 1122 19572 * if the receiver shrinks the window, i.e. moves the right window to the 19573 * left, the we should not send new data, but should retransmit normally the 19574 * old unacked data between suna and suna + swnd. We might has sent data 19575 * that is now outside the new window, pretend that we didn't send it. 19576 */ 19577 static void 19578 tcp_process_shrunk_swnd(tcp_t *tcp, uint32_t shrunk_count) 19579 { 19580 uint32_t snxt = tcp->tcp_snxt; 19581 mblk_t *xmit_tail; 19582 int32_t offset; 19583 19584 ASSERT(shrunk_count > 0); 19585 19586 /* Pretend we didn't send the data outside the window */ 19587 snxt -= shrunk_count; 19588 19589 /* Get the mblk and the offset in it per the shrunk window */ 19590 xmit_tail = tcp_get_seg_mp(tcp, snxt, &offset); 19591 19592 ASSERT(xmit_tail != NULL); 19593 19594 /* Reset all the values per the now shrunk window */ 19595 tcp->tcp_snxt = snxt; 19596 tcp->tcp_xmit_tail = xmit_tail; 19597 tcp->tcp_xmit_tail_unsent = xmit_tail->b_wptr - xmit_tail->b_rptr - 19598 offset; 19599 tcp->tcp_unsent += shrunk_count; 19600 19601 if (tcp->tcp_suna == tcp->tcp_snxt && tcp->tcp_swnd == 0) 19602 /* 19603 * Make sure the timer is running so that we will probe a zero 19604 * window. 19605 */ 19606 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 19607 } 19608 19609 19610 /* 19611 * The TCP normal data output path. 19612 * NOTE: the logic of the fast path is duplicated from this function. 19613 */ 19614 static void 19615 tcp_wput_data(tcp_t *tcp, mblk_t *mp, boolean_t urgent) 19616 { 19617 int len; 19618 mblk_t *local_time; 19619 mblk_t *mp1; 19620 uint32_t snxt; 19621 int tail_unsent; 19622 int tcpstate; 19623 int usable = 0; 19624 mblk_t *xmit_tail; 19625 queue_t *q = tcp->tcp_wq; 19626 int32_t mss; 19627 int32_t num_sack_blk = 0; 19628 int32_t tcp_hdr_len; 19629 int32_t tcp_tcp_hdr_len; 19630 int mdt_thres; 19631 int rc; 19632 tcp_stack_t *tcps = tcp->tcp_tcps; 19633 ip_stack_t *ipst; 19634 19635 tcpstate = tcp->tcp_state; 19636 if (mp == NULL) { 19637 /* 19638 * tcp_wput_data() with NULL mp should only be called when 19639 * there is unsent data. 19640 */ 19641 ASSERT(tcp->tcp_unsent > 0); 19642 /* Really tacky... but we need this for detached closes. */ 19643 len = tcp->tcp_unsent; 19644 goto data_null; 19645 } 19646 19647 #if CCS_STATS 19648 wrw_stats.tot.count++; 19649 wrw_stats.tot.bytes += msgdsize(mp); 19650 #endif 19651 ASSERT(mp->b_datap->db_type == M_DATA); 19652 /* 19653 * Don't allow data after T_ORDREL_REQ or T_DISCON_REQ, 19654 * or before a connection attempt has begun. 19655 */ 19656 if (tcpstate < TCPS_SYN_SENT || tcpstate > TCPS_CLOSE_WAIT || 19657 (tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) { 19658 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) != 0) { 19659 #ifdef DEBUG 19660 cmn_err(CE_WARN, 19661 "tcp_wput_data: data after ordrel, %s", 19662 tcp_display(tcp, NULL, 19663 DISP_ADDR_AND_PORT)); 19664 #else 19665 if (tcp->tcp_debug) { 19666 (void) strlog(TCP_MOD_ID, 0, 1, 19667 SL_TRACE|SL_ERROR, 19668 "tcp_wput_data: data after ordrel, %s\n", 19669 tcp_display(tcp, NULL, 19670 DISP_ADDR_AND_PORT)); 19671 } 19672 #endif /* DEBUG */ 19673 } 19674 if (tcp->tcp_snd_zcopy_aware && 19675 (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY) != 0) 19676 tcp_zcopy_notify(tcp); 19677 freemsg(mp); 19678 mutex_enter(&tcp->tcp_non_sq_lock); 19679 if (tcp->tcp_flow_stopped && 19680 TCP_UNSENT_BYTES(tcp) <= tcp->tcp_xmit_lowater) { 19681 tcp_clrqfull(tcp); 19682 } 19683 mutex_exit(&tcp->tcp_non_sq_lock); 19684 return; 19685 } 19686 19687 /* Strip empties */ 19688 for (;;) { 19689 ASSERT((uintptr_t)(mp->b_wptr - mp->b_rptr) <= 19690 (uintptr_t)INT_MAX); 19691 len = (int)(mp->b_wptr - mp->b_rptr); 19692 if (len > 0) 19693 break; 19694 mp1 = mp; 19695 mp = mp->b_cont; 19696 freeb(mp1); 19697 if (!mp) { 19698 return; 19699 } 19700 } 19701 19702 /* If we are the first on the list ... */ 19703 if (tcp->tcp_xmit_head == NULL) { 19704 tcp->tcp_xmit_head = mp; 19705 tcp->tcp_xmit_tail = mp; 19706 tcp->tcp_xmit_tail_unsent = len; 19707 } else { 19708 /* If tiny tx and room in txq tail, pullup to save mblks. */ 19709 struct datab *dp; 19710 19711 mp1 = tcp->tcp_xmit_last; 19712 if (len < tcp_tx_pull_len && 19713 (dp = mp1->b_datap)->db_ref == 1 && 19714 dp->db_lim - mp1->b_wptr >= len) { 19715 ASSERT(len > 0); 19716 ASSERT(!mp1->b_cont); 19717 if (len == 1) { 19718 *mp1->b_wptr++ = *mp->b_rptr; 19719 } else { 19720 bcopy(mp->b_rptr, mp1->b_wptr, len); 19721 mp1->b_wptr += len; 19722 } 19723 if (mp1 == tcp->tcp_xmit_tail) 19724 tcp->tcp_xmit_tail_unsent += len; 19725 mp1->b_cont = mp->b_cont; 19726 if (tcp->tcp_snd_zcopy_aware && 19727 (mp->b_datap->db_struioflag & STRUIO_ZCNOTIFY)) 19728 mp1->b_datap->db_struioflag |= STRUIO_ZCNOTIFY; 19729 freeb(mp); 19730 mp = mp1; 19731 } else { 19732 tcp->tcp_xmit_last->b_cont = mp; 19733 } 19734 len += tcp->tcp_unsent; 19735 } 19736 19737 /* Tack on however many more positive length mblks we have */ 19738 if ((mp1 = mp->b_cont) != NULL) { 19739 do { 19740 int tlen; 19741 ASSERT((uintptr_t)(mp1->b_wptr - mp1->b_rptr) <= 19742 (uintptr_t)INT_MAX); 19743 tlen = (int)(mp1->b_wptr - mp1->b_rptr); 19744 if (tlen <= 0) { 19745 mp->b_cont = mp1->b_cont; 19746 freeb(mp1); 19747 } else { 19748 len += tlen; 19749 mp = mp1; 19750 } 19751 } while ((mp1 = mp->b_cont) != NULL); 19752 } 19753 tcp->tcp_xmit_last = mp; 19754 tcp->tcp_unsent = len; 19755 19756 if (urgent) 19757 usable = 1; 19758 19759 data_null: 19760 snxt = tcp->tcp_snxt; 19761 xmit_tail = tcp->tcp_xmit_tail; 19762 tail_unsent = tcp->tcp_xmit_tail_unsent; 19763 19764 /* 19765 * Note that tcp_mss has been adjusted to take into account the 19766 * timestamp option if applicable. Because SACK options do not 19767 * appear in every TCP segments and they are of variable lengths, 19768 * they cannot be included in tcp_mss. Thus we need to calculate 19769 * the actual segment length when we need to send a segment which 19770 * includes SACK options. 19771 */ 19772 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) { 19773 int32_t opt_len; 19774 19775 num_sack_blk = MIN(tcp->tcp_max_sack_blk, 19776 tcp->tcp_num_sack_blk); 19777 opt_len = num_sack_blk * sizeof (sack_blk_t) + TCPOPT_NOP_LEN * 19778 2 + TCPOPT_HEADER_LEN; 19779 mss = tcp->tcp_mss - opt_len; 19780 tcp_hdr_len = tcp->tcp_hdr_len + opt_len; 19781 tcp_tcp_hdr_len = tcp->tcp_tcp_hdr_len + opt_len; 19782 } else { 19783 mss = tcp->tcp_mss; 19784 tcp_hdr_len = tcp->tcp_hdr_len; 19785 tcp_tcp_hdr_len = tcp->tcp_tcp_hdr_len; 19786 } 19787 19788 if ((tcp->tcp_suna == snxt) && !tcp->tcp_localnet && 19789 (TICK_TO_MSEC(lbolt - tcp->tcp_last_recv_time) >= tcp->tcp_rto)) { 19790 SET_TCP_INIT_CWND(tcp, mss, tcps->tcps_slow_start_after_idle); 19791 } 19792 if (tcpstate == TCPS_SYN_RCVD) { 19793 /* 19794 * The three-way connection establishment handshake is not 19795 * complete yet. We want to queue the data for transmission 19796 * after entering ESTABLISHED state (RFC793). A jump to 19797 * "done" label effectively leaves data on the queue. 19798 */ 19799 goto done; 19800 } else { 19801 int usable_r; 19802 19803 /* 19804 * In the special case when cwnd is zero, which can only 19805 * happen if the connection is ECN capable, return now. 19806 * New segments is sent using tcp_timer(). The timer 19807 * is set in tcp_rput_data(). 19808 */ 19809 if (tcp->tcp_cwnd == 0) { 19810 /* 19811 * Note that tcp_cwnd is 0 before 3-way handshake is 19812 * finished. 19813 */ 19814 ASSERT(tcp->tcp_ecn_ok || 19815 tcp->tcp_state < TCPS_ESTABLISHED); 19816 return; 19817 } 19818 19819 /* NOTE: trouble if xmitting while SYN not acked? */ 19820 usable_r = snxt - tcp->tcp_suna; 19821 usable_r = tcp->tcp_swnd - usable_r; 19822 19823 /* 19824 * Check if the receiver has shrunk the window. If 19825 * tcp_wput_data() with NULL mp is called, tcp_fin_sent 19826 * cannot be set as there is unsent data, so FIN cannot 19827 * be sent out. Otherwise, we need to take into account 19828 * of FIN as it consumes an "invisible" sequence number. 19829 */ 19830 ASSERT(tcp->tcp_fin_sent == 0); 19831 if (usable_r < 0) { 19832 /* 19833 * The receiver has shrunk the window and we have sent 19834 * -usable_r date beyond the window, re-adjust. 19835 * 19836 * If TCP window scaling is enabled, there can be 19837 * round down error as the advertised receive window 19838 * is actually right shifted n bits. This means that 19839 * the lower n bits info is wiped out. It will look 19840 * like the window is shrunk. Do a check here to 19841 * see if the shrunk amount is actually within the 19842 * error in window calculation. If it is, just 19843 * return. Note that this check is inside the 19844 * shrunk window check. This makes sure that even 19845 * though tcp_process_shrunk_swnd() is not called, 19846 * we will stop further processing. 19847 */ 19848 if ((-usable_r >> tcp->tcp_snd_ws) > 0) { 19849 tcp_process_shrunk_swnd(tcp, -usable_r); 19850 } 19851 return; 19852 } 19853 19854 /* usable = MIN(swnd, cwnd) - unacked_bytes */ 19855 if (tcp->tcp_swnd > tcp->tcp_cwnd) 19856 usable_r -= tcp->tcp_swnd - tcp->tcp_cwnd; 19857 19858 /* usable = MIN(usable, unsent) */ 19859 if (usable_r > len) 19860 usable_r = len; 19861 19862 /* usable = MAX(usable, {1 for urgent, 0 for data}) */ 19863 if (usable_r > 0) { 19864 usable = usable_r; 19865 } else { 19866 /* Bypass all other unnecessary processing. */ 19867 goto done; 19868 } 19869 } 19870 19871 local_time = (mblk_t *)lbolt; 19872 19873 /* 19874 * "Our" Nagle Algorithm. This is not the same as in the old 19875 * BSD. This is more in line with the true intent of Nagle. 19876 * 19877 * The conditions are: 19878 * 1. The amount of unsent data (or amount of data which can be 19879 * sent, whichever is smaller) is less than Nagle limit. 19880 * 2. The last sent size is also less than Nagle limit. 19881 * 3. There is unack'ed data. 19882 * 4. Urgent pointer is not set. Send urgent data ignoring the 19883 * Nagle algorithm. This reduces the probability that urgent 19884 * bytes get "merged" together. 19885 * 5. The app has not closed the connection. This eliminates the 19886 * wait time of the receiving side waiting for the last piece of 19887 * (small) data. 19888 * 19889 * If all are satisified, exit without sending anything. Note 19890 * that Nagle limit can be smaller than 1 MSS. Nagle limit is 19891 * the smaller of 1 MSS and global tcp_naglim_def (default to be 19892 * 4095). 19893 */ 19894 if (usable < (int)tcp->tcp_naglim && 19895 tcp->tcp_naglim > tcp->tcp_last_sent_len && 19896 snxt != tcp->tcp_suna && 19897 !(tcp->tcp_valid_bits & TCP_URG_VALID) && 19898 !(tcp->tcp_valid_bits & TCP_FSS_VALID)) { 19899 goto done; 19900 } 19901 19902 if (tcp->tcp_cork) { 19903 /* 19904 * if the tcp->tcp_cork option is set, then we have to force 19905 * TCP not to send partial segment (smaller than MSS bytes). 19906 * We are calculating the usable now based on full mss and 19907 * will save the rest of remaining data for later. 19908 */ 19909 if (usable < mss) 19910 goto done; 19911 usable = (usable / mss) * mss; 19912 } 19913 19914 /* Update the latest receive window size in TCP header. */ 19915 U32_TO_ABE16(tcp->tcp_rwnd >> tcp->tcp_rcv_ws, 19916 tcp->tcp_tcph->th_win); 19917 19918 /* 19919 * Determine if it's worthwhile to attempt LSO or MDT, based on: 19920 * 19921 * 1. Simple TCP/IP{v4,v6} (no options). 19922 * 2. IPSEC/IPQoS processing is not needed for the TCP connection. 19923 * 3. If the TCP connection is in ESTABLISHED state. 19924 * 4. The TCP is not detached. 19925 * 19926 * If any of the above conditions have changed during the 19927 * connection, stop using LSO/MDT and restore the stream head 19928 * parameters accordingly. 19929 */ 19930 ipst = tcps->tcps_netstack->netstack_ip; 19931 19932 if ((tcp->tcp_lso || tcp->tcp_mdt) && 19933 ((tcp->tcp_ipversion == IPV4_VERSION && 19934 tcp->tcp_ip_hdr_len != IP_SIMPLE_HDR_LENGTH) || 19935 (tcp->tcp_ipversion == IPV6_VERSION && 19936 tcp->tcp_ip_hdr_len != IPV6_HDR_LEN) || 19937 tcp->tcp_state != TCPS_ESTABLISHED || 19938 TCP_IS_DETACHED(tcp) || !CONN_IS_LSO_MD_FASTPATH(tcp->tcp_connp) || 19939 CONN_IPSEC_OUT_ENCAPSULATED(tcp->tcp_connp) || 19940 IPP_ENABLED(IPP_LOCAL_OUT, ipst))) { 19941 if (tcp->tcp_lso) { 19942 tcp->tcp_connp->conn_lso_ok = B_FALSE; 19943 tcp->tcp_lso = B_FALSE; 19944 } else { 19945 tcp->tcp_connp->conn_mdt_ok = B_FALSE; 19946 tcp->tcp_mdt = B_FALSE; 19947 } 19948 19949 /* Anything other than detached is considered pathological */ 19950 if (!TCP_IS_DETACHED(tcp)) { 19951 if (tcp->tcp_lso) 19952 TCP_STAT(tcps, tcp_lso_disabled); 19953 else 19954 TCP_STAT(tcps, tcp_mdt_conn_halted1); 19955 (void) tcp_maxpsz_set(tcp, B_TRUE); 19956 } 19957 } 19958 19959 /* Use MDT if sendable amount is greater than the threshold */ 19960 if (tcp->tcp_mdt && 19961 (mdt_thres = mss << tcp_mdt_smss_threshold, usable > mdt_thres) && 19962 (tail_unsent > mdt_thres || (xmit_tail->b_cont != NULL && 19963 MBLKL(xmit_tail->b_cont) > mdt_thres)) && 19964 (tcp->tcp_valid_bits == 0 || 19965 tcp->tcp_valid_bits == TCP_FSS_VALID)) { 19966 ASSERT(tcp->tcp_connp->conn_mdt_ok); 19967 rc = tcp_multisend(q, tcp, mss, tcp_hdr_len, tcp_tcp_hdr_len, 19968 num_sack_blk, &usable, &snxt, &tail_unsent, &xmit_tail, 19969 local_time, mdt_thres); 19970 } else { 19971 rc = tcp_send(q, tcp, mss, tcp_hdr_len, tcp_tcp_hdr_len, 19972 num_sack_blk, &usable, &snxt, &tail_unsent, &xmit_tail, 19973 local_time, INT_MAX); 19974 } 19975 19976 /* Pretend that all we were trying to send really got sent */ 19977 if (rc < 0 && tail_unsent < 0) { 19978 do { 19979 xmit_tail = xmit_tail->b_cont; 19980 xmit_tail->b_prev = local_time; 19981 ASSERT((uintptr_t)(xmit_tail->b_wptr - 19982 xmit_tail->b_rptr) <= (uintptr_t)INT_MAX); 19983 tail_unsent += (int)(xmit_tail->b_wptr - 19984 xmit_tail->b_rptr); 19985 } while (tail_unsent < 0); 19986 } 19987 done:; 19988 tcp->tcp_xmit_tail = xmit_tail; 19989 tcp->tcp_xmit_tail_unsent = tail_unsent; 19990 len = tcp->tcp_snxt - snxt; 19991 if (len) { 19992 /* 19993 * If new data was sent, need to update the notsack 19994 * list, which is, afterall, data blocks that have 19995 * not been sack'ed by the receiver. New data is 19996 * not sack'ed. 19997 */ 19998 if (tcp->tcp_snd_sack_ok && tcp->tcp_notsack_list != NULL) { 19999 /* len is a negative value. */ 20000 tcp->tcp_pipe -= len; 20001 tcp_notsack_update(&(tcp->tcp_notsack_list), 20002 tcp->tcp_snxt, snxt, 20003 &(tcp->tcp_num_notsack_blk), 20004 &(tcp->tcp_cnt_notsack_list)); 20005 } 20006 tcp->tcp_snxt = snxt + tcp->tcp_fin_sent; 20007 tcp->tcp_rack = tcp->tcp_rnxt; 20008 tcp->tcp_rack_cnt = 0; 20009 if ((snxt + len) == tcp->tcp_suna) { 20010 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 20011 } 20012 } else if (snxt == tcp->tcp_suna && tcp->tcp_swnd == 0) { 20013 /* 20014 * Didn't send anything. Make sure the timer is running 20015 * so that we will probe a zero window. 20016 */ 20017 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 20018 } 20019 /* Note that len is the amount we just sent but with a negative sign */ 20020 tcp->tcp_unsent += len; 20021 mutex_enter(&tcp->tcp_non_sq_lock); 20022 if (tcp->tcp_flow_stopped) { 20023 if (TCP_UNSENT_BYTES(tcp) <= tcp->tcp_xmit_lowater) { 20024 tcp_clrqfull(tcp); 20025 } 20026 } else if (TCP_UNSENT_BYTES(tcp) >= tcp->tcp_xmit_hiwater) { 20027 tcp_setqfull(tcp); 20028 } 20029 mutex_exit(&tcp->tcp_non_sq_lock); 20030 } 20031 20032 /* 20033 * tcp_fill_header is called by tcp_send() and tcp_multisend() to fill the 20034 * outgoing TCP header with the template header, as well as other 20035 * options such as time-stamp, ECN and/or SACK. 20036 */ 20037 static void 20038 tcp_fill_header(tcp_t *tcp, uchar_t *rptr, clock_t now, int num_sack_blk) 20039 { 20040 tcph_t *tcp_tmpl, *tcp_h; 20041 uint32_t *dst, *src; 20042 int hdrlen; 20043 20044 ASSERT(OK_32PTR(rptr)); 20045 20046 /* Template header */ 20047 tcp_tmpl = tcp->tcp_tcph; 20048 20049 /* Header of outgoing packet */ 20050 tcp_h = (tcph_t *)(rptr + tcp->tcp_ip_hdr_len); 20051 20052 /* dst and src are opaque 32-bit fields, used for copying */ 20053 dst = (uint32_t *)rptr; 20054 src = (uint32_t *)tcp->tcp_iphc; 20055 hdrlen = tcp->tcp_hdr_len; 20056 20057 /* Fill time-stamp option if needed */ 20058 if (tcp->tcp_snd_ts_ok) { 20059 U32_TO_BE32((uint32_t)now, 20060 (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 4); 20061 U32_TO_BE32(tcp->tcp_ts_recent, 20062 (char *)tcp_tmpl + TCP_MIN_HEADER_LENGTH + 8); 20063 } else { 20064 ASSERT(tcp->tcp_tcp_hdr_len == TCP_MIN_HEADER_LENGTH); 20065 } 20066 20067 /* 20068 * Copy the template header; is this really more efficient than 20069 * calling bcopy()? For simple IPv4/TCP, it may be the case, 20070 * but perhaps not for other scenarios. 20071 */ 20072 dst[0] = src[0]; 20073 dst[1] = src[1]; 20074 dst[2] = src[2]; 20075 dst[3] = src[3]; 20076 dst[4] = src[4]; 20077 dst[5] = src[5]; 20078 dst[6] = src[6]; 20079 dst[7] = src[7]; 20080 dst[8] = src[8]; 20081 dst[9] = src[9]; 20082 if (hdrlen -= 40) { 20083 hdrlen >>= 2; 20084 dst += 10; 20085 src += 10; 20086 do { 20087 *dst++ = *src++; 20088 } while (--hdrlen); 20089 } 20090 20091 /* 20092 * Set the ECN info in the TCP header if it is not a zero 20093 * window probe. Zero window probe is only sent in 20094 * tcp_wput_data() and tcp_timer(). 20095 */ 20096 if (tcp->tcp_ecn_ok && !tcp->tcp_zero_win_probe) { 20097 SET_ECT(tcp, rptr); 20098 20099 if (tcp->tcp_ecn_echo_on) 20100 tcp_h->th_flags[0] |= TH_ECE; 20101 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) { 20102 tcp_h->th_flags[0] |= TH_CWR; 20103 tcp->tcp_ecn_cwr_sent = B_TRUE; 20104 } 20105 } 20106 20107 /* Fill in SACK options */ 20108 if (num_sack_blk > 0) { 20109 uchar_t *wptr = rptr + tcp->tcp_hdr_len; 20110 sack_blk_t *tmp; 20111 int32_t i; 20112 20113 wptr[0] = TCPOPT_NOP; 20114 wptr[1] = TCPOPT_NOP; 20115 wptr[2] = TCPOPT_SACK; 20116 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk * 20117 sizeof (sack_blk_t); 20118 wptr += TCPOPT_REAL_SACK_LEN; 20119 20120 tmp = tcp->tcp_sack_list; 20121 for (i = 0; i < num_sack_blk; i++) { 20122 U32_TO_BE32(tmp[i].begin, wptr); 20123 wptr += sizeof (tcp_seq); 20124 U32_TO_BE32(tmp[i].end, wptr); 20125 wptr += sizeof (tcp_seq); 20126 } 20127 tcp_h->th_offset_and_rsrvd[0] += 20128 ((num_sack_blk * 2 + 1) << 4); 20129 } 20130 } 20131 20132 /* 20133 * tcp_mdt_add_attrs() is called by tcp_multisend() in order to attach 20134 * the destination address and SAP attribute, and if necessary, the 20135 * hardware checksum offload attribute to a Multidata message. 20136 */ 20137 static int 20138 tcp_mdt_add_attrs(multidata_t *mmd, const mblk_t *dlmp, const boolean_t hwcksum, 20139 const uint32_t start, const uint32_t stuff, const uint32_t end, 20140 const uint32_t flags, tcp_stack_t *tcps) 20141 { 20142 /* Add global destination address & SAP attribute */ 20143 if (dlmp == NULL || !ip_md_addr_attr(mmd, NULL, dlmp)) { 20144 ip1dbg(("tcp_mdt_add_attrs: can't add global physical " 20145 "destination address+SAP\n")); 20146 20147 if (dlmp != NULL) 20148 TCP_STAT(tcps, tcp_mdt_allocfail); 20149 return (-1); 20150 } 20151 20152 /* Add global hwcksum attribute */ 20153 if (hwcksum && 20154 !ip_md_hcksum_attr(mmd, NULL, start, stuff, end, flags)) { 20155 ip1dbg(("tcp_mdt_add_attrs: can't add global hardware " 20156 "checksum attribute\n")); 20157 20158 TCP_STAT(tcps, tcp_mdt_allocfail); 20159 return (-1); 20160 } 20161 20162 return (0); 20163 } 20164 20165 /* 20166 * Smaller and private version of pdescinfo_t used specifically for TCP, 20167 * which allows for only two payload spans per packet. 20168 */ 20169 typedef struct tcp_pdescinfo_s PDESCINFO_STRUCT(2) tcp_pdescinfo_t; 20170 20171 /* 20172 * tcp_multisend() is called by tcp_wput_data() for Multidata Transmit 20173 * scheme, and returns one the following: 20174 * 20175 * -1 = failed allocation. 20176 * 0 = success; burst count reached, or usable send window is too small, 20177 * and that we'd rather wait until later before sending again. 20178 */ 20179 static int 20180 tcp_multisend(queue_t *q, tcp_t *tcp, const int mss, const int tcp_hdr_len, 20181 const int tcp_tcp_hdr_len, const int num_sack_blk, int *usable, 20182 uint_t *snxt, int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time, 20183 const int mdt_thres) 20184 { 20185 mblk_t *md_mp_head, *md_mp, *md_pbuf, *md_pbuf_nxt, *md_hbuf; 20186 multidata_t *mmd; 20187 uint_t obsegs, obbytes, hdr_frag_sz; 20188 uint_t cur_hdr_off, cur_pld_off, base_pld_off, first_snxt; 20189 int num_burst_seg, max_pld; 20190 pdesc_t *pkt; 20191 tcp_pdescinfo_t tcp_pkt_info; 20192 pdescinfo_t *pkt_info; 20193 int pbuf_idx, pbuf_idx_nxt; 20194 int seg_len, len, spill, af; 20195 boolean_t add_buffer, zcopy, clusterwide; 20196 boolean_t buf_trunked = B_FALSE; 20197 boolean_t rconfirm = B_FALSE; 20198 boolean_t done = B_FALSE; 20199 uint32_t cksum; 20200 uint32_t hwcksum_flags; 20201 ire_t *ire = NULL; 20202 ill_t *ill; 20203 ipha_t *ipha; 20204 ip6_t *ip6h; 20205 ipaddr_t src, dst; 20206 ill_zerocopy_capab_t *zc_cap = NULL; 20207 uint16_t *up; 20208 int err; 20209 conn_t *connp; 20210 mblk_t *mp, *mp1, *fw_mp_head = NULL; 20211 uchar_t *pld_start; 20212 tcp_stack_t *tcps = tcp->tcp_tcps; 20213 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 20214 20215 #ifdef _BIG_ENDIAN 20216 #define IPVER(ip6h) ((((uint32_t *)ip6h)[0] >> 28) & 0x7) 20217 #else 20218 #define IPVER(ip6h) ((((uint32_t *)ip6h)[0] >> 4) & 0x7) 20219 #endif 20220 20221 #define PREP_NEW_MULTIDATA() { \ 20222 mmd = NULL; \ 20223 md_mp = md_hbuf = NULL; \ 20224 cur_hdr_off = 0; \ 20225 max_pld = tcp->tcp_mdt_max_pld; \ 20226 pbuf_idx = pbuf_idx_nxt = -1; \ 20227 add_buffer = B_TRUE; \ 20228 zcopy = B_FALSE; \ 20229 } 20230 20231 #define PREP_NEW_PBUF() { \ 20232 md_pbuf = md_pbuf_nxt = NULL; \ 20233 pbuf_idx = pbuf_idx_nxt = -1; \ 20234 cur_pld_off = 0; \ 20235 first_snxt = *snxt; \ 20236 ASSERT(*tail_unsent > 0); \ 20237 base_pld_off = MBLKL(*xmit_tail) - *tail_unsent; \ 20238 } 20239 20240 ASSERT(mdt_thres >= mss); 20241 ASSERT(*usable > 0 && *usable > mdt_thres); 20242 ASSERT(tcp->tcp_state == TCPS_ESTABLISHED); 20243 ASSERT(!TCP_IS_DETACHED(tcp)); 20244 ASSERT(tcp->tcp_valid_bits == 0 || 20245 tcp->tcp_valid_bits == TCP_FSS_VALID); 20246 ASSERT((tcp->tcp_ipversion == IPV4_VERSION && 20247 tcp->tcp_ip_hdr_len == IP_SIMPLE_HDR_LENGTH) || 20248 (tcp->tcp_ipversion == IPV6_VERSION && 20249 tcp->tcp_ip_hdr_len == IPV6_HDR_LEN)); 20250 20251 connp = tcp->tcp_connp; 20252 ASSERT(connp != NULL); 20253 ASSERT(CONN_IS_LSO_MD_FASTPATH(connp)); 20254 ASSERT(!CONN_IPSEC_OUT_ENCAPSULATED(connp)); 20255 20256 /* 20257 * Note that tcp will only declare at most 2 payload spans per 20258 * packet, which is much lower than the maximum allowable number 20259 * of packet spans per Multidata. For this reason, we use the 20260 * privately declared and smaller descriptor info structure, in 20261 * order to save some stack space. 20262 */ 20263 pkt_info = (pdescinfo_t *)&tcp_pkt_info; 20264 20265 af = (tcp->tcp_ipversion == IPV4_VERSION) ? AF_INET : AF_INET6; 20266 if (af == AF_INET) { 20267 dst = tcp->tcp_ipha->ipha_dst; 20268 src = tcp->tcp_ipha->ipha_src; 20269 ASSERT(!CLASSD(dst)); 20270 } 20271 ASSERT(af == AF_INET || 20272 !IN6_IS_ADDR_MULTICAST(&tcp->tcp_ip6h->ip6_dst)); 20273 20274 obsegs = obbytes = 0; 20275 num_burst_seg = tcp->tcp_snd_burst; 20276 md_mp_head = NULL; 20277 PREP_NEW_MULTIDATA(); 20278 20279 /* 20280 * Before we go on further, make sure there is an IRE that we can 20281 * use, and that the ILL supports MDT. Otherwise, there's no point 20282 * in proceeding any further, and we should just hand everything 20283 * off to the legacy path. 20284 */ 20285 if (!tcp_send_find_ire(tcp, (af == AF_INET) ? &dst : NULL, &ire)) 20286 goto legacy_send_no_md; 20287 20288 ASSERT(ire != NULL); 20289 ASSERT(af != AF_INET || ire->ire_ipversion == IPV4_VERSION); 20290 ASSERT(af == AF_INET || !IN6_IS_ADDR_V4MAPPED(&(ire->ire_addr_v6))); 20291 ASSERT(af == AF_INET || ire->ire_nce != NULL); 20292 ASSERT(!(ire->ire_type & IRE_BROADCAST)); 20293 /* 20294 * If we do support loopback for MDT (which requires modifications 20295 * to the receiving paths), the following assertions should go away, 20296 * and we would be sending the Multidata to loopback conn later on. 20297 */ 20298 ASSERT(!IRE_IS_LOCAL(ire)); 20299 ASSERT(ire->ire_stq != NULL); 20300 20301 ill = ire_to_ill(ire); 20302 ASSERT(ill != NULL); 20303 ASSERT(!ILL_MDT_CAPABLE(ill) || ill->ill_mdt_capab != NULL); 20304 20305 if (!tcp->tcp_ire_ill_check_done) { 20306 tcp_ire_ill_check(tcp, ire, ill, B_TRUE); 20307 tcp->tcp_ire_ill_check_done = B_TRUE; 20308 } 20309 20310 /* 20311 * If the underlying interface conditions have changed, or if the 20312 * new interface does not support MDT, go back to legacy path. 20313 */ 20314 if (!ILL_MDT_USABLE(ill) || (ire->ire_flags & RTF_MULTIRT) != 0) { 20315 /* don't go through this path anymore for this connection */ 20316 TCP_STAT(tcps, tcp_mdt_conn_halted2); 20317 tcp->tcp_mdt = B_FALSE; 20318 ip1dbg(("tcp_multisend: disabling MDT for connp %p on " 20319 "interface %s\n", (void *)connp, ill->ill_name)); 20320 /* IRE will be released prior to returning */ 20321 goto legacy_send_no_md; 20322 } 20323 20324 if (ill->ill_capabilities & ILL_CAPAB_ZEROCOPY) 20325 zc_cap = ill->ill_zerocopy_capab; 20326 20327 /* 20328 * Check if we can take tcp fast-path. Note that "incomplete" 20329 * ire's (where the link-layer for next hop is not resolved 20330 * or where the fast-path header in nce_fp_mp is not available 20331 * yet) are sent down the legacy (slow) path. 20332 * NOTE: We should fix ip_xmit_v4 to handle M_MULTIDATA 20333 */ 20334 if (ire->ire_nce && ire->ire_nce->nce_state != ND_REACHABLE) { 20335 /* IRE will be released prior to returning */ 20336 goto legacy_send_no_md; 20337 } 20338 20339 /* go to legacy path if interface doesn't support zerocopy */ 20340 if (tcp->tcp_snd_zcopy_aware && do_tcpzcopy != 2 && 20341 (zc_cap == NULL || zc_cap->ill_zerocopy_flags == 0)) { 20342 /* IRE will be released prior to returning */ 20343 goto legacy_send_no_md; 20344 } 20345 20346 /* does the interface support hardware checksum offload? */ 20347 hwcksum_flags = 0; 20348 if (ILL_HCKSUM_CAPABLE(ill) && 20349 (ill->ill_hcksum_capab->ill_hcksum_txflags & 20350 (HCKSUM_INET_FULL_V4 | HCKSUM_INET_FULL_V6 | HCKSUM_INET_PARTIAL | 20351 HCKSUM_IPHDRCKSUM)) && dohwcksum) { 20352 if (ill->ill_hcksum_capab->ill_hcksum_txflags & 20353 HCKSUM_IPHDRCKSUM) 20354 hwcksum_flags = HCK_IPV4_HDRCKSUM; 20355 20356 if (ill->ill_hcksum_capab->ill_hcksum_txflags & 20357 (HCKSUM_INET_FULL_V4 | HCKSUM_INET_FULL_V6)) 20358 hwcksum_flags |= HCK_FULLCKSUM; 20359 else if (ill->ill_hcksum_capab->ill_hcksum_txflags & 20360 HCKSUM_INET_PARTIAL) 20361 hwcksum_flags |= HCK_PARTIALCKSUM; 20362 } 20363 20364 /* 20365 * Each header fragment consists of the leading extra space, 20366 * followed by the TCP/IP header, and the trailing extra space. 20367 * We make sure that each header fragment begins on a 32-bit 20368 * aligned memory address (tcp_mdt_hdr_head is already 32-bit 20369 * aligned in tcp_mdt_update). 20370 */ 20371 hdr_frag_sz = roundup((tcp->tcp_mdt_hdr_head + tcp_hdr_len + 20372 tcp->tcp_mdt_hdr_tail), 4); 20373 20374 /* are we starting from the beginning of data block? */ 20375 if (*tail_unsent == 0) { 20376 *xmit_tail = (*xmit_tail)->b_cont; 20377 ASSERT((uintptr_t)MBLKL(*xmit_tail) <= (uintptr_t)INT_MAX); 20378 *tail_unsent = (int)MBLKL(*xmit_tail); 20379 } 20380 20381 /* 20382 * Here we create one or more Multidata messages, each made up of 20383 * one header buffer and up to N payload buffers. This entire 20384 * operation is done within two loops: 20385 * 20386 * The outer loop mostly deals with creating the Multidata message, 20387 * as well as the header buffer that gets added to it. It also 20388 * links the Multidata messages together such that all of them can 20389 * be sent down to the lower layer in a single putnext call; this 20390 * linking behavior depends on the tcp_mdt_chain tunable. 20391 * 20392 * The inner loop takes an existing Multidata message, and adds 20393 * one or more (up to tcp_mdt_max_pld) payload buffers to it. It 20394 * packetizes those buffers by filling up the corresponding header 20395 * buffer fragments with the proper IP and TCP headers, and by 20396 * describing the layout of each packet in the packet descriptors 20397 * that get added to the Multidata. 20398 */ 20399 do { 20400 /* 20401 * If usable send window is too small, or data blocks in 20402 * transmit list are smaller than our threshold (i.e. app 20403 * performs large writes followed by small ones), we hand 20404 * off the control over to the legacy path. Note that we'll 20405 * get back the control once it encounters a large block. 20406 */ 20407 if (*usable < mss || (*tail_unsent <= mdt_thres && 20408 (*xmit_tail)->b_cont != NULL && 20409 MBLKL((*xmit_tail)->b_cont) <= mdt_thres)) { 20410 /* send down what we've got so far */ 20411 if (md_mp_head != NULL) { 20412 tcp_multisend_data(tcp, ire, ill, md_mp_head, 20413 obsegs, obbytes, &rconfirm); 20414 } 20415 /* 20416 * Pass control over to tcp_send(), but tell it to 20417 * return to us once a large-size transmission is 20418 * possible. 20419 */ 20420 TCP_STAT(tcps, tcp_mdt_legacy_small); 20421 if ((err = tcp_send(q, tcp, mss, tcp_hdr_len, 20422 tcp_tcp_hdr_len, num_sack_blk, usable, snxt, 20423 tail_unsent, xmit_tail, local_time, 20424 mdt_thres)) <= 0) { 20425 /* burst count reached, or alloc failed */ 20426 IRE_REFRELE(ire); 20427 return (err); 20428 } 20429 20430 /* tcp_send() may have sent everything, so check */ 20431 if (*usable <= 0) { 20432 IRE_REFRELE(ire); 20433 return (0); 20434 } 20435 20436 TCP_STAT(tcps, tcp_mdt_legacy_ret); 20437 /* 20438 * We may have delivered the Multidata, so make sure 20439 * to re-initialize before the next round. 20440 */ 20441 md_mp_head = NULL; 20442 obsegs = obbytes = 0; 20443 num_burst_seg = tcp->tcp_snd_burst; 20444 PREP_NEW_MULTIDATA(); 20445 20446 /* are we starting from the beginning of data block? */ 20447 if (*tail_unsent == 0) { 20448 *xmit_tail = (*xmit_tail)->b_cont; 20449 ASSERT((uintptr_t)MBLKL(*xmit_tail) <= 20450 (uintptr_t)INT_MAX); 20451 *tail_unsent = (int)MBLKL(*xmit_tail); 20452 } 20453 } 20454 20455 /* 20456 * max_pld limits the number of mblks in tcp's transmit 20457 * queue that can be added to a Multidata message. Once 20458 * this counter reaches zero, no more additional mblks 20459 * can be added to it. What happens afterwards depends 20460 * on whether or not we are set to chain the Multidata 20461 * messages. If we are to link them together, reset 20462 * max_pld to its original value (tcp_mdt_max_pld) and 20463 * prepare to create a new Multidata message which will 20464 * get linked to md_mp_head. Else, leave it alone and 20465 * let the inner loop break on its own. 20466 */ 20467 if (tcp_mdt_chain && max_pld == 0) 20468 PREP_NEW_MULTIDATA(); 20469 20470 /* adding a payload buffer; re-initialize values */ 20471 if (add_buffer) 20472 PREP_NEW_PBUF(); 20473 20474 /* 20475 * If we don't have a Multidata, either because we just 20476 * (re)entered this outer loop, or after we branched off 20477 * to tcp_send above, setup the Multidata and header 20478 * buffer to be used. 20479 */ 20480 if (md_mp == NULL) { 20481 int md_hbuflen; 20482 uint32_t start, stuff; 20483 20484 /* 20485 * Calculate Multidata header buffer size large enough 20486 * to hold all of the headers that can possibly be 20487 * sent at this moment. We'd rather over-estimate 20488 * the size than running out of space; this is okay 20489 * since this buffer is small anyway. 20490 */ 20491 md_hbuflen = (howmany(*usable, mss) + 1) * hdr_frag_sz; 20492 20493 /* 20494 * Start and stuff offset for partial hardware 20495 * checksum offload; these are currently for IPv4. 20496 * For full checksum offload, they are set to zero. 20497 */ 20498 if ((hwcksum_flags & HCK_PARTIALCKSUM)) { 20499 if (af == AF_INET) { 20500 start = IP_SIMPLE_HDR_LENGTH; 20501 stuff = IP_SIMPLE_HDR_LENGTH + 20502 TCP_CHECKSUM_OFFSET; 20503 } else { 20504 start = IPV6_HDR_LEN; 20505 stuff = IPV6_HDR_LEN + 20506 TCP_CHECKSUM_OFFSET; 20507 } 20508 } else { 20509 start = stuff = 0; 20510 } 20511 20512 /* 20513 * Create the header buffer, Multidata, as well as 20514 * any necessary attributes (destination address, 20515 * SAP and hardware checksum offload) that should 20516 * be associated with the Multidata message. 20517 */ 20518 ASSERT(cur_hdr_off == 0); 20519 if ((md_hbuf = allocb(md_hbuflen, BPRI_HI)) == NULL || 20520 ((md_hbuf->b_wptr += md_hbuflen), 20521 (mmd = mmd_alloc(md_hbuf, &md_mp, 20522 KM_NOSLEEP)) == NULL) || (tcp_mdt_add_attrs(mmd, 20523 /* fastpath mblk */ 20524 ire->ire_nce->nce_res_mp, 20525 /* hardware checksum enabled */ 20526 (hwcksum_flags & (HCK_FULLCKSUM|HCK_PARTIALCKSUM)), 20527 /* hardware checksum offsets */ 20528 start, stuff, 0, 20529 /* hardware checksum flag */ 20530 hwcksum_flags, tcps) != 0)) { 20531 legacy_send: 20532 if (md_mp != NULL) { 20533 /* Unlink message from the chain */ 20534 if (md_mp_head != NULL) { 20535 err = (intptr_t)rmvb(md_mp_head, 20536 md_mp); 20537 /* 20538 * We can't assert that rmvb 20539 * did not return -1, since we 20540 * may get here before linkb 20541 * happens. We do, however, 20542 * check if we just removed the 20543 * only element in the list. 20544 */ 20545 if (err == 0) 20546 md_mp_head = NULL; 20547 } 20548 /* md_hbuf gets freed automatically */ 20549 TCP_STAT(tcps, tcp_mdt_discarded); 20550 freeb(md_mp); 20551 } else { 20552 /* Either allocb or mmd_alloc failed */ 20553 TCP_STAT(tcps, tcp_mdt_allocfail); 20554 if (md_hbuf != NULL) 20555 freeb(md_hbuf); 20556 } 20557 20558 /* send down what we've got so far */ 20559 if (md_mp_head != NULL) { 20560 tcp_multisend_data(tcp, ire, ill, 20561 md_mp_head, obsegs, obbytes, 20562 &rconfirm); 20563 } 20564 legacy_send_no_md: 20565 if (ire != NULL) 20566 IRE_REFRELE(ire); 20567 /* 20568 * Too bad; let the legacy path handle this. 20569 * We specify INT_MAX for the threshold, since 20570 * we gave up with the Multidata processings 20571 * and let the old path have it all. 20572 */ 20573 TCP_STAT(tcps, tcp_mdt_legacy_all); 20574 return (tcp_send(q, tcp, mss, tcp_hdr_len, 20575 tcp_tcp_hdr_len, num_sack_blk, usable, 20576 snxt, tail_unsent, xmit_tail, local_time, 20577 INT_MAX)); 20578 } 20579 20580 /* link to any existing ones, if applicable */ 20581 TCP_STAT(tcps, tcp_mdt_allocd); 20582 if (md_mp_head == NULL) { 20583 md_mp_head = md_mp; 20584 } else if (tcp_mdt_chain) { 20585 TCP_STAT(tcps, tcp_mdt_linked); 20586 linkb(md_mp_head, md_mp); 20587 } 20588 } 20589 20590 ASSERT(md_mp_head != NULL); 20591 ASSERT(tcp_mdt_chain || md_mp_head->b_cont == NULL); 20592 ASSERT(md_mp != NULL && mmd != NULL); 20593 ASSERT(md_hbuf != NULL); 20594 20595 /* 20596 * Packetize the transmittable portion of the data block; 20597 * each data block is essentially added to the Multidata 20598 * as a payload buffer. We also deal with adding more 20599 * than one payload buffers, which happens when the remaining 20600 * packetized portion of the current payload buffer is less 20601 * than MSS, while the next data block in transmit queue 20602 * has enough data to make up for one. This "spillover" 20603 * case essentially creates a split-packet, where portions 20604 * of the packet's payload fragments may span across two 20605 * virtually discontiguous address blocks. 20606 */ 20607 seg_len = mss; 20608 do { 20609 len = seg_len; 20610 20611 /* one must remain NULL for DTRACE_IP_FASTPATH */ 20612 ipha = NULL; 20613 ip6h = NULL; 20614 20615 ASSERT(len > 0); 20616 ASSERT(max_pld >= 0); 20617 ASSERT(!add_buffer || cur_pld_off == 0); 20618 20619 /* 20620 * First time around for this payload buffer; note 20621 * in the case of a spillover, the following has 20622 * been done prior to adding the split-packet 20623 * descriptor to Multidata, and we don't want to 20624 * repeat the process. 20625 */ 20626 if (add_buffer) { 20627 ASSERT(mmd != NULL); 20628 ASSERT(md_pbuf == NULL); 20629 ASSERT(md_pbuf_nxt == NULL); 20630 ASSERT(pbuf_idx == -1 && pbuf_idx_nxt == -1); 20631 20632 /* 20633 * Have we reached the limit? We'd get to 20634 * this case when we're not chaining the 20635 * Multidata messages together, and since 20636 * we're done, terminate this loop. 20637 */ 20638 if (max_pld == 0) 20639 break; /* done */ 20640 20641 if ((md_pbuf = dupb(*xmit_tail)) == NULL) { 20642 TCP_STAT(tcps, tcp_mdt_allocfail); 20643 goto legacy_send; /* out_of_mem */ 20644 } 20645 20646 if (IS_VMLOANED_MBLK(md_pbuf) && !zcopy && 20647 zc_cap != NULL) { 20648 if (!ip_md_zcopy_attr(mmd, NULL, 20649 zc_cap->ill_zerocopy_flags)) { 20650 freeb(md_pbuf); 20651 TCP_STAT(tcps, 20652 tcp_mdt_allocfail); 20653 /* out_of_mem */ 20654 goto legacy_send; 20655 } 20656 zcopy = B_TRUE; 20657 } 20658 20659 md_pbuf->b_rptr += base_pld_off; 20660 20661 /* 20662 * Add a payload buffer to the Multidata; this 20663 * operation must not fail, or otherwise our 20664 * logic in this routine is broken. There 20665 * is no memory allocation done by the 20666 * routine, so any returned failure simply 20667 * tells us that we've done something wrong. 20668 * 20669 * A failure tells us that either we're adding 20670 * the same payload buffer more than once, or 20671 * we're trying to add more buffers than 20672 * allowed (max_pld calculation is wrong). 20673 * None of the above cases should happen, and 20674 * we panic because either there's horrible 20675 * heap corruption, and/or programming mistake. 20676 */ 20677 pbuf_idx = mmd_addpldbuf(mmd, md_pbuf); 20678 if (pbuf_idx < 0) { 20679 cmn_err(CE_PANIC, "tcp_multisend: " 20680 "payload buffer logic error " 20681 "detected for tcp %p mmd %p " 20682 "pbuf %p (%d)\n", 20683 (void *)tcp, (void *)mmd, 20684 (void *)md_pbuf, pbuf_idx); 20685 } 20686 20687 ASSERT(max_pld > 0); 20688 --max_pld; 20689 add_buffer = B_FALSE; 20690 } 20691 20692 ASSERT(md_mp_head != NULL); 20693 ASSERT(md_pbuf != NULL); 20694 ASSERT(md_pbuf_nxt == NULL); 20695 ASSERT(pbuf_idx != -1); 20696 ASSERT(pbuf_idx_nxt == -1); 20697 ASSERT(*usable > 0); 20698 20699 /* 20700 * We spillover to the next payload buffer only 20701 * if all of the following is true: 20702 * 20703 * 1. There is not enough data on the current 20704 * payload buffer to make up `len', 20705 * 2. We are allowed to send `len', 20706 * 3. The next payload buffer length is large 20707 * enough to accomodate `spill'. 20708 */ 20709 if ((spill = len - *tail_unsent) > 0 && 20710 *usable >= len && 20711 MBLKL((*xmit_tail)->b_cont) >= spill && 20712 max_pld > 0) { 20713 md_pbuf_nxt = dupb((*xmit_tail)->b_cont); 20714 if (md_pbuf_nxt == NULL) { 20715 TCP_STAT(tcps, tcp_mdt_allocfail); 20716 goto legacy_send; /* out_of_mem */ 20717 } 20718 20719 if (IS_VMLOANED_MBLK(md_pbuf_nxt) && !zcopy && 20720 zc_cap != NULL) { 20721 if (!ip_md_zcopy_attr(mmd, NULL, 20722 zc_cap->ill_zerocopy_flags)) { 20723 freeb(md_pbuf_nxt); 20724 TCP_STAT(tcps, 20725 tcp_mdt_allocfail); 20726 /* out_of_mem */ 20727 goto legacy_send; 20728 } 20729 zcopy = B_TRUE; 20730 } 20731 20732 /* 20733 * See comments above on the first call to 20734 * mmd_addpldbuf for explanation on the panic. 20735 */ 20736 pbuf_idx_nxt = mmd_addpldbuf(mmd, md_pbuf_nxt); 20737 if (pbuf_idx_nxt < 0) { 20738 panic("tcp_multisend: " 20739 "next payload buffer logic error " 20740 "detected for tcp %p mmd %p " 20741 "pbuf %p (%d)\n", 20742 (void *)tcp, (void *)mmd, 20743 (void *)md_pbuf_nxt, pbuf_idx_nxt); 20744 } 20745 20746 ASSERT(max_pld > 0); 20747 --max_pld; 20748 } else if (spill > 0) { 20749 /* 20750 * If there's a spillover, but the following 20751 * xmit_tail couldn't give us enough octets 20752 * to reach "len", then stop the current 20753 * Multidata creation and let the legacy 20754 * tcp_send() path take over. We don't want 20755 * to send the tiny segment as part of this 20756 * Multidata for performance reasons; instead, 20757 * we let the legacy path deal with grouping 20758 * it with the subsequent small mblks. 20759 */ 20760 if (*usable >= len && 20761 MBLKL((*xmit_tail)->b_cont) < spill) { 20762 max_pld = 0; 20763 break; /* done */ 20764 } 20765 20766 /* 20767 * We can't spillover, and we are near 20768 * the end of the current payload buffer, 20769 * so send what's left. 20770 */ 20771 ASSERT(*tail_unsent > 0); 20772 len = *tail_unsent; 20773 } 20774 20775 /* tail_unsent is negated if there is a spillover */ 20776 *tail_unsent -= len; 20777 *usable -= len; 20778 ASSERT(*usable >= 0); 20779 20780 if (*usable < mss) 20781 seg_len = *usable; 20782 /* 20783 * Sender SWS avoidance; see comments in tcp_send(); 20784 * everything else is the same, except that we only 20785 * do this here if there is no more data to be sent 20786 * following the current xmit_tail. We don't check 20787 * for 1-byte urgent data because we shouldn't get 20788 * here if TCP_URG_VALID is set. 20789 */ 20790 if (*usable > 0 && *usable < mss && 20791 ((md_pbuf_nxt == NULL && 20792 (*xmit_tail)->b_cont == NULL) || 20793 (md_pbuf_nxt != NULL && 20794 (*xmit_tail)->b_cont->b_cont == NULL)) && 20795 seg_len < (tcp->tcp_max_swnd >> 1) && 20796 (tcp->tcp_unsent - 20797 ((*snxt + len) - tcp->tcp_snxt)) > seg_len && 20798 !tcp->tcp_zero_win_probe) { 20799 if ((*snxt + len) == tcp->tcp_snxt && 20800 (*snxt + len) == tcp->tcp_suna) { 20801 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 20802 } 20803 done = B_TRUE; 20804 } 20805 20806 /* 20807 * Prime pump for IP's checksumming on our behalf; 20808 * include the adjustment for a source route if any. 20809 * Do this only for software/partial hardware checksum 20810 * offload, as this field gets zeroed out later for 20811 * the full hardware checksum offload case. 20812 */ 20813 if (!(hwcksum_flags & HCK_FULLCKSUM)) { 20814 cksum = len + tcp_tcp_hdr_len + tcp->tcp_sum; 20815 cksum = (cksum >> 16) + (cksum & 0xFFFF); 20816 U16_TO_ABE16(cksum, tcp->tcp_tcph->th_sum); 20817 } 20818 20819 U32_TO_ABE32(*snxt, tcp->tcp_tcph->th_seq); 20820 *snxt += len; 20821 20822 tcp->tcp_tcph->th_flags[0] = TH_ACK; 20823 /* 20824 * We set the PUSH bit only if TCP has no more buffered 20825 * data to be transmitted (or if sender SWS avoidance 20826 * takes place), as opposed to setting it for every 20827 * last packet in the burst. 20828 */ 20829 if (done || 20830 (tcp->tcp_unsent - (*snxt - tcp->tcp_snxt)) == 0) 20831 tcp->tcp_tcph->th_flags[0] |= TH_PUSH; 20832 20833 /* 20834 * Set FIN bit if this is our last segment; snxt 20835 * already includes its length, and it will not 20836 * be adjusted after this point. 20837 */ 20838 if (tcp->tcp_valid_bits == TCP_FSS_VALID && 20839 *snxt == tcp->tcp_fss) { 20840 if (!tcp->tcp_fin_acked) { 20841 tcp->tcp_tcph->th_flags[0] |= TH_FIN; 20842 BUMP_MIB(&tcps->tcps_mib, 20843 tcpOutControl); 20844 } 20845 if (!tcp->tcp_fin_sent) { 20846 tcp->tcp_fin_sent = B_TRUE; 20847 /* 20848 * tcp state must be ESTABLISHED 20849 * in order for us to get here in 20850 * the first place. 20851 */ 20852 tcp->tcp_state = TCPS_FIN_WAIT_1; 20853 20854 /* 20855 * Upon returning from this routine, 20856 * tcp_wput_data() will set tcp_snxt 20857 * to be equal to snxt + tcp_fin_sent. 20858 * This is essentially the same as 20859 * setting it to tcp_fss + 1. 20860 */ 20861 } 20862 } 20863 20864 tcp->tcp_last_sent_len = (ushort_t)len; 20865 20866 len += tcp_hdr_len; 20867 if (tcp->tcp_ipversion == IPV4_VERSION) 20868 tcp->tcp_ipha->ipha_length = htons(len); 20869 else 20870 tcp->tcp_ip6h->ip6_plen = htons(len - 20871 ((char *)&tcp->tcp_ip6h[1] - 20872 tcp->tcp_iphc)); 20873 20874 pkt_info->flags = (PDESC_HBUF_REF | PDESC_PBUF_REF); 20875 20876 /* setup header fragment */ 20877 PDESC_HDR_ADD(pkt_info, 20878 md_hbuf->b_rptr + cur_hdr_off, /* base */ 20879 tcp->tcp_mdt_hdr_head, /* head room */ 20880 tcp_hdr_len, /* len */ 20881 tcp->tcp_mdt_hdr_tail); /* tail room */ 20882 20883 ASSERT(pkt_info->hdr_lim - pkt_info->hdr_base == 20884 hdr_frag_sz); 20885 ASSERT(MBLKIN(md_hbuf, 20886 (pkt_info->hdr_base - md_hbuf->b_rptr), 20887 PDESC_HDRSIZE(pkt_info))); 20888 20889 /* setup first payload fragment */ 20890 PDESC_PLD_INIT(pkt_info); 20891 PDESC_PLD_SPAN_ADD(pkt_info, 20892 pbuf_idx, /* index */ 20893 md_pbuf->b_rptr + cur_pld_off, /* start */ 20894 tcp->tcp_last_sent_len); /* len */ 20895 20896 /* create a split-packet in case of a spillover */ 20897 if (md_pbuf_nxt != NULL) { 20898 ASSERT(spill > 0); 20899 ASSERT(pbuf_idx_nxt > pbuf_idx); 20900 ASSERT(!add_buffer); 20901 20902 md_pbuf = md_pbuf_nxt; 20903 md_pbuf_nxt = NULL; 20904 pbuf_idx = pbuf_idx_nxt; 20905 pbuf_idx_nxt = -1; 20906 cur_pld_off = spill; 20907 20908 /* trim out first payload fragment */ 20909 PDESC_PLD_SPAN_TRIM(pkt_info, 0, spill); 20910 20911 /* setup second payload fragment */ 20912 PDESC_PLD_SPAN_ADD(pkt_info, 20913 pbuf_idx, /* index */ 20914 md_pbuf->b_rptr, /* start */ 20915 spill); /* len */ 20916 20917 if ((*xmit_tail)->b_next == NULL) { 20918 /* 20919 * Store the lbolt used for RTT 20920 * estimation. We can only record one 20921 * timestamp per mblk so we do it when 20922 * we reach the end of the payload 20923 * buffer. Also we only take a new 20924 * timestamp sample when the previous 20925 * timed data from the same mblk has 20926 * been ack'ed. 20927 */ 20928 (*xmit_tail)->b_prev = local_time; 20929 (*xmit_tail)->b_next = 20930 (mblk_t *)(uintptr_t)first_snxt; 20931 } 20932 20933 first_snxt = *snxt - spill; 20934 20935 /* 20936 * Advance xmit_tail; usable could be 0 by 20937 * the time we got here, but we made sure 20938 * above that we would only spillover to 20939 * the next data block if usable includes 20940 * the spilled-over amount prior to the 20941 * subtraction. Therefore, we are sure 20942 * that xmit_tail->b_cont can't be NULL. 20943 */ 20944 ASSERT((*xmit_tail)->b_cont != NULL); 20945 *xmit_tail = (*xmit_tail)->b_cont; 20946 ASSERT((uintptr_t)MBLKL(*xmit_tail) <= 20947 (uintptr_t)INT_MAX); 20948 *tail_unsent = (int)MBLKL(*xmit_tail) - spill; 20949 } else { 20950 cur_pld_off += tcp->tcp_last_sent_len; 20951 } 20952 20953 /* 20954 * Fill in the header using the template header, and 20955 * add options such as time-stamp, ECN and/or SACK, 20956 * as needed. 20957 */ 20958 tcp_fill_header(tcp, pkt_info->hdr_rptr, 20959 (clock_t)local_time, num_sack_blk); 20960 20961 /* take care of some IP header businesses */ 20962 if (af == AF_INET) { 20963 ipha = (ipha_t *)pkt_info->hdr_rptr; 20964 20965 ASSERT(OK_32PTR((uchar_t *)ipha)); 20966 ASSERT(PDESC_HDRL(pkt_info) >= 20967 IP_SIMPLE_HDR_LENGTH); 20968 ASSERT(ipha->ipha_version_and_hdr_length == 20969 IP_SIMPLE_HDR_VERSION); 20970 20971 /* 20972 * Assign ident value for current packet; see 20973 * related comments in ip_wput_ire() about the 20974 * contract private interface with clustering 20975 * group. 20976 */ 20977 clusterwide = B_FALSE; 20978 if (cl_inet_ipident != NULL) { 20979 ASSERT(cl_inet_isclusterwide != NULL); 20980 if ((*cl_inet_isclusterwide)(IPPROTO_IP, 20981 AF_INET, 20982 (uint8_t *)(uintptr_t)src)) { 20983 ipha->ipha_ident = 20984 (*cl_inet_ipident) 20985 (IPPROTO_IP, AF_INET, 20986 (uint8_t *)(uintptr_t)src, 20987 (uint8_t *)(uintptr_t)dst); 20988 clusterwide = B_TRUE; 20989 } 20990 } 20991 20992 if (!clusterwide) { 20993 ipha->ipha_ident = (uint16_t) 20994 atomic_add_32_nv( 20995 &ire->ire_ident, 1); 20996 } 20997 #ifndef _BIG_ENDIAN 20998 ipha->ipha_ident = (ipha->ipha_ident << 8) | 20999 (ipha->ipha_ident >> 8); 21000 #endif 21001 } else { 21002 ip6h = (ip6_t *)pkt_info->hdr_rptr; 21003 21004 ASSERT(OK_32PTR((uchar_t *)ip6h)); 21005 ASSERT(IPVER(ip6h) == IPV6_VERSION); 21006 ASSERT(ip6h->ip6_nxt == IPPROTO_TCP); 21007 ASSERT(PDESC_HDRL(pkt_info) >= 21008 (IPV6_HDR_LEN + TCP_CHECKSUM_OFFSET + 21009 TCP_CHECKSUM_SIZE)); 21010 ASSERT(tcp->tcp_ipversion == IPV6_VERSION); 21011 21012 if (tcp->tcp_ip_forward_progress) { 21013 rconfirm = B_TRUE; 21014 tcp->tcp_ip_forward_progress = B_FALSE; 21015 } 21016 } 21017 21018 /* at least one payload span, and at most two */ 21019 ASSERT(pkt_info->pld_cnt > 0 && pkt_info->pld_cnt < 3); 21020 21021 /* add the packet descriptor to Multidata */ 21022 if ((pkt = mmd_addpdesc(mmd, pkt_info, &err, 21023 KM_NOSLEEP)) == NULL) { 21024 /* 21025 * Any failure other than ENOMEM indicates 21026 * that we have passed in invalid pkt_info 21027 * or parameters to mmd_addpdesc, which must 21028 * not happen. 21029 * 21030 * EINVAL is a result of failure on boundary 21031 * checks against the pkt_info contents. It 21032 * should not happen, and we panic because 21033 * either there's horrible heap corruption, 21034 * and/or programming mistake. 21035 */ 21036 if (err != ENOMEM) { 21037 cmn_err(CE_PANIC, "tcp_multisend: " 21038 "pdesc logic error detected for " 21039 "tcp %p mmd %p pinfo %p (%d)\n", 21040 (void *)tcp, (void *)mmd, 21041 (void *)pkt_info, err); 21042 } 21043 TCP_STAT(tcps, tcp_mdt_addpdescfail); 21044 goto legacy_send; /* out_of_mem */ 21045 } 21046 ASSERT(pkt != NULL); 21047 21048 /* calculate IP header and TCP checksums */ 21049 if (af == AF_INET) { 21050 /* calculate pseudo-header checksum */ 21051 cksum = (dst >> 16) + (dst & 0xFFFF) + 21052 (src >> 16) + (src & 0xFFFF); 21053 21054 /* offset for TCP header checksum */ 21055 up = IPH_TCPH_CHECKSUMP(ipha, 21056 IP_SIMPLE_HDR_LENGTH); 21057 } else { 21058 up = (uint16_t *)&ip6h->ip6_src; 21059 21060 /* calculate pseudo-header checksum */ 21061 cksum = up[0] + up[1] + up[2] + up[3] + 21062 up[4] + up[5] + up[6] + up[7] + 21063 up[8] + up[9] + up[10] + up[11] + 21064 up[12] + up[13] + up[14] + up[15]; 21065 21066 /* Fold the initial sum */ 21067 cksum = (cksum & 0xffff) + (cksum >> 16); 21068 21069 up = (uint16_t *)(((uchar_t *)ip6h) + 21070 IPV6_HDR_LEN + TCP_CHECKSUM_OFFSET); 21071 } 21072 21073 if (hwcksum_flags & HCK_FULLCKSUM) { 21074 /* clear checksum field for hardware */ 21075 *up = 0; 21076 } else if (hwcksum_flags & HCK_PARTIALCKSUM) { 21077 uint32_t sum; 21078 21079 /* pseudo-header checksumming */ 21080 sum = *up + cksum + IP_TCP_CSUM_COMP; 21081 sum = (sum & 0xFFFF) + (sum >> 16); 21082 *up = (sum & 0xFFFF) + (sum >> 16); 21083 } else { 21084 /* software checksumming */ 21085 TCP_STAT(tcps, tcp_out_sw_cksum); 21086 TCP_STAT_UPDATE(tcps, tcp_out_sw_cksum_bytes, 21087 tcp->tcp_hdr_len + tcp->tcp_last_sent_len); 21088 *up = IP_MD_CSUM(pkt, tcp->tcp_ip_hdr_len, 21089 cksum + IP_TCP_CSUM_COMP); 21090 if (*up == 0) 21091 *up = 0xFFFF; 21092 } 21093 21094 /* IPv4 header checksum */ 21095 if (af == AF_INET) { 21096 ipha->ipha_fragment_offset_and_flags |= 21097 (uint32_t)htons(ire->ire_frag_flag); 21098 21099 if (hwcksum_flags & HCK_IPV4_HDRCKSUM) { 21100 ipha->ipha_hdr_checksum = 0; 21101 } else { 21102 IP_HDR_CKSUM(ipha, cksum, 21103 ((uint32_t *)ipha)[0], 21104 ((uint16_t *)ipha)[4]); 21105 } 21106 } 21107 21108 if (af == AF_INET && 21109 HOOKS4_INTERESTED_PHYSICAL_OUT(ipst) || 21110 af == AF_INET6 && 21111 HOOKS6_INTERESTED_PHYSICAL_OUT(ipst)) { 21112 /* build header(IP/TCP) mblk for this segment */ 21113 if ((mp = dupb(md_hbuf)) == NULL) 21114 goto legacy_send; 21115 21116 mp->b_rptr = pkt_info->hdr_rptr; 21117 mp->b_wptr = pkt_info->hdr_wptr; 21118 21119 /* build payload mblk for this segment */ 21120 if ((mp1 = dupb(*xmit_tail)) == NULL) { 21121 freemsg(mp); 21122 goto legacy_send; 21123 } 21124 mp1->b_wptr = md_pbuf->b_rptr + cur_pld_off; 21125 mp1->b_rptr = mp1->b_wptr - 21126 tcp->tcp_last_sent_len; 21127 linkb(mp, mp1); 21128 21129 pld_start = mp1->b_rptr; 21130 21131 if (af == AF_INET) { 21132 DTRACE_PROBE4( 21133 ip4__physical__out__start, 21134 ill_t *, NULL, 21135 ill_t *, ill, 21136 ipha_t *, ipha, 21137 mblk_t *, mp); 21138 FW_HOOKS( 21139 ipst->ips_ip4_physical_out_event, 21140 ipst->ips_ipv4firewall_physical_out, 21141 NULL, ill, ipha, mp, mp, 0, ipst); 21142 DTRACE_PROBE1( 21143 ip4__physical__out__end, 21144 mblk_t *, mp); 21145 } else { 21146 DTRACE_PROBE4( 21147 ip6__physical__out_start, 21148 ill_t *, NULL, 21149 ill_t *, ill, 21150 ip6_t *, ip6h, 21151 mblk_t *, mp); 21152 FW_HOOKS6( 21153 ipst->ips_ip6_physical_out_event, 21154 ipst->ips_ipv6firewall_physical_out, 21155 NULL, ill, ip6h, mp, mp, 0, ipst); 21156 DTRACE_PROBE1( 21157 ip6__physical__out__end, 21158 mblk_t *, mp); 21159 } 21160 21161 if (buf_trunked && mp != NULL) { 21162 /* 21163 * Need to pass it to normal path. 21164 */ 21165 CALL_IP_WPUT(tcp->tcp_connp, q, mp); 21166 mp = NULL; 21167 } else if (mp == NULL || 21168 mp->b_rptr != pkt_info->hdr_rptr || 21169 mp->b_wptr != pkt_info->hdr_wptr || 21170 (mp1 = mp->b_cont) == NULL || 21171 mp1->b_rptr != pld_start || 21172 mp1->b_wptr != pld_start + 21173 tcp->tcp_last_sent_len || 21174 mp1->b_cont != NULL) { 21175 /* 21176 * Need to pass all packets of this 21177 * buffer to normal path, either when 21178 * packet is blocked, or when boundary 21179 * of header buffer or payload buffer 21180 * has been changed by FW_HOOKS[6]. 21181 */ 21182 buf_trunked = B_TRUE; 21183 if (md_mp_head != NULL) { 21184 err = (intptr_t)rmvb(md_mp_head, 21185 md_mp); 21186 if (err == 0) 21187 md_mp_head = NULL; 21188 } 21189 21190 /* send down what we've got so far */ 21191 if (md_mp_head != NULL) { 21192 tcp_multisend_data(tcp, ire, 21193 ill, md_mp_head, obsegs, 21194 obbytes, &rconfirm); 21195 } 21196 md_mp_head = NULL; 21197 21198 if (mp != NULL) 21199 CALL_IP_WPUT(tcp->tcp_connp, 21200 q, mp); 21201 21202 mp1 = fw_mp_head; 21203 do { 21204 mp = mp1; 21205 mp1 = mp1->b_next; 21206 mp->b_next = NULL; 21207 mp->b_prev = NULL; 21208 CALL_IP_WPUT(tcp->tcp_connp, 21209 q, mp); 21210 } while (mp1 != NULL); 21211 21212 fw_mp_head = mp = NULL; 21213 } else { 21214 if (fw_mp_head == NULL) 21215 fw_mp_head = mp; 21216 else 21217 fw_mp_head->b_prev->b_next = mp; 21218 fw_mp_head->b_prev = mp; 21219 } 21220 } 21221 21222 if (mp != NULL) { 21223 DTRACE_IP_FASTPATH(md_hbuf, pkt_info->hdr_rptr, 21224 ill, ipha, ip6h); 21225 } 21226 21227 /* advance header offset */ 21228 cur_hdr_off += hdr_frag_sz; 21229 21230 obbytes += tcp->tcp_last_sent_len; 21231 ++obsegs; 21232 } while (!done && *usable > 0 && --num_burst_seg > 0 && 21233 *tail_unsent > 0); 21234 21235 if ((*xmit_tail)->b_next == NULL) { 21236 /* 21237 * Store the lbolt used for RTT estimation. We can only 21238 * record one timestamp per mblk so we do it when we 21239 * reach the end of the payload buffer. Also we only 21240 * take a new timestamp sample when the previous timed 21241 * data from the same mblk has been ack'ed. 21242 */ 21243 (*xmit_tail)->b_prev = local_time; 21244 (*xmit_tail)->b_next = (mblk_t *)(uintptr_t)first_snxt; 21245 } 21246 21247 ASSERT(*tail_unsent >= 0); 21248 if (*tail_unsent > 0) { 21249 /* 21250 * We got here because we broke out of the above 21251 * loop due to of one of the following cases: 21252 * 21253 * 1. len < adjusted MSS (i.e. small), 21254 * 2. Sender SWS avoidance, 21255 * 3. max_pld is zero. 21256 * 21257 * We are done for this Multidata, so trim our 21258 * last payload buffer (if any) accordingly. 21259 */ 21260 if (md_pbuf != NULL) 21261 md_pbuf->b_wptr -= *tail_unsent; 21262 } else if (*usable > 0) { 21263 *xmit_tail = (*xmit_tail)->b_cont; 21264 ASSERT((uintptr_t)MBLKL(*xmit_tail) <= 21265 (uintptr_t)INT_MAX); 21266 *tail_unsent = (int)MBLKL(*xmit_tail); 21267 add_buffer = B_TRUE; 21268 } 21269 21270 while (fw_mp_head) { 21271 mp = fw_mp_head; 21272 fw_mp_head = fw_mp_head->b_next; 21273 mp->b_prev = mp->b_next = NULL; 21274 freemsg(mp); 21275 } 21276 if (buf_trunked) { 21277 TCP_STAT(tcps, tcp_mdt_discarded); 21278 freeb(md_mp); 21279 buf_trunked = B_FALSE; 21280 } 21281 } while (!done && *usable > 0 && num_burst_seg > 0 && 21282 (tcp_mdt_chain || max_pld > 0)); 21283 21284 if (md_mp_head != NULL) { 21285 /* send everything down */ 21286 tcp_multisend_data(tcp, ire, ill, md_mp_head, obsegs, obbytes, 21287 &rconfirm); 21288 } 21289 21290 #undef PREP_NEW_MULTIDATA 21291 #undef PREP_NEW_PBUF 21292 #undef IPVER 21293 21294 IRE_REFRELE(ire); 21295 return (0); 21296 } 21297 21298 /* 21299 * A wrapper function for sending one or more Multidata messages down to 21300 * the module below ip; this routine does not release the reference of the 21301 * IRE (caller does that). This routine is analogous to tcp_send_data(). 21302 */ 21303 static void 21304 tcp_multisend_data(tcp_t *tcp, ire_t *ire, const ill_t *ill, mblk_t *md_mp_head, 21305 const uint_t obsegs, const uint_t obbytes, boolean_t *rconfirm) 21306 { 21307 uint64_t delta; 21308 nce_t *nce; 21309 tcp_stack_t *tcps = tcp->tcp_tcps; 21310 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 21311 21312 ASSERT(ire != NULL && ill != NULL); 21313 ASSERT(ire->ire_stq != NULL); 21314 ASSERT(md_mp_head != NULL); 21315 ASSERT(rconfirm != NULL); 21316 21317 /* adjust MIBs and IRE timestamp */ 21318 DTRACE_PROBE2(tcp__trace__send, mblk_t *, md_mp_head, tcp_t *, tcp); 21319 tcp->tcp_obsegs += obsegs; 21320 UPDATE_MIB(&tcps->tcps_mib, tcpOutDataSegs, obsegs); 21321 UPDATE_MIB(&tcps->tcps_mib, tcpOutDataBytes, obbytes); 21322 TCP_STAT_UPDATE(tcps, tcp_mdt_pkt_out, obsegs); 21323 21324 if (tcp->tcp_ipversion == IPV4_VERSION) { 21325 TCP_STAT_UPDATE(tcps, tcp_mdt_pkt_out_v4, obsegs); 21326 } else { 21327 TCP_STAT_UPDATE(tcps, tcp_mdt_pkt_out_v6, obsegs); 21328 } 21329 UPDATE_MIB(ill->ill_ip_mib, ipIfStatsHCOutRequests, obsegs); 21330 UPDATE_MIB(ill->ill_ip_mib, ipIfStatsHCOutTransmits, obsegs); 21331 UPDATE_MIB(ill->ill_ip_mib, ipIfStatsHCOutOctets, obbytes); 21332 21333 ire->ire_ob_pkt_count += obsegs; 21334 if (ire->ire_ipif != NULL) 21335 atomic_add_32(&ire->ire_ipif->ipif_ob_pkt_count, obsegs); 21336 ire->ire_last_used_time = lbolt; 21337 21338 /* send it down */ 21339 if (ILL_DLS_CAPABLE(ill)) { 21340 ill_dls_capab_t *ill_dls = ill->ill_dls_capab; 21341 ill_dls->ill_tx(ill_dls->ill_tx_handle, md_mp_head); 21342 } else { 21343 putnext(ire->ire_stq, md_mp_head); 21344 } 21345 21346 /* we're done for TCP/IPv4 */ 21347 if (tcp->tcp_ipversion == IPV4_VERSION) 21348 return; 21349 21350 nce = ire->ire_nce; 21351 21352 ASSERT(nce != NULL); 21353 ASSERT(!(nce->nce_flags & (NCE_F_NONUD|NCE_F_PERMANENT))); 21354 ASSERT(nce->nce_state != ND_INCOMPLETE); 21355 21356 /* reachability confirmation? */ 21357 if (*rconfirm) { 21358 nce->nce_last = TICK_TO_MSEC(lbolt64); 21359 if (nce->nce_state != ND_REACHABLE) { 21360 mutex_enter(&nce->nce_lock); 21361 nce->nce_state = ND_REACHABLE; 21362 nce->nce_pcnt = ND_MAX_UNICAST_SOLICIT; 21363 mutex_exit(&nce->nce_lock); 21364 (void) untimeout(nce->nce_timeout_id); 21365 if (ip_debug > 2) { 21366 /* ip1dbg */ 21367 pr_addr_dbg("tcp_multisend_data: state " 21368 "for %s changed to REACHABLE\n", 21369 AF_INET6, &ire->ire_addr_v6); 21370 } 21371 } 21372 /* reset transport reachability confirmation */ 21373 *rconfirm = B_FALSE; 21374 } 21375 21376 delta = TICK_TO_MSEC(lbolt64) - nce->nce_last; 21377 ip1dbg(("tcp_multisend_data: delta = %" PRId64 21378 " ill_reachable_time = %d \n", delta, ill->ill_reachable_time)); 21379 21380 if (delta > (uint64_t)ill->ill_reachable_time) { 21381 mutex_enter(&nce->nce_lock); 21382 switch (nce->nce_state) { 21383 case ND_REACHABLE: 21384 case ND_STALE: 21385 /* 21386 * ND_REACHABLE is identical to ND_STALE in this 21387 * specific case. If reachable time has expired for 21388 * this neighbor (delta is greater than reachable 21389 * time), conceptually, the neighbor cache is no 21390 * longer in REACHABLE state, but already in STALE 21391 * state. So the correct transition here is to 21392 * ND_DELAY. 21393 */ 21394 nce->nce_state = ND_DELAY; 21395 mutex_exit(&nce->nce_lock); 21396 NDP_RESTART_TIMER(nce, 21397 ipst->ips_delay_first_probe_time); 21398 if (ip_debug > 3) { 21399 /* ip2dbg */ 21400 pr_addr_dbg("tcp_multisend_data: state " 21401 "for %s changed to DELAY\n", 21402 AF_INET6, &ire->ire_addr_v6); 21403 } 21404 break; 21405 case ND_DELAY: 21406 case ND_PROBE: 21407 mutex_exit(&nce->nce_lock); 21408 /* Timers have already started */ 21409 break; 21410 case ND_UNREACHABLE: 21411 /* 21412 * ndp timer has detected that this nce is 21413 * unreachable and initiated deleting this nce 21414 * and all its associated IREs. This is a race 21415 * where we found the ire before it was deleted 21416 * and have just sent out a packet using this 21417 * unreachable nce. 21418 */ 21419 mutex_exit(&nce->nce_lock); 21420 break; 21421 default: 21422 ASSERT(0); 21423 } 21424 } 21425 } 21426 21427 /* 21428 * Derived from tcp_send_data(). 21429 */ 21430 static void 21431 tcp_lsosend_data(tcp_t *tcp, mblk_t *mp, ire_t *ire, ill_t *ill, const int mss, 21432 int num_lso_seg) 21433 { 21434 ipha_t *ipha; 21435 mblk_t *ire_fp_mp; 21436 uint_t ire_fp_mp_len; 21437 uint32_t hcksum_txflags = 0; 21438 ipaddr_t src; 21439 ipaddr_t dst; 21440 uint32_t cksum; 21441 uint16_t *up; 21442 tcp_stack_t *tcps = tcp->tcp_tcps; 21443 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 21444 21445 ASSERT(DB_TYPE(mp) == M_DATA); 21446 ASSERT(tcp->tcp_state == TCPS_ESTABLISHED); 21447 ASSERT(tcp->tcp_ipversion == IPV4_VERSION); 21448 ASSERT(tcp->tcp_connp != NULL); 21449 ASSERT(CONN_IS_LSO_MD_FASTPATH(tcp->tcp_connp)); 21450 21451 ipha = (ipha_t *)mp->b_rptr; 21452 src = ipha->ipha_src; 21453 dst = ipha->ipha_dst; 21454 21455 DTRACE_PROBE2(tcp__trace__send, mblk_t *, mp, tcp_t *, tcp); 21456 21457 ASSERT(ipha->ipha_ident == 0 || ipha->ipha_ident == IP_HDR_INCLUDED); 21458 ipha->ipha_ident = (uint16_t)atomic_add_32_nv(&ire->ire_ident, 21459 num_lso_seg); 21460 #ifndef _BIG_ENDIAN 21461 ipha->ipha_ident = (ipha->ipha_ident << 8) | (ipha->ipha_ident >> 8); 21462 #endif 21463 if (tcp->tcp_snd_zcopy_aware) { 21464 if ((ill->ill_capabilities & ILL_CAPAB_ZEROCOPY) == 0 || 21465 (ill->ill_zerocopy_capab->ill_zerocopy_flags == 0)) 21466 mp = tcp_zcopy_disable(tcp, mp); 21467 } 21468 21469 if (ILL_HCKSUM_CAPABLE(ill) && dohwcksum) { 21470 ASSERT(ill->ill_hcksum_capab != NULL); 21471 hcksum_txflags = ill->ill_hcksum_capab->ill_hcksum_txflags; 21472 } 21473 21474 /* 21475 * Since the TCP checksum should be recalculated by h/w, we can just 21476 * zero the checksum field for HCK_FULLCKSUM, or calculate partial 21477 * pseudo-header checksum for HCK_PARTIALCKSUM. 21478 * The partial pseudo-header excludes TCP length, that was calculated 21479 * in tcp_send(), so to zero *up before further processing. 21480 */ 21481 cksum = (dst >> 16) + (dst & 0xFFFF) + (src >> 16) + (src & 0xFFFF); 21482 21483 up = IPH_TCPH_CHECKSUMP(ipha, IP_SIMPLE_HDR_LENGTH); 21484 *up = 0; 21485 21486 IP_CKSUM_XMIT_FAST(ire->ire_ipversion, hcksum_txflags, mp, ipha, up, 21487 IPPROTO_TCP, IP_SIMPLE_HDR_LENGTH, ntohs(ipha->ipha_length), cksum); 21488 21489 /* 21490 * Append LSO flag to DB_LSOFLAGS(mp) and set the mss to DB_LSOMSS(mp). 21491 */ 21492 DB_LSOFLAGS(mp) |= HW_LSO; 21493 DB_LSOMSS(mp) = mss; 21494 21495 ipha->ipha_fragment_offset_and_flags |= 21496 (uint32_t)htons(ire->ire_frag_flag); 21497 21498 ire_fp_mp = ire->ire_nce->nce_fp_mp; 21499 ire_fp_mp_len = MBLKL(ire_fp_mp); 21500 ASSERT(DB_TYPE(ire_fp_mp) == M_DATA); 21501 mp->b_rptr = (uchar_t *)ipha - ire_fp_mp_len; 21502 bcopy(ire_fp_mp->b_rptr, mp->b_rptr, ire_fp_mp_len); 21503 21504 UPDATE_OB_PKT_COUNT(ire); 21505 ire->ire_last_used_time = lbolt; 21506 BUMP_MIB(ill->ill_ip_mib, ipIfStatsHCOutRequests); 21507 BUMP_MIB(ill->ill_ip_mib, ipIfStatsHCOutTransmits); 21508 UPDATE_MIB(ill->ill_ip_mib, ipIfStatsHCOutOctets, 21509 ntohs(ipha->ipha_length)); 21510 21511 if (ILL_DLS_CAPABLE(ill)) { 21512 /* 21513 * Send the packet directly to DLD, where it may be queued 21514 * depending on the availability of transmit resources at 21515 * the media layer. 21516 */ 21517 IP_DLS_ILL_TX(ill, ipha, mp, ipst); 21518 } else { 21519 ill_t *out_ill = (ill_t *)ire->ire_stq->q_ptr; 21520 DTRACE_PROBE4(ip4__physical__out__start, 21521 ill_t *, NULL, ill_t *, out_ill, 21522 ipha_t *, ipha, mblk_t *, mp); 21523 FW_HOOKS(ipst->ips_ip4_physical_out_event, 21524 ipst->ips_ipv4firewall_physical_out, 21525 NULL, out_ill, ipha, mp, mp, 0, ipst); 21526 DTRACE_PROBE1(ip4__physical__out__end, mblk_t *, mp); 21527 21528 if (mp != NULL) { 21529 DTRACE_IP_FASTPATH(mp, ipha, out_ill, ipha, NULL); 21530 putnext(ire->ire_stq, mp); 21531 } 21532 } 21533 } 21534 21535 /* 21536 * tcp_send() is called by tcp_wput_data() for non-Multidata transmission 21537 * scheme, and returns one of the following: 21538 * 21539 * -1 = failed allocation. 21540 * 0 = success; burst count reached, or usable send window is too small, 21541 * and that we'd rather wait until later before sending again. 21542 * 1 = success; we are called from tcp_multisend(), and both usable send 21543 * window and tail_unsent are greater than the MDT threshold, and thus 21544 * Multidata Transmit should be used instead. 21545 */ 21546 static int 21547 tcp_send(queue_t *q, tcp_t *tcp, const int mss, const int tcp_hdr_len, 21548 const int tcp_tcp_hdr_len, const int num_sack_blk, int *usable, 21549 uint_t *snxt, int *tail_unsent, mblk_t **xmit_tail, mblk_t *local_time, 21550 const int mdt_thres) 21551 { 21552 int num_burst_seg = tcp->tcp_snd_burst; 21553 ire_t *ire = NULL; 21554 ill_t *ill = NULL; 21555 mblk_t *ire_fp_mp = NULL; 21556 uint_t ire_fp_mp_len = 0; 21557 int num_lso_seg = 1; 21558 uint_t lso_usable; 21559 boolean_t do_lso_send = B_FALSE; 21560 tcp_stack_t *tcps = tcp->tcp_tcps; 21561 21562 /* 21563 * Check LSO capability before any further work. And the similar check 21564 * need to be done in for(;;) loop. 21565 * LSO will be deployed when therer is more than one mss of available 21566 * data and a burst transmission is allowed. 21567 */ 21568 if (tcp->tcp_lso && 21569 (tcp->tcp_valid_bits == 0 || 21570 tcp->tcp_valid_bits == TCP_FSS_VALID) && 21571 num_burst_seg >= 2 && (*usable - 1) / mss >= 1) { 21572 /* 21573 * Try to find usable IRE/ILL and do basic check to the ILL. 21574 */ 21575 if (tcp_send_find_ire_ill(tcp, NULL, &ire, &ill)) { 21576 /* 21577 * Enable LSO with this transmission. 21578 * Since IRE has been hold in 21579 * tcp_send_find_ire_ill(), IRE_REFRELE(ire) 21580 * should be called before return. 21581 */ 21582 do_lso_send = B_TRUE; 21583 ire_fp_mp = ire->ire_nce->nce_fp_mp; 21584 ire_fp_mp_len = MBLKL(ire_fp_mp); 21585 /* Round up to multiple of 4 */ 21586 ire_fp_mp_len = ((ire_fp_mp_len + 3) / 4) * 4; 21587 } else { 21588 do_lso_send = B_FALSE; 21589 ill = NULL; 21590 } 21591 } 21592 21593 for (;;) { 21594 struct datab *db; 21595 tcph_t *tcph; 21596 uint32_t sum; 21597 mblk_t *mp, *mp1; 21598 uchar_t *rptr; 21599 int len; 21600 21601 /* 21602 * If we're called by tcp_multisend(), and the amount of 21603 * sendable data as well as the size of current xmit_tail 21604 * is beyond the MDT threshold, return to the caller and 21605 * let the large data transmit be done using MDT. 21606 */ 21607 if (*usable > 0 && *usable > mdt_thres && 21608 (*tail_unsent > mdt_thres || (*tail_unsent == 0 && 21609 MBLKL((*xmit_tail)->b_cont) > mdt_thres))) { 21610 ASSERT(tcp->tcp_mdt); 21611 return (1); /* success; do large send */ 21612 } 21613 21614 if (num_burst_seg == 0) 21615 break; /* success; burst count reached */ 21616 21617 /* 21618 * Calculate the maximum payload length we can send in *one* 21619 * time. 21620 */ 21621 if (do_lso_send) { 21622 /* 21623 * Check whether need to do LSO any more. 21624 */ 21625 if (num_burst_seg >= 2 && (*usable - 1) / mss >= 1) { 21626 lso_usable = MIN(tcp->tcp_lso_max, *usable); 21627 lso_usable = MIN(lso_usable, 21628 num_burst_seg * mss); 21629 21630 num_lso_seg = lso_usable / mss; 21631 if (lso_usable % mss) { 21632 num_lso_seg++; 21633 tcp->tcp_last_sent_len = (ushort_t) 21634 (lso_usable % mss); 21635 } else { 21636 tcp->tcp_last_sent_len = (ushort_t)mss; 21637 } 21638 } else { 21639 do_lso_send = B_FALSE; 21640 num_lso_seg = 1; 21641 lso_usable = mss; 21642 } 21643 } 21644 21645 ASSERT(num_lso_seg <= IP_MAXPACKET / mss + 1); 21646 21647 /* 21648 * Adjust num_burst_seg here. 21649 */ 21650 num_burst_seg -= num_lso_seg; 21651 21652 len = mss; 21653 if (len > *usable) { 21654 ASSERT(do_lso_send == B_FALSE); 21655 21656 len = *usable; 21657 if (len <= 0) { 21658 /* Terminate the loop */ 21659 break; /* success; too small */ 21660 } 21661 /* 21662 * Sender silly-window avoidance. 21663 * Ignore this if we are going to send a 21664 * zero window probe out. 21665 * 21666 * TODO: force data into microscopic window? 21667 * ==> (!pushed || (unsent > usable)) 21668 */ 21669 if (len < (tcp->tcp_max_swnd >> 1) && 21670 (tcp->tcp_unsent - (*snxt - tcp->tcp_snxt)) > len && 21671 !((tcp->tcp_valid_bits & TCP_URG_VALID) && 21672 len == 1) && (! tcp->tcp_zero_win_probe)) { 21673 /* 21674 * If the retransmit timer is not running 21675 * we start it so that we will retransmit 21676 * in the case when the the receiver has 21677 * decremented the window. 21678 */ 21679 if (*snxt == tcp->tcp_snxt && 21680 *snxt == tcp->tcp_suna) { 21681 /* 21682 * We are not supposed to send 21683 * anything. So let's wait a little 21684 * bit longer before breaking SWS 21685 * avoidance. 21686 * 21687 * What should the value be? 21688 * Suggestion: MAX(init rexmit time, 21689 * tcp->tcp_rto) 21690 */ 21691 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 21692 } 21693 break; /* success; too small */ 21694 } 21695 } 21696 21697 tcph = tcp->tcp_tcph; 21698 21699 /* 21700 * The reason to adjust len here is that we need to set flags 21701 * and calculate checksum. 21702 */ 21703 if (do_lso_send) 21704 len = lso_usable; 21705 21706 *usable -= len; /* Approximate - can be adjusted later */ 21707 if (*usable > 0) 21708 tcph->th_flags[0] = TH_ACK; 21709 else 21710 tcph->th_flags[0] = (TH_ACK | TH_PUSH); 21711 21712 /* 21713 * Prime pump for IP's checksumming on our behalf 21714 * Include the adjustment for a source route if any. 21715 */ 21716 sum = len + tcp_tcp_hdr_len + tcp->tcp_sum; 21717 sum = (sum >> 16) + (sum & 0xFFFF); 21718 U16_TO_ABE16(sum, tcph->th_sum); 21719 21720 U32_TO_ABE32(*snxt, tcph->th_seq); 21721 21722 /* 21723 * Branch off to tcp_xmit_mp() if any of the VALID bits is 21724 * set. For the case when TCP_FSS_VALID is the only valid 21725 * bit (normal active close), branch off only when we think 21726 * that the FIN flag needs to be set. Note for this case, 21727 * that (snxt + len) may not reflect the actual seg_len, 21728 * as len may be further reduced in tcp_xmit_mp(). If len 21729 * gets modified, we will end up here again. 21730 */ 21731 if (tcp->tcp_valid_bits != 0 && 21732 (tcp->tcp_valid_bits != TCP_FSS_VALID || 21733 ((*snxt + len) == tcp->tcp_fss))) { 21734 uchar_t *prev_rptr; 21735 uint32_t prev_snxt = tcp->tcp_snxt; 21736 21737 if (*tail_unsent == 0) { 21738 ASSERT((*xmit_tail)->b_cont != NULL); 21739 *xmit_tail = (*xmit_tail)->b_cont; 21740 prev_rptr = (*xmit_tail)->b_rptr; 21741 *tail_unsent = (int)((*xmit_tail)->b_wptr - 21742 (*xmit_tail)->b_rptr); 21743 } else { 21744 prev_rptr = (*xmit_tail)->b_rptr; 21745 (*xmit_tail)->b_rptr = (*xmit_tail)->b_wptr - 21746 *tail_unsent; 21747 } 21748 mp = tcp_xmit_mp(tcp, *xmit_tail, len, NULL, NULL, 21749 *snxt, B_FALSE, (uint32_t *)&len, B_FALSE); 21750 /* Restore tcp_snxt so we get amount sent right. */ 21751 tcp->tcp_snxt = prev_snxt; 21752 if (prev_rptr == (*xmit_tail)->b_rptr) { 21753 /* 21754 * If the previous timestamp is still in use, 21755 * don't stomp on it. 21756 */ 21757 if ((*xmit_tail)->b_next == NULL) { 21758 (*xmit_tail)->b_prev = local_time; 21759 (*xmit_tail)->b_next = 21760 (mblk_t *)(uintptr_t)(*snxt); 21761 } 21762 } else 21763 (*xmit_tail)->b_rptr = prev_rptr; 21764 21765 if (mp == NULL) { 21766 if (ire != NULL) 21767 IRE_REFRELE(ire); 21768 return (-1); 21769 } 21770 mp1 = mp->b_cont; 21771 21772 if (len <= mss) /* LSO is unusable (!do_lso_send) */ 21773 tcp->tcp_last_sent_len = (ushort_t)len; 21774 while (mp1->b_cont) { 21775 *xmit_tail = (*xmit_tail)->b_cont; 21776 (*xmit_tail)->b_prev = local_time; 21777 (*xmit_tail)->b_next = 21778 (mblk_t *)(uintptr_t)(*snxt); 21779 mp1 = mp1->b_cont; 21780 } 21781 *snxt += len; 21782 *tail_unsent = (*xmit_tail)->b_wptr - mp1->b_wptr; 21783 BUMP_LOCAL(tcp->tcp_obsegs); 21784 BUMP_MIB(&tcps->tcps_mib, tcpOutDataSegs); 21785 UPDATE_MIB(&tcps->tcps_mib, tcpOutDataBytes, len); 21786 tcp_send_data(tcp, q, mp); 21787 continue; 21788 } 21789 21790 *snxt += len; /* Adjust later if we don't send all of len */ 21791 BUMP_MIB(&tcps->tcps_mib, tcpOutDataSegs); 21792 UPDATE_MIB(&tcps->tcps_mib, tcpOutDataBytes, len); 21793 21794 if (*tail_unsent) { 21795 /* Are the bytes above us in flight? */ 21796 rptr = (*xmit_tail)->b_wptr - *tail_unsent; 21797 if (rptr != (*xmit_tail)->b_rptr) { 21798 *tail_unsent -= len; 21799 if (len <= mss) /* LSO is unusable */ 21800 tcp->tcp_last_sent_len = (ushort_t)len; 21801 len += tcp_hdr_len; 21802 if (tcp->tcp_ipversion == IPV4_VERSION) 21803 tcp->tcp_ipha->ipha_length = htons(len); 21804 else 21805 tcp->tcp_ip6h->ip6_plen = 21806 htons(len - 21807 ((char *)&tcp->tcp_ip6h[1] - 21808 tcp->tcp_iphc)); 21809 mp = dupb(*xmit_tail); 21810 if (mp == NULL) { 21811 if (ire != NULL) 21812 IRE_REFRELE(ire); 21813 return (-1); /* out_of_mem */ 21814 } 21815 mp->b_rptr = rptr; 21816 /* 21817 * If the old timestamp is no longer in use, 21818 * sample a new timestamp now. 21819 */ 21820 if ((*xmit_tail)->b_next == NULL) { 21821 (*xmit_tail)->b_prev = local_time; 21822 (*xmit_tail)->b_next = 21823 (mblk_t *)(uintptr_t)(*snxt-len); 21824 } 21825 goto must_alloc; 21826 } 21827 } else { 21828 *xmit_tail = (*xmit_tail)->b_cont; 21829 ASSERT((uintptr_t)((*xmit_tail)->b_wptr - 21830 (*xmit_tail)->b_rptr) <= (uintptr_t)INT_MAX); 21831 *tail_unsent = (int)((*xmit_tail)->b_wptr - 21832 (*xmit_tail)->b_rptr); 21833 } 21834 21835 (*xmit_tail)->b_prev = local_time; 21836 (*xmit_tail)->b_next = (mblk_t *)(uintptr_t)(*snxt - len); 21837 21838 *tail_unsent -= len; 21839 if (len <= mss) /* LSO is unusable (!do_lso_send) */ 21840 tcp->tcp_last_sent_len = (ushort_t)len; 21841 21842 len += tcp_hdr_len; 21843 if (tcp->tcp_ipversion == IPV4_VERSION) 21844 tcp->tcp_ipha->ipha_length = htons(len); 21845 else 21846 tcp->tcp_ip6h->ip6_plen = htons(len - 21847 ((char *)&tcp->tcp_ip6h[1] - tcp->tcp_iphc)); 21848 21849 mp = dupb(*xmit_tail); 21850 if (mp == NULL) { 21851 if (ire != NULL) 21852 IRE_REFRELE(ire); 21853 return (-1); /* out_of_mem */ 21854 } 21855 21856 len = tcp_hdr_len; 21857 /* 21858 * There are four reasons to allocate a new hdr mblk: 21859 * 1) The bytes above us are in use by another packet 21860 * 2) We don't have good alignment 21861 * 3) The mblk is being shared 21862 * 4) We don't have enough room for a header 21863 */ 21864 rptr = mp->b_rptr - len; 21865 if (!OK_32PTR(rptr) || 21866 ((db = mp->b_datap), db->db_ref != 2) || 21867 rptr < db->db_base + ire_fp_mp_len) { 21868 /* NOTE: we assume allocb returns an OK_32PTR */ 21869 21870 must_alloc:; 21871 mp1 = allocb(tcp->tcp_ip_hdr_len + TCP_MAX_HDR_LENGTH + 21872 tcps->tcps_wroff_xtra + ire_fp_mp_len, BPRI_MED); 21873 if (mp1 == NULL) { 21874 freemsg(mp); 21875 if (ire != NULL) 21876 IRE_REFRELE(ire); 21877 return (-1); /* out_of_mem */ 21878 } 21879 mp1->b_cont = mp; 21880 mp = mp1; 21881 /* Leave room for Link Level header */ 21882 len = tcp_hdr_len; 21883 rptr = 21884 &mp->b_rptr[tcps->tcps_wroff_xtra + ire_fp_mp_len]; 21885 mp->b_wptr = &rptr[len]; 21886 } 21887 21888 /* 21889 * Fill in the header using the template header, and add 21890 * options such as time-stamp, ECN and/or SACK, as needed. 21891 */ 21892 tcp_fill_header(tcp, rptr, (clock_t)local_time, num_sack_blk); 21893 21894 mp->b_rptr = rptr; 21895 21896 if (*tail_unsent) { 21897 int spill = *tail_unsent; 21898 21899 mp1 = mp->b_cont; 21900 if (mp1 == NULL) 21901 mp1 = mp; 21902 21903 /* 21904 * If we're a little short, tack on more mblks until 21905 * there is no more spillover. 21906 */ 21907 while (spill < 0) { 21908 mblk_t *nmp; 21909 int nmpsz; 21910 21911 nmp = (*xmit_tail)->b_cont; 21912 nmpsz = MBLKL(nmp); 21913 21914 /* 21915 * Excess data in mblk; can we split it? 21916 * If MDT is enabled for the connection, 21917 * keep on splitting as this is a transient 21918 * send path. 21919 */ 21920 if (!do_lso_send && !tcp->tcp_mdt && 21921 (spill + nmpsz > 0)) { 21922 /* 21923 * Don't split if stream head was 21924 * told to break up larger writes 21925 * into smaller ones. 21926 */ 21927 if (tcp->tcp_maxpsz > 0) 21928 break; 21929 21930 /* 21931 * Next mblk is less than SMSS/2 21932 * rounded up to nearest 64-byte; 21933 * let it get sent as part of the 21934 * next segment. 21935 */ 21936 if (tcp->tcp_localnet && 21937 !tcp->tcp_cork && 21938 (nmpsz < roundup((mss >> 1), 64))) 21939 break; 21940 } 21941 21942 *xmit_tail = nmp; 21943 ASSERT((uintptr_t)nmpsz <= (uintptr_t)INT_MAX); 21944 /* Stash for rtt use later */ 21945 (*xmit_tail)->b_prev = local_time; 21946 (*xmit_tail)->b_next = 21947 (mblk_t *)(uintptr_t)(*snxt - len); 21948 mp1->b_cont = dupb(*xmit_tail); 21949 mp1 = mp1->b_cont; 21950 21951 spill += nmpsz; 21952 if (mp1 == NULL) { 21953 *tail_unsent = spill; 21954 freemsg(mp); 21955 if (ire != NULL) 21956 IRE_REFRELE(ire); 21957 return (-1); /* out_of_mem */ 21958 } 21959 } 21960 21961 /* Trim back any surplus on the last mblk */ 21962 if (spill >= 0) { 21963 mp1->b_wptr -= spill; 21964 *tail_unsent = spill; 21965 } else { 21966 /* 21967 * We did not send everything we could in 21968 * order to remain within the b_cont limit. 21969 */ 21970 *usable -= spill; 21971 *snxt += spill; 21972 tcp->tcp_last_sent_len += spill; 21973 UPDATE_MIB(&tcps->tcps_mib, 21974 tcpOutDataBytes, spill); 21975 /* 21976 * Adjust the checksum 21977 */ 21978 tcph = (tcph_t *)(rptr + tcp->tcp_ip_hdr_len); 21979 sum += spill; 21980 sum = (sum >> 16) + (sum & 0xFFFF); 21981 U16_TO_ABE16(sum, tcph->th_sum); 21982 if (tcp->tcp_ipversion == IPV4_VERSION) { 21983 sum = ntohs( 21984 ((ipha_t *)rptr)->ipha_length) + 21985 spill; 21986 ((ipha_t *)rptr)->ipha_length = 21987 htons(sum); 21988 } else { 21989 sum = ntohs( 21990 ((ip6_t *)rptr)->ip6_plen) + 21991 spill; 21992 ((ip6_t *)rptr)->ip6_plen = 21993 htons(sum); 21994 } 21995 *tail_unsent = 0; 21996 } 21997 } 21998 if (tcp->tcp_ip_forward_progress) { 21999 ASSERT(tcp->tcp_ipversion == IPV6_VERSION); 22000 *(uint32_t *)mp->b_rptr |= IP_FORWARD_PROG; 22001 tcp->tcp_ip_forward_progress = B_FALSE; 22002 } 22003 22004 if (do_lso_send) { 22005 tcp_lsosend_data(tcp, mp, ire, ill, mss, 22006 num_lso_seg); 22007 tcp->tcp_obsegs += num_lso_seg; 22008 22009 TCP_STAT(tcps, tcp_lso_times); 22010 TCP_STAT_UPDATE(tcps, tcp_lso_pkt_out, num_lso_seg); 22011 } else { 22012 tcp_send_data(tcp, q, mp); 22013 BUMP_LOCAL(tcp->tcp_obsegs); 22014 } 22015 } 22016 22017 if (ire != NULL) 22018 IRE_REFRELE(ire); 22019 return (0); 22020 } 22021 22022 /* Unlink and return any mblk that looks like it contains a MDT info */ 22023 static mblk_t * 22024 tcp_mdt_info_mp(mblk_t *mp) 22025 { 22026 mblk_t *prev_mp; 22027 22028 for (;;) { 22029 prev_mp = mp; 22030 /* no more to process? */ 22031 if ((mp = mp->b_cont) == NULL) 22032 break; 22033 22034 switch (DB_TYPE(mp)) { 22035 case M_CTL: 22036 if (*(uint32_t *)mp->b_rptr != MDT_IOC_INFO_UPDATE) 22037 continue; 22038 ASSERT(prev_mp != NULL); 22039 prev_mp->b_cont = mp->b_cont; 22040 mp->b_cont = NULL; 22041 return (mp); 22042 default: 22043 break; 22044 } 22045 } 22046 return (mp); 22047 } 22048 22049 /* MDT info update routine, called when IP notifies us about MDT */ 22050 static void 22051 tcp_mdt_update(tcp_t *tcp, ill_mdt_capab_t *mdt_capab, boolean_t first) 22052 { 22053 boolean_t prev_state; 22054 tcp_stack_t *tcps = tcp->tcp_tcps; 22055 22056 /* 22057 * IP is telling us to abort MDT on this connection? We know 22058 * this because the capability is only turned off when IP 22059 * encounters some pathological cases, e.g. link-layer change 22060 * where the new driver doesn't support MDT, or in situation 22061 * where MDT usage on the link-layer has been switched off. 22062 * IP would not have sent us the initial MDT_IOC_INFO_UPDATE 22063 * if the link-layer doesn't support MDT, and if it does, it 22064 * will indicate that the feature is to be turned on. 22065 */ 22066 prev_state = tcp->tcp_mdt; 22067 tcp->tcp_mdt = (mdt_capab->ill_mdt_on != 0); 22068 if (!tcp->tcp_mdt && !first) { 22069 TCP_STAT(tcps, tcp_mdt_conn_halted3); 22070 ip1dbg(("tcp_mdt_update: disabling MDT for connp %p\n", 22071 (void *)tcp->tcp_connp)); 22072 } 22073 22074 /* 22075 * We currently only support MDT on simple TCP/{IPv4,IPv6}, 22076 * so disable MDT otherwise. The checks are done here 22077 * and in tcp_wput_data(). 22078 */ 22079 if (tcp->tcp_mdt && 22080 (tcp->tcp_ipversion == IPV4_VERSION && 22081 tcp->tcp_ip_hdr_len != IP_SIMPLE_HDR_LENGTH) || 22082 (tcp->tcp_ipversion == IPV6_VERSION && 22083 tcp->tcp_ip_hdr_len != IPV6_HDR_LEN)) 22084 tcp->tcp_mdt = B_FALSE; 22085 22086 if (tcp->tcp_mdt) { 22087 if (mdt_capab->ill_mdt_version != MDT_VERSION_2) { 22088 cmn_err(CE_NOTE, "tcp_mdt_update: unknown MDT " 22089 "version (%d), expected version is %d", 22090 mdt_capab->ill_mdt_version, MDT_VERSION_2); 22091 tcp->tcp_mdt = B_FALSE; 22092 return; 22093 } 22094 22095 /* 22096 * We need the driver to be able to handle at least three 22097 * spans per packet in order for tcp MDT to be utilized. 22098 * The first is for the header portion, while the rest are 22099 * needed to handle a packet that straddles across two 22100 * virtually non-contiguous buffers; a typical tcp packet 22101 * therefore consists of only two spans. Note that we take 22102 * a zero as "don't care". 22103 */ 22104 if (mdt_capab->ill_mdt_span_limit > 0 && 22105 mdt_capab->ill_mdt_span_limit < 3) { 22106 tcp->tcp_mdt = B_FALSE; 22107 return; 22108 } 22109 22110 /* a zero means driver wants default value */ 22111 tcp->tcp_mdt_max_pld = MIN(mdt_capab->ill_mdt_max_pld, 22112 tcps->tcps_mdt_max_pbufs); 22113 if (tcp->tcp_mdt_max_pld == 0) 22114 tcp->tcp_mdt_max_pld = tcps->tcps_mdt_max_pbufs; 22115 22116 /* ensure 32-bit alignment */ 22117 tcp->tcp_mdt_hdr_head = roundup(MAX(tcps->tcps_mdt_hdr_head_min, 22118 mdt_capab->ill_mdt_hdr_head), 4); 22119 tcp->tcp_mdt_hdr_tail = roundup(MAX(tcps->tcps_mdt_hdr_tail_min, 22120 mdt_capab->ill_mdt_hdr_tail), 4); 22121 22122 if (!first && !prev_state) { 22123 TCP_STAT(tcps, tcp_mdt_conn_resumed2); 22124 ip1dbg(("tcp_mdt_update: reenabling MDT for connp %p\n", 22125 (void *)tcp->tcp_connp)); 22126 } 22127 } 22128 } 22129 22130 /* Unlink and return any mblk that looks like it contains a LSO info */ 22131 static mblk_t * 22132 tcp_lso_info_mp(mblk_t *mp) 22133 { 22134 mblk_t *prev_mp; 22135 22136 for (;;) { 22137 prev_mp = mp; 22138 /* no more to process? */ 22139 if ((mp = mp->b_cont) == NULL) 22140 break; 22141 22142 switch (DB_TYPE(mp)) { 22143 case M_CTL: 22144 if (*(uint32_t *)mp->b_rptr != LSO_IOC_INFO_UPDATE) 22145 continue; 22146 ASSERT(prev_mp != NULL); 22147 prev_mp->b_cont = mp->b_cont; 22148 mp->b_cont = NULL; 22149 return (mp); 22150 default: 22151 break; 22152 } 22153 } 22154 22155 return (mp); 22156 } 22157 22158 /* LSO info update routine, called when IP notifies us about LSO */ 22159 static void 22160 tcp_lso_update(tcp_t *tcp, ill_lso_capab_t *lso_capab) 22161 { 22162 tcp_stack_t *tcps = tcp->tcp_tcps; 22163 22164 /* 22165 * IP is telling us to abort LSO on this connection? We know 22166 * this because the capability is only turned off when IP 22167 * encounters some pathological cases, e.g. link-layer change 22168 * where the new NIC/driver doesn't support LSO, or in situation 22169 * where LSO usage on the link-layer has been switched off. 22170 * IP would not have sent us the initial LSO_IOC_INFO_UPDATE 22171 * if the link-layer doesn't support LSO, and if it does, it 22172 * will indicate that the feature is to be turned on. 22173 */ 22174 tcp->tcp_lso = (lso_capab->ill_lso_on != 0); 22175 TCP_STAT(tcps, tcp_lso_enabled); 22176 22177 /* 22178 * We currently only support LSO on simple TCP/IPv4, 22179 * so disable LSO otherwise. The checks are done here 22180 * and in tcp_wput_data(). 22181 */ 22182 if (tcp->tcp_lso && 22183 (tcp->tcp_ipversion == IPV4_VERSION && 22184 tcp->tcp_ip_hdr_len != IP_SIMPLE_HDR_LENGTH) || 22185 (tcp->tcp_ipversion == IPV6_VERSION)) { 22186 tcp->tcp_lso = B_FALSE; 22187 TCP_STAT(tcps, tcp_lso_disabled); 22188 } else { 22189 tcp->tcp_lso_max = MIN(TCP_MAX_LSO_LENGTH, 22190 lso_capab->ill_lso_max); 22191 } 22192 } 22193 22194 static void 22195 tcp_ire_ill_check(tcp_t *tcp, ire_t *ire, ill_t *ill, boolean_t check_lso_mdt) 22196 { 22197 conn_t *connp = tcp->tcp_connp; 22198 tcp_stack_t *tcps = tcp->tcp_tcps; 22199 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 22200 22201 ASSERT(ire != NULL); 22202 22203 /* 22204 * We may be in the fastpath here, and although we essentially do 22205 * similar checks as in ip_bind_connected{_v6}/ip_xxinfo_return, 22206 * we try to keep things as brief as possible. After all, these 22207 * are only best-effort checks, and we do more thorough ones prior 22208 * to calling tcp_send()/tcp_multisend(). 22209 */ 22210 if ((ipst->ips_ip_lso_outbound || ipst->ips_ip_multidata_outbound) && 22211 check_lso_mdt && !(ire->ire_type & (IRE_LOCAL | IRE_LOOPBACK)) && 22212 ill != NULL && !CONN_IPSEC_OUT_ENCAPSULATED(connp) && 22213 !(ire->ire_flags & RTF_MULTIRT) && 22214 !IPP_ENABLED(IPP_LOCAL_OUT, ipst) && 22215 CONN_IS_LSO_MD_FASTPATH(connp)) { 22216 if (ipst->ips_ip_lso_outbound && ILL_LSO_CAPABLE(ill)) { 22217 /* Cache the result */ 22218 connp->conn_lso_ok = B_TRUE; 22219 22220 ASSERT(ill->ill_lso_capab != NULL); 22221 if (!ill->ill_lso_capab->ill_lso_on) { 22222 ill->ill_lso_capab->ill_lso_on = 1; 22223 ip1dbg(("tcp_ire_ill_check: connp %p enables " 22224 "LSO for interface %s\n", (void *)connp, 22225 ill->ill_name)); 22226 } 22227 tcp_lso_update(tcp, ill->ill_lso_capab); 22228 } else if (ipst->ips_ip_multidata_outbound && 22229 ILL_MDT_CAPABLE(ill)) { 22230 /* Cache the result */ 22231 connp->conn_mdt_ok = B_TRUE; 22232 22233 ASSERT(ill->ill_mdt_capab != NULL); 22234 if (!ill->ill_mdt_capab->ill_mdt_on) { 22235 ill->ill_mdt_capab->ill_mdt_on = 1; 22236 ip1dbg(("tcp_ire_ill_check: connp %p enables " 22237 "MDT for interface %s\n", (void *)connp, 22238 ill->ill_name)); 22239 } 22240 tcp_mdt_update(tcp, ill->ill_mdt_capab, B_TRUE); 22241 } 22242 } 22243 22244 /* 22245 * The goal is to reduce the number of generated tcp segments by 22246 * setting the maxpsz multiplier to 0; this will have an affect on 22247 * tcp_maxpsz_set(). With this behavior, tcp will pack more data 22248 * into each packet, up to SMSS bytes. Doing this reduces the number 22249 * of outbound segments and incoming ACKs, thus allowing for better 22250 * network and system performance. In contrast the legacy behavior 22251 * may result in sending less than SMSS size, because the last mblk 22252 * for some packets may have more data than needed to make up SMSS, 22253 * and the legacy code refused to "split" it. 22254 * 22255 * We apply the new behavior on following situations: 22256 * 22257 * 1) Loopback connections, 22258 * 2) Connections in which the remote peer is not on local subnet, 22259 * 3) Local subnet connections over the bge interface (see below). 22260 * 22261 * Ideally, we would like this behavior to apply for interfaces other 22262 * than bge. However, doing so would negatively impact drivers which 22263 * perform dynamic mapping and unmapping of DMA resources, which are 22264 * increased by setting the maxpsz multiplier to 0 (more mblks per 22265 * packet will be generated by tcp). The bge driver does not suffer 22266 * from this, as it copies the mblks into pre-mapped buffers, and 22267 * therefore does not require more I/O resources than before. 22268 * 22269 * Otherwise, this behavior is present on all network interfaces when 22270 * the destination endpoint is non-local, since reducing the number 22271 * of packets in general is good for the network. 22272 * 22273 * TODO We need to remove this hard-coded conditional for bge once 22274 * a better "self-tuning" mechanism, or a way to comprehend 22275 * the driver transmit strategy is devised. Until the solution 22276 * is found and well understood, we live with this hack. 22277 */ 22278 if (!tcp_static_maxpsz && 22279 (tcp->tcp_loopback || !tcp->tcp_localnet || 22280 (ill->ill_name_length > 3 && bcmp(ill->ill_name, "bge", 3) == 0))) { 22281 /* override the default value */ 22282 tcp->tcp_maxpsz = 0; 22283 22284 ip3dbg(("tcp_ire_ill_check: connp %p tcp_maxpsz %d on " 22285 "interface %s\n", (void *)connp, tcp->tcp_maxpsz, 22286 ill != NULL ? ill->ill_name : ipif_loopback_name)); 22287 } 22288 22289 /* set the stream head parameters accordingly */ 22290 (void) tcp_maxpsz_set(tcp, B_TRUE); 22291 } 22292 22293 /* tcp_wput_flush is called by tcp_wput_nondata to handle M_FLUSH messages. */ 22294 static void 22295 tcp_wput_flush(tcp_t *tcp, mblk_t *mp) 22296 { 22297 uchar_t fval = *mp->b_rptr; 22298 mblk_t *tail; 22299 queue_t *q = tcp->tcp_wq; 22300 22301 /* TODO: How should flush interact with urgent data? */ 22302 if ((fval & FLUSHW) && tcp->tcp_xmit_head && 22303 !(tcp->tcp_valid_bits & TCP_URG_VALID)) { 22304 /* 22305 * Flush only data that has not yet been put on the wire. If 22306 * we flush data that we have already transmitted, life, as we 22307 * know it, may come to an end. 22308 */ 22309 tail = tcp->tcp_xmit_tail; 22310 tail->b_wptr -= tcp->tcp_xmit_tail_unsent; 22311 tcp->tcp_xmit_tail_unsent = 0; 22312 tcp->tcp_unsent = 0; 22313 if (tail->b_wptr != tail->b_rptr) 22314 tail = tail->b_cont; 22315 if (tail) { 22316 mblk_t **excess = &tcp->tcp_xmit_head; 22317 for (;;) { 22318 mblk_t *mp1 = *excess; 22319 if (mp1 == tail) 22320 break; 22321 tcp->tcp_xmit_tail = mp1; 22322 tcp->tcp_xmit_last = mp1; 22323 excess = &mp1->b_cont; 22324 } 22325 *excess = NULL; 22326 tcp_close_mpp(&tail); 22327 if (tcp->tcp_snd_zcopy_aware) 22328 tcp_zcopy_notify(tcp); 22329 } 22330 /* 22331 * We have no unsent data, so unsent must be less than 22332 * tcp_xmit_lowater, so re-enable flow. 22333 */ 22334 mutex_enter(&tcp->tcp_non_sq_lock); 22335 if (tcp->tcp_flow_stopped) { 22336 tcp_clrqfull(tcp); 22337 } 22338 mutex_exit(&tcp->tcp_non_sq_lock); 22339 } 22340 /* 22341 * TODO: you can't just flush these, you have to increase rwnd for one 22342 * thing. For another, how should urgent data interact? 22343 */ 22344 if (fval & FLUSHR) { 22345 *mp->b_rptr = fval & ~FLUSHW; 22346 /* XXX */ 22347 qreply(q, mp); 22348 return; 22349 } 22350 freemsg(mp); 22351 } 22352 22353 /* 22354 * tcp_wput_iocdata is called by tcp_wput_nondata to handle all M_IOCDATA 22355 * messages. 22356 */ 22357 static void 22358 tcp_wput_iocdata(tcp_t *tcp, mblk_t *mp) 22359 { 22360 mblk_t *mp1; 22361 struct iocblk *iocp = (struct iocblk *)mp->b_rptr; 22362 STRUCT_HANDLE(strbuf, sb); 22363 queue_t *q = tcp->tcp_wq; 22364 int error; 22365 uint_t addrlen; 22366 22367 /* Make sure it is one of ours. */ 22368 switch (iocp->ioc_cmd) { 22369 case TI_GETMYNAME: 22370 case TI_GETPEERNAME: 22371 break; 22372 default: 22373 CALL_IP_WPUT(tcp->tcp_connp, q, mp); 22374 return; 22375 } 22376 switch (mi_copy_state(q, mp, &mp1)) { 22377 case -1: 22378 return; 22379 case MI_COPY_CASE(MI_COPY_IN, 1): 22380 break; 22381 case MI_COPY_CASE(MI_COPY_OUT, 1): 22382 /* Copy out the strbuf. */ 22383 mi_copyout(q, mp); 22384 return; 22385 case MI_COPY_CASE(MI_COPY_OUT, 2): 22386 /* All done. */ 22387 mi_copy_done(q, mp, 0); 22388 return; 22389 default: 22390 mi_copy_done(q, mp, EPROTO); 22391 return; 22392 } 22393 /* Check alignment of the strbuf */ 22394 if (!OK_32PTR(mp1->b_rptr)) { 22395 mi_copy_done(q, mp, EINVAL); 22396 return; 22397 } 22398 22399 STRUCT_SET_HANDLE(sb, iocp->ioc_flag, (void *)mp1->b_rptr); 22400 addrlen = tcp->tcp_family == AF_INET ? sizeof (sin_t) : sizeof (sin6_t); 22401 if (STRUCT_FGET(sb, maxlen) < addrlen) { 22402 mi_copy_done(q, mp, EINVAL); 22403 return; 22404 } 22405 22406 mp1 = mi_copyout_alloc(q, mp, STRUCT_FGETP(sb, buf), addrlen, B_TRUE); 22407 if (mp1 == NULL) 22408 return; 22409 22410 switch (iocp->ioc_cmd) { 22411 case TI_GETMYNAME: 22412 error = tcp_getmyname(tcp, (void *)mp1->b_rptr, &addrlen); 22413 break; 22414 case TI_GETPEERNAME: 22415 error = tcp_getpeername(tcp, (void *)mp1->b_rptr, &addrlen); 22416 break; 22417 } 22418 22419 if (error != 0) { 22420 mi_copy_done(q, mp, error); 22421 } else { 22422 mp1->b_wptr += addrlen; 22423 STRUCT_FSET(sb, len, addrlen); 22424 22425 /* Copy out the address */ 22426 mi_copyout(q, mp); 22427 } 22428 } 22429 22430 /* 22431 * tcp_wput_ioctl is called by tcp_wput_nondata() to handle all M_IOCTL 22432 * messages. 22433 */ 22434 /* ARGSUSED */ 22435 static void 22436 tcp_wput_ioctl(void *arg, mblk_t *mp, void *arg2) 22437 { 22438 conn_t *connp = (conn_t *)arg; 22439 tcp_t *tcp = connp->conn_tcp; 22440 queue_t *q = tcp->tcp_wq; 22441 struct iocblk *iocp; 22442 tcp_stack_t *tcps = tcp->tcp_tcps; 22443 22444 ASSERT(DB_TYPE(mp) == M_IOCTL); 22445 /* 22446 * Try and ASSERT the minimum possible references on the 22447 * conn early enough. Since we are executing on write side, 22448 * the connection is obviously not detached and that means 22449 * there is a ref each for TCP and IP. Since we are behind 22450 * the squeue, the minimum references needed are 3. If the 22451 * conn is in classifier hash list, there should be an 22452 * extra ref for that (we check both the possibilities). 22453 */ 22454 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) || 22455 (connp->conn_fanout == NULL && connp->conn_ref >= 3)); 22456 22457 iocp = (struct iocblk *)mp->b_rptr; 22458 switch (iocp->ioc_cmd) { 22459 case TCP_IOC_DEFAULT_Q: 22460 /* Wants to be the default wq. */ 22461 if (secpolicy_ip_config(iocp->ioc_cr, B_FALSE) != 0) { 22462 iocp->ioc_error = EPERM; 22463 iocp->ioc_count = 0; 22464 mp->b_datap->db_type = M_IOCACK; 22465 qreply(q, mp); 22466 return; 22467 } 22468 tcp_def_q_set(tcp, mp); 22469 return; 22470 case _SIOCSOCKFALLBACK: 22471 /* 22472 * Either sockmod is about to be popped and the socket 22473 * would now be treated as a plain stream, or a module 22474 * is about to be pushed so we could no longer use read- 22475 * side synchronous streams for fused loopback tcp. 22476 * Drain any queued data and disable direct sockfs 22477 * interface from now on. 22478 */ 22479 if (!tcp->tcp_issocket) { 22480 DB_TYPE(mp) = M_IOCNAK; 22481 iocp->ioc_error = EINVAL; 22482 } else { 22483 #ifdef _ILP32 22484 tcp->tcp_acceptor_id = (t_uscalar_t)RD(q); 22485 #else 22486 tcp->tcp_acceptor_id = tcp->tcp_connp->conn_dev; 22487 #endif 22488 /* 22489 * Insert this socket into the acceptor hash. 22490 * We might need it for T_CONN_RES message 22491 */ 22492 tcp_acceptor_hash_insert(tcp->tcp_acceptor_id, tcp); 22493 22494 if (tcp->tcp_fused) { 22495 /* 22496 * This is a fused loopback tcp; disable 22497 * read-side synchronous streams interface 22498 * and drain any queued data. It is okay 22499 * to do this for non-synchronous streams 22500 * fused tcp as well. 22501 */ 22502 tcp_fuse_disable_pair(tcp, B_FALSE); 22503 } 22504 tcp->tcp_issocket = B_FALSE; 22505 tcp->tcp_sodirect = NULL; 22506 TCP_STAT(tcps, tcp_sock_fallback); 22507 22508 DB_TYPE(mp) = M_IOCACK; 22509 iocp->ioc_error = 0; 22510 } 22511 iocp->ioc_count = 0; 22512 iocp->ioc_rval = 0; 22513 qreply(q, mp); 22514 return; 22515 } 22516 CALL_IP_WPUT(connp, q, mp); 22517 } 22518 22519 /* 22520 * This routine is called by tcp_wput() to handle all TPI requests. 22521 */ 22522 /* ARGSUSED */ 22523 static void 22524 tcp_wput_proto(void *arg, mblk_t *mp, void *arg2) 22525 { 22526 conn_t *connp = (conn_t *)arg; 22527 tcp_t *tcp = connp->conn_tcp; 22528 union T_primitives *tprim = (union T_primitives *)mp->b_rptr; 22529 uchar_t *rptr; 22530 t_scalar_t type; 22531 int len; 22532 cred_t *cr = DB_CREDDEF(mp, tcp->tcp_cred); 22533 22534 /* 22535 * Try and ASSERT the minimum possible references on the 22536 * conn early enough. Since we are executing on write side, 22537 * the connection is obviously not detached and that means 22538 * there is a ref each for TCP and IP. Since we are behind 22539 * the squeue, the minimum references needed are 3. If the 22540 * conn is in classifier hash list, there should be an 22541 * extra ref for that (we check both the possibilities). 22542 */ 22543 ASSERT((connp->conn_fanout != NULL && connp->conn_ref >= 4) || 22544 (connp->conn_fanout == NULL && connp->conn_ref >= 3)); 22545 22546 rptr = mp->b_rptr; 22547 ASSERT((uintptr_t)(mp->b_wptr - rptr) <= (uintptr_t)INT_MAX); 22548 if ((mp->b_wptr - rptr) >= sizeof (t_scalar_t)) { 22549 type = ((union T_primitives *)rptr)->type; 22550 if (type == T_EXDATA_REQ) { 22551 uint32_t msize = msgdsize(mp->b_cont); 22552 22553 len = msize - 1; 22554 if (len < 0) { 22555 freemsg(mp); 22556 return; 22557 } 22558 /* 22559 * Try to force urgent data out on the wire. 22560 * Even if we have unsent data this will 22561 * at least send the urgent flag. 22562 * XXX does not handle more flag correctly. 22563 */ 22564 len += tcp->tcp_unsent; 22565 len += tcp->tcp_snxt; 22566 tcp->tcp_urg = len; 22567 tcp->tcp_valid_bits |= TCP_URG_VALID; 22568 22569 /* Bypass tcp protocol for fused tcp loopback */ 22570 if (tcp->tcp_fused && tcp_fuse_output(tcp, mp, msize)) 22571 return; 22572 } else if (type != T_DATA_REQ) { 22573 goto non_urgent_data; 22574 } 22575 /* TODO: options, flags, ... from user */ 22576 /* Set length to zero for reclamation below */ 22577 tcp_wput_data(tcp, mp->b_cont, B_TRUE); 22578 freeb(mp); 22579 return; 22580 } else { 22581 if (tcp->tcp_debug) { 22582 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE, 22583 "tcp_wput_proto, dropping one..."); 22584 } 22585 freemsg(mp); 22586 return; 22587 } 22588 22589 non_urgent_data: 22590 22591 switch ((int)tprim->type) { 22592 case T_SSL_PROXY_BIND_REQ: /* an SSL proxy endpoint bind request */ 22593 /* 22594 * save the kssl_ent_t from the next block, and convert this 22595 * back to a normal bind_req. 22596 */ 22597 if (mp->b_cont != NULL) { 22598 ASSERT(MBLKL(mp->b_cont) >= sizeof (kssl_ent_t)); 22599 22600 if (tcp->tcp_kssl_ent != NULL) { 22601 kssl_release_ent(tcp->tcp_kssl_ent, NULL, 22602 KSSL_NO_PROXY); 22603 tcp->tcp_kssl_ent = NULL; 22604 } 22605 bcopy(mp->b_cont->b_rptr, &tcp->tcp_kssl_ent, 22606 sizeof (kssl_ent_t)); 22607 kssl_hold_ent(tcp->tcp_kssl_ent); 22608 freemsg(mp->b_cont); 22609 mp->b_cont = NULL; 22610 } 22611 tprim->type = T_BIND_REQ; 22612 22613 /* FALLTHROUGH */ 22614 case O_T_BIND_REQ: /* bind request */ 22615 case T_BIND_REQ: /* new semantics bind request */ 22616 tcp_bind(tcp, mp); 22617 break; 22618 case T_UNBIND_REQ: /* unbind request */ 22619 tcp_unbind(tcp, mp); 22620 break; 22621 case O_T_CONN_RES: /* old connection response XXX */ 22622 case T_CONN_RES: /* connection response */ 22623 tcp_accept(tcp, mp); 22624 break; 22625 case T_CONN_REQ: /* connection request */ 22626 tcp_connect(tcp, mp); 22627 break; 22628 case T_DISCON_REQ: /* disconnect request */ 22629 tcp_disconnect(tcp, mp); 22630 break; 22631 case T_CAPABILITY_REQ: 22632 tcp_capability_req(tcp, mp); /* capability request */ 22633 break; 22634 case T_INFO_REQ: /* information request */ 22635 tcp_info_req(tcp, mp); 22636 break; 22637 case T_SVR4_OPTMGMT_REQ: /* manage options req */ 22638 (void) svr4_optcom_req(tcp->tcp_wq, mp, cr, 22639 &tcp_opt_obj, B_TRUE); 22640 break; 22641 case T_OPTMGMT_REQ: 22642 /* 22643 * Note: no support for snmpcom_req() through new 22644 * T_OPTMGMT_REQ. See comments in ip.c 22645 */ 22646 /* Only IP is allowed to return meaningful value */ 22647 (void) tpi_optcom_req(tcp->tcp_wq, mp, cr, &tcp_opt_obj, 22648 B_TRUE); 22649 break; 22650 22651 case T_UNITDATA_REQ: /* unitdata request */ 22652 tcp_err_ack(tcp, mp, TNOTSUPPORT, 0); 22653 break; 22654 case T_ORDREL_REQ: /* orderly release req */ 22655 freemsg(mp); 22656 22657 if (tcp->tcp_fused) 22658 tcp_unfuse(tcp); 22659 22660 if (tcp_xmit_end(tcp) != 0) { 22661 /* 22662 * We were crossing FINs and got a reset from 22663 * the other side. Just ignore it. 22664 */ 22665 if (tcp->tcp_debug) { 22666 (void) strlog(TCP_MOD_ID, 0, 1, 22667 SL_ERROR|SL_TRACE, 22668 "tcp_wput_proto, T_ORDREL_REQ out of " 22669 "state %s", 22670 tcp_display(tcp, NULL, 22671 DISP_ADDR_AND_PORT)); 22672 } 22673 } 22674 break; 22675 case T_ADDR_REQ: 22676 tcp_addr_req(tcp, mp); 22677 break; 22678 default: 22679 if (tcp->tcp_debug) { 22680 (void) strlog(TCP_MOD_ID, 0, 1, SL_ERROR|SL_TRACE, 22681 "tcp_wput_proto, bogus TPI msg, type %d", 22682 tprim->type); 22683 } 22684 /* 22685 * We used to M_ERROR. Sending TNOTSUPPORT gives the user 22686 * to recover. 22687 */ 22688 tcp_err_ack(tcp, mp, TNOTSUPPORT, 0); 22689 break; 22690 } 22691 } 22692 22693 /* 22694 * The TCP write service routine should never be called... 22695 */ 22696 /* ARGSUSED */ 22697 static void 22698 tcp_wsrv(queue_t *q) 22699 { 22700 tcp_stack_t *tcps = Q_TO_TCP(q)->tcp_tcps; 22701 22702 TCP_STAT(tcps, tcp_wsrv_called); 22703 } 22704 22705 /* Non overlapping byte exchanger */ 22706 static void 22707 tcp_xchg(uchar_t *a, uchar_t *b, int len) 22708 { 22709 uchar_t uch; 22710 22711 while (len-- > 0) { 22712 uch = a[len]; 22713 a[len] = b[len]; 22714 b[len] = uch; 22715 } 22716 } 22717 22718 /* 22719 * Send out a control packet on the tcp connection specified. This routine 22720 * is typically called where we need a simple ACK or RST generated. 22721 */ 22722 static void 22723 tcp_xmit_ctl(char *str, tcp_t *tcp, uint32_t seq, uint32_t ack, int ctl) 22724 { 22725 uchar_t *rptr; 22726 tcph_t *tcph; 22727 ipha_t *ipha = NULL; 22728 ip6_t *ip6h = NULL; 22729 uint32_t sum; 22730 int tcp_hdr_len; 22731 int tcp_ip_hdr_len; 22732 mblk_t *mp; 22733 tcp_stack_t *tcps = tcp->tcp_tcps; 22734 22735 /* 22736 * Save sum for use in source route later. 22737 */ 22738 ASSERT(tcp != NULL); 22739 sum = tcp->tcp_tcp_hdr_len + tcp->tcp_sum; 22740 tcp_hdr_len = tcp->tcp_hdr_len; 22741 tcp_ip_hdr_len = tcp->tcp_ip_hdr_len; 22742 22743 /* If a text string is passed in with the request, pass it to strlog. */ 22744 if (str != NULL && tcp->tcp_debug) { 22745 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE, 22746 "tcp_xmit_ctl: '%s', seq 0x%x, ack 0x%x, ctl 0x%x", 22747 str, seq, ack, ctl); 22748 } 22749 mp = allocb(tcp_ip_hdr_len + TCP_MAX_HDR_LENGTH + tcps->tcps_wroff_xtra, 22750 BPRI_MED); 22751 if (mp == NULL) { 22752 return; 22753 } 22754 rptr = &mp->b_rptr[tcps->tcps_wroff_xtra]; 22755 mp->b_rptr = rptr; 22756 mp->b_wptr = &rptr[tcp_hdr_len]; 22757 bcopy(tcp->tcp_iphc, rptr, tcp_hdr_len); 22758 22759 if (tcp->tcp_ipversion == IPV4_VERSION) { 22760 ipha = (ipha_t *)rptr; 22761 ipha->ipha_length = htons(tcp_hdr_len); 22762 } else { 22763 ip6h = (ip6_t *)rptr; 22764 ASSERT(tcp != NULL); 22765 ip6h->ip6_plen = htons(tcp->tcp_hdr_len - 22766 ((char *)&tcp->tcp_ip6h[1] - tcp->tcp_iphc)); 22767 } 22768 tcph = (tcph_t *)&rptr[tcp_ip_hdr_len]; 22769 tcph->th_flags[0] = (uint8_t)ctl; 22770 if (ctl & TH_RST) { 22771 BUMP_MIB(&tcps->tcps_mib, tcpOutRsts); 22772 BUMP_MIB(&tcps->tcps_mib, tcpOutControl); 22773 /* 22774 * Don't send TSopt w/ TH_RST packets per RFC 1323. 22775 */ 22776 if (tcp->tcp_snd_ts_ok && 22777 tcp->tcp_state > TCPS_SYN_SENT) { 22778 mp->b_wptr = &rptr[tcp_hdr_len - TCPOPT_REAL_TS_LEN]; 22779 *(mp->b_wptr) = TCPOPT_EOL; 22780 if (tcp->tcp_ipversion == IPV4_VERSION) { 22781 ipha->ipha_length = htons(tcp_hdr_len - 22782 TCPOPT_REAL_TS_LEN); 22783 } else { 22784 ip6h->ip6_plen = htons(ntohs(ip6h->ip6_plen) - 22785 TCPOPT_REAL_TS_LEN); 22786 } 22787 tcph->th_offset_and_rsrvd[0] -= (3 << 4); 22788 sum -= TCPOPT_REAL_TS_LEN; 22789 } 22790 } 22791 if (ctl & TH_ACK) { 22792 if (tcp->tcp_snd_ts_ok) { 22793 U32_TO_BE32(lbolt, 22794 (char *)tcph+TCP_MIN_HEADER_LENGTH+4); 22795 U32_TO_BE32(tcp->tcp_ts_recent, 22796 (char *)tcph+TCP_MIN_HEADER_LENGTH+8); 22797 } 22798 22799 /* Update the latest receive window size in TCP header. */ 22800 U32_TO_ABE16(tcp->tcp_rwnd >> tcp->tcp_rcv_ws, 22801 tcph->th_win); 22802 tcp->tcp_rack = ack; 22803 tcp->tcp_rack_cnt = 0; 22804 BUMP_MIB(&tcps->tcps_mib, tcpOutAck); 22805 } 22806 BUMP_LOCAL(tcp->tcp_obsegs); 22807 U32_TO_BE32(seq, tcph->th_seq); 22808 U32_TO_BE32(ack, tcph->th_ack); 22809 /* 22810 * Include the adjustment for a source route if any. 22811 */ 22812 sum = (sum >> 16) + (sum & 0xFFFF); 22813 U16_TO_BE16(sum, tcph->th_sum); 22814 tcp_send_data(tcp, tcp->tcp_wq, mp); 22815 } 22816 22817 /* 22818 * If this routine returns B_TRUE, TCP can generate a RST in response 22819 * to a segment. If it returns B_FALSE, TCP should not respond. 22820 */ 22821 static boolean_t 22822 tcp_send_rst_chk(tcp_stack_t *tcps) 22823 { 22824 clock_t now; 22825 22826 /* 22827 * TCP needs to protect itself from generating too many RSTs. 22828 * This can be a DoS attack by sending us random segments 22829 * soliciting RSTs. 22830 * 22831 * What we do here is to have a limit of tcp_rst_sent_rate RSTs 22832 * in each 1 second interval. In this way, TCP still generate 22833 * RSTs in normal cases but when under attack, the impact is 22834 * limited. 22835 */ 22836 if (tcps->tcps_rst_sent_rate_enabled != 0) { 22837 now = lbolt; 22838 /* lbolt can wrap around. */ 22839 if ((tcps->tcps_last_rst_intrvl > now) || 22840 (TICK_TO_MSEC(now - tcps->tcps_last_rst_intrvl) > 22841 1*SECONDS)) { 22842 tcps->tcps_last_rst_intrvl = now; 22843 tcps->tcps_rst_cnt = 1; 22844 } else if (++tcps->tcps_rst_cnt > tcps->tcps_rst_sent_rate) { 22845 return (B_FALSE); 22846 } 22847 } 22848 return (B_TRUE); 22849 } 22850 22851 /* 22852 * Send down the advice IP ioctl to tell IP to mark an IRE temporary. 22853 */ 22854 static void 22855 tcp_ip_ire_mark_advice(tcp_t *tcp) 22856 { 22857 mblk_t *mp; 22858 ipic_t *ipic; 22859 22860 if (tcp->tcp_ipversion == IPV4_VERSION) { 22861 mp = tcp_ip_advise_mblk(&tcp->tcp_ipha->ipha_dst, IP_ADDR_LEN, 22862 &ipic); 22863 } else { 22864 mp = tcp_ip_advise_mblk(&tcp->tcp_ip6h->ip6_dst, IPV6_ADDR_LEN, 22865 &ipic); 22866 } 22867 if (mp == NULL) 22868 return; 22869 ipic->ipic_ire_marks |= IRE_MARK_TEMPORARY; 22870 CALL_IP_WPUT(tcp->tcp_connp, tcp->tcp_wq, mp); 22871 } 22872 22873 /* 22874 * Return an IP advice ioctl mblk and set ipic to be the pointer 22875 * to the advice structure. 22876 */ 22877 static mblk_t * 22878 tcp_ip_advise_mblk(void *addr, int addr_len, ipic_t **ipic) 22879 { 22880 struct iocblk *ioc; 22881 mblk_t *mp, *mp1; 22882 22883 mp = allocb(sizeof (ipic_t) + addr_len, BPRI_HI); 22884 if (mp == NULL) 22885 return (NULL); 22886 bzero(mp->b_rptr, sizeof (ipic_t) + addr_len); 22887 *ipic = (ipic_t *)mp->b_rptr; 22888 (*ipic)->ipic_cmd = IP_IOC_IRE_ADVISE_NO_REPLY; 22889 (*ipic)->ipic_addr_offset = sizeof (ipic_t); 22890 22891 bcopy(addr, *ipic + 1, addr_len); 22892 22893 (*ipic)->ipic_addr_length = addr_len; 22894 mp->b_wptr = &mp->b_rptr[sizeof (ipic_t) + addr_len]; 22895 22896 mp1 = mkiocb(IP_IOCTL); 22897 if (mp1 == NULL) { 22898 freemsg(mp); 22899 return (NULL); 22900 } 22901 mp1->b_cont = mp; 22902 ioc = (struct iocblk *)mp1->b_rptr; 22903 ioc->ioc_count = sizeof (ipic_t) + addr_len; 22904 22905 return (mp1); 22906 } 22907 22908 /* 22909 * Generate a reset based on an inbound packet, connp is set by caller 22910 * when RST is in response to an unexpected inbound packet for which 22911 * there is active tcp state in the system. 22912 * 22913 * IPSEC NOTE : Try to send the reply with the same protection as it came 22914 * in. We still have the ipsec_mp that the packet was attached to. Thus 22915 * the packet will go out at the same level of protection as it came in by 22916 * converting the IPSEC_IN to IPSEC_OUT. 22917 */ 22918 static void 22919 tcp_xmit_early_reset(char *str, mblk_t *mp, uint32_t seq, 22920 uint32_t ack, int ctl, uint_t ip_hdr_len, zoneid_t zoneid, 22921 tcp_stack_t *tcps, conn_t *connp) 22922 { 22923 ipha_t *ipha = NULL; 22924 ip6_t *ip6h = NULL; 22925 ushort_t len; 22926 tcph_t *tcph; 22927 int i; 22928 mblk_t *ipsec_mp; 22929 boolean_t mctl_present; 22930 ipic_t *ipic; 22931 ipaddr_t v4addr; 22932 in6_addr_t v6addr; 22933 int addr_len; 22934 void *addr; 22935 queue_t *q = tcps->tcps_g_q; 22936 tcp_t *tcp; 22937 cred_t *cr; 22938 mblk_t *nmp; 22939 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 22940 22941 if (tcps->tcps_g_q == NULL) { 22942 /* 22943 * For non-zero stackids the default queue isn't created 22944 * until the first open, thus there can be a need to send 22945 * a reset before then. But we can't do that, hence we just 22946 * drop the packet. Later during boot, when the default queue 22947 * has been setup, a retransmitted packet from the peer 22948 * will result in a reset. 22949 */ 22950 ASSERT(tcps->tcps_netstack->netstack_stackid != 22951 GLOBAL_NETSTACKID); 22952 freemsg(mp); 22953 return; 22954 } 22955 22956 if (connp != NULL) 22957 tcp = connp->conn_tcp; 22958 else 22959 tcp = Q_TO_TCP(q); 22960 22961 if (!tcp_send_rst_chk(tcps)) { 22962 tcps->tcps_rst_unsent++; 22963 freemsg(mp); 22964 return; 22965 } 22966 22967 if (mp->b_datap->db_type == M_CTL) { 22968 ipsec_mp = mp; 22969 mp = mp->b_cont; 22970 mctl_present = B_TRUE; 22971 } else { 22972 ipsec_mp = mp; 22973 mctl_present = B_FALSE; 22974 } 22975 22976 if (str && q && tcps->tcps_dbg) { 22977 (void) strlog(TCP_MOD_ID, 0, 1, SL_TRACE, 22978 "tcp_xmit_early_reset: '%s', seq 0x%x, ack 0x%x, " 22979 "flags 0x%x", 22980 str, seq, ack, ctl); 22981 } 22982 if (mp->b_datap->db_ref != 1) { 22983 mblk_t *mp1 = copyb(mp); 22984 freemsg(mp); 22985 mp = mp1; 22986 if (!mp) { 22987 if (mctl_present) 22988 freeb(ipsec_mp); 22989 return; 22990 } else { 22991 if (mctl_present) { 22992 ipsec_mp->b_cont = mp; 22993 } else { 22994 ipsec_mp = mp; 22995 } 22996 } 22997 } else if (mp->b_cont) { 22998 freemsg(mp->b_cont); 22999 mp->b_cont = NULL; 23000 } 23001 /* 23002 * We skip reversing source route here. 23003 * (for now we replace all IP options with EOL) 23004 */ 23005 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) { 23006 ipha = (ipha_t *)mp->b_rptr; 23007 for (i = IP_SIMPLE_HDR_LENGTH; i < (int)ip_hdr_len; i++) 23008 mp->b_rptr[i] = IPOPT_EOL; 23009 /* 23010 * Make sure that src address isn't flagrantly invalid. 23011 * Not all broadcast address checking for the src address 23012 * is possible, since we don't know the netmask of the src 23013 * addr. No check for destination address is done, since 23014 * IP will not pass up a packet with a broadcast dest 23015 * address to TCP. Similar checks are done below for IPv6. 23016 */ 23017 if (ipha->ipha_src == 0 || ipha->ipha_src == INADDR_BROADCAST || 23018 CLASSD(ipha->ipha_src)) { 23019 freemsg(ipsec_mp); 23020 BUMP_MIB(&ipst->ips_ip_mib, ipIfStatsInDiscards); 23021 return; 23022 } 23023 } else { 23024 ip6h = (ip6_t *)mp->b_rptr; 23025 23026 if (IN6_IS_ADDR_UNSPECIFIED(&ip6h->ip6_src) || 23027 IN6_IS_ADDR_MULTICAST(&ip6h->ip6_src)) { 23028 freemsg(ipsec_mp); 23029 BUMP_MIB(&ipst->ips_ip6_mib, ipIfStatsInDiscards); 23030 return; 23031 } 23032 23033 /* Remove any extension headers assuming partial overlay */ 23034 if (ip_hdr_len > IPV6_HDR_LEN) { 23035 uint8_t *to; 23036 23037 to = mp->b_rptr + ip_hdr_len - IPV6_HDR_LEN; 23038 ovbcopy(ip6h, to, IPV6_HDR_LEN); 23039 mp->b_rptr += ip_hdr_len - IPV6_HDR_LEN; 23040 ip_hdr_len = IPV6_HDR_LEN; 23041 ip6h = (ip6_t *)mp->b_rptr; 23042 ip6h->ip6_nxt = IPPROTO_TCP; 23043 } 23044 } 23045 tcph = (tcph_t *)&mp->b_rptr[ip_hdr_len]; 23046 if (tcph->th_flags[0] & TH_RST) { 23047 freemsg(ipsec_mp); 23048 return; 23049 } 23050 tcph->th_offset_and_rsrvd[0] = (5 << 4); 23051 len = ip_hdr_len + sizeof (tcph_t); 23052 mp->b_wptr = &mp->b_rptr[len]; 23053 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) { 23054 ipha->ipha_length = htons(len); 23055 /* Swap addresses */ 23056 v4addr = ipha->ipha_src; 23057 ipha->ipha_src = ipha->ipha_dst; 23058 ipha->ipha_dst = v4addr; 23059 ipha->ipha_ident = 0; 23060 ipha->ipha_ttl = (uchar_t)tcps->tcps_ipv4_ttl; 23061 addr_len = IP_ADDR_LEN; 23062 addr = &v4addr; 23063 } else { 23064 /* No ip6i_t in this case */ 23065 ip6h->ip6_plen = htons(len - IPV6_HDR_LEN); 23066 /* Swap addresses */ 23067 v6addr = ip6h->ip6_src; 23068 ip6h->ip6_src = ip6h->ip6_dst; 23069 ip6h->ip6_dst = v6addr; 23070 ip6h->ip6_hops = (uchar_t)tcps->tcps_ipv6_hoplimit; 23071 addr_len = IPV6_ADDR_LEN; 23072 addr = &v6addr; 23073 } 23074 tcp_xchg(tcph->th_fport, tcph->th_lport, 2); 23075 U32_TO_BE32(ack, tcph->th_ack); 23076 U32_TO_BE32(seq, tcph->th_seq); 23077 U16_TO_BE16(0, tcph->th_win); 23078 U16_TO_BE16(sizeof (tcph_t), tcph->th_sum); 23079 tcph->th_flags[0] = (uint8_t)ctl; 23080 if (ctl & TH_RST) { 23081 BUMP_MIB(&tcps->tcps_mib, tcpOutRsts); 23082 BUMP_MIB(&tcps->tcps_mib, tcpOutControl); 23083 } 23084 23085 /* IP trusts us to set up labels when required. */ 23086 if (is_system_labeled() && (cr = DB_CRED(mp)) != NULL && 23087 crgetlabel(cr) != NULL) { 23088 int err; 23089 23090 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) 23091 err = tsol_check_label(cr, &mp, 23092 tcp->tcp_connp->conn_mac_exempt, 23093 tcps->tcps_netstack->netstack_ip); 23094 else 23095 err = tsol_check_label_v6(cr, &mp, 23096 tcp->tcp_connp->conn_mac_exempt, 23097 tcps->tcps_netstack->netstack_ip); 23098 if (mctl_present) 23099 ipsec_mp->b_cont = mp; 23100 else 23101 ipsec_mp = mp; 23102 if (err != 0) { 23103 freemsg(ipsec_mp); 23104 return; 23105 } 23106 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) { 23107 ipha = (ipha_t *)mp->b_rptr; 23108 } else { 23109 ip6h = (ip6_t *)mp->b_rptr; 23110 } 23111 } 23112 23113 if (mctl_present) { 23114 ipsec_in_t *ii = (ipsec_in_t *)ipsec_mp->b_rptr; 23115 23116 ASSERT(ii->ipsec_in_type == IPSEC_IN); 23117 if (!ipsec_in_to_out(ipsec_mp, ipha, ip6h)) { 23118 return; 23119 } 23120 } 23121 if (zoneid == ALL_ZONES) 23122 zoneid = GLOBAL_ZONEID; 23123 23124 /* Add the zoneid so ip_output routes it properly */ 23125 if ((nmp = ip_prepend_zoneid(ipsec_mp, zoneid, ipst)) == NULL) { 23126 freemsg(ipsec_mp); 23127 return; 23128 } 23129 ipsec_mp = nmp; 23130 23131 /* 23132 * NOTE: one might consider tracing a TCP packet here, but 23133 * this function has no active TCP state and no tcp structure 23134 * that has a trace buffer. If we traced here, we would have 23135 * to keep a local trace buffer in tcp_record_trace(). 23136 * 23137 * TSol note: The mblk that contains the incoming packet was 23138 * reused by tcp_xmit_listener_reset, so it already contains 23139 * the right credentials and we don't need to call mblk_setcred. 23140 * Also the conn's cred is not right since it is associated 23141 * with tcps_g_q. 23142 */ 23143 CALL_IP_WPUT(tcp->tcp_connp, tcp->tcp_wq, ipsec_mp); 23144 23145 /* 23146 * Tell IP to mark the IRE used for this destination temporary. 23147 * This way, we can limit our exposure to DoS attack because IP 23148 * creates an IRE for each destination. If there are too many, 23149 * the time to do any routing lookup will be extremely long. And 23150 * the lookup can be in interrupt context. 23151 * 23152 * Note that in normal circumstances, this marking should not 23153 * affect anything. It would be nice if only 1 message is 23154 * needed to inform IP that the IRE created for this RST should 23155 * not be added to the cache table. But there is currently 23156 * not such communication mechanism between TCP and IP. So 23157 * the best we can do now is to send the advice ioctl to IP 23158 * to mark the IRE temporary. 23159 */ 23160 if ((mp = tcp_ip_advise_mblk(addr, addr_len, &ipic)) != NULL) { 23161 ipic->ipic_ire_marks |= IRE_MARK_TEMPORARY; 23162 CALL_IP_WPUT(tcp->tcp_connp, tcp->tcp_wq, mp); 23163 } 23164 } 23165 23166 /* 23167 * Initiate closedown sequence on an active connection. (May be called as 23168 * writer.) Return value zero for OK return, non-zero for error return. 23169 */ 23170 static int 23171 tcp_xmit_end(tcp_t *tcp) 23172 { 23173 ipic_t *ipic; 23174 mblk_t *mp; 23175 tcp_stack_t *tcps = tcp->tcp_tcps; 23176 23177 if (tcp->tcp_state < TCPS_SYN_RCVD || 23178 tcp->tcp_state > TCPS_CLOSE_WAIT) { 23179 /* 23180 * Invalid state, only states TCPS_SYN_RCVD, 23181 * TCPS_ESTABLISHED and TCPS_CLOSE_WAIT are valid 23182 */ 23183 return (-1); 23184 } 23185 23186 tcp->tcp_fss = tcp->tcp_snxt + tcp->tcp_unsent; 23187 tcp->tcp_valid_bits |= TCP_FSS_VALID; 23188 /* 23189 * If there is nothing more unsent, send the FIN now. 23190 * Otherwise, it will go out with the last segment. 23191 */ 23192 if (tcp->tcp_unsent == 0) { 23193 mp = tcp_xmit_mp(tcp, NULL, 0, NULL, NULL, 23194 tcp->tcp_fss, B_FALSE, NULL, B_FALSE); 23195 23196 if (mp) { 23197 tcp_send_data(tcp, tcp->tcp_wq, mp); 23198 } else { 23199 /* 23200 * Couldn't allocate msg. Pretend we got it out. 23201 * Wait for rexmit timeout. 23202 */ 23203 tcp->tcp_snxt = tcp->tcp_fss + 1; 23204 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 23205 } 23206 23207 /* 23208 * If needed, update tcp_rexmit_snxt as tcp_snxt is 23209 * changed. 23210 */ 23211 if (tcp->tcp_rexmit && tcp->tcp_rexmit_nxt == tcp->tcp_fss) { 23212 tcp->tcp_rexmit_nxt = tcp->tcp_snxt; 23213 } 23214 } else { 23215 /* 23216 * If tcp->tcp_cork is set, then the data will not get sent, 23217 * so we have to check that and unset it first. 23218 */ 23219 if (tcp->tcp_cork) 23220 tcp->tcp_cork = B_FALSE; 23221 tcp_wput_data(tcp, NULL, B_FALSE); 23222 } 23223 23224 /* 23225 * If TCP does not get enough samples of RTT or tcp_rtt_updates 23226 * is 0, don't update the cache. 23227 */ 23228 if (tcps->tcps_rtt_updates == 0 || 23229 tcp->tcp_rtt_update < tcps->tcps_rtt_updates) 23230 return (0); 23231 23232 /* 23233 * NOTE: should not update if source routes i.e. if tcp_remote if 23234 * different from the destination. 23235 */ 23236 if (tcp->tcp_ipversion == IPV4_VERSION) { 23237 if (tcp->tcp_remote != tcp->tcp_ipha->ipha_dst) { 23238 return (0); 23239 } 23240 mp = tcp_ip_advise_mblk(&tcp->tcp_ipha->ipha_dst, IP_ADDR_LEN, 23241 &ipic); 23242 } else { 23243 if (!(IN6_ARE_ADDR_EQUAL(&tcp->tcp_remote_v6, 23244 &tcp->tcp_ip6h->ip6_dst))) { 23245 return (0); 23246 } 23247 mp = tcp_ip_advise_mblk(&tcp->tcp_ip6h->ip6_dst, IPV6_ADDR_LEN, 23248 &ipic); 23249 } 23250 23251 /* Record route attributes in the IRE for use by future connections. */ 23252 if (mp == NULL) 23253 return (0); 23254 23255 /* 23256 * We do not have a good algorithm to update ssthresh at this time. 23257 * So don't do any update. 23258 */ 23259 ipic->ipic_rtt = tcp->tcp_rtt_sa; 23260 ipic->ipic_rtt_sd = tcp->tcp_rtt_sd; 23261 23262 CALL_IP_WPUT(tcp->tcp_connp, tcp->tcp_wq, mp); 23263 return (0); 23264 } 23265 23266 /* 23267 * Generate a "no listener here" RST in response to an "unknown" segment. 23268 * connp is set by caller when RST is in response to an unexpected 23269 * inbound packet for which there is active tcp state in the system. 23270 * Note that we are reusing the incoming mp to construct the outgoing RST. 23271 */ 23272 void 23273 tcp_xmit_listeners_reset(mblk_t *mp, uint_t ip_hdr_len, zoneid_t zoneid, 23274 tcp_stack_t *tcps, conn_t *connp) 23275 { 23276 uchar_t *rptr; 23277 uint32_t seg_len; 23278 tcph_t *tcph; 23279 uint32_t seg_seq; 23280 uint32_t seg_ack; 23281 uint_t flags; 23282 mblk_t *ipsec_mp; 23283 ipha_t *ipha; 23284 ip6_t *ip6h; 23285 boolean_t mctl_present = B_FALSE; 23286 boolean_t check = B_TRUE; 23287 boolean_t policy_present; 23288 ipsec_stack_t *ipss = tcps->tcps_netstack->netstack_ipsec; 23289 23290 TCP_STAT(tcps, tcp_no_listener); 23291 23292 ipsec_mp = mp; 23293 23294 if (mp->b_datap->db_type == M_CTL) { 23295 ipsec_in_t *ii; 23296 23297 mctl_present = B_TRUE; 23298 mp = mp->b_cont; 23299 23300 ii = (ipsec_in_t *)ipsec_mp->b_rptr; 23301 ASSERT(ii->ipsec_in_type == IPSEC_IN); 23302 if (ii->ipsec_in_dont_check) { 23303 check = B_FALSE; 23304 if (!ii->ipsec_in_secure) { 23305 freeb(ipsec_mp); 23306 mctl_present = B_FALSE; 23307 ipsec_mp = mp; 23308 } 23309 } 23310 } 23311 23312 if (IPH_HDR_VERSION(mp->b_rptr) == IPV4_VERSION) { 23313 policy_present = ipss->ipsec_inbound_v4_policy_present; 23314 ipha = (ipha_t *)mp->b_rptr; 23315 ip6h = NULL; 23316 } else { 23317 policy_present = ipss->ipsec_inbound_v6_policy_present; 23318 ipha = NULL; 23319 ip6h = (ip6_t *)mp->b_rptr; 23320 } 23321 23322 if (check && policy_present) { 23323 /* 23324 * The conn_t parameter is NULL because we already know 23325 * nobody's home. 23326 */ 23327 ipsec_mp = ipsec_check_global_policy( 23328 ipsec_mp, (conn_t *)NULL, ipha, ip6h, mctl_present, 23329 tcps->tcps_netstack); 23330 if (ipsec_mp == NULL) 23331 return; 23332 } 23333 if (is_system_labeled() && !tsol_can_reply_error(mp)) { 23334 DTRACE_PROBE2( 23335 tx__ip__log__error__nolistener__tcp, 23336 char *, "Could not reply with RST to mp(1)", 23337 mblk_t *, mp); 23338 ip2dbg(("tcp_xmit_listeners_reset: not permitted to reply\n")); 23339 freemsg(ipsec_mp); 23340 return; 23341 } 23342 23343 rptr = mp->b_rptr; 23344 23345 tcph = (tcph_t *)&rptr[ip_hdr_len]; 23346 seg_seq = BE32_TO_U32(tcph->th_seq); 23347 seg_ack = BE32_TO_U32(tcph->th_ack); 23348 flags = tcph->th_flags[0]; 23349 23350 seg_len = msgdsize(mp) - (TCP_HDR_LENGTH(tcph) + ip_hdr_len); 23351 if (flags & TH_RST) { 23352 freemsg(ipsec_mp); 23353 } else if (flags & TH_ACK) { 23354 tcp_xmit_early_reset("no tcp, reset", 23355 ipsec_mp, seg_ack, 0, TH_RST, ip_hdr_len, zoneid, tcps, 23356 connp); 23357 } else { 23358 if (flags & TH_SYN) { 23359 seg_len++; 23360 } else { 23361 /* 23362 * Here we violate the RFC. Note that a normal 23363 * TCP will never send a segment without the ACK 23364 * flag, except for RST or SYN segment. This 23365 * segment is neither. Just drop it on the 23366 * floor. 23367 */ 23368 freemsg(ipsec_mp); 23369 tcps->tcps_rst_unsent++; 23370 return; 23371 } 23372 23373 tcp_xmit_early_reset("no tcp, reset/ack", 23374 ipsec_mp, 0, seg_seq + seg_len, 23375 TH_RST | TH_ACK, ip_hdr_len, zoneid, tcps, connp); 23376 } 23377 } 23378 23379 /* 23380 * tcp_xmit_mp is called to return a pointer to an mblk chain complete with 23381 * ip and tcp header ready to pass down to IP. If the mp passed in is 23382 * non-NULL, then up to max_to_send bytes of data will be dup'ed off that 23383 * mblk. (If sendall is not set the dup'ing will stop at an mblk boundary 23384 * otherwise it will dup partial mblks.) 23385 * Otherwise, an appropriate ACK packet will be generated. This 23386 * routine is not usually called to send new data for the first time. It 23387 * is mostly called out of the timer for retransmits, and to generate ACKs. 23388 * 23389 * If offset is not NULL, the returned mblk chain's first mblk's b_rptr will 23390 * be adjusted by *offset. And after dupb(), the offset and the ending mblk 23391 * of the original mblk chain will be returned in *offset and *end_mp. 23392 */ 23393 mblk_t * 23394 tcp_xmit_mp(tcp_t *tcp, mblk_t *mp, int32_t max_to_send, int32_t *offset, 23395 mblk_t **end_mp, uint32_t seq, boolean_t sendall, uint32_t *seg_len, 23396 boolean_t rexmit) 23397 { 23398 int data_length; 23399 int32_t off = 0; 23400 uint_t flags; 23401 mblk_t *mp1; 23402 mblk_t *mp2; 23403 uchar_t *rptr; 23404 tcph_t *tcph; 23405 int32_t num_sack_blk = 0; 23406 int32_t sack_opt_len = 0; 23407 tcp_stack_t *tcps = tcp->tcp_tcps; 23408 23409 /* Allocate for our maximum TCP header + link-level */ 23410 mp1 = allocb(tcp->tcp_ip_hdr_len + TCP_MAX_HDR_LENGTH + 23411 tcps->tcps_wroff_xtra, BPRI_MED); 23412 if (!mp1) 23413 return (NULL); 23414 data_length = 0; 23415 23416 /* 23417 * Note that tcp_mss has been adjusted to take into account the 23418 * timestamp option if applicable. Because SACK options do not 23419 * appear in every TCP segments and they are of variable lengths, 23420 * they cannot be included in tcp_mss. Thus we need to calculate 23421 * the actual segment length when we need to send a segment which 23422 * includes SACK options. 23423 */ 23424 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) { 23425 num_sack_blk = MIN(tcp->tcp_max_sack_blk, 23426 tcp->tcp_num_sack_blk); 23427 sack_opt_len = num_sack_blk * sizeof (sack_blk_t) + 23428 TCPOPT_NOP_LEN * 2 + TCPOPT_HEADER_LEN; 23429 if (max_to_send + sack_opt_len > tcp->tcp_mss) 23430 max_to_send -= sack_opt_len; 23431 } 23432 23433 if (offset != NULL) { 23434 off = *offset; 23435 /* We use offset as an indicator that end_mp is not NULL. */ 23436 *end_mp = NULL; 23437 } 23438 for (mp2 = mp1; mp && data_length != max_to_send; mp = mp->b_cont) { 23439 /* This could be faster with cooperation from downstream */ 23440 if (mp2 != mp1 && !sendall && 23441 data_length + (int)(mp->b_wptr - mp->b_rptr) > 23442 max_to_send) 23443 /* 23444 * Don't send the next mblk since the whole mblk 23445 * does not fit. 23446 */ 23447 break; 23448 mp2->b_cont = dupb(mp); 23449 mp2 = mp2->b_cont; 23450 if (!mp2) { 23451 freemsg(mp1); 23452 return (NULL); 23453 } 23454 mp2->b_rptr += off; 23455 ASSERT((uintptr_t)(mp2->b_wptr - mp2->b_rptr) <= 23456 (uintptr_t)INT_MAX); 23457 23458 data_length += (int)(mp2->b_wptr - mp2->b_rptr); 23459 if (data_length > max_to_send) { 23460 mp2->b_wptr -= data_length - max_to_send; 23461 data_length = max_to_send; 23462 off = mp2->b_wptr - mp->b_rptr; 23463 break; 23464 } else { 23465 off = 0; 23466 } 23467 } 23468 if (offset != NULL) { 23469 *offset = off; 23470 *end_mp = mp; 23471 } 23472 if (seg_len != NULL) { 23473 *seg_len = data_length; 23474 } 23475 23476 /* Update the latest receive window size in TCP header. */ 23477 U32_TO_ABE16(tcp->tcp_rwnd >> tcp->tcp_rcv_ws, 23478 tcp->tcp_tcph->th_win); 23479 23480 rptr = mp1->b_rptr + tcps->tcps_wroff_xtra; 23481 mp1->b_rptr = rptr; 23482 mp1->b_wptr = rptr + tcp->tcp_hdr_len + sack_opt_len; 23483 bcopy(tcp->tcp_iphc, rptr, tcp->tcp_hdr_len); 23484 tcph = (tcph_t *)&rptr[tcp->tcp_ip_hdr_len]; 23485 U32_TO_ABE32(seq, tcph->th_seq); 23486 23487 /* 23488 * Use tcp_unsent to determine if the PUSH bit should be used assumes 23489 * that this function was called from tcp_wput_data. Thus, when called 23490 * to retransmit data the setting of the PUSH bit may appear some 23491 * what random in that it might get set when it should not. This 23492 * should not pose any performance issues. 23493 */ 23494 if (data_length != 0 && (tcp->tcp_unsent == 0 || 23495 tcp->tcp_unsent == data_length)) { 23496 flags = TH_ACK | TH_PUSH; 23497 } else { 23498 flags = TH_ACK; 23499 } 23500 23501 if (tcp->tcp_ecn_ok) { 23502 if (tcp->tcp_ecn_echo_on) 23503 flags |= TH_ECE; 23504 23505 /* 23506 * Only set ECT bit and ECN_CWR if a segment contains new data. 23507 * There is no TCP flow control for non-data segments, and 23508 * only data segment is transmitted reliably. 23509 */ 23510 if (data_length > 0 && !rexmit) { 23511 SET_ECT(tcp, rptr); 23512 if (tcp->tcp_cwr && !tcp->tcp_ecn_cwr_sent) { 23513 flags |= TH_CWR; 23514 tcp->tcp_ecn_cwr_sent = B_TRUE; 23515 } 23516 } 23517 } 23518 23519 if (tcp->tcp_valid_bits) { 23520 uint32_t u1; 23521 23522 if ((tcp->tcp_valid_bits & TCP_ISS_VALID) && 23523 seq == tcp->tcp_iss) { 23524 uchar_t *wptr; 23525 23526 /* 23527 * If TCP_ISS_VALID and the seq number is tcp_iss, 23528 * TCP can only be in SYN-SENT, SYN-RCVD or 23529 * FIN-WAIT-1 state. It can be FIN-WAIT-1 if 23530 * our SYN is not ack'ed but the app closes this 23531 * TCP connection. 23532 */ 23533 ASSERT(tcp->tcp_state == TCPS_SYN_SENT || 23534 tcp->tcp_state == TCPS_SYN_RCVD || 23535 tcp->tcp_state == TCPS_FIN_WAIT_1); 23536 23537 /* 23538 * Tack on the MSS option. It is always needed 23539 * for both active and passive open. 23540 * 23541 * MSS option value should be interface MTU - MIN 23542 * TCP/IP header according to RFC 793 as it means 23543 * the maximum segment size TCP can receive. But 23544 * to get around some broken middle boxes/end hosts 23545 * out there, we allow the option value to be the 23546 * same as the MSS option size on the peer side. 23547 * In this way, the other side will not send 23548 * anything larger than they can receive. 23549 * 23550 * Note that for SYN_SENT state, the ndd param 23551 * tcp_use_smss_as_mss_opt has no effect as we 23552 * don't know the peer's MSS option value. So 23553 * the only case we need to take care of is in 23554 * SYN_RCVD state, which is done later. 23555 */ 23556 wptr = mp1->b_wptr; 23557 wptr[0] = TCPOPT_MAXSEG; 23558 wptr[1] = TCPOPT_MAXSEG_LEN; 23559 wptr += 2; 23560 u1 = tcp->tcp_if_mtu - 23561 (tcp->tcp_ipversion == IPV4_VERSION ? 23562 IP_SIMPLE_HDR_LENGTH : IPV6_HDR_LEN) - 23563 TCP_MIN_HEADER_LENGTH; 23564 U16_TO_BE16(u1, wptr); 23565 mp1->b_wptr = wptr + 2; 23566 /* Update the offset to cover the additional word */ 23567 tcph->th_offset_and_rsrvd[0] += (1 << 4); 23568 23569 /* 23570 * Note that the following way of filling in 23571 * TCP options are not optimal. Some NOPs can 23572 * be saved. But there is no need at this time 23573 * to optimize it. When it is needed, we will 23574 * do it. 23575 */ 23576 switch (tcp->tcp_state) { 23577 case TCPS_SYN_SENT: 23578 flags = TH_SYN; 23579 23580 if (tcp->tcp_snd_ts_ok) { 23581 uint32_t llbolt = (uint32_t)lbolt; 23582 23583 wptr = mp1->b_wptr; 23584 wptr[0] = TCPOPT_NOP; 23585 wptr[1] = TCPOPT_NOP; 23586 wptr[2] = TCPOPT_TSTAMP; 23587 wptr[3] = TCPOPT_TSTAMP_LEN; 23588 wptr += 4; 23589 U32_TO_BE32(llbolt, wptr); 23590 wptr += 4; 23591 ASSERT(tcp->tcp_ts_recent == 0); 23592 U32_TO_BE32(0L, wptr); 23593 mp1->b_wptr += TCPOPT_REAL_TS_LEN; 23594 tcph->th_offset_and_rsrvd[0] += 23595 (3 << 4); 23596 } 23597 23598 /* 23599 * Set up all the bits to tell other side 23600 * we are ECN capable. 23601 */ 23602 if (tcp->tcp_ecn_ok) { 23603 flags |= (TH_ECE | TH_CWR); 23604 } 23605 break; 23606 case TCPS_SYN_RCVD: 23607 flags |= TH_SYN; 23608 23609 /* 23610 * Reset the MSS option value to be SMSS 23611 * We should probably add back the bytes 23612 * for timestamp option and IPsec. We 23613 * don't do that as this is a workaround 23614 * for broken middle boxes/end hosts, it 23615 * is better for us to be more cautious. 23616 * They may not take these things into 23617 * account in their SMSS calculation. Thus 23618 * the peer's calculated SMSS may be smaller 23619 * than what it can be. This should be OK. 23620 */ 23621 if (tcps->tcps_use_smss_as_mss_opt) { 23622 u1 = tcp->tcp_mss; 23623 U16_TO_BE16(u1, wptr); 23624 } 23625 23626 /* 23627 * If the other side is ECN capable, reply 23628 * that we are also ECN capable. 23629 */ 23630 if (tcp->tcp_ecn_ok) 23631 flags |= TH_ECE; 23632 break; 23633 default: 23634 /* 23635 * The above ASSERT() makes sure that this 23636 * must be FIN-WAIT-1 state. Our SYN has 23637 * not been ack'ed so retransmit it. 23638 */ 23639 flags |= TH_SYN; 23640 break; 23641 } 23642 23643 if (tcp->tcp_snd_ws_ok) { 23644 wptr = mp1->b_wptr; 23645 wptr[0] = TCPOPT_NOP; 23646 wptr[1] = TCPOPT_WSCALE; 23647 wptr[2] = TCPOPT_WS_LEN; 23648 wptr[3] = (uchar_t)tcp->tcp_rcv_ws; 23649 mp1->b_wptr += TCPOPT_REAL_WS_LEN; 23650 tcph->th_offset_and_rsrvd[0] += (1 << 4); 23651 } 23652 23653 if (tcp->tcp_snd_sack_ok) { 23654 wptr = mp1->b_wptr; 23655 wptr[0] = TCPOPT_NOP; 23656 wptr[1] = TCPOPT_NOP; 23657 wptr[2] = TCPOPT_SACK_PERMITTED; 23658 wptr[3] = TCPOPT_SACK_OK_LEN; 23659 mp1->b_wptr += TCPOPT_REAL_SACK_OK_LEN; 23660 tcph->th_offset_and_rsrvd[0] += (1 << 4); 23661 } 23662 23663 /* allocb() of adequate mblk assures space */ 23664 ASSERT((uintptr_t)(mp1->b_wptr - mp1->b_rptr) <= 23665 (uintptr_t)INT_MAX); 23666 u1 = (int)(mp1->b_wptr - mp1->b_rptr); 23667 /* 23668 * Get IP set to checksum on our behalf 23669 * Include the adjustment for a source route if any. 23670 */ 23671 u1 += tcp->tcp_sum; 23672 u1 = (u1 >> 16) + (u1 & 0xFFFF); 23673 U16_TO_BE16(u1, tcph->th_sum); 23674 BUMP_MIB(&tcps->tcps_mib, tcpOutControl); 23675 } 23676 if ((tcp->tcp_valid_bits & TCP_FSS_VALID) && 23677 (seq + data_length) == tcp->tcp_fss) { 23678 if (!tcp->tcp_fin_acked) { 23679 flags |= TH_FIN; 23680 BUMP_MIB(&tcps->tcps_mib, tcpOutControl); 23681 } 23682 if (!tcp->tcp_fin_sent) { 23683 tcp->tcp_fin_sent = B_TRUE; 23684 switch (tcp->tcp_state) { 23685 case TCPS_SYN_RCVD: 23686 case TCPS_ESTABLISHED: 23687 tcp->tcp_state = TCPS_FIN_WAIT_1; 23688 break; 23689 case TCPS_CLOSE_WAIT: 23690 tcp->tcp_state = TCPS_LAST_ACK; 23691 break; 23692 } 23693 if (tcp->tcp_suna == tcp->tcp_snxt) 23694 TCP_TIMER_RESTART(tcp, tcp->tcp_rto); 23695 tcp->tcp_snxt = tcp->tcp_fss + 1; 23696 } 23697 } 23698 /* 23699 * Note the trick here. u1 is unsigned. When tcp_urg 23700 * is smaller than seq, u1 will become a very huge value. 23701 * So the comparison will fail. Also note that tcp_urp 23702 * should be positive, see RFC 793 page 17. 23703 */ 23704 u1 = tcp->tcp_urg - seq + TCP_OLD_URP_INTERPRETATION; 23705 if ((tcp->tcp_valid_bits & TCP_URG_VALID) && u1 != 0 && 23706 u1 < (uint32_t)(64 * 1024)) { 23707 flags |= TH_URG; 23708 BUMP_MIB(&tcps->tcps_mib, tcpOutUrg); 23709 U32_TO_ABE16(u1, tcph->th_urp); 23710 } 23711 } 23712 tcph->th_flags[0] = (uchar_t)flags; 23713 tcp->tcp_rack = tcp->tcp_rnxt; 23714 tcp->tcp_rack_cnt = 0; 23715 23716 if (tcp->tcp_snd_ts_ok) { 23717 if (tcp->tcp_state != TCPS_SYN_SENT) { 23718 uint32_t llbolt = (uint32_t)lbolt; 23719 23720 U32_TO_BE32(llbolt, 23721 (char *)tcph+TCP_MIN_HEADER_LENGTH+4); 23722 U32_TO_BE32(tcp->tcp_ts_recent, 23723 (char *)tcph+TCP_MIN_HEADER_LENGTH+8); 23724 } 23725 } 23726 23727 if (num_sack_blk > 0) { 23728 uchar_t *wptr = (uchar_t *)tcph + tcp->tcp_tcp_hdr_len; 23729 sack_blk_t *tmp; 23730 int32_t i; 23731 23732 wptr[0] = TCPOPT_NOP; 23733 wptr[1] = TCPOPT_NOP; 23734 wptr[2] = TCPOPT_SACK; 23735 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk * 23736 sizeof (sack_blk_t); 23737 wptr += TCPOPT_REAL_SACK_LEN; 23738 23739 tmp = tcp->tcp_sack_list; 23740 for (i = 0; i < num_sack_blk; i++) { 23741 U32_TO_BE32(tmp[i].begin, wptr); 23742 wptr += sizeof (tcp_seq); 23743 U32_TO_BE32(tmp[i].end, wptr); 23744 wptr += sizeof (tcp_seq); 23745 } 23746 tcph->th_offset_and_rsrvd[0] += ((num_sack_blk * 2 + 1) << 4); 23747 } 23748 ASSERT((uintptr_t)(mp1->b_wptr - rptr) <= (uintptr_t)INT_MAX); 23749 data_length += (int)(mp1->b_wptr - rptr); 23750 if (tcp->tcp_ipversion == IPV4_VERSION) { 23751 ((ipha_t *)rptr)->ipha_length = htons(data_length); 23752 } else { 23753 ip6_t *ip6 = (ip6_t *)(rptr + 23754 (((ip6_t *)rptr)->ip6_nxt == IPPROTO_RAW ? 23755 sizeof (ip6i_t) : 0)); 23756 23757 ip6->ip6_plen = htons(data_length - 23758 ((char *)&tcp->tcp_ip6h[1] - tcp->tcp_iphc)); 23759 } 23760 23761 /* 23762 * Prime pump for IP 23763 * Include the adjustment for a source route if any. 23764 */ 23765 data_length -= tcp->tcp_ip_hdr_len; 23766 data_length += tcp->tcp_sum; 23767 data_length = (data_length >> 16) + (data_length & 0xFFFF); 23768 U16_TO_ABE16(data_length, tcph->th_sum); 23769 if (tcp->tcp_ip_forward_progress) { 23770 ASSERT(tcp->tcp_ipversion == IPV6_VERSION); 23771 *(uint32_t *)mp1->b_rptr |= IP_FORWARD_PROG; 23772 tcp->tcp_ip_forward_progress = B_FALSE; 23773 } 23774 return (mp1); 23775 } 23776 23777 /* This function handles the push timeout. */ 23778 void 23779 tcp_push_timer(void *arg) 23780 { 23781 conn_t *connp = (conn_t *)arg; 23782 tcp_t *tcp = connp->conn_tcp; 23783 tcp_stack_t *tcps = tcp->tcp_tcps; 23784 uint_t flags; 23785 sodirect_t *sodp; 23786 23787 TCP_DBGSTAT(tcps, tcp_push_timer_cnt); 23788 23789 ASSERT(tcp->tcp_listener == NULL); 23790 23791 /* 23792 * We need to plug synchronous streams during our drain to prevent 23793 * a race with tcp_fuse_rrw() or tcp_fusion_rinfop(). 23794 */ 23795 TCP_FUSE_SYNCSTR_PLUG_DRAIN(tcp); 23796 tcp->tcp_push_tid = 0; 23797 23798 SOD_PTR_ENTER(tcp, sodp); 23799 if (sodp != NULL) { 23800 flags = tcp_rcv_sod_wakeup(tcp, sodp); 23801 /* sod_wakeup() does the mutex_exit() */ 23802 } else if (tcp->tcp_rcv_list != NULL) { 23803 flags = tcp_rcv_drain(tcp->tcp_rq, tcp); 23804 } 23805 if (flags == TH_ACK_NEEDED) 23806 tcp_xmit_ctl(NULL, tcp, tcp->tcp_snxt, tcp->tcp_rnxt, TH_ACK); 23807 23808 TCP_FUSE_SYNCSTR_UNPLUG_DRAIN(tcp); 23809 } 23810 23811 /* 23812 * This function handles delayed ACK timeout. 23813 */ 23814 static void 23815 tcp_ack_timer(void *arg) 23816 { 23817 conn_t *connp = (conn_t *)arg; 23818 tcp_t *tcp = connp->conn_tcp; 23819 mblk_t *mp; 23820 tcp_stack_t *tcps = tcp->tcp_tcps; 23821 23822 TCP_DBGSTAT(tcps, tcp_ack_timer_cnt); 23823 23824 tcp->tcp_ack_tid = 0; 23825 23826 if (tcp->tcp_fused) 23827 return; 23828 23829 /* 23830 * Do not send ACK if there is no outstanding unack'ed data. 23831 */ 23832 if (tcp->tcp_rnxt == tcp->tcp_rack) { 23833 return; 23834 } 23835 23836 if ((tcp->tcp_rnxt - tcp->tcp_rack) > tcp->tcp_mss) { 23837 /* 23838 * Make sure we don't allow deferred ACKs to result in 23839 * timer-based ACKing. If we have held off an ACK 23840 * when there was more than an mss here, and the timer 23841 * goes off, we have to worry about the possibility 23842 * that the sender isn't doing slow-start, or is out 23843 * of step with us for some other reason. We fall 23844 * permanently back in the direction of 23845 * ACK-every-other-packet as suggested in RFC 1122. 23846 */ 23847 if (tcp->tcp_rack_abs_max > 2) 23848 tcp->tcp_rack_abs_max--; 23849 tcp->tcp_rack_cur_max = 2; 23850 } 23851 mp = tcp_ack_mp(tcp); 23852 23853 if (mp != NULL) { 23854 BUMP_LOCAL(tcp->tcp_obsegs); 23855 BUMP_MIB(&tcps->tcps_mib, tcpOutAck); 23856 BUMP_MIB(&tcps->tcps_mib, tcpOutAckDelayed); 23857 tcp_send_data(tcp, tcp->tcp_wq, mp); 23858 } 23859 } 23860 23861 23862 /* Generate an ACK-only (no data) segment for a TCP endpoint */ 23863 static mblk_t * 23864 tcp_ack_mp(tcp_t *tcp) 23865 { 23866 uint32_t seq_no; 23867 tcp_stack_t *tcps = tcp->tcp_tcps; 23868 23869 /* 23870 * There are a few cases to be considered while setting the sequence no. 23871 * Essentially, we can come here while processing an unacceptable pkt 23872 * in the TCPS_SYN_RCVD state, in which case we set the sequence number 23873 * to snxt (per RFC 793), note the swnd wouldn't have been set yet. 23874 * If we are here for a zero window probe, stick with suna. In all 23875 * other cases, we check if suna + swnd encompasses snxt and set 23876 * the sequence number to snxt, if so. If snxt falls outside the 23877 * window (the receiver probably shrunk its window), we will go with 23878 * suna + swnd, otherwise the sequence no will be unacceptable to the 23879 * receiver. 23880 */ 23881 if (tcp->tcp_zero_win_probe) { 23882 seq_no = tcp->tcp_suna; 23883 } else if (tcp->tcp_state == TCPS_SYN_RCVD) { 23884 ASSERT(tcp->tcp_swnd == 0); 23885 seq_no = tcp->tcp_snxt; 23886 } else { 23887 seq_no = SEQ_GT(tcp->tcp_snxt, 23888 (tcp->tcp_suna + tcp->tcp_swnd)) ? 23889 (tcp->tcp_suna + tcp->tcp_swnd) : tcp->tcp_snxt; 23890 } 23891 23892 if (tcp->tcp_valid_bits) { 23893 /* 23894 * For the complex case where we have to send some 23895 * controls (FIN or SYN), let tcp_xmit_mp do it. 23896 */ 23897 return (tcp_xmit_mp(tcp, NULL, 0, NULL, NULL, seq_no, B_FALSE, 23898 NULL, B_FALSE)); 23899 } else { 23900 /* Generate a simple ACK */ 23901 int data_length; 23902 uchar_t *rptr; 23903 tcph_t *tcph; 23904 mblk_t *mp1; 23905 int32_t tcp_hdr_len; 23906 int32_t tcp_tcp_hdr_len; 23907 int32_t num_sack_blk = 0; 23908 int32_t sack_opt_len; 23909 23910 /* 23911 * Allocate space for TCP + IP headers 23912 * and link-level header 23913 */ 23914 if (tcp->tcp_snd_sack_ok && tcp->tcp_num_sack_blk > 0) { 23915 num_sack_blk = MIN(tcp->tcp_max_sack_blk, 23916 tcp->tcp_num_sack_blk); 23917 sack_opt_len = num_sack_blk * sizeof (sack_blk_t) + 23918 TCPOPT_NOP_LEN * 2 + TCPOPT_HEADER_LEN; 23919 tcp_hdr_len = tcp->tcp_hdr_len + sack_opt_len; 23920 tcp_tcp_hdr_len = tcp->tcp_tcp_hdr_len + sack_opt_len; 23921 } else { 23922 tcp_hdr_len = tcp->tcp_hdr_len; 23923 tcp_tcp_hdr_len = tcp->tcp_tcp_hdr_len; 23924 } 23925 mp1 = allocb(tcp_hdr_len + tcps->tcps_wroff_xtra, BPRI_MED); 23926 if (!mp1) 23927 return (NULL); 23928 23929 /* Update the latest receive window size in TCP header. */ 23930 U32_TO_ABE16(tcp->tcp_rwnd >> tcp->tcp_rcv_ws, 23931 tcp->tcp_tcph->th_win); 23932 /* copy in prototype TCP + IP header */ 23933 rptr = mp1->b_rptr + tcps->tcps_wroff_xtra; 23934 mp1->b_rptr = rptr; 23935 mp1->b_wptr = rptr + tcp_hdr_len; 23936 bcopy(tcp->tcp_iphc, rptr, tcp->tcp_hdr_len); 23937 23938 tcph = (tcph_t *)&rptr[tcp->tcp_ip_hdr_len]; 23939 23940 /* Set the TCP sequence number. */ 23941 U32_TO_ABE32(seq_no, tcph->th_seq); 23942 23943 /* Set up the TCP flag field. */ 23944 tcph->th_flags[0] = (uchar_t)TH_ACK; 23945 if (tcp->tcp_ecn_echo_on) 23946 tcph->th_flags[0] |= TH_ECE; 23947 23948 tcp->tcp_rack = tcp->tcp_rnxt; 23949 tcp->tcp_rack_cnt = 0; 23950 23951 /* fill in timestamp option if in use */ 23952 if (tcp->tcp_snd_ts_ok) { 23953 uint32_t llbolt = (uint32_t)lbolt; 23954 23955 U32_TO_BE32(llbolt, 23956 (char *)tcph+TCP_MIN_HEADER_LENGTH+4); 23957 U32_TO_BE32(tcp->tcp_ts_recent, 23958 (char *)tcph+TCP_MIN_HEADER_LENGTH+8); 23959 } 23960 23961 /* Fill in SACK options */ 23962 if (num_sack_blk > 0) { 23963 uchar_t *wptr = (uchar_t *)tcph + tcp->tcp_tcp_hdr_len; 23964 sack_blk_t *tmp; 23965 int32_t i; 23966 23967 wptr[0] = TCPOPT_NOP; 23968 wptr[1] = TCPOPT_NOP; 23969 wptr[2] = TCPOPT_SACK; 23970 wptr[3] = TCPOPT_HEADER_LEN + num_sack_blk * 23971 sizeof (sack_blk_t); 23972 wptr += TCPOPT_REAL_SACK_LEN; 23973 23974 tmp = tcp->tcp_sack_list; 23975 for (i = 0; i < num_sack_blk; i++) { 23976 U32_TO_BE32(tmp[i].begin, wptr); 23977 wptr += sizeof (tcp_seq); 23978 U32_TO_BE32(tmp[i].end, wptr); 23979 wptr += sizeof (tcp_seq); 23980 } 23981 tcph->th_offset_and_rsrvd[0] += ((num_sack_blk * 2 + 1) 23982 << 4); 23983 } 23984 23985 if (tcp->tcp_ipversion == IPV4_VERSION) { 23986 ((ipha_t *)rptr)->ipha_length = htons(tcp_hdr_len); 23987 } else { 23988 /* Check for ip6i_t header in sticky hdrs */ 23989 ip6_t *ip6 = (ip6_t *)(rptr + 23990 (((ip6_t *)rptr)->ip6_nxt == IPPROTO_RAW ? 23991 sizeof (ip6i_t) : 0)); 23992 23993 ip6->ip6_plen = htons(tcp_hdr_len - 23994 ((char *)&tcp->tcp_ip6h[1] - tcp->tcp_iphc)); 23995 } 23996 23997 /* 23998 * Prime pump for checksum calculation in IP. Include the 23999 * adjustment for a source route if any. 24000 */ 24001 data_length = tcp_tcp_hdr_len + tcp->tcp_sum; 24002 data_length = (data_length >> 16) + (data_length & 0xFFFF); 24003 U16_TO_ABE16(data_length, tcph->th_sum); 24004 24005 if (tcp->tcp_ip_forward_progress) { 24006 ASSERT(tcp->tcp_ipversion == IPV6_VERSION); 24007 *(uint32_t *)mp1->b_rptr |= IP_FORWARD_PROG; 24008 tcp->tcp_ip_forward_progress = B_FALSE; 24009 } 24010 return (mp1); 24011 } 24012 } 24013 24014 /* 24015 * Hash list insertion routine for tcp_t structures. 24016 * Inserts entries with the ones bound to a specific IP address first 24017 * followed by those bound to INADDR_ANY. 24018 */ 24019 static void 24020 tcp_bind_hash_insert(tf_t *tbf, tcp_t *tcp, int caller_holds_lock) 24021 { 24022 tcp_t **tcpp; 24023 tcp_t *tcpnext; 24024 24025 if (tcp->tcp_ptpbhn != NULL) { 24026 ASSERT(!caller_holds_lock); 24027 tcp_bind_hash_remove(tcp); 24028 } 24029 tcpp = &tbf->tf_tcp; 24030 if (!caller_holds_lock) { 24031 mutex_enter(&tbf->tf_lock); 24032 } else { 24033 ASSERT(MUTEX_HELD(&tbf->tf_lock)); 24034 } 24035 tcpnext = tcpp[0]; 24036 if (tcpnext) { 24037 /* 24038 * If the new tcp bound to the INADDR_ANY address 24039 * and the first one in the list is not bound to 24040 * INADDR_ANY we skip all entries until we find the 24041 * first one bound to INADDR_ANY. 24042 * This makes sure that applications binding to a 24043 * specific address get preference over those binding to 24044 * INADDR_ANY. 24045 */ 24046 if (V6_OR_V4_INADDR_ANY(tcp->tcp_bound_source_v6) && 24047 !V6_OR_V4_INADDR_ANY(tcpnext->tcp_bound_source_v6)) { 24048 while ((tcpnext = tcpp[0]) != NULL && 24049 !V6_OR_V4_INADDR_ANY(tcpnext->tcp_bound_source_v6)) 24050 tcpp = &(tcpnext->tcp_bind_hash); 24051 if (tcpnext) 24052 tcpnext->tcp_ptpbhn = &tcp->tcp_bind_hash; 24053 } else 24054 tcpnext->tcp_ptpbhn = &tcp->tcp_bind_hash; 24055 } 24056 tcp->tcp_bind_hash = tcpnext; 24057 tcp->tcp_ptpbhn = tcpp; 24058 tcpp[0] = tcp; 24059 if (!caller_holds_lock) 24060 mutex_exit(&tbf->tf_lock); 24061 } 24062 24063 /* 24064 * Hash list removal routine for tcp_t structures. 24065 */ 24066 static void 24067 tcp_bind_hash_remove(tcp_t *tcp) 24068 { 24069 tcp_t *tcpnext; 24070 kmutex_t *lockp; 24071 tcp_stack_t *tcps = tcp->tcp_tcps; 24072 24073 if (tcp->tcp_ptpbhn == NULL) 24074 return; 24075 24076 /* 24077 * Extract the lock pointer in case there are concurrent 24078 * hash_remove's for this instance. 24079 */ 24080 ASSERT(tcp->tcp_lport != 0); 24081 lockp = &tcps->tcps_bind_fanout[TCP_BIND_HASH(tcp->tcp_lport)].tf_lock; 24082 24083 ASSERT(lockp != NULL); 24084 mutex_enter(lockp); 24085 if (tcp->tcp_ptpbhn) { 24086 tcpnext = tcp->tcp_bind_hash; 24087 if (tcpnext) { 24088 tcpnext->tcp_ptpbhn = tcp->tcp_ptpbhn; 24089 tcp->tcp_bind_hash = NULL; 24090 } 24091 *tcp->tcp_ptpbhn = tcpnext; 24092 tcp->tcp_ptpbhn = NULL; 24093 } 24094 mutex_exit(lockp); 24095 } 24096 24097 24098 /* 24099 * Hash list lookup routine for tcp_t structures. 24100 * Returns with a CONN_INC_REF tcp structure. Caller must do a CONN_DEC_REF. 24101 */ 24102 static tcp_t * 24103 tcp_acceptor_hash_lookup(t_uscalar_t id, tcp_stack_t *tcps) 24104 { 24105 tf_t *tf; 24106 tcp_t *tcp; 24107 24108 tf = &tcps->tcps_acceptor_fanout[TCP_ACCEPTOR_HASH(id)]; 24109 mutex_enter(&tf->tf_lock); 24110 for (tcp = tf->tf_tcp; tcp != NULL; 24111 tcp = tcp->tcp_acceptor_hash) { 24112 if (tcp->tcp_acceptor_id == id) { 24113 CONN_INC_REF(tcp->tcp_connp); 24114 mutex_exit(&tf->tf_lock); 24115 return (tcp); 24116 } 24117 } 24118 mutex_exit(&tf->tf_lock); 24119 return (NULL); 24120 } 24121 24122 24123 /* 24124 * Hash list insertion routine for tcp_t structures. 24125 */ 24126 void 24127 tcp_acceptor_hash_insert(t_uscalar_t id, tcp_t *tcp) 24128 { 24129 tf_t *tf; 24130 tcp_t **tcpp; 24131 tcp_t *tcpnext; 24132 tcp_stack_t *tcps = tcp->tcp_tcps; 24133 24134 tf = &tcps->tcps_acceptor_fanout[TCP_ACCEPTOR_HASH(id)]; 24135 24136 if (tcp->tcp_ptpahn != NULL) 24137 tcp_acceptor_hash_remove(tcp); 24138 tcpp = &tf->tf_tcp; 24139 mutex_enter(&tf->tf_lock); 24140 tcpnext = tcpp[0]; 24141 if (tcpnext) 24142 tcpnext->tcp_ptpahn = &tcp->tcp_acceptor_hash; 24143 tcp->tcp_acceptor_hash = tcpnext; 24144 tcp->tcp_ptpahn = tcpp; 24145 tcpp[0] = tcp; 24146 tcp->tcp_acceptor_lockp = &tf->tf_lock; /* For tcp_*_hash_remove */ 24147 mutex_exit(&tf->tf_lock); 24148 } 24149 24150 /* 24151 * Hash list removal routine for tcp_t structures. 24152 */ 24153 static void 24154 tcp_acceptor_hash_remove(tcp_t *tcp) 24155 { 24156 tcp_t *tcpnext; 24157 kmutex_t *lockp; 24158 24159 /* 24160 * Extract the lock pointer in case there are concurrent 24161 * hash_remove's for this instance. 24162 */ 24163 lockp = tcp->tcp_acceptor_lockp; 24164 24165 if (tcp->tcp_ptpahn == NULL) 24166 return; 24167 24168 ASSERT(lockp != NULL); 24169 mutex_enter(lockp); 24170 if (tcp->tcp_ptpahn) { 24171 tcpnext = tcp->tcp_acceptor_hash; 24172 if (tcpnext) { 24173 tcpnext->tcp_ptpahn = tcp->tcp_ptpahn; 24174 tcp->tcp_acceptor_hash = NULL; 24175 } 24176 *tcp->tcp_ptpahn = tcpnext; 24177 tcp->tcp_ptpahn = NULL; 24178 } 24179 mutex_exit(lockp); 24180 tcp->tcp_acceptor_lockp = NULL; 24181 } 24182 24183 /* Data for fast netmask macro used by tcp_hsp_lookup */ 24184 24185 static ipaddr_t netmasks[] = { 24186 IN_CLASSA_NET, IN_CLASSA_NET, IN_CLASSB_NET, 24187 IN_CLASSC_NET | IN_CLASSD_NET /* Class C,D,E */ 24188 }; 24189 24190 #define netmask(addr) (netmasks[(ipaddr_t)(addr) >> 30]) 24191 24192 /* 24193 * XXX This routine should go away and instead we should use the metrics 24194 * associated with the routes to determine the default sndspace and rcvspace. 24195 */ 24196 static tcp_hsp_t * 24197 tcp_hsp_lookup(ipaddr_t addr, tcp_stack_t *tcps) 24198 { 24199 tcp_hsp_t *hsp = NULL; 24200 24201 /* Quick check without acquiring the lock. */ 24202 if (tcps->tcps_hsp_hash == NULL) 24203 return (NULL); 24204 24205 rw_enter(&tcps->tcps_hsp_lock, RW_READER); 24206 24207 /* This routine finds the best-matching HSP for address addr. */ 24208 24209 if (tcps->tcps_hsp_hash) { 24210 int i; 24211 ipaddr_t srchaddr; 24212 tcp_hsp_t *hsp_net; 24213 24214 /* We do three passes: host, network, and subnet. */ 24215 24216 srchaddr = addr; 24217 24218 for (i = 1; i <= 3; i++) { 24219 /* Look for exact match on srchaddr */ 24220 24221 hsp = tcps->tcps_hsp_hash[TCP_HSP_HASH(srchaddr)]; 24222 while (hsp) { 24223 if (hsp->tcp_hsp_vers == IPV4_VERSION && 24224 hsp->tcp_hsp_addr == srchaddr) 24225 break; 24226 hsp = hsp->tcp_hsp_next; 24227 } 24228 ASSERT(hsp == NULL || 24229 hsp->tcp_hsp_vers == IPV4_VERSION); 24230 24231 /* 24232 * If this is the first pass: 24233 * If we found a match, great, return it. 24234 * If not, search for the network on the second pass. 24235 */ 24236 24237 if (i == 1) 24238 if (hsp) 24239 break; 24240 else 24241 { 24242 srchaddr = addr & netmask(addr); 24243 continue; 24244 } 24245 24246 /* 24247 * If this is the second pass: 24248 * If we found a match, but there's a subnet mask, 24249 * save the match but try again using the subnet 24250 * mask on the third pass. 24251 * Otherwise, return whatever we found. 24252 */ 24253 24254 if (i == 2) { 24255 if (hsp && hsp->tcp_hsp_subnet) { 24256 hsp_net = hsp; 24257 srchaddr = addr & hsp->tcp_hsp_subnet; 24258 continue; 24259 } else { 24260 break; 24261 } 24262 } 24263 24264 /* 24265 * This must be the third pass. If we didn't find 24266 * anything, return the saved network HSP instead. 24267 */ 24268 24269 if (!hsp) 24270 hsp = hsp_net; 24271 } 24272 } 24273 24274 rw_exit(&tcps->tcps_hsp_lock); 24275 return (hsp); 24276 } 24277 24278 /* 24279 * XXX Equally broken as the IPv4 routine. Doesn't handle longest 24280 * match lookup. 24281 */ 24282 static tcp_hsp_t * 24283 tcp_hsp_lookup_ipv6(in6_addr_t *v6addr, tcp_stack_t *tcps) 24284 { 24285 tcp_hsp_t *hsp = NULL; 24286 24287 /* Quick check without acquiring the lock. */ 24288 if (tcps->tcps_hsp_hash == NULL) 24289 return (NULL); 24290 24291 rw_enter(&tcps->tcps_hsp_lock, RW_READER); 24292 24293 /* This routine finds the best-matching HSP for address addr. */ 24294 24295 if (tcps->tcps_hsp_hash) { 24296 int i; 24297 in6_addr_t v6srchaddr; 24298 tcp_hsp_t *hsp_net; 24299 24300 /* We do three passes: host, network, and subnet. */ 24301 24302 v6srchaddr = *v6addr; 24303 24304 for (i = 1; i <= 3; i++) { 24305 /* Look for exact match on srchaddr */ 24306 24307 hsp = tcps->tcps_hsp_hash[TCP_HSP_HASH( 24308 V4_PART_OF_V6(v6srchaddr))]; 24309 while (hsp) { 24310 if (hsp->tcp_hsp_vers == IPV6_VERSION && 24311 IN6_ARE_ADDR_EQUAL(&hsp->tcp_hsp_addr_v6, 24312 &v6srchaddr)) 24313 break; 24314 hsp = hsp->tcp_hsp_next; 24315 } 24316 24317 /* 24318 * If this is the first pass: 24319 * If we found a match, great, return it. 24320 * If not, search for the network on the second pass. 24321 */ 24322 24323 if (i == 1) 24324 if (hsp) 24325 break; 24326 else { 24327 /* Assume a 64 bit mask */ 24328 v6srchaddr.s6_addr32[0] = 24329 v6addr->s6_addr32[0]; 24330 v6srchaddr.s6_addr32[1] = 24331 v6addr->s6_addr32[1]; 24332 v6srchaddr.s6_addr32[2] = 0; 24333 v6srchaddr.s6_addr32[3] = 0; 24334 continue; 24335 } 24336 24337 /* 24338 * If this is the second pass: 24339 * If we found a match, but there's a subnet mask, 24340 * save the match but try again using the subnet 24341 * mask on the third pass. 24342 * Otherwise, return whatever we found. 24343 */ 24344 24345 if (i == 2) { 24346 ASSERT(hsp == NULL || 24347 hsp->tcp_hsp_vers == IPV6_VERSION); 24348 if (hsp && 24349 !IN6_IS_ADDR_UNSPECIFIED( 24350 &hsp->tcp_hsp_subnet_v6)) { 24351 hsp_net = hsp; 24352 V6_MASK_COPY(*v6addr, 24353 hsp->tcp_hsp_subnet_v6, v6srchaddr); 24354 continue; 24355 } else { 24356 break; 24357 } 24358 } 24359 24360 /* 24361 * This must be the third pass. If we didn't find 24362 * anything, return the saved network HSP instead. 24363 */ 24364 24365 if (!hsp) 24366 hsp = hsp_net; 24367 } 24368 } 24369 24370 rw_exit(&tcps->tcps_hsp_lock); 24371 return (hsp); 24372 } 24373 24374 /* 24375 * Type three generator adapted from the random() function in 4.4 BSD: 24376 */ 24377 24378 /* 24379 * Copyright (c) 1983, 1993 24380 * The Regents of the University of California. All rights reserved. 24381 * 24382 * Redistribution and use in source and binary forms, with or without 24383 * modification, are permitted provided that the following conditions 24384 * are met: 24385 * 1. Redistributions of source code must retain the above copyright 24386 * notice, this list of conditions and the following disclaimer. 24387 * 2. Redistributions in binary form must reproduce the above copyright 24388 * notice, this list of conditions and the following disclaimer in the 24389 * documentation and/or other materials provided with the distribution. 24390 * 3. All advertising materials mentioning features or use of this software 24391 * must display the following acknowledgement: 24392 * This product includes software developed by the University of 24393 * California, Berkeley and its contributors. 24394 * 4. Neither the name of the University nor the names of its contributors 24395 * may be used to endorse or promote products derived from this software 24396 * without specific prior written permission. 24397 * 24398 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 24399 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24400 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24401 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24402 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24403 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24404 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24405 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24406 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24407 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24408 * SUCH DAMAGE. 24409 */ 24410 24411 /* Type 3 -- x**31 + x**3 + 1 */ 24412 #define DEG_3 31 24413 #define SEP_3 3 24414 24415 24416 /* Protected by tcp_random_lock */ 24417 static int tcp_randtbl[DEG_3 + 1]; 24418 24419 static int *tcp_random_fptr = &tcp_randtbl[SEP_3 + 1]; 24420 static int *tcp_random_rptr = &tcp_randtbl[1]; 24421 24422 static int *tcp_random_state = &tcp_randtbl[1]; 24423 static int *tcp_random_end_ptr = &tcp_randtbl[DEG_3 + 1]; 24424 24425 kmutex_t tcp_random_lock; 24426 24427 void 24428 tcp_random_init(void) 24429 { 24430 int i; 24431 hrtime_t hrt; 24432 time_t wallclock; 24433 uint64_t result; 24434 24435 /* 24436 * Use high-res timer and current time for seed. Gethrtime() returns 24437 * a longlong, which may contain resolution down to nanoseconds. 24438 * The current time will either be a 32-bit or a 64-bit quantity. 24439 * XOR the two together in a 64-bit result variable. 24440 * Convert the result to a 32-bit value by multiplying the high-order 24441 * 32-bits by the low-order 32-bits. 24442 */ 24443 24444 hrt = gethrtime(); 24445 (void) drv_getparm(TIME, &wallclock); 24446 result = (uint64_t)wallclock ^ (uint64_t)hrt; 24447 mutex_enter(&tcp_random_lock); 24448 tcp_random_state[0] = ((result >> 32) & 0xffffffff) * 24449 (result & 0xffffffff); 24450 24451 for (i = 1; i < DEG_3; i++) 24452 tcp_random_state[i] = 1103515245 * tcp_random_state[i - 1] 24453 + 12345; 24454 tcp_random_fptr = &tcp_random_state[SEP_3]; 24455 tcp_random_rptr = &tcp_random_state[0]; 24456 mutex_exit(&tcp_random_lock); 24457 for (i = 0; i < 10 * DEG_3; i++) 24458 (void) tcp_random(); 24459 } 24460 24461 /* 24462 * tcp_random: Return a random number in the range [1 - (128K + 1)]. 24463 * This range is selected to be approximately centered on TCP_ISS / 2, 24464 * and easy to compute. We get this value by generating a 32-bit random 24465 * number, selecting out the high-order 17 bits, and then adding one so 24466 * that we never return zero. 24467 */ 24468 int 24469 tcp_random(void) 24470 { 24471 int i; 24472 24473 mutex_enter(&tcp_random_lock); 24474 *tcp_random_fptr += *tcp_random_rptr; 24475 24476 /* 24477 * The high-order bits are more random than the low-order bits, 24478 * so we select out the high-order 17 bits and add one so that 24479 * we never return zero. 24480 */ 24481 i = ((*tcp_random_fptr >> 15) & 0x1ffff) + 1; 24482 if (++tcp_random_fptr >= tcp_random_end_ptr) { 24483 tcp_random_fptr = tcp_random_state; 24484 ++tcp_random_rptr; 24485 } else if (++tcp_random_rptr >= tcp_random_end_ptr) 24486 tcp_random_rptr = tcp_random_state; 24487 24488 mutex_exit(&tcp_random_lock); 24489 return (i); 24490 } 24491 24492 /* 24493 * XXX This will go away when TPI is extended to send 24494 * info reqs to sockfs/timod ..... 24495 * Given a queue, set the max packet size for the write 24496 * side of the queue below stream head. This value is 24497 * cached on the stream head. 24498 * Returns 1 on success, 0 otherwise. 24499 */ 24500 static int 24501 setmaxps(queue_t *q, int maxpsz) 24502 { 24503 struct stdata *stp; 24504 queue_t *wq; 24505 stp = STREAM(q); 24506 24507 /* 24508 * At this point change of a queue parameter is not allowed 24509 * when a multiplexor is sitting on top. 24510 */ 24511 if (stp->sd_flag & STPLEX) 24512 return (0); 24513 24514 claimstr(stp->sd_wrq); 24515 wq = stp->sd_wrq->q_next; 24516 ASSERT(wq != NULL); 24517 (void) strqset(wq, QMAXPSZ, 0, maxpsz); 24518 releasestr(stp->sd_wrq); 24519 return (1); 24520 } 24521 24522 static int 24523 tcp_conprim_opt_process(tcp_t *tcp, mblk_t *mp, int *do_disconnectp, 24524 int *t_errorp, int *sys_errorp) 24525 { 24526 int error; 24527 int is_absreq_failure; 24528 t_scalar_t *opt_lenp; 24529 t_scalar_t opt_offset; 24530 int prim_type; 24531 struct T_conn_req *tcreqp; 24532 struct T_conn_res *tcresp; 24533 cred_t *cr; 24534 24535 cr = DB_CREDDEF(mp, tcp->tcp_cred); 24536 24537 prim_type = ((union T_primitives *)mp->b_rptr)->type; 24538 ASSERT(prim_type == T_CONN_REQ || prim_type == O_T_CONN_RES || 24539 prim_type == T_CONN_RES); 24540 24541 switch (prim_type) { 24542 case T_CONN_REQ: 24543 tcreqp = (struct T_conn_req *)mp->b_rptr; 24544 opt_offset = tcreqp->OPT_offset; 24545 opt_lenp = (t_scalar_t *)&tcreqp->OPT_length; 24546 break; 24547 case O_T_CONN_RES: 24548 case T_CONN_RES: 24549 tcresp = (struct T_conn_res *)mp->b_rptr; 24550 opt_offset = tcresp->OPT_offset; 24551 opt_lenp = (t_scalar_t *)&tcresp->OPT_length; 24552 break; 24553 } 24554 24555 *t_errorp = 0; 24556 *sys_errorp = 0; 24557 *do_disconnectp = 0; 24558 24559 error = tpi_optcom_buf(tcp->tcp_wq, mp, opt_lenp, 24560 opt_offset, cr, &tcp_opt_obj, 24561 NULL, &is_absreq_failure); 24562 24563 switch (error) { 24564 case 0: /* no error */ 24565 ASSERT(is_absreq_failure == 0); 24566 return (0); 24567 case ENOPROTOOPT: 24568 *t_errorp = TBADOPT; 24569 break; 24570 case EACCES: 24571 *t_errorp = TACCES; 24572 break; 24573 default: 24574 *t_errorp = TSYSERR; *sys_errorp = error; 24575 break; 24576 } 24577 if (is_absreq_failure != 0) { 24578 /* 24579 * The connection request should get the local ack 24580 * T_OK_ACK and then a T_DISCON_IND. 24581 */ 24582 *do_disconnectp = 1; 24583 } 24584 return (-1); 24585 } 24586 24587 /* 24588 * Split this function out so that if the secret changes, I'm okay. 24589 * 24590 * Initialize the tcp_iss_cookie and tcp_iss_key. 24591 */ 24592 24593 #define PASSWD_SIZE 16 /* MUST be multiple of 4 */ 24594 24595 static void 24596 tcp_iss_key_init(uint8_t *phrase, int len, tcp_stack_t *tcps) 24597 { 24598 struct { 24599 int32_t current_time; 24600 uint32_t randnum; 24601 uint16_t pad; 24602 uint8_t ether[6]; 24603 uint8_t passwd[PASSWD_SIZE]; 24604 } tcp_iss_cookie; 24605 time_t t; 24606 24607 /* 24608 * Start with the current absolute time. 24609 */ 24610 (void) drv_getparm(TIME, &t); 24611 tcp_iss_cookie.current_time = t; 24612 24613 /* 24614 * XXX - Need a more random number per RFC 1750, not this crap. 24615 * OTOH, if what follows is pretty random, then I'm in better shape. 24616 */ 24617 tcp_iss_cookie.randnum = (uint32_t)(gethrtime() + tcp_random()); 24618 tcp_iss_cookie.pad = 0x365c; /* Picked from HMAC pad values. */ 24619 24620 /* 24621 * The cpu_type_info is pretty non-random. Ugggh. It does serve 24622 * as a good template. 24623 */ 24624 bcopy(&cpu_list->cpu_type_info, &tcp_iss_cookie.passwd, 24625 min(PASSWD_SIZE, sizeof (cpu_list->cpu_type_info))); 24626 24627 /* 24628 * The pass-phrase. Normally this is supplied by user-called NDD. 24629 */ 24630 bcopy(phrase, &tcp_iss_cookie.passwd, min(PASSWD_SIZE, len)); 24631 24632 /* 24633 * See 4010593 if this section becomes a problem again, 24634 * but the local ethernet address is useful here. 24635 */ 24636 (void) localetheraddr(NULL, 24637 (struct ether_addr *)&tcp_iss_cookie.ether); 24638 24639 /* 24640 * Hash 'em all together. The MD5Final is called per-connection. 24641 */ 24642 mutex_enter(&tcps->tcps_iss_key_lock); 24643 MD5Init(&tcps->tcps_iss_key); 24644 MD5Update(&tcps->tcps_iss_key, (uchar_t *)&tcp_iss_cookie, 24645 sizeof (tcp_iss_cookie)); 24646 mutex_exit(&tcps->tcps_iss_key_lock); 24647 } 24648 24649 /* 24650 * Set the RFC 1948 pass phrase 24651 */ 24652 /* ARGSUSED */ 24653 static int 24654 tcp_1948_phrase_set(queue_t *q, mblk_t *mp, char *value, caddr_t cp, 24655 cred_t *cr) 24656 { 24657 tcp_stack_t *tcps = Q_TO_TCP(q)->tcp_tcps; 24658 24659 /* 24660 * Basically, value contains a new pass phrase. Pass it along! 24661 */ 24662 tcp_iss_key_init((uint8_t *)value, strlen(value), tcps); 24663 return (0); 24664 } 24665 24666 /* ARGSUSED */ 24667 static int 24668 tcp_sack_info_constructor(void *buf, void *cdrarg, int kmflags) 24669 { 24670 bzero(buf, sizeof (tcp_sack_info_t)); 24671 return (0); 24672 } 24673 24674 /* ARGSUSED */ 24675 static int 24676 tcp_iphc_constructor(void *buf, void *cdrarg, int kmflags) 24677 { 24678 bzero(buf, TCP_MAX_COMBINED_HEADER_LENGTH); 24679 return (0); 24680 } 24681 24682 /* 24683 * Make sure we wait until the default queue is setup, yet allow 24684 * tcp_g_q_create() to open a TCP stream. 24685 * We need to allow tcp_g_q_create() do do an open 24686 * of tcp, hence we compare curhread. 24687 * All others have to wait until the tcps_g_q has been 24688 * setup. 24689 */ 24690 void 24691 tcp_g_q_setup(tcp_stack_t *tcps) 24692 { 24693 mutex_enter(&tcps->tcps_g_q_lock); 24694 if (tcps->tcps_g_q != NULL) { 24695 mutex_exit(&tcps->tcps_g_q_lock); 24696 return; 24697 } 24698 if (tcps->tcps_g_q_creator == NULL) { 24699 /* This thread will set it up */ 24700 tcps->tcps_g_q_creator = curthread; 24701 mutex_exit(&tcps->tcps_g_q_lock); 24702 tcp_g_q_create(tcps); 24703 mutex_enter(&tcps->tcps_g_q_lock); 24704 ASSERT(tcps->tcps_g_q_creator == curthread); 24705 tcps->tcps_g_q_creator = NULL; 24706 cv_signal(&tcps->tcps_g_q_cv); 24707 ASSERT(tcps->tcps_g_q != NULL); 24708 mutex_exit(&tcps->tcps_g_q_lock); 24709 return; 24710 } 24711 /* Everybody but the creator has to wait */ 24712 if (tcps->tcps_g_q_creator != curthread) { 24713 while (tcps->tcps_g_q == NULL) 24714 cv_wait(&tcps->tcps_g_q_cv, &tcps->tcps_g_q_lock); 24715 } 24716 mutex_exit(&tcps->tcps_g_q_lock); 24717 } 24718 24719 #define IP "ip" 24720 24721 #define TCP6DEV "/devices/pseudo/tcp6@0:tcp6" 24722 24723 /* 24724 * Create a default tcp queue here instead of in strplumb 24725 */ 24726 void 24727 tcp_g_q_create(tcp_stack_t *tcps) 24728 { 24729 int error; 24730 ldi_handle_t lh = NULL; 24731 ldi_ident_t li = NULL; 24732 int rval; 24733 cred_t *cr; 24734 major_t IP_MAJ; 24735 24736 #ifdef NS_DEBUG 24737 (void) printf("tcp_g_q_create()\n"); 24738 #endif 24739 24740 IP_MAJ = ddi_name_to_major(IP); 24741 24742 ASSERT(tcps->tcps_g_q_creator == curthread); 24743 24744 error = ldi_ident_from_major(IP_MAJ, &li); 24745 if (error) { 24746 #ifdef DEBUG 24747 printf("tcp_g_q_create: lyr ident get failed error %d\n", 24748 error); 24749 #endif 24750 return; 24751 } 24752 24753 cr = zone_get_kcred(netstackid_to_zoneid( 24754 tcps->tcps_netstack->netstack_stackid)); 24755 ASSERT(cr != NULL); 24756 /* 24757 * We set the tcp default queue to IPv6 because IPv4 falls 24758 * back to IPv6 when it can't find a client, but 24759 * IPv6 does not fall back to IPv4. 24760 */ 24761 error = ldi_open_by_name(TCP6DEV, FREAD|FWRITE, cr, &lh, li); 24762 if (error) { 24763 #ifdef DEBUG 24764 printf("tcp_g_q_create: open of TCP6DEV failed error %d\n", 24765 error); 24766 #endif 24767 goto out; 24768 } 24769 24770 /* 24771 * This ioctl causes the tcp framework to cache a pointer to 24772 * this stream, so we don't want to close the stream after 24773 * this operation. 24774 * Use the kernel credentials that are for the zone we're in. 24775 */ 24776 error = ldi_ioctl(lh, TCP_IOC_DEFAULT_Q, 24777 (intptr_t)0, FKIOCTL, cr, &rval); 24778 if (error) { 24779 #ifdef DEBUG 24780 printf("tcp_g_q_create: ioctl TCP_IOC_DEFAULT_Q failed " 24781 "error %d\n", error); 24782 #endif 24783 goto out; 24784 } 24785 tcps->tcps_g_q_lh = lh; /* For tcp_g_q_close */ 24786 lh = NULL; 24787 out: 24788 /* Close layered handles */ 24789 if (li) 24790 ldi_ident_release(li); 24791 /* Keep cred around until _inactive needs it */ 24792 tcps->tcps_g_q_cr = cr; 24793 } 24794 24795 /* 24796 * We keep tcp_g_q set until all other tcp_t's in the zone 24797 * has gone away, and then when tcp_g_q_inactive() is called 24798 * we clear it. 24799 */ 24800 void 24801 tcp_g_q_destroy(tcp_stack_t *tcps) 24802 { 24803 #ifdef NS_DEBUG 24804 (void) printf("tcp_g_q_destroy()for stack %d\n", 24805 tcps->tcps_netstack->netstack_stackid); 24806 #endif 24807 24808 if (tcps->tcps_g_q == NULL) { 24809 return; /* Nothing to cleanup */ 24810 } 24811 /* 24812 * Drop reference corresponding to the default queue. 24813 * This reference was added from tcp_open when the default queue 24814 * was created, hence we compensate for this extra drop in 24815 * tcp_g_q_close. If the refcnt drops to zero here it means 24816 * the default queue was the last one to be open, in which 24817 * case, then tcp_g_q_inactive will be 24818 * called as a result of the refrele. 24819 */ 24820 TCPS_REFRELE(tcps); 24821 } 24822 24823 /* 24824 * Called when last tcp_t drops reference count using TCPS_REFRELE. 24825 * Run by tcp_q_q_inactive using a taskq. 24826 */ 24827 static void 24828 tcp_g_q_close(void *arg) 24829 { 24830 tcp_stack_t *tcps = arg; 24831 int error; 24832 ldi_handle_t lh = NULL; 24833 ldi_ident_t li = NULL; 24834 cred_t *cr; 24835 major_t IP_MAJ; 24836 24837 IP_MAJ = ddi_name_to_major(IP); 24838 24839 #ifdef NS_DEBUG 24840 (void) printf("tcp_g_q_inactive() for stack %d refcnt %d\n", 24841 tcps->tcps_netstack->netstack_stackid, 24842 tcps->tcps_netstack->netstack_refcnt); 24843 #endif 24844 lh = tcps->tcps_g_q_lh; 24845 if (lh == NULL) 24846 return; /* Nothing to cleanup */ 24847 24848 ASSERT(tcps->tcps_refcnt == 1); 24849 ASSERT(tcps->tcps_g_q != NULL); 24850 24851 error = ldi_ident_from_major(IP_MAJ, &li); 24852 if (error) { 24853 #ifdef DEBUG 24854 printf("tcp_g_q_inactive: lyr ident get failed error %d\n", 24855 error); 24856 #endif 24857 return; 24858 } 24859 24860 cr = tcps->tcps_g_q_cr; 24861 tcps->tcps_g_q_cr = NULL; 24862 ASSERT(cr != NULL); 24863 24864 /* 24865 * Make sure we can break the recursion when tcp_close decrements 24866 * the reference count causing g_q_inactive to be called again. 24867 */ 24868 tcps->tcps_g_q_lh = NULL; 24869 24870 /* close the default queue */ 24871 (void) ldi_close(lh, FREAD|FWRITE, cr); 24872 /* 24873 * At this point in time tcps and the rest of netstack_t might 24874 * have been deleted. 24875 */ 24876 tcps = NULL; 24877 24878 /* Close layered handles */ 24879 ldi_ident_release(li); 24880 crfree(cr); 24881 } 24882 24883 /* 24884 * Called when last tcp_t drops reference count using TCPS_REFRELE. 24885 * 24886 * Have to ensure that the ldi routines are not used by an 24887 * interrupt thread by using a taskq. 24888 */ 24889 void 24890 tcp_g_q_inactive(tcp_stack_t *tcps) 24891 { 24892 if (tcps->tcps_g_q_lh == NULL) 24893 return; /* Nothing to cleanup */ 24894 24895 ASSERT(tcps->tcps_refcnt == 0); 24896 TCPS_REFHOLD(tcps); /* Compensate for what g_q_destroy did */ 24897 24898 if (servicing_interrupt()) { 24899 (void) taskq_dispatch(tcp_taskq, tcp_g_q_close, 24900 (void *) tcps, TQ_SLEEP); 24901 } else { 24902 tcp_g_q_close(tcps); 24903 } 24904 } 24905 24906 /* 24907 * Called by IP when IP is loaded into the kernel 24908 */ 24909 void 24910 tcp_ddi_g_init(void) 24911 { 24912 tcp_timercache = kmem_cache_create("tcp_timercache", 24913 sizeof (tcp_timer_t) + sizeof (mblk_t), 0, 24914 NULL, NULL, NULL, NULL, NULL, 0); 24915 24916 tcp_sack_info_cache = kmem_cache_create("tcp_sack_info_cache", 24917 sizeof (tcp_sack_info_t), 0, 24918 tcp_sack_info_constructor, NULL, NULL, NULL, NULL, 0); 24919 24920 tcp_iphc_cache = kmem_cache_create("tcp_iphc_cache", 24921 TCP_MAX_COMBINED_HEADER_LENGTH, 0, 24922 tcp_iphc_constructor, NULL, NULL, NULL, NULL, 0); 24923 24924 mutex_init(&tcp_random_lock, NULL, MUTEX_DEFAULT, NULL); 24925 24926 /* Initialize the random number generator */ 24927 tcp_random_init(); 24928 24929 tcp_squeue_wput_proc = tcp_squeue_switch(tcp_squeue_wput); 24930 tcp_squeue_close_proc = tcp_squeue_switch(tcp_squeue_close); 24931 24932 /* A single callback independently of how many netstacks we have */ 24933 ip_squeue_init(tcp_squeue_add); 24934 24935 tcp_g_kstat = tcp_g_kstat_init(&tcp_g_statistics); 24936 24937 tcp_taskq = taskq_create("tcp_taskq", 1, minclsyspri, 1, 1, 24938 TASKQ_PREPOPULATE); 24939 24940 /* 24941 * We want to be informed each time a stack is created or 24942 * destroyed in the kernel, so we can maintain the 24943 * set of tcp_stack_t's. 24944 */ 24945 netstack_register(NS_TCP, tcp_stack_init, tcp_stack_shutdown, 24946 tcp_stack_fini); 24947 } 24948 24949 24950 /* 24951 * Initialize the TCP stack instance. 24952 */ 24953 static void * 24954 tcp_stack_init(netstackid_t stackid, netstack_t *ns) 24955 { 24956 tcp_stack_t *tcps; 24957 tcpparam_t *pa; 24958 int i; 24959 24960 tcps = (tcp_stack_t *)kmem_zalloc(sizeof (*tcps), KM_SLEEP); 24961 tcps->tcps_netstack = ns; 24962 24963 /* Initialize locks */ 24964 rw_init(&tcps->tcps_hsp_lock, NULL, RW_DEFAULT, NULL); 24965 mutex_init(&tcps->tcps_g_q_lock, NULL, MUTEX_DEFAULT, NULL); 24966 cv_init(&tcps->tcps_g_q_cv, NULL, CV_DEFAULT, NULL); 24967 mutex_init(&tcps->tcps_iss_key_lock, NULL, MUTEX_DEFAULT, NULL); 24968 mutex_init(&tcps->tcps_epriv_port_lock, NULL, MUTEX_DEFAULT, NULL); 24969 24970 tcps->tcps_g_num_epriv_ports = TCP_NUM_EPRIV_PORTS; 24971 tcps->tcps_g_epriv_ports[0] = 2049; 24972 tcps->tcps_g_epriv_ports[1] = 4045; 24973 tcps->tcps_min_anonpriv_port = 512; 24974 24975 tcps->tcps_bind_fanout = kmem_zalloc(sizeof (tf_t) * 24976 TCP_BIND_FANOUT_SIZE, KM_SLEEP); 24977 tcps->tcps_acceptor_fanout = kmem_zalloc(sizeof (tf_t) * 24978 TCP_FANOUT_SIZE, KM_SLEEP); 24979 24980 for (i = 0; i < TCP_BIND_FANOUT_SIZE; i++) { 24981 mutex_init(&tcps->tcps_bind_fanout[i].tf_lock, NULL, 24982 MUTEX_DEFAULT, NULL); 24983 } 24984 24985 for (i = 0; i < TCP_FANOUT_SIZE; i++) { 24986 mutex_init(&tcps->tcps_acceptor_fanout[i].tf_lock, NULL, 24987 MUTEX_DEFAULT, NULL); 24988 } 24989 24990 /* TCP's IPsec code calls the packet dropper. */ 24991 ip_drop_register(&tcps->tcps_dropper, "TCP IPsec policy enforcement"); 24992 24993 pa = (tcpparam_t *)kmem_alloc(sizeof (lcl_tcp_param_arr), KM_SLEEP); 24994 tcps->tcps_params = pa; 24995 bcopy(lcl_tcp_param_arr, tcps->tcps_params, sizeof (lcl_tcp_param_arr)); 24996 24997 (void) tcp_param_register(&tcps->tcps_g_nd, tcps->tcps_params, 24998 A_CNT(lcl_tcp_param_arr), tcps); 24999 25000 /* 25001 * Note: To really walk the device tree you need the devinfo 25002 * pointer to your device which is only available after probe/attach. 25003 * The following is safe only because it uses ddi_root_node() 25004 */ 25005 tcp_max_optsize = optcom_max_optsize(tcp_opt_obj.odb_opt_des_arr, 25006 tcp_opt_obj.odb_opt_arr_cnt); 25007 25008 /* 25009 * Initialize RFC 1948 secret values. This will probably be reset once 25010 * by the boot scripts. 25011 * 25012 * Use NULL name, as the name is caught by the new lockstats. 25013 * 25014 * Initialize with some random, non-guessable string, like the global 25015 * T_INFO_ACK. 25016 */ 25017 25018 tcp_iss_key_init((uint8_t *)&tcp_g_t_info_ack, 25019 sizeof (tcp_g_t_info_ack), tcps); 25020 25021 tcps->tcps_kstat = tcp_kstat2_init(stackid, &tcps->tcps_statistics); 25022 tcps->tcps_mibkp = tcp_kstat_init(stackid, tcps); 25023 25024 return (tcps); 25025 } 25026 25027 /* 25028 * Called when the IP module is about to be unloaded. 25029 */ 25030 void 25031 tcp_ddi_g_destroy(void) 25032 { 25033 tcp_g_kstat_fini(tcp_g_kstat); 25034 tcp_g_kstat = NULL; 25035 bzero(&tcp_g_statistics, sizeof (tcp_g_statistics)); 25036 25037 mutex_destroy(&tcp_random_lock); 25038 25039 kmem_cache_destroy(tcp_timercache); 25040 kmem_cache_destroy(tcp_sack_info_cache); 25041 kmem_cache_destroy(tcp_iphc_cache); 25042 25043 netstack_unregister(NS_TCP); 25044 taskq_destroy(tcp_taskq); 25045 } 25046 25047 /* 25048 * Shut down the TCP stack instance. 25049 */ 25050 /* ARGSUSED */ 25051 static void 25052 tcp_stack_shutdown(netstackid_t stackid, void *arg) 25053 { 25054 tcp_stack_t *tcps = (tcp_stack_t *)arg; 25055 25056 tcp_g_q_destroy(tcps); 25057 } 25058 25059 /* 25060 * Free the TCP stack instance. 25061 */ 25062 static void 25063 tcp_stack_fini(netstackid_t stackid, void *arg) 25064 { 25065 tcp_stack_t *tcps = (tcp_stack_t *)arg; 25066 int i; 25067 25068 nd_free(&tcps->tcps_g_nd); 25069 kmem_free(tcps->tcps_params, sizeof (lcl_tcp_param_arr)); 25070 tcps->tcps_params = NULL; 25071 kmem_free(tcps->tcps_wroff_xtra_param, sizeof (tcpparam_t)); 25072 tcps->tcps_wroff_xtra_param = NULL; 25073 kmem_free(tcps->tcps_mdt_head_param, sizeof (tcpparam_t)); 25074 tcps->tcps_mdt_head_param = NULL; 25075 kmem_free(tcps->tcps_mdt_tail_param, sizeof (tcpparam_t)); 25076 tcps->tcps_mdt_tail_param = NULL; 25077 kmem_free(tcps->tcps_mdt_max_pbufs_param, sizeof (tcpparam_t)); 25078 tcps->tcps_mdt_max_pbufs_param = NULL; 25079 25080 for (i = 0; i < TCP_BIND_FANOUT_SIZE; i++) { 25081 ASSERT(tcps->tcps_bind_fanout[i].tf_tcp == NULL); 25082 mutex_destroy(&tcps->tcps_bind_fanout[i].tf_lock); 25083 } 25084 25085 for (i = 0; i < TCP_FANOUT_SIZE; i++) { 25086 ASSERT(tcps->tcps_acceptor_fanout[i].tf_tcp == NULL); 25087 mutex_destroy(&tcps->tcps_acceptor_fanout[i].tf_lock); 25088 } 25089 25090 kmem_free(tcps->tcps_bind_fanout, sizeof (tf_t) * TCP_BIND_FANOUT_SIZE); 25091 tcps->tcps_bind_fanout = NULL; 25092 25093 kmem_free(tcps->tcps_acceptor_fanout, sizeof (tf_t) * TCP_FANOUT_SIZE); 25094 tcps->tcps_acceptor_fanout = NULL; 25095 25096 mutex_destroy(&tcps->tcps_iss_key_lock); 25097 rw_destroy(&tcps->tcps_hsp_lock); 25098 mutex_destroy(&tcps->tcps_g_q_lock); 25099 cv_destroy(&tcps->tcps_g_q_cv); 25100 mutex_destroy(&tcps->tcps_epriv_port_lock); 25101 25102 ip_drop_unregister(&tcps->tcps_dropper); 25103 25104 tcp_kstat2_fini(stackid, tcps->tcps_kstat); 25105 tcps->tcps_kstat = NULL; 25106 bzero(&tcps->tcps_statistics, sizeof (tcps->tcps_statistics)); 25107 25108 tcp_kstat_fini(stackid, tcps->tcps_mibkp); 25109 tcps->tcps_mibkp = NULL; 25110 25111 kmem_free(tcps, sizeof (*tcps)); 25112 } 25113 25114 /* 25115 * Generate ISS, taking into account NDD changes may happen halfway through. 25116 * (If the iss is not zero, set it.) 25117 */ 25118 25119 static void 25120 tcp_iss_init(tcp_t *tcp) 25121 { 25122 MD5_CTX context; 25123 struct { uint32_t ports; in6_addr_t src; in6_addr_t dst; } arg; 25124 uint32_t answer[4]; 25125 tcp_stack_t *tcps = tcp->tcp_tcps; 25126 25127 tcps->tcps_iss_incr_extra += (ISS_INCR >> 1); 25128 tcp->tcp_iss = tcps->tcps_iss_incr_extra; 25129 switch (tcps->tcps_strong_iss) { 25130 case 2: 25131 mutex_enter(&tcps->tcps_iss_key_lock); 25132 context = tcps->tcps_iss_key; 25133 mutex_exit(&tcps->tcps_iss_key_lock); 25134 arg.ports = tcp->tcp_ports; 25135 if (tcp->tcp_ipversion == IPV4_VERSION) { 25136 IN6_IPADDR_TO_V4MAPPED(tcp->tcp_ipha->ipha_src, 25137 &arg.src); 25138 IN6_IPADDR_TO_V4MAPPED(tcp->tcp_ipha->ipha_dst, 25139 &arg.dst); 25140 } else { 25141 arg.src = tcp->tcp_ip6h->ip6_src; 25142 arg.dst = tcp->tcp_ip6h->ip6_dst; 25143 } 25144 MD5Update(&context, (uchar_t *)&arg, sizeof (arg)); 25145 MD5Final((uchar_t *)answer, &context); 25146 tcp->tcp_iss += answer[0] ^ answer[1] ^ answer[2] ^ answer[3]; 25147 /* 25148 * Now that we've hashed into a unique per-connection sequence 25149 * space, add a random increment per strong_iss == 1. So I 25150 * guess we'll have to... 25151 */ 25152 /* FALLTHRU */ 25153 case 1: 25154 tcp->tcp_iss += (gethrtime() >> ISS_NSEC_SHT) + tcp_random(); 25155 break; 25156 default: 25157 tcp->tcp_iss += (uint32_t)gethrestime_sec() * ISS_INCR; 25158 break; 25159 } 25160 tcp->tcp_valid_bits = TCP_ISS_VALID; 25161 tcp->tcp_fss = tcp->tcp_iss - 1; 25162 tcp->tcp_suna = tcp->tcp_iss; 25163 tcp->tcp_snxt = tcp->tcp_iss + 1; 25164 tcp->tcp_rexmit_nxt = tcp->tcp_snxt; 25165 tcp->tcp_csuna = tcp->tcp_snxt; 25166 } 25167 25168 /* 25169 * Exported routine for extracting active tcp connection status. 25170 * 25171 * This is used by the Solaris Cluster Networking software to 25172 * gather a list of connections that need to be forwarded to 25173 * specific nodes in the cluster when configuration changes occur. 25174 * 25175 * The callback is invoked for each tcp_t structure. Returning 25176 * non-zero from the callback routine terminates the search. 25177 */ 25178 int 25179 cl_tcp_walk_list(int (*cl_callback)(cl_tcp_info_t *, void *), 25180 void *arg) 25181 { 25182 netstack_handle_t nh; 25183 netstack_t *ns; 25184 int ret = 0; 25185 25186 netstack_next_init(&nh); 25187 while ((ns = netstack_next(&nh)) != NULL) { 25188 ret = cl_tcp_walk_list_stack(cl_callback, arg, 25189 ns->netstack_tcp); 25190 netstack_rele(ns); 25191 } 25192 netstack_next_fini(&nh); 25193 return (ret); 25194 } 25195 25196 static int 25197 cl_tcp_walk_list_stack(int (*callback)(cl_tcp_info_t *, void *), void *arg, 25198 tcp_stack_t *tcps) 25199 { 25200 tcp_t *tcp; 25201 cl_tcp_info_t cl_tcpi; 25202 connf_t *connfp; 25203 conn_t *connp; 25204 int i; 25205 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 25206 25207 ASSERT(callback != NULL); 25208 25209 for (i = 0; i < CONN_G_HASH_SIZE; i++) { 25210 connfp = &ipst->ips_ipcl_globalhash_fanout[i]; 25211 connp = NULL; 25212 25213 while ((connp = 25214 ipcl_get_next_conn(connfp, connp, IPCL_TCP)) != NULL) { 25215 25216 tcp = connp->conn_tcp; 25217 cl_tcpi.cl_tcpi_version = CL_TCPI_V1; 25218 cl_tcpi.cl_tcpi_ipversion = tcp->tcp_ipversion; 25219 cl_tcpi.cl_tcpi_state = tcp->tcp_state; 25220 cl_tcpi.cl_tcpi_lport = tcp->tcp_lport; 25221 cl_tcpi.cl_tcpi_fport = tcp->tcp_fport; 25222 /* 25223 * The macros tcp_laddr and tcp_faddr give the IPv4 25224 * addresses. They are copied implicitly below as 25225 * mapped addresses. 25226 */ 25227 cl_tcpi.cl_tcpi_laddr_v6 = tcp->tcp_ip_src_v6; 25228 if (tcp->tcp_ipversion == IPV4_VERSION) { 25229 cl_tcpi.cl_tcpi_faddr = 25230 tcp->tcp_ipha->ipha_dst; 25231 } else { 25232 cl_tcpi.cl_tcpi_faddr_v6 = 25233 tcp->tcp_ip6h->ip6_dst; 25234 } 25235 25236 /* 25237 * If the callback returns non-zero 25238 * we terminate the traversal. 25239 */ 25240 if ((*callback)(&cl_tcpi, arg) != 0) { 25241 CONN_DEC_REF(tcp->tcp_connp); 25242 return (1); 25243 } 25244 } 25245 } 25246 25247 return (0); 25248 } 25249 25250 /* 25251 * Macros used for accessing the different types of sockaddr 25252 * structures inside a tcp_ioc_abort_conn_t. 25253 */ 25254 #define TCP_AC_V4LADDR(acp) ((sin_t *)&(acp)->ac_local) 25255 #define TCP_AC_V4RADDR(acp) ((sin_t *)&(acp)->ac_remote) 25256 #define TCP_AC_V4LOCAL(acp) (TCP_AC_V4LADDR(acp)->sin_addr.s_addr) 25257 #define TCP_AC_V4REMOTE(acp) (TCP_AC_V4RADDR(acp)->sin_addr.s_addr) 25258 #define TCP_AC_V4LPORT(acp) (TCP_AC_V4LADDR(acp)->sin_port) 25259 #define TCP_AC_V4RPORT(acp) (TCP_AC_V4RADDR(acp)->sin_port) 25260 #define TCP_AC_V6LADDR(acp) ((sin6_t *)&(acp)->ac_local) 25261 #define TCP_AC_V6RADDR(acp) ((sin6_t *)&(acp)->ac_remote) 25262 #define TCP_AC_V6LOCAL(acp) (TCP_AC_V6LADDR(acp)->sin6_addr) 25263 #define TCP_AC_V6REMOTE(acp) (TCP_AC_V6RADDR(acp)->sin6_addr) 25264 #define TCP_AC_V6LPORT(acp) (TCP_AC_V6LADDR(acp)->sin6_port) 25265 #define TCP_AC_V6RPORT(acp) (TCP_AC_V6RADDR(acp)->sin6_port) 25266 25267 /* 25268 * Return the correct error code to mimic the behavior 25269 * of a connection reset. 25270 */ 25271 #define TCP_AC_GET_ERRCODE(state, err) { \ 25272 switch ((state)) { \ 25273 case TCPS_SYN_SENT: \ 25274 case TCPS_SYN_RCVD: \ 25275 (err) = ECONNREFUSED; \ 25276 break; \ 25277 case TCPS_ESTABLISHED: \ 25278 case TCPS_FIN_WAIT_1: \ 25279 case TCPS_FIN_WAIT_2: \ 25280 case TCPS_CLOSE_WAIT: \ 25281 (err) = ECONNRESET; \ 25282 break; \ 25283 case TCPS_CLOSING: \ 25284 case TCPS_LAST_ACK: \ 25285 case TCPS_TIME_WAIT: \ 25286 (err) = 0; \ 25287 break; \ 25288 default: \ 25289 (err) = ENXIO; \ 25290 } \ 25291 } 25292 25293 /* 25294 * Check if a tcp structure matches the info in acp. 25295 */ 25296 #define TCP_AC_ADDR_MATCH(acp, tcp) \ 25297 (((acp)->ac_local.ss_family == AF_INET) ? \ 25298 ((TCP_AC_V4LOCAL((acp)) == INADDR_ANY || \ 25299 TCP_AC_V4LOCAL((acp)) == (tcp)->tcp_ip_src) && \ 25300 (TCP_AC_V4REMOTE((acp)) == INADDR_ANY || \ 25301 TCP_AC_V4REMOTE((acp)) == (tcp)->tcp_remote) && \ 25302 (TCP_AC_V4LPORT((acp)) == 0 || \ 25303 TCP_AC_V4LPORT((acp)) == (tcp)->tcp_lport) && \ 25304 (TCP_AC_V4RPORT((acp)) == 0 || \ 25305 TCP_AC_V4RPORT((acp)) == (tcp)->tcp_fport) && \ 25306 (acp)->ac_start <= (tcp)->tcp_state && \ 25307 (acp)->ac_end >= (tcp)->tcp_state) : \ 25308 ((IN6_IS_ADDR_UNSPECIFIED(&TCP_AC_V6LOCAL((acp))) || \ 25309 IN6_ARE_ADDR_EQUAL(&TCP_AC_V6LOCAL((acp)), \ 25310 &(tcp)->tcp_ip_src_v6)) && \ 25311 (IN6_IS_ADDR_UNSPECIFIED(&TCP_AC_V6REMOTE((acp))) || \ 25312 IN6_ARE_ADDR_EQUAL(&TCP_AC_V6REMOTE((acp)), \ 25313 &(tcp)->tcp_remote_v6)) && \ 25314 (TCP_AC_V6LPORT((acp)) == 0 || \ 25315 TCP_AC_V6LPORT((acp)) == (tcp)->tcp_lport) && \ 25316 (TCP_AC_V6RPORT((acp)) == 0 || \ 25317 TCP_AC_V6RPORT((acp)) == (tcp)->tcp_fport) && \ 25318 (acp)->ac_start <= (tcp)->tcp_state && \ 25319 (acp)->ac_end >= (tcp)->tcp_state)) 25320 25321 #define TCP_AC_MATCH(acp, tcp) \ 25322 (((acp)->ac_zoneid == ALL_ZONES || \ 25323 (acp)->ac_zoneid == tcp->tcp_connp->conn_zoneid) ? \ 25324 TCP_AC_ADDR_MATCH(acp, tcp) : 0) 25325 25326 /* 25327 * Build a message containing a tcp_ioc_abort_conn_t structure 25328 * which is filled in with information from acp and tp. 25329 */ 25330 static mblk_t * 25331 tcp_ioctl_abort_build_msg(tcp_ioc_abort_conn_t *acp, tcp_t *tp) 25332 { 25333 mblk_t *mp; 25334 tcp_ioc_abort_conn_t *tacp; 25335 25336 mp = allocb(sizeof (uint32_t) + sizeof (*acp), BPRI_LO); 25337 if (mp == NULL) 25338 return (NULL); 25339 25340 mp->b_datap->db_type = M_CTL; 25341 25342 *((uint32_t *)mp->b_rptr) = TCP_IOC_ABORT_CONN; 25343 tacp = (tcp_ioc_abort_conn_t *)((uchar_t *)mp->b_rptr + 25344 sizeof (uint32_t)); 25345 25346 tacp->ac_start = acp->ac_start; 25347 tacp->ac_end = acp->ac_end; 25348 tacp->ac_zoneid = acp->ac_zoneid; 25349 25350 if (acp->ac_local.ss_family == AF_INET) { 25351 tacp->ac_local.ss_family = AF_INET; 25352 tacp->ac_remote.ss_family = AF_INET; 25353 TCP_AC_V4LOCAL(tacp) = tp->tcp_ip_src; 25354 TCP_AC_V4REMOTE(tacp) = tp->tcp_remote; 25355 TCP_AC_V4LPORT(tacp) = tp->tcp_lport; 25356 TCP_AC_V4RPORT(tacp) = tp->tcp_fport; 25357 } else { 25358 tacp->ac_local.ss_family = AF_INET6; 25359 tacp->ac_remote.ss_family = AF_INET6; 25360 TCP_AC_V6LOCAL(tacp) = tp->tcp_ip_src_v6; 25361 TCP_AC_V6REMOTE(tacp) = tp->tcp_remote_v6; 25362 TCP_AC_V6LPORT(tacp) = tp->tcp_lport; 25363 TCP_AC_V6RPORT(tacp) = tp->tcp_fport; 25364 } 25365 mp->b_wptr = (uchar_t *)mp->b_rptr + sizeof (uint32_t) + sizeof (*acp); 25366 return (mp); 25367 } 25368 25369 /* 25370 * Print a tcp_ioc_abort_conn_t structure. 25371 */ 25372 static void 25373 tcp_ioctl_abort_dump(tcp_ioc_abort_conn_t *acp) 25374 { 25375 char lbuf[128]; 25376 char rbuf[128]; 25377 sa_family_t af; 25378 in_port_t lport, rport; 25379 ushort_t logflags; 25380 25381 af = acp->ac_local.ss_family; 25382 25383 if (af == AF_INET) { 25384 (void) inet_ntop(af, (const void *)&TCP_AC_V4LOCAL(acp), 25385 lbuf, 128); 25386 (void) inet_ntop(af, (const void *)&TCP_AC_V4REMOTE(acp), 25387 rbuf, 128); 25388 lport = ntohs(TCP_AC_V4LPORT(acp)); 25389 rport = ntohs(TCP_AC_V4RPORT(acp)); 25390 } else { 25391 (void) inet_ntop(af, (const void *)&TCP_AC_V6LOCAL(acp), 25392 lbuf, 128); 25393 (void) inet_ntop(af, (const void *)&TCP_AC_V6REMOTE(acp), 25394 rbuf, 128); 25395 lport = ntohs(TCP_AC_V6LPORT(acp)); 25396 rport = ntohs(TCP_AC_V6RPORT(acp)); 25397 } 25398 25399 logflags = SL_TRACE | SL_NOTE; 25400 /* 25401 * Don't print this message to the console if the operation was done 25402 * to a non-global zone. 25403 */ 25404 if (acp->ac_zoneid == GLOBAL_ZONEID || acp->ac_zoneid == ALL_ZONES) 25405 logflags |= SL_CONSOLE; 25406 (void) strlog(TCP_MOD_ID, 0, 1, logflags, 25407 "TCP_IOC_ABORT_CONN: local = %s:%d, remote = %s:%d, " 25408 "start = %d, end = %d\n", lbuf, lport, rbuf, rport, 25409 acp->ac_start, acp->ac_end); 25410 } 25411 25412 /* 25413 * Called inside tcp_rput when a message built using 25414 * tcp_ioctl_abort_build_msg is put into a queue. 25415 * Note that when we get here there is no wildcard in acp any more. 25416 */ 25417 static void 25418 tcp_ioctl_abort_handler(tcp_t *tcp, mblk_t *mp) 25419 { 25420 tcp_ioc_abort_conn_t *acp; 25421 25422 acp = (tcp_ioc_abort_conn_t *)(mp->b_rptr + sizeof (uint32_t)); 25423 if (tcp->tcp_state <= acp->ac_end) { 25424 /* 25425 * If we get here, we are already on the correct 25426 * squeue. This ioctl follows the following path 25427 * tcp_wput -> tcp_wput_ioctl -> tcp_ioctl_abort_conn 25428 * ->tcp_ioctl_abort->squeue_fill (if on a 25429 * different squeue) 25430 */ 25431 int errcode; 25432 25433 TCP_AC_GET_ERRCODE(tcp->tcp_state, errcode); 25434 (void) tcp_clean_death(tcp, errcode, 26); 25435 } 25436 freemsg(mp); 25437 } 25438 25439 /* 25440 * Abort all matching connections on a hash chain. 25441 */ 25442 static int 25443 tcp_ioctl_abort_bucket(tcp_ioc_abort_conn_t *acp, int index, int *count, 25444 boolean_t exact, tcp_stack_t *tcps) 25445 { 25446 int nmatch, err = 0; 25447 tcp_t *tcp; 25448 MBLKP mp, last, listhead = NULL; 25449 conn_t *tconnp; 25450 connf_t *connfp; 25451 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 25452 25453 connfp = &ipst->ips_ipcl_conn_fanout[index]; 25454 25455 startover: 25456 nmatch = 0; 25457 25458 mutex_enter(&connfp->connf_lock); 25459 for (tconnp = connfp->connf_head; tconnp != NULL; 25460 tconnp = tconnp->conn_next) { 25461 tcp = tconnp->conn_tcp; 25462 if (TCP_AC_MATCH(acp, tcp)) { 25463 CONN_INC_REF(tcp->tcp_connp); 25464 mp = tcp_ioctl_abort_build_msg(acp, tcp); 25465 if (mp == NULL) { 25466 err = ENOMEM; 25467 CONN_DEC_REF(tcp->tcp_connp); 25468 break; 25469 } 25470 mp->b_prev = (mblk_t *)tcp; 25471 25472 if (listhead == NULL) { 25473 listhead = mp; 25474 last = mp; 25475 } else { 25476 last->b_next = mp; 25477 last = mp; 25478 } 25479 nmatch++; 25480 if (exact) 25481 break; 25482 } 25483 25484 /* Avoid holding lock for too long. */ 25485 if (nmatch >= 500) 25486 break; 25487 } 25488 mutex_exit(&connfp->connf_lock); 25489 25490 /* Pass mp into the correct tcp */ 25491 while ((mp = listhead) != NULL) { 25492 listhead = listhead->b_next; 25493 tcp = (tcp_t *)mp->b_prev; 25494 mp->b_next = mp->b_prev = NULL; 25495 squeue_fill(tcp->tcp_connp->conn_sqp, mp, 25496 tcp_input, tcp->tcp_connp, SQTAG_TCP_ABORT_BUCKET); 25497 } 25498 25499 *count += nmatch; 25500 if (nmatch >= 500 && err == 0) 25501 goto startover; 25502 return (err); 25503 } 25504 25505 /* 25506 * Abort all connections that matches the attributes specified in acp. 25507 */ 25508 static int 25509 tcp_ioctl_abort(tcp_ioc_abort_conn_t *acp, tcp_stack_t *tcps) 25510 { 25511 sa_family_t af; 25512 uint32_t ports; 25513 uint16_t *pports; 25514 int err = 0, count = 0; 25515 boolean_t exact = B_FALSE; /* set when there is no wildcard */ 25516 int index = -1; 25517 ushort_t logflags; 25518 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 25519 25520 af = acp->ac_local.ss_family; 25521 25522 if (af == AF_INET) { 25523 if (TCP_AC_V4REMOTE(acp) != INADDR_ANY && 25524 TCP_AC_V4LPORT(acp) != 0 && TCP_AC_V4RPORT(acp) != 0) { 25525 pports = (uint16_t *)&ports; 25526 pports[1] = TCP_AC_V4LPORT(acp); 25527 pports[0] = TCP_AC_V4RPORT(acp); 25528 exact = (TCP_AC_V4LOCAL(acp) != INADDR_ANY); 25529 } 25530 } else { 25531 if (!IN6_IS_ADDR_UNSPECIFIED(&TCP_AC_V6REMOTE(acp)) && 25532 TCP_AC_V6LPORT(acp) != 0 && TCP_AC_V6RPORT(acp) != 0) { 25533 pports = (uint16_t *)&ports; 25534 pports[1] = TCP_AC_V6LPORT(acp); 25535 pports[0] = TCP_AC_V6RPORT(acp); 25536 exact = !IN6_IS_ADDR_UNSPECIFIED(&TCP_AC_V6LOCAL(acp)); 25537 } 25538 } 25539 25540 /* 25541 * For cases where remote addr, local port, and remote port are non- 25542 * wildcards, tcp_ioctl_abort_bucket will only be called once. 25543 */ 25544 if (index != -1) { 25545 err = tcp_ioctl_abort_bucket(acp, index, 25546 &count, exact, tcps); 25547 } else { 25548 /* 25549 * loop through all entries for wildcard case 25550 */ 25551 for (index = 0; 25552 index < ipst->ips_ipcl_conn_fanout_size; 25553 index++) { 25554 err = tcp_ioctl_abort_bucket(acp, index, 25555 &count, exact, tcps); 25556 if (err != 0) 25557 break; 25558 } 25559 } 25560 25561 logflags = SL_TRACE | SL_NOTE; 25562 /* 25563 * Don't print this message to the console if the operation was done 25564 * to a non-global zone. 25565 */ 25566 if (acp->ac_zoneid == GLOBAL_ZONEID || acp->ac_zoneid == ALL_ZONES) 25567 logflags |= SL_CONSOLE; 25568 (void) strlog(TCP_MOD_ID, 0, 1, logflags, "TCP_IOC_ABORT_CONN: " 25569 "aborted %d connection%c\n", count, ((count > 1) ? 's' : ' ')); 25570 if (err == 0 && count == 0) 25571 err = ENOENT; 25572 return (err); 25573 } 25574 25575 /* 25576 * Process the TCP_IOC_ABORT_CONN ioctl request. 25577 */ 25578 static void 25579 tcp_ioctl_abort_conn(queue_t *q, mblk_t *mp) 25580 { 25581 int err; 25582 IOCP iocp; 25583 MBLKP mp1; 25584 sa_family_t laf, raf; 25585 tcp_ioc_abort_conn_t *acp; 25586 zone_t *zptr; 25587 conn_t *connp = Q_TO_CONN(q); 25588 zoneid_t zoneid = connp->conn_zoneid; 25589 tcp_t *tcp = connp->conn_tcp; 25590 tcp_stack_t *tcps = tcp->tcp_tcps; 25591 25592 iocp = (IOCP)mp->b_rptr; 25593 25594 if ((mp1 = mp->b_cont) == NULL || 25595 iocp->ioc_count != sizeof (tcp_ioc_abort_conn_t)) { 25596 err = EINVAL; 25597 goto out; 25598 } 25599 25600 /* check permissions */ 25601 if (secpolicy_ip_config(iocp->ioc_cr, B_FALSE) != 0) { 25602 err = EPERM; 25603 goto out; 25604 } 25605 25606 if (mp1->b_cont != NULL) { 25607 freemsg(mp1->b_cont); 25608 mp1->b_cont = NULL; 25609 } 25610 25611 acp = (tcp_ioc_abort_conn_t *)mp1->b_rptr; 25612 laf = acp->ac_local.ss_family; 25613 raf = acp->ac_remote.ss_family; 25614 25615 /* check that a zone with the supplied zoneid exists */ 25616 if (acp->ac_zoneid != GLOBAL_ZONEID && acp->ac_zoneid != ALL_ZONES) { 25617 zptr = zone_find_by_id(zoneid); 25618 if (zptr != NULL) { 25619 zone_rele(zptr); 25620 } else { 25621 err = EINVAL; 25622 goto out; 25623 } 25624 } 25625 25626 /* 25627 * For exclusive stacks we set the zoneid to zero 25628 * to make TCP operate as if in the global zone. 25629 */ 25630 if (tcps->tcps_netstack->netstack_stackid != GLOBAL_NETSTACKID) 25631 acp->ac_zoneid = GLOBAL_ZONEID; 25632 25633 if (acp->ac_start < TCPS_SYN_SENT || acp->ac_end > TCPS_TIME_WAIT || 25634 acp->ac_start > acp->ac_end || laf != raf || 25635 (laf != AF_INET && laf != AF_INET6)) { 25636 err = EINVAL; 25637 goto out; 25638 } 25639 25640 tcp_ioctl_abort_dump(acp); 25641 err = tcp_ioctl_abort(acp, tcps); 25642 25643 out: 25644 if (mp1 != NULL) { 25645 freemsg(mp1); 25646 mp->b_cont = NULL; 25647 } 25648 25649 if (err != 0) 25650 miocnak(q, mp, 0, err); 25651 else 25652 miocack(q, mp, 0, 0); 25653 } 25654 25655 /* 25656 * tcp_time_wait_processing() handles processing of incoming packets when 25657 * the tcp is in the TIME_WAIT state. 25658 * A TIME_WAIT tcp that has an associated open TCP stream is never put 25659 * on the time wait list. 25660 */ 25661 void 25662 tcp_time_wait_processing(tcp_t *tcp, mblk_t *mp, uint32_t seg_seq, 25663 uint32_t seg_ack, int seg_len, tcph_t *tcph) 25664 { 25665 int32_t bytes_acked; 25666 int32_t gap; 25667 int32_t rgap; 25668 tcp_opt_t tcpopt; 25669 uint_t flags; 25670 uint32_t new_swnd = 0; 25671 conn_t *connp; 25672 tcp_stack_t *tcps = tcp->tcp_tcps; 25673 25674 BUMP_LOCAL(tcp->tcp_ibsegs); 25675 DTRACE_PROBE2(tcp__trace__recv, mblk_t *, mp, tcp_t *, tcp); 25676 25677 flags = (unsigned int)tcph->th_flags[0] & 0xFF; 25678 new_swnd = BE16_TO_U16(tcph->th_win) << 25679 ((tcph->th_flags[0] & TH_SYN) ? 0 : tcp->tcp_snd_ws); 25680 if (tcp->tcp_snd_ts_ok) { 25681 if (!tcp_paws_check(tcp, tcph, &tcpopt)) { 25682 tcp_xmit_ctl(NULL, tcp, tcp->tcp_snxt, 25683 tcp->tcp_rnxt, TH_ACK); 25684 goto done; 25685 } 25686 } 25687 gap = seg_seq - tcp->tcp_rnxt; 25688 rgap = tcp->tcp_rwnd - (gap + seg_len); 25689 if (gap < 0) { 25690 BUMP_MIB(&tcps->tcps_mib, tcpInDataDupSegs); 25691 UPDATE_MIB(&tcps->tcps_mib, tcpInDataDupBytes, 25692 (seg_len > -gap ? -gap : seg_len)); 25693 seg_len += gap; 25694 if (seg_len < 0 || (seg_len == 0 && !(flags & TH_FIN))) { 25695 if (flags & TH_RST) { 25696 goto done; 25697 } 25698 if ((flags & TH_FIN) && seg_len == -1) { 25699 /* 25700 * When TCP receives a duplicate FIN in 25701 * TIME_WAIT state, restart the 2 MSL timer. 25702 * See page 73 in RFC 793. Make sure this TCP 25703 * is already on the TIME_WAIT list. If not, 25704 * just restart the timer. 25705 */ 25706 if (TCP_IS_DETACHED(tcp)) { 25707 if (tcp_time_wait_remove(tcp, NULL) == 25708 B_TRUE) { 25709 tcp_time_wait_append(tcp); 25710 TCP_DBGSTAT(tcps, 25711 tcp_rput_time_wait); 25712 } 25713 } else { 25714 ASSERT(tcp != NULL); 25715 TCP_TIMER_RESTART(tcp, 25716 tcps->tcps_time_wait_interval); 25717 } 25718 tcp_xmit_ctl(NULL, tcp, tcp->tcp_snxt, 25719 tcp->tcp_rnxt, TH_ACK); 25720 goto done; 25721 } 25722 flags |= TH_ACK_NEEDED; 25723 seg_len = 0; 25724 goto process_ack; 25725 } 25726 25727 /* Fix seg_seq, and chew the gap off the front. */ 25728 seg_seq = tcp->tcp_rnxt; 25729 } 25730 25731 if ((flags & TH_SYN) && gap > 0 && rgap < 0) { 25732 /* 25733 * Make sure that when we accept the connection, pick 25734 * an ISS greater than (tcp_snxt + ISS_INCR/2) for the 25735 * old connection. 25736 * 25737 * The next ISS generated is equal to tcp_iss_incr_extra 25738 * + ISS_INCR/2 + other components depending on the 25739 * value of tcp_strong_iss. We pre-calculate the new 25740 * ISS here and compare with tcp_snxt to determine if 25741 * we need to make adjustment to tcp_iss_incr_extra. 25742 * 25743 * The above calculation is ugly and is a 25744 * waste of CPU cycles... 25745 */ 25746 uint32_t new_iss = tcps->tcps_iss_incr_extra; 25747 int32_t adj; 25748 ip_stack_t *ipst = tcps->tcps_netstack->netstack_ip; 25749 25750 switch (tcps->tcps_strong_iss) { 25751 case 2: { 25752 /* Add time and MD5 components. */ 25753 uint32_t answer[4]; 25754 struct { 25755 uint32_t ports; 25756 in6_addr_t src; 25757 in6_addr_t dst; 25758 } arg; 25759 MD5_CTX context; 25760 25761 mutex_enter(&tcps->tcps_iss_key_lock); 25762 context = tcps->tcps_iss_key; 25763 mutex_exit(&tcps->tcps_iss_key_lock); 25764 arg.ports = tcp->tcp_ports; 25765 /* We use MAPPED addresses in tcp_iss_init */ 25766 arg.src = tcp->tcp_ip_src_v6; 25767 if (tcp->tcp_ipversion == IPV4_VERSION) { 25768 IN6_IPADDR_TO_V4MAPPED( 25769 tcp->tcp_ipha->ipha_dst, 25770 &arg.dst); 25771 } else { 25772 arg.dst = 25773 tcp->tcp_ip6h->ip6_dst; 25774 } 25775 MD5Update(&context, (uchar_t *)&arg, 25776 sizeof (arg)); 25777 MD5Final((uchar_t *)answer, &context); 25778 answer[0] ^= answer[1] ^ answer[2] ^ answer[3]; 25779 new_iss += (gethrtime() >> ISS_NSEC_SHT) + answer[0]; 25780 break; 25781 } 25782 case 1: 25783 /* Add time component and min random (i.e. 1). */ 25784 new_iss += (gethrtime() >> ISS_NSEC_SHT) + 1; 25785 break; 25786 default: 25787 /* Add only time component. */ 25788 new_iss += (uint32_t)gethrestime_sec() * ISS_INCR; 25789 break; 25790 } 25791 if ((adj = (int32_t)(tcp->tcp_snxt - new_iss)) > 0) { 25792 /* 25793 * New ISS not guaranteed to be ISS_INCR/2 25794 * ahead of the current tcp_snxt, so add the 25795 * difference to tcp_iss_incr_extra. 25796 */ 25797 tcps->tcps_iss_incr_extra += adj; 25798 } 25799 /* 25800 * If tcp_clean_death() can not perform the task now, 25801 * drop the SYN packet and let the other side re-xmit. 25802 * Otherwise pass the SYN packet back in, since the 25803 * old tcp state has been cleaned up or freed. 25804 */ 25805 if (tcp_clean_death(tcp, 0, 27) == -1) 25806 goto done; 25807 /* 25808 * We will come back to tcp_rput_data 25809 * on the global queue. Packets destined 25810 * for the global queue will be checked 25811 * with global policy. But the policy for 25812 * this packet has already been checked as 25813 * this was destined for the detached 25814 * connection. We need to bypass policy 25815 * check this time by attaching a dummy 25816 * ipsec_in with ipsec_in_dont_check set. 25817 */ 25818 connp = ipcl_classify(mp, tcp->tcp_connp->conn_zoneid, ipst); 25819 if (connp != NULL) { 25820 TCP_STAT(tcps, tcp_time_wait_syn_success); 25821 tcp_reinput(connp, mp, tcp->tcp_connp->conn_sqp); 25822 return; 25823 } 25824 goto done; 25825 } 25826 25827 /* 25828 * rgap is the amount of stuff received out of window. A negative 25829 * value is the amount out of window. 25830 */ 25831 if (rgap < 0) { 25832 BUMP_MIB(&tcps->tcps_mib, tcpInDataPastWinSegs); 25833 UPDATE_MIB(&tcps->tcps_mib, tcpInDataPastWinBytes, -rgap); 25834 /* Fix seg_len and make sure there is something left. */ 25835 seg_len += rgap; 25836 if (seg_len <= 0) { 25837 if (flags & TH_RST) { 25838 goto done; 25839 } 25840 flags |= TH_ACK_NEEDED; 25841 seg_len = 0; 25842 goto process_ack; 25843 } 25844 } 25845 /* 25846 * Check whether we can update tcp_ts_recent. This test is 25847 * NOT the one in RFC 1323 3.4. It is from Braden, 1993, "TCP 25848 * Extensions for High Performance: An Update", Internet Draft. 25849 */ 25850 if (tcp->tcp_snd_ts_ok && 25851 TSTMP_GEQ(tcpopt.tcp_opt_ts_val, tcp->tcp_ts_recent) && 25852 SEQ_LEQ(seg_seq, tcp->tcp_rack)) { 25853 tcp->tcp_ts_recent = tcpopt.tcp_opt_ts_val; 25854 tcp->tcp_last_rcv_lbolt = lbolt64; 25855 } 25856 25857 if (seg_seq != tcp->tcp_rnxt && seg_len > 0) { 25858 /* Always ack out of order packets */ 25859 flags |= TH_ACK_NEEDED; 25860 seg_len = 0; 25861 } else if (seg_len > 0) { 25862 BUMP_MIB(&tcps->tcps_mib, tcpInClosed); 25863 BUMP_MIB(&tcps->tcps_mib, tcpInDataInorderSegs); 25864 UPDATE_MIB(&tcps->tcps_mib, tcpInDataInorderBytes, seg_len); 25865 } 25866 if (flags & TH_RST) { 25867 (void) tcp_clean_death(tcp, 0, 28); 25868 goto done; 25869 } 25870 if (flags & TH_SYN) { 25871 tcp_xmit_ctl("TH_SYN", tcp, seg_ack, seg_seq + 1, 25872 TH_RST|TH_ACK); 25873 /* 25874 * Do not delete the TCP structure if it is in 25875 * TIME_WAIT state. Refer to RFC 1122, 4.2.2.13. 25876 */ 25877 goto done; 25878 } 25879 process_ack: 25880 if (flags & TH_ACK) { 25881 bytes_acked = (int)(seg_ack - tcp->tcp_suna); 25882 if (bytes_acked <= 0) { 25883 if (bytes_acked == 0 && seg_len == 0 && 25884 new_swnd == tcp->tcp_swnd) 25885 BUMP_MIB(&tcps->tcps_mib, tcpInDupAck); 25886 } else { 25887 /* Acks something not sent */ 25888 flags |= TH_ACK_NEEDED; 25889 } 25890 } 25891 if (flags & TH_ACK_NEEDED) { 25892 /* 25893 * Time to send an ack for some reason. 25894 */ 25895 tcp_xmit_ctl(NULL, tcp, tcp->tcp_snxt, 25896 tcp->tcp_rnxt, TH_ACK); 25897 } 25898 done: 25899 if ((mp->b_datap->db_struioflag & STRUIO_EAGER) != 0) { 25900 DB_CKSUMSTART(mp) = 0; 25901 mp->b_datap->db_struioflag &= ~STRUIO_EAGER; 25902 TCP_STAT(tcps, tcp_time_wait_syn_fail); 25903 } 25904 freemsg(mp); 25905 } 25906 25907 /* 25908 * Allocate a T_SVR4_OPTMGMT_REQ. 25909 * The caller needs to increment tcp_drop_opt_ack_cnt when sending these so 25910 * that tcp_rput_other can drop the acks. 25911 */ 25912 static mblk_t * 25913 tcp_setsockopt_mp(int level, int cmd, char *opt, int optlen) 25914 { 25915 mblk_t *mp; 25916 struct T_optmgmt_req *tor; 25917 struct opthdr *oh; 25918 uint_t size; 25919 char *optptr; 25920 25921 size = sizeof (*tor) + sizeof (*oh) + optlen; 25922 mp = allocb(size, BPRI_MED); 25923 if (mp == NULL) 25924 return (NULL); 25925 25926 mp->b_wptr += size; 25927 mp->b_datap->db_type = M_PROTO; 25928 tor = (struct T_optmgmt_req *)mp->b_rptr; 25929 tor->PRIM_type = T_SVR4_OPTMGMT_REQ; 25930 tor->MGMT_flags = T_NEGOTIATE; 25931 tor->OPT_length = sizeof (*oh) + optlen; 25932 tor->OPT_offset = (t_scalar_t)sizeof (*tor); 25933 25934 oh = (struct opthdr *)&tor[1]; 25935 oh->level = level; 25936 oh->name = cmd; 25937 oh->len = optlen; 25938 if (optlen != 0) { 25939 optptr = (char *)&oh[1]; 25940 bcopy(opt, optptr, optlen); 25941 } 25942 return (mp); 25943 } 25944 25945 /* 25946 * TCP Timers Implementation. 25947 */ 25948 timeout_id_t 25949 tcp_timeout(conn_t *connp, void (*f)(void *), clock_t tim) 25950 { 25951 mblk_t *mp; 25952 tcp_timer_t *tcpt; 25953 tcp_t *tcp = connp->conn_tcp; 25954 tcp_stack_t *tcps = tcp->tcp_tcps; 25955 25956 ASSERT(connp->conn_sqp != NULL); 25957 25958 TCP_DBGSTAT(tcps, tcp_timeout_calls); 25959 25960 if (tcp->tcp_timercache == NULL) { 25961 mp = tcp_timermp_alloc(KM_NOSLEEP | KM_PANIC); 25962 } else { 25963 TCP_DBGSTAT(tcps, tcp_timeout_cached_alloc); 25964 mp = tcp->tcp_timercache; 25965 tcp->tcp_timercache = mp->b_next; 25966 mp->b_next = NULL; 25967 ASSERT(mp->b_wptr == NULL); 25968 } 25969 25970 CONN_INC_REF(connp); 25971 tcpt = (tcp_timer_t *)mp->b_rptr; 25972 tcpt->connp = connp; 25973 tcpt->tcpt_proc = f; 25974 tcpt->tcpt_tid = timeout(tcp_timer_callback, mp, tim); 25975 return ((timeout_id_t)mp); 25976 } 25977 25978 static void 25979 tcp_timer_callback(void *arg) 25980 { 25981 mblk_t *mp = (mblk_t *)arg; 25982 tcp_timer_t *tcpt; 25983 conn_t *connp; 25984 25985 tcpt = (tcp_timer_t *)mp->b_rptr; 25986 connp = tcpt->connp; 25987 squeue_fill(connp->conn_sqp, mp, 25988 tcp_timer_handler, connp, SQTAG_TCP_TIMER); 25989 } 25990 25991 static void 25992 tcp_timer_handler(void *arg, mblk_t *mp, void *arg2) 25993 { 25994 tcp_timer_t *tcpt; 25995 conn_t *connp = (conn_t *)arg; 25996 tcp_t *tcp = connp->conn_tcp; 25997 25998 tcpt = (tcp_timer_t *)mp->b_rptr; 25999 ASSERT(connp == tcpt->connp); 26000 ASSERT((squeue_t *)arg2 == connp->conn_sqp); 26001 26002 /* 26003 * If the TCP has reached the closed state, don't proceed any 26004 * further. This TCP logically does not exist on the system. 26005 * tcpt_proc could for example access queues, that have already 26006 * been qprocoff'ed off. Also see comments at the start of tcp_input 26007 */ 26008 if (tcp->tcp_state != TCPS_CLOSED) { 26009 (*tcpt->tcpt_proc)(connp); 26010 } else { 26011 tcp->tcp_timer_tid = 0; 26012 } 26013 tcp_timer_free(connp->conn_tcp, mp); 26014 } 26015 26016 /* 26017 * There is potential race with untimeout and the handler firing at the same 26018 * time. The mblock may be freed by the handler while we are trying to use 26019 * it. But since both should execute on the same squeue, this race should not 26020 * occur. 26021 */ 26022 clock_t 26023 tcp_timeout_cancel(conn_t *connp, timeout_id_t id) 26024 { 26025 mblk_t *mp = (mblk_t *)id; 26026 tcp_timer_t *tcpt; 26027 clock_t delta; 26028 tcp_stack_t *tcps = connp->conn_tcp->tcp_tcps; 26029 26030 TCP_DBGSTAT(tcps, tcp_timeout_cancel_reqs); 26031 26032 if (mp == NULL) 26033 return (-1); 26034 26035 tcpt = (tcp_timer_t *)mp->b_rptr; 26036 ASSERT(tcpt->connp == connp); 26037 26038 delta = untimeout(tcpt->tcpt_tid); 26039 26040 if (delta >= 0) { 26041 TCP_DBGSTAT(tcps, tcp_timeout_canceled); 26042 tcp_timer_free(connp->conn_tcp, mp); 26043 CONN_DEC_REF(connp); 26044 } 26045 26046 return (delta); 26047 } 26048 26049 /* 26050 * Allocate space for the timer event. The allocation looks like mblk, but it is 26051 * not a proper mblk. To avoid confusion we set b_wptr to NULL. 26052 * 26053 * Dealing with failures: If we can't allocate from the timer cache we try 26054 * allocating from dblock caches using allocb_tryhard(). In this case b_wptr 26055 * points to b_rptr. 26056 * If we can't allocate anything using allocb_tryhard(), we perform a last 26057 * attempt and use kmem_alloc_tryhard(). In this case we set b_wptr to -1 and 26058 * save the actual allocation size in b_datap. 26059 */ 26060 mblk_t * 26061 tcp_timermp_alloc(int kmflags) 26062 { 26063 mblk_t *mp = (mblk_t *)kmem_cache_alloc(tcp_timercache, 26064 kmflags & ~KM_PANIC); 26065 26066 if (mp != NULL) { 26067 mp->b_next = mp->b_prev = NULL; 26068 mp->b_rptr = (uchar_t *)(&mp[1]); 26069 mp->b_wptr = NULL; 26070 mp->b_datap = NULL; 26071 mp->b_queue = NULL; 26072 mp->b_cont = NULL; 26073 } else if (kmflags & KM_PANIC) { 26074 /* 26075 * Failed to allocate memory for the timer. Try allocating from 26076 * dblock caches. 26077 */ 26078 /* ipclassifier calls this from a constructor - hence no tcps */ 26079 TCP_G_STAT(tcp_timermp_allocfail); 26080 mp = allocb_tryhard(sizeof (tcp_timer_t)); 26081 if (mp == NULL) { 26082 size_t size = 0; 26083 /* 26084 * Memory is really low. Try tryhard allocation. 26085 * 26086 * ipclassifier calls this from a constructor - 26087 * hence no tcps 26088 */ 26089 TCP_G_STAT(tcp_timermp_allocdblfail); 26090 mp = kmem_alloc_tryhard(sizeof (mblk_t) + 26091 sizeof (tcp_timer_t), &size, kmflags); 26092 mp->b_rptr = (uchar_t *)(&mp[1]); 26093 mp->b_next = mp->b_prev = NULL; 26094 mp->b_wptr = (uchar_t *)-1; 26095 mp->b_datap = (dblk_t *)size; 26096 mp->b_queue = NULL; 26097 mp->b_cont = NULL; 26098 } 26099 ASSERT(mp->b_wptr != NULL); 26100 } 26101 /* ipclassifier calls this from a constructor - hence no tcps */ 26102 TCP_G_DBGSTAT(tcp_timermp_alloced); 26103 26104 return (mp); 26105 } 26106 26107 /* 26108 * Free per-tcp timer cache. 26109 * It can only contain entries from tcp_timercache. 26110 */ 26111 void 26112 tcp_timermp_free(tcp_t *tcp) 26113 { 26114 mblk_t *mp; 26115 26116 while ((mp = tcp->tcp_timercache) != NULL) { 26117 ASSERT(mp->b_wptr == NULL); 26118 tcp->tcp_timercache = tcp->tcp_timercache->b_next; 26119 kmem_cache_free(tcp_timercache, mp); 26120 } 26121 } 26122 26123 /* 26124 * Free timer event. Put it on the per-tcp timer cache if there is not too many 26125 * events there already (currently at most two events are cached). 26126 * If the event is not allocated from the timer cache, free it right away. 26127 */ 26128 static void 26129 tcp_timer_free(tcp_t *tcp, mblk_t *mp) 26130 { 26131 mblk_t *mp1 = tcp->tcp_timercache; 26132 tcp_stack_t *tcps = tcp->tcp_tcps; 26133 26134 if (mp->b_wptr != NULL) { 26135 /* 26136 * This allocation is not from a timer cache, free it right 26137 * away. 26138 */ 26139 if (mp->b_wptr != (uchar_t *)-1) 26140 freeb(mp); 26141 else 26142 kmem_free(mp, (size_t)mp->b_datap); 26143 } else if (mp1 == NULL || mp1->b_next == NULL) { 26144 /* Cache this timer block for future allocations */ 26145 mp->b_rptr = (uchar_t *)(&mp[1]); 26146 mp->b_next = mp1; 26147 tcp->tcp_timercache = mp; 26148 } else { 26149 kmem_cache_free(tcp_timercache, mp); 26150 TCP_DBGSTAT(tcps, tcp_timermp_freed); 26151 } 26152 } 26153 26154 /* 26155 * End of TCP Timers implementation. 26156 */ 26157 26158 /* 26159 * tcp_{set,clr}qfull() functions are used to either set or clear QFULL 26160 * on the specified backing STREAMS q. Note, the caller may make the 26161 * decision to call based on the tcp_t.tcp_flow_stopped value which 26162 * when check outside the q's lock is only an advisory check ... 26163 */ 26164 26165 void 26166 tcp_setqfull(tcp_t *tcp) 26167 { 26168 queue_t *q = tcp->tcp_wq; 26169 tcp_stack_t *tcps = tcp->tcp_tcps; 26170 26171 if (!(q->q_flag & QFULL)) { 26172 mutex_enter(QLOCK(q)); 26173 if (!(q->q_flag & QFULL)) { 26174 /* still need to set QFULL */ 26175 q->q_flag |= QFULL; 26176 tcp->tcp_flow_stopped = B_TRUE; 26177 mutex_exit(QLOCK(q)); 26178 TCP_STAT(tcps, tcp_flwctl_on); 26179 } else { 26180 mutex_exit(QLOCK(q)); 26181 } 26182 } 26183 } 26184 26185 void 26186 tcp_clrqfull(tcp_t *tcp) 26187 { 26188 queue_t *q = tcp->tcp_wq; 26189 26190 if (q->q_flag & QFULL) { 26191 mutex_enter(QLOCK(q)); 26192 if (q->q_flag & QFULL) { 26193 q->q_flag &= ~QFULL; 26194 tcp->tcp_flow_stopped = B_FALSE; 26195 mutex_exit(QLOCK(q)); 26196 if (q->q_flag & QWANTW) 26197 qbackenable(q, 0); 26198 } else { 26199 mutex_exit(QLOCK(q)); 26200 } 26201 } 26202 } 26203 26204 26205 /* 26206 * kstats related to squeues i.e. not per IP instance 26207 */ 26208 static void * 26209 tcp_g_kstat_init(tcp_g_stat_t *tcp_g_statp) 26210 { 26211 kstat_t *ksp; 26212 26213 tcp_g_stat_t template = { 26214 { "tcp_timermp_alloced", KSTAT_DATA_UINT64 }, 26215 { "tcp_timermp_allocfail", KSTAT_DATA_UINT64 }, 26216 { "tcp_timermp_allocdblfail", KSTAT_DATA_UINT64 }, 26217 { "tcp_freelist_cleanup", KSTAT_DATA_UINT64 }, 26218 }; 26219 26220 ksp = kstat_create(TCP_MOD_NAME, 0, "tcpstat_g", "net", 26221 KSTAT_TYPE_NAMED, sizeof (template) / sizeof (kstat_named_t), 26222 KSTAT_FLAG_VIRTUAL); 26223 26224 if (ksp == NULL) 26225 return (NULL); 26226 26227 bcopy(&template, tcp_g_statp, sizeof (template)); 26228 ksp->ks_data = (void *)tcp_g_statp; 26229 26230 kstat_install(ksp); 26231 return (ksp); 26232 } 26233 26234 static void 26235 tcp_g_kstat_fini(kstat_t *ksp) 26236 { 26237 if (ksp != NULL) { 26238 kstat_delete(ksp); 26239 } 26240 } 26241 26242 26243 static void * 26244 tcp_kstat2_init(netstackid_t stackid, tcp_stat_t *tcps_statisticsp) 26245 { 26246 kstat_t *ksp; 26247 26248 tcp_stat_t template = { 26249 { "tcp_time_wait", KSTAT_DATA_UINT64 }, 26250 { "tcp_time_wait_syn", KSTAT_DATA_UINT64 }, 26251 { "tcp_time_wait_success", KSTAT_DATA_UINT64 }, 26252 { "tcp_time_wait_fail", KSTAT_DATA_UINT64 }, 26253 { "tcp_reinput_syn", KSTAT_DATA_UINT64 }, 26254 { "tcp_ip_output", KSTAT_DATA_UINT64 }, 26255 { "tcp_detach_non_time_wait", KSTAT_DATA_UINT64 }, 26256 { "tcp_detach_time_wait", KSTAT_DATA_UINT64 }, 26257 { "tcp_time_wait_reap", KSTAT_DATA_UINT64 }, 26258 { "tcp_clean_death_nondetached", KSTAT_DATA_UINT64 }, 26259 { "tcp_reinit_calls", KSTAT_DATA_UINT64 }, 26260 { "tcp_eager_err1", KSTAT_DATA_UINT64 }, 26261 { "tcp_eager_err2", KSTAT_DATA_UINT64 }, 26262 { "tcp_eager_blowoff_calls", KSTAT_DATA_UINT64 }, 26263 { "tcp_eager_blowoff_q", KSTAT_DATA_UINT64 }, 26264 { "tcp_eager_blowoff_q0", KSTAT_DATA_UINT64 }, 26265 { "tcp_not_hard_bound", KSTAT_DATA_UINT64 }, 26266 { "tcp_no_listener", KSTAT_DATA_UINT64 }, 26267 { "tcp_found_eager", KSTAT_DATA_UINT64 }, 26268 { "tcp_wrong_queue", KSTAT_DATA_UINT64 }, 26269 { "tcp_found_eager_binding1", KSTAT_DATA_UINT64 }, 26270 { "tcp_found_eager_bound1", KSTAT_DATA_UINT64 }, 26271 { "tcp_eager_has_listener1", KSTAT_DATA_UINT64 }, 26272 { "tcp_open_alloc", KSTAT_DATA_UINT64 }, 26273 { "tcp_open_detached_alloc", KSTAT_DATA_UINT64 }, 26274 { "tcp_rput_time_wait", KSTAT_DATA_UINT64 }, 26275 { "tcp_listendrop", KSTAT_DATA_UINT64 }, 26276 { "tcp_listendropq0", KSTAT_DATA_UINT64 }, 26277 { "tcp_wrong_rq", KSTAT_DATA_UINT64 }, 26278 { "tcp_rsrv_calls", KSTAT_DATA_UINT64 }, 26279 { "tcp_eagerfree2", KSTAT_DATA_UINT64 }, 26280 { "tcp_eagerfree3", KSTAT_DATA_UINT64 }, 26281 { "tcp_eagerfree4", KSTAT_DATA_UINT64 }, 26282 { "tcp_eagerfree5", KSTAT_DATA_UINT64 }, 26283 { "tcp_timewait_syn_fail", KSTAT_DATA_UINT64 }, 26284 { "tcp_listen_badflags", KSTAT_DATA_UINT64 }, 26285 { "tcp_timeout_calls", KSTAT_DATA_UINT64 }, 26286 { "tcp_timeout_cached_alloc", KSTAT_DATA_UINT64 }, 26287 { "tcp_timeout_cancel_reqs", KSTAT_DATA_UINT64 }, 26288 { "tcp_timeout_canceled", KSTAT_DATA_UINT64 }, 26289 { "tcp_timermp_freed", KSTAT_DATA_UINT64 }, 26290 { "tcp_push_timer_cnt", KSTAT_DATA_UINT64 }, 26291 { "tcp_ack_timer_cnt", KSTAT_DATA_UINT64 }, 26292 { "tcp_ire_null1", KSTAT_DATA_UINT64 }, 26293 { "tcp_ire_null", KSTAT_DATA_UINT64 }, 26294 { "tcp_ip_send", KSTAT_DATA_UINT64 }, 26295 { "tcp_ip_ire_send", KSTAT_DATA_UINT64 }, 26296 { "tcp_wsrv_called", KSTAT_DATA_UINT64 }, 26297 { "tcp_flwctl_on", KSTAT_DATA_UINT64 }, 26298 { "tcp_timer_fire_early", KSTAT_DATA_UINT64 }, 26299 { "tcp_timer_fire_miss", KSTAT_DATA_UINT64 }, 26300 { "tcp_rput_v6_error", KSTAT_DATA_UINT64 }, 26301 { "tcp_out_sw_cksum", KSTAT_DATA_UINT64 }, 26302 { "tcp_out_sw_cksum_bytes", KSTAT_DATA_UINT64 }, 26303 { "tcp_zcopy_on", KSTAT_DATA_UINT64 }, 26304 { "tcp_zcopy_off", KSTAT_DATA_UINT64 }, 26305 { "tcp_zcopy_backoff", KSTAT_DATA_UINT64 }, 26306 { "tcp_zcopy_disable", KSTAT_DATA_UINT64 }, 26307 { "tcp_mdt_pkt_out", KSTAT_DATA_UINT64 }, 26308 { "tcp_mdt_pkt_out_v4", KSTAT_DATA_UINT64 }, 26309 { "tcp_mdt_pkt_out_v6", KSTAT_DATA_UINT64 }, 26310 { "tcp_mdt_discarded", KSTAT_DATA_UINT64 }, 26311 { "tcp_mdt_conn_halted1", KSTAT_DATA_UINT64 }, 26312 { "tcp_mdt_conn_halted2", KSTAT_DATA_UINT64 }, 26313 { "tcp_mdt_conn_halted3", KSTAT_DATA_UINT64 }, 26314 { "tcp_mdt_conn_resumed1", KSTAT_DATA_UINT64 }, 26315 { "tcp_mdt_conn_resumed2", KSTAT_DATA_UINT64 }, 26316 { "tcp_mdt_legacy_small", KSTAT_DATA_UINT64 }, 26317 { "tcp_mdt_legacy_all", KSTAT_DATA_UINT64 }, 26318 { "tcp_mdt_legacy_ret", KSTAT_DATA_UINT64 }, 26319 { "tcp_mdt_allocfail", KSTAT_DATA_UINT64 }, 26320 { "tcp_mdt_addpdescfail", KSTAT_DATA_UINT64 }, 26321 { "tcp_mdt_allocd", KSTAT_DATA_UINT64 }, 26322 { "tcp_mdt_linked", KSTAT_DATA_UINT64 }, 26323 { "tcp_fusion_flowctl", KSTAT_DATA_UINT64 }, 26324 { "tcp_fusion_backenabled", KSTAT_DATA_UINT64 }, 26325 { "tcp_fusion_urg", KSTAT_DATA_UINT64 }, 26326 { "tcp_fusion_putnext", KSTAT_DATA_UINT64 }, 26327 { "tcp_fusion_unfusable", KSTAT_DATA_UINT64 }, 26328 { "tcp_fusion_aborted", KSTAT_DATA_UINT64 }, 26329 { "tcp_fusion_unqualified", KSTAT_DATA_UINT64 }, 26330 { "tcp_fusion_rrw_busy", KSTAT_DATA_UINT64 }, 26331 { "tcp_fusion_rrw_msgcnt", KSTAT_DATA_UINT64 }, 26332 { "tcp_fusion_rrw_plugged", KSTAT_DATA_UINT64 }, 26333 { "tcp_in_ack_unsent_drop", KSTAT_DATA_UINT64 }, 26334 { "tcp_sock_fallback", KSTAT_DATA_UINT64 }, 26335 { "tcp_lso_enabled", KSTAT_DATA_UINT64 }, 26336 { "tcp_lso_disabled", KSTAT_DATA_UINT64 }, 26337 { "tcp_lso_times", KSTAT_DATA_UINT64 }, 26338 { "tcp_lso_pkt_out", KSTAT_DATA_UINT64 }, 26339 }; 26340 26341 ksp = kstat_create_netstack(TCP_MOD_NAME, 0, "tcpstat", "net", 26342 KSTAT_TYPE_NAMED, sizeof (template) / sizeof (kstat_named_t), 26343 KSTAT_FLAG_VIRTUAL, stackid); 26344 26345 if (ksp == NULL) 26346 return (NULL); 26347 26348 bcopy(&template, tcps_statisticsp, sizeof (template)); 26349 ksp->ks_data = (void *)tcps_statisticsp; 26350 ksp->ks_private = (void *)(uintptr_t)stackid; 26351 26352 kstat_install(ksp); 26353 return (ksp); 26354 } 26355 26356 static void 26357 tcp_kstat2_fini(netstackid_t stackid, kstat_t *ksp) 26358 { 26359 if (ksp != NULL) { 26360 ASSERT(stackid == (netstackid_t)(uintptr_t)ksp->ks_private); 26361 kstat_delete_netstack(ksp, stackid); 26362 } 26363 } 26364 26365 /* 26366 * TCP Kstats implementation 26367 */ 26368 static void * 26369 tcp_kstat_init(netstackid_t stackid, tcp_stack_t *tcps) 26370 { 26371 kstat_t *ksp; 26372 26373 tcp_named_kstat_t template = { 26374 { "rtoAlgorithm", KSTAT_DATA_INT32, 0 }, 26375 { "rtoMin", KSTAT_DATA_INT32, 0 }, 26376 { "rtoMax", KSTAT_DATA_INT32, 0 }, 26377 { "maxConn", KSTAT_DATA_INT32, 0 }, 26378 { "activeOpens", KSTAT_DATA_UINT32, 0 }, 26379 { "passiveOpens", KSTAT_DATA_UINT32, 0 }, 26380 { "attemptFails", KSTAT_DATA_UINT32, 0 }, 26381 { "estabResets", KSTAT_DATA_UINT32, 0 }, 26382 { "currEstab", KSTAT_DATA_UINT32, 0 }, 26383 { "inSegs", KSTAT_DATA_UINT64, 0 }, 26384 { "outSegs", KSTAT_DATA_UINT64, 0 }, 26385 { "retransSegs", KSTAT_DATA_UINT32, 0 }, 26386 { "connTableSize", KSTAT_DATA_INT32, 0 }, 26387 { "outRsts", KSTAT_DATA_UINT32, 0 }, 26388 { "outDataSegs", KSTAT_DATA_UINT32, 0 }, 26389 { "outDataBytes", KSTAT_DATA_UINT32, 0 }, 26390 { "retransBytes", KSTAT_DATA_UINT32, 0 }, 26391 { "outAck", KSTAT_DATA_UINT32, 0 }, 26392 { "outAckDelayed", KSTAT_DATA_UINT32, 0 }, 26393 { "outUrg", KSTAT_DATA_UINT32, 0 }, 26394 { "outWinUpdate", KSTAT_DATA_UINT32, 0 }, 26395 { "outWinProbe", KSTAT_DATA_UINT32, 0 }, 26396 { "outControl", KSTAT_DATA_UINT32, 0 }, 26397 { "outFastRetrans", KSTAT_DATA_UINT32, 0 }, 26398 { "inAckSegs", KSTAT_DATA_UINT32, 0 }, 26399 { "inAckBytes", KSTAT_DATA_UINT32, 0 }, 26400 { "inDupAck", KSTAT_DATA_UINT32, 0 }, 26401 { "inAckUnsent", KSTAT_DATA_UINT32, 0 }, 26402 { "inDataInorderSegs", KSTAT_DATA_UINT32, 0 }, 26403 { "inDataInorderBytes", KSTAT_DATA_UINT32, 0 }, 26404 { "inDataUnorderSegs", KSTAT_DATA_UINT32, 0 }, 26405 { "inDataUnorderBytes", KSTAT_DATA_UINT32, 0 }, 26406 { "inDataDupSegs", KSTAT_DATA_UINT32, 0 }, 26407 { "inDataDupBytes", KSTAT_DATA_UINT32, 0 }, 26408 { "inDataPartDupSegs", KSTAT_DATA_UINT32, 0 }, 26409 { "inDataPartDupBytes", KSTAT_DATA_UINT32, 0 }, 26410 { "inDataPastWinSegs", KSTAT_DATA_UINT32, 0 }, 26411 { "inDataPastWinBytes", KSTAT_DATA_UINT32, 0 }, 26412 { "inWinProbe", KSTAT_DATA_UINT32, 0 }, 26413 { "inWinUpdate", KSTAT_DATA_UINT32, 0 }, 26414 { "inClosed", KSTAT_DATA_UINT32, 0 }, 26415 { "rttUpdate", KSTAT_DATA_UINT32, 0 }, 26416 { "rttNoUpdate", KSTAT_DATA_UINT32, 0 }, 26417 { "timRetrans", KSTAT_DATA_UINT32, 0 }, 26418 { "timRetransDrop", KSTAT_DATA_UINT32, 0 }, 26419 { "timKeepalive", KSTAT_DATA_UINT32, 0 }, 26420 { "timKeepaliveProbe", KSTAT_DATA_UINT32, 0 }, 26421 { "timKeepaliveDrop", KSTAT_DATA_UINT32, 0 }, 26422 { "listenDrop", KSTAT_DATA_UINT32, 0 }, 26423 { "listenDropQ0", KSTAT_DATA_UINT32, 0 }, 26424 { "halfOpenDrop", KSTAT_DATA_UINT32, 0 }, 26425 { "outSackRetransSegs", KSTAT_DATA_UINT32, 0 }, 26426 { "connTableSize6", KSTAT_DATA_INT32, 0 } 26427 }; 26428 26429 ksp = kstat_create_netstack(TCP_MOD_NAME, 0, TCP_MOD_NAME, "mib2", 26430 KSTAT_TYPE_NAMED, NUM_OF_FIELDS(tcp_named_kstat_t), 0, stackid); 26431 26432 if (ksp == NULL) 26433 return (NULL); 26434 26435 template.rtoAlgorithm.value.ui32 = 4; 26436 template.rtoMin.value.ui32 = tcps->tcps_rexmit_interval_min; 26437 template.rtoMax.value.ui32 = tcps->tcps_rexmit_interval_max; 26438 template.maxConn.value.i32 = -1; 26439 26440 bcopy(&template, ksp->ks_data, sizeof (template)); 26441 ksp->ks_update = tcp_kstat_update; 26442 ksp->ks_private = (void *)(uintptr_t)stackid; 26443 26444 kstat_install(ksp); 26445 return (ksp); 26446 } 26447 26448 static void 26449 tcp_kstat_fini(netstackid_t stackid, kstat_t *ksp) 26450 { 26451 if (ksp != NULL) { 26452 ASSERT(stackid == (netstackid_t)(uintptr_t)ksp->ks_private); 26453 kstat_delete_netstack(ksp, stackid); 26454 } 26455 } 26456 26457 static int 26458 tcp_kstat_update(kstat_t *kp, int rw) 26459 { 26460 tcp_named_kstat_t *tcpkp; 26461 tcp_t *tcp; 26462 connf_t *connfp; 26463 conn_t *connp; 26464 int i; 26465 netstackid_t stackid = (netstackid_t)(uintptr_t)kp->ks_private; 26466 netstack_t *ns; 26467 tcp_stack_t *tcps; 26468 ip_stack_t *ipst; 26469 26470 if ((kp == NULL) || (kp->ks_data == NULL)) 26471 return (EIO); 26472 26473 if (rw == KSTAT_WRITE) 26474 return (EACCES); 26475 26476 ns = netstack_find_by_stackid(stackid); 26477 if (ns == NULL) 26478 return (-1); 26479 tcps = ns->netstack_tcp; 26480 if (tcps == NULL) { 26481 netstack_rele(ns); 26482 return (-1); 26483 } 26484 tcpkp = (tcp_named_kstat_t *)kp->ks_data; 26485 26486 tcpkp->currEstab.value.ui32 = 0; 26487 26488 ipst = ns->netstack_ip; 26489 26490 for (i = 0; i < CONN_G_HASH_SIZE; i++) { 26491 connfp = &ipst->ips_ipcl_globalhash_fanout[i]; 26492 connp = NULL; 26493 while ((connp = 26494 ipcl_get_next_conn(connfp, connp, IPCL_TCP)) != NULL) { 26495 tcp = connp->conn_tcp; 26496 switch (tcp_snmp_state(tcp)) { 26497 case MIB2_TCP_established: 26498 case MIB2_TCP_closeWait: 26499 tcpkp->currEstab.value.ui32++; 26500 break; 26501 } 26502 } 26503 } 26504 26505 tcpkp->activeOpens.value.ui32 = tcps->tcps_mib.tcpActiveOpens; 26506 tcpkp->passiveOpens.value.ui32 = tcps->tcps_mib.tcpPassiveOpens; 26507 tcpkp->attemptFails.value.ui32 = tcps->tcps_mib.tcpAttemptFails; 26508 tcpkp->estabResets.value.ui32 = tcps->tcps_mib.tcpEstabResets; 26509 tcpkp->inSegs.value.ui64 = tcps->tcps_mib.tcpHCInSegs; 26510 tcpkp->outSegs.value.ui64 = tcps->tcps_mib.tcpHCOutSegs; 26511 tcpkp->retransSegs.value.ui32 = tcps->tcps_mib.tcpRetransSegs; 26512 tcpkp->connTableSize.value.i32 = tcps->tcps_mib.tcpConnTableSize; 26513 tcpkp->outRsts.value.ui32 = tcps->tcps_mib.tcpOutRsts; 26514 tcpkp->outDataSegs.value.ui32 = tcps->tcps_mib.tcpOutDataSegs; 26515 tcpkp->outDataBytes.value.ui32 = tcps->tcps_mib.tcpOutDataBytes; 26516 tcpkp->retransBytes.value.ui32 = tcps->tcps_mib.tcpRetransBytes; 26517 tcpkp->outAck.value.ui32 = tcps->tcps_mib.tcpOutAck; 26518 tcpkp->outAckDelayed.value.ui32 = tcps->tcps_mib.tcpOutAckDelayed; 26519 tcpkp->outUrg.value.ui32 = tcps->tcps_mib.tcpOutUrg; 26520 tcpkp->outWinUpdate.value.ui32 = tcps->tcps_mib.tcpOutWinUpdate; 26521 tcpkp->outWinProbe.value.ui32 = tcps->tcps_mib.tcpOutWinProbe; 26522 tcpkp->outControl.value.ui32 = tcps->tcps_mib.tcpOutControl; 26523 tcpkp->outFastRetrans.value.ui32 = tcps->tcps_mib.tcpOutFastRetrans; 26524 tcpkp->inAckSegs.value.ui32 = tcps->tcps_mib.tcpInAckSegs; 26525 tcpkp->inAckBytes.value.ui32 = tcps->tcps_mib.tcpInAckBytes; 26526 tcpkp->inDupAck.value.ui32 = tcps->tcps_mib.tcpInDupAck; 26527 tcpkp->inAckUnsent.value.ui32 = tcps->tcps_mib.tcpInAckUnsent; 26528 tcpkp->inDataInorderSegs.value.ui32 = 26529 tcps->tcps_mib.tcpInDataInorderSegs; 26530 tcpkp->inDataInorderBytes.value.ui32 = 26531 tcps->tcps_mib.tcpInDataInorderBytes; 26532 tcpkp->inDataUnorderSegs.value.ui32 = 26533 tcps->tcps_mib.tcpInDataUnorderSegs; 26534 tcpkp->inDataUnorderBytes.value.ui32 = 26535 tcps->tcps_mib.tcpInDataUnorderBytes; 26536 tcpkp->inDataDupSegs.value.ui32 = tcps->tcps_mib.tcpInDataDupSegs; 26537 tcpkp->inDataDupBytes.value.ui32 = tcps->tcps_mib.tcpInDataDupBytes; 26538 tcpkp->inDataPartDupSegs.value.ui32 = 26539 tcps->tcps_mib.tcpInDataPartDupSegs; 26540 tcpkp->inDataPartDupBytes.value.ui32 = 26541 tcps->tcps_mib.tcpInDataPartDupBytes; 26542 tcpkp->inDataPastWinSegs.value.ui32 = 26543 tcps->tcps_mib.tcpInDataPastWinSegs; 26544 tcpkp->inDataPastWinBytes.value.ui32 = 26545 tcps->tcps_mib.tcpInDataPastWinBytes; 26546 tcpkp->inWinProbe.value.ui32 = tcps->tcps_mib.tcpInWinProbe; 26547 tcpkp->inWinUpdate.value.ui32 = tcps->tcps_mib.tcpInWinUpdate; 26548 tcpkp->inClosed.value.ui32 = tcps->tcps_mib.tcpInClosed; 26549 tcpkp->rttNoUpdate.value.ui32 = tcps->tcps_mib.tcpRttNoUpdate; 26550 tcpkp->rttUpdate.value.ui32 = tcps->tcps_mib.tcpRttUpdate; 26551 tcpkp->timRetrans.value.ui32 = tcps->tcps_mib.tcpTimRetrans; 26552 tcpkp->timRetransDrop.value.ui32 = tcps->tcps_mib.tcpTimRetransDrop; 26553 tcpkp->timKeepalive.value.ui32 = tcps->tcps_mib.tcpTimKeepalive; 26554 tcpkp->timKeepaliveProbe.value.ui32 = 26555 tcps->tcps_mib.tcpTimKeepaliveProbe; 26556 tcpkp->timKeepaliveDrop.value.ui32 = 26557 tcps->tcps_mib.tcpTimKeepaliveDrop; 26558 tcpkp->listenDrop.value.ui32 = tcps->tcps_mib.tcpListenDrop; 26559 tcpkp->listenDropQ0.value.ui32 = tcps->tcps_mib.tcpListenDropQ0; 26560 tcpkp->halfOpenDrop.value.ui32 = tcps->tcps_mib.tcpHalfOpenDrop; 26561 tcpkp->outSackRetransSegs.value.ui32 = 26562 tcps->tcps_mib.tcpOutSackRetransSegs; 26563 tcpkp->connTableSize6.value.i32 = tcps->tcps_mib.tcp6ConnTableSize; 26564 26565 netstack_rele(ns); 26566 return (0); 26567 } 26568 26569 void 26570 tcp_reinput(conn_t *connp, mblk_t *mp, squeue_t *sqp) 26571 { 26572 uint16_t hdr_len; 26573 ipha_t *ipha; 26574 uint8_t *nexthdrp; 26575 tcph_t *tcph; 26576 tcp_stack_t *tcps = connp->conn_tcp->tcp_tcps; 26577 26578 /* Already has an eager */ 26579 if ((mp->b_datap->db_struioflag & STRUIO_EAGER) != 0) { 26580 TCP_STAT(tcps, tcp_reinput_syn); 26581 squeue_enter(connp->conn_sqp, mp, connp->conn_recv, 26582 connp, SQTAG_TCP_REINPUT_EAGER); 26583 return; 26584 } 26585 26586 switch (IPH_HDR_VERSION(mp->b_rptr)) { 26587 case IPV4_VERSION: 26588 ipha = (ipha_t *)mp->b_rptr; 26589 hdr_len = IPH_HDR_LENGTH(ipha); 26590 break; 26591 case IPV6_VERSION: 26592 if (!ip_hdr_length_nexthdr_v6(mp, (ip6_t *)mp->b_rptr, 26593 &hdr_len, &nexthdrp)) { 26594 CONN_DEC_REF(connp); 26595 freemsg(mp); 26596 return; 26597 } 26598 break; 26599 } 26600 26601 tcph = (tcph_t *)&mp->b_rptr[hdr_len]; 26602 if ((tcph->th_flags[0] & (TH_SYN|TH_ACK|TH_RST|TH_URG)) == TH_SYN) { 26603 mp->b_datap->db_struioflag |= STRUIO_EAGER; 26604 DB_CKSUMSTART(mp) = (intptr_t)sqp; 26605 } 26606 26607 squeue_fill(connp->conn_sqp, mp, connp->conn_recv, connp, 26608 SQTAG_TCP_REINPUT); 26609 } 26610 26611 static squeue_func_t 26612 tcp_squeue_switch(int val) 26613 { 26614 squeue_func_t rval = squeue_fill; 26615 26616 switch (val) { 26617 case 1: 26618 rval = squeue_enter_nodrain; 26619 break; 26620 case 2: 26621 rval = squeue_enter; 26622 break; 26623 default: 26624 break; 26625 } 26626 return (rval); 26627 } 26628 26629 /* 26630 * This is called once for each squeue - globally for all stack 26631 * instances. 26632 */ 26633 static void 26634 tcp_squeue_add(squeue_t *sqp) 26635 { 26636 tcp_squeue_priv_t *tcp_time_wait = kmem_zalloc( 26637 sizeof (tcp_squeue_priv_t), KM_SLEEP); 26638 26639 *squeue_getprivate(sqp, SQPRIVATE_TCP) = (intptr_t)tcp_time_wait; 26640 tcp_time_wait->tcp_time_wait_tid = timeout(tcp_time_wait_collector, 26641 sqp, TCP_TIME_WAIT_DELAY); 26642 if (tcp_free_list_max_cnt == 0) { 26643 int tcp_ncpus = ((boot_max_ncpus == -1) ? 26644 max_ncpus : boot_max_ncpus); 26645 26646 /* 26647 * Limit number of entries to 1% of availble memory / tcp_ncpus 26648 */ 26649 tcp_free_list_max_cnt = (freemem * PAGESIZE) / 26650 (tcp_ncpus * sizeof (tcp_t) * 100); 26651 } 26652 tcp_time_wait->tcp_free_list_cnt = 0; 26653 } 26654