1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Implementation of the Transmission Control Protocol(TCP). 7 * 8 * Version: $Id: tcp.c,v 1.216 2002/02/01 22:01:04 davem Exp $ 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Mark Evans, <evansmp@uhura.aston.ac.uk> 13 * Corey Minyard <wf-rch!minyard@relay.EU.net> 14 * Florian La Roche, <flla@stud.uni-sb.de> 15 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 16 * Linus Torvalds, <torvalds@cs.helsinki.fi> 17 * Alan Cox, <gw4pts@gw4pts.ampr.org> 18 * Matthew Dillon, <dillon@apollo.west.oic.com> 19 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 20 * Jorge Cwik, <jorge@laser.satlink.net> 21 * 22 * Fixes: 23 * Alan Cox : Numerous verify_area() calls 24 * Alan Cox : Set the ACK bit on a reset 25 * Alan Cox : Stopped it crashing if it closed while 26 * sk->inuse=1 and was trying to connect 27 * (tcp_err()). 28 * Alan Cox : All icmp error handling was broken 29 * pointers passed where wrong and the 30 * socket was looked up backwards. Nobody 31 * tested any icmp error code obviously. 32 * Alan Cox : tcp_err() now handled properly. It 33 * wakes people on errors. poll 34 * behaves and the icmp error race 35 * has gone by moving it into sock.c 36 * Alan Cox : tcp_send_reset() fixed to work for 37 * everything not just packets for 38 * unknown sockets. 39 * Alan Cox : tcp option processing. 40 * Alan Cox : Reset tweaked (still not 100%) [Had 41 * syn rule wrong] 42 * Herp Rosmanith : More reset fixes 43 * Alan Cox : No longer acks invalid rst frames. 44 * Acking any kind of RST is right out. 45 * Alan Cox : Sets an ignore me flag on an rst 46 * receive otherwise odd bits of prattle 47 * escape still 48 * Alan Cox : Fixed another acking RST frame bug. 49 * Should stop LAN workplace lockups. 50 * Alan Cox : Some tidyups using the new skb list 51 * facilities 52 * Alan Cox : sk->keepopen now seems to work 53 * Alan Cox : Pulls options out correctly on accepts 54 * Alan Cox : Fixed assorted sk->rqueue->next errors 55 * Alan Cox : PSH doesn't end a TCP read. Switched a 56 * bit to skb ops. 57 * Alan Cox : Tidied tcp_data to avoid a potential 58 * nasty. 59 * Alan Cox : Added some better commenting, as the 60 * tcp is hard to follow 61 * Alan Cox : Removed incorrect check for 20 * psh 62 * Michael O'Reilly : ack < copied bug fix. 63 * Johannes Stille : Misc tcp fixes (not all in yet). 64 * Alan Cox : FIN with no memory -> CRASH 65 * Alan Cox : Added socket option proto entries. 66 * Also added awareness of them to accept. 67 * Alan Cox : Added TCP options (SOL_TCP) 68 * Alan Cox : Switched wakeup calls to callbacks, 69 * so the kernel can layer network 70 * sockets. 71 * Alan Cox : Use ip_tos/ip_ttl settings. 72 * Alan Cox : Handle FIN (more) properly (we hope). 73 * Alan Cox : RST frames sent on unsynchronised 74 * state ack error. 75 * Alan Cox : Put in missing check for SYN bit. 76 * Alan Cox : Added tcp_select_window() aka NET2E 77 * window non shrink trick. 78 * Alan Cox : Added a couple of small NET2E timer 79 * fixes 80 * Charles Hedrick : TCP fixes 81 * Toomas Tamm : TCP window fixes 82 * Alan Cox : Small URG fix to rlogin ^C ack fight 83 * Charles Hedrick : Rewrote most of it to actually work 84 * Linus : Rewrote tcp_read() and URG handling 85 * completely 86 * Gerhard Koerting: Fixed some missing timer handling 87 * Matthew Dillon : Reworked TCP machine states as per RFC 88 * Gerhard Koerting: PC/TCP workarounds 89 * Adam Caldwell : Assorted timer/timing errors 90 * Matthew Dillon : Fixed another RST bug 91 * Alan Cox : Move to kernel side addressing changes. 92 * Alan Cox : Beginning work on TCP fastpathing 93 * (not yet usable) 94 * Arnt Gulbrandsen: Turbocharged tcp_check() routine. 95 * Alan Cox : TCP fast path debugging 96 * Alan Cox : Window clamping 97 * Michael Riepe : Bug in tcp_check() 98 * Matt Dillon : More TCP improvements and RST bug fixes 99 * Matt Dillon : Yet more small nasties remove from the 100 * TCP code (Be very nice to this man if 101 * tcp finally works 100%) 8) 102 * Alan Cox : BSD accept semantics. 103 * Alan Cox : Reset on closedown bug. 104 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto(). 105 * Michael Pall : Handle poll() after URG properly in 106 * all cases. 107 * Michael Pall : Undo the last fix in tcp_read_urg() 108 * (multi URG PUSH broke rlogin). 109 * Michael Pall : Fix the multi URG PUSH problem in 110 * tcp_readable(), poll() after URG 111 * works now. 112 * Michael Pall : recv(...,MSG_OOB) never blocks in the 113 * BSD api. 114 * Alan Cox : Changed the semantics of sk->socket to 115 * fix a race and a signal problem with 116 * accept() and async I/O. 117 * Alan Cox : Relaxed the rules on tcp_sendto(). 118 * Yury Shevchuk : Really fixed accept() blocking problem. 119 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for 120 * clients/servers which listen in on 121 * fixed ports. 122 * Alan Cox : Cleaned the above up and shrank it to 123 * a sensible code size. 124 * Alan Cox : Self connect lockup fix. 125 * Alan Cox : No connect to multicast. 126 * Ross Biro : Close unaccepted children on master 127 * socket close. 128 * Alan Cox : Reset tracing code. 129 * Alan Cox : Spurious resets on shutdown. 130 * Alan Cox : Giant 15 minute/60 second timer error 131 * Alan Cox : Small whoops in polling before an 132 * accept. 133 * Alan Cox : Kept the state trace facility since 134 * it's handy for debugging. 135 * Alan Cox : More reset handler fixes. 136 * Alan Cox : Started rewriting the code based on 137 * the RFC's for other useful protocol 138 * references see: Comer, KA9Q NOS, and 139 * for a reference on the difference 140 * between specifications and how BSD 141 * works see the 4.4lite source. 142 * A.N.Kuznetsov : Don't time wait on completion of tidy 143 * close. 144 * Linus Torvalds : Fin/Shutdown & copied_seq changes. 145 * Linus Torvalds : Fixed BSD port reuse to work first syn 146 * Alan Cox : Reimplemented timers as per the RFC 147 * and using multiple timers for sanity. 148 * Alan Cox : Small bug fixes, and a lot of new 149 * comments. 150 * Alan Cox : Fixed dual reader crash by locking 151 * the buffers (much like datagram.c) 152 * Alan Cox : Fixed stuck sockets in probe. A probe 153 * now gets fed up of retrying without 154 * (even a no space) answer. 155 * Alan Cox : Extracted closing code better 156 * Alan Cox : Fixed the closing state machine to 157 * resemble the RFC. 158 * Alan Cox : More 'per spec' fixes. 159 * Jorge Cwik : Even faster checksumming. 160 * Alan Cox : tcp_data() doesn't ack illegal PSH 161 * only frames. At least one pc tcp stack 162 * generates them. 163 * Alan Cox : Cache last socket. 164 * Alan Cox : Per route irtt. 165 * Matt Day : poll()->select() match BSD precisely on error 166 * Alan Cox : New buffers 167 * Marc Tamsky : Various sk->prot->retransmits and 168 * sk->retransmits misupdating fixed. 169 * Fixed tcp_write_timeout: stuck close, 170 * and TCP syn retries gets used now. 171 * Mark Yarvis : In tcp_read_wakeup(), don't send an 172 * ack if state is TCP_CLOSED. 173 * Alan Cox : Look up device on a retransmit - routes may 174 * change. Doesn't yet cope with MSS shrink right 175 * but it's a start! 176 * Marc Tamsky : Closing in closing fixes. 177 * Mike Shaver : RFC1122 verifications. 178 * Alan Cox : rcv_saddr errors. 179 * Alan Cox : Block double connect(). 180 * Alan Cox : Small hooks for enSKIP. 181 * Alexey Kuznetsov: Path MTU discovery. 182 * Alan Cox : Support soft errors. 183 * Alan Cox : Fix MTU discovery pathological case 184 * when the remote claims no mtu! 185 * Marc Tamsky : TCP_CLOSE fix. 186 * Colin (G3TNE) : Send a reset on syn ack replies in 187 * window but wrong (fixes NT lpd problems) 188 * Pedro Roque : Better TCP window handling, delayed ack. 189 * Joerg Reuter : No modification of locked buffers in 190 * tcp_do_retransmit() 191 * Eric Schenk : Changed receiver side silly window 192 * avoidance algorithm to BSD style 193 * algorithm. This doubles throughput 194 * against machines running Solaris, 195 * and seems to result in general 196 * improvement. 197 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD 198 * Willy Konynenberg : Transparent proxying support. 199 * Mike McLagan : Routing by source 200 * Keith Owens : Do proper merging with partial SKB's in 201 * tcp_do_sendmsg to avoid burstiness. 202 * Eric Schenk : Fix fast close down bug with 203 * shutdown() followed by close(). 204 * Andi Kleen : Make poll agree with SIGIO 205 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and 206 * lingertime == 0 (RFC 793 ABORT Call) 207 * Hirokazu Takahashi : Use copy_from_user() instead of 208 * csum_and_copy_from_user() if possible. 209 * 210 * This program is free software; you can redistribute it and/or 211 * modify it under the terms of the GNU General Public License 212 * as published by the Free Software Foundation; either version 213 * 2 of the License, or(at your option) any later version. 214 * 215 * Description of States: 216 * 217 * TCP_SYN_SENT sent a connection request, waiting for ack 218 * 219 * TCP_SYN_RECV received a connection request, sent ack, 220 * waiting for final ack in three-way handshake. 221 * 222 * TCP_ESTABLISHED connection established 223 * 224 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete 225 * transmission of remaining buffered data 226 * 227 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote 228 * to shutdown 229 * 230 * TCP_CLOSING both sides have shutdown but we still have 231 * data we have to finish sending 232 * 233 * TCP_TIME_WAIT timeout to catch resent junk before entering 234 * closed, can only be entered from FIN_WAIT2 235 * or CLOSING. Required because the other end 236 * may not have gotten our last ACK causing it 237 * to retransmit the data packet (which we ignore) 238 * 239 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for 240 * us to finish writing our data and to shutdown 241 * (we have to close() to move on to LAST_ACK) 242 * 243 * TCP_LAST_ACK out side has shutdown after remote has 244 * shutdown. There may still be data in our 245 * buffer that we have to finish sending 246 * 247 * TCP_CLOSE socket is finished 248 */ 249 250 #include <linux/config.h> 251 #include <linux/module.h> 252 #include <linux/types.h> 253 #include <linux/fcntl.h> 254 #include <linux/poll.h> 255 #include <linux/init.h> 256 #include <linux/smp_lock.h> 257 #include <linux/fs.h> 258 #include <linux/random.h> 259 #include <linux/bootmem.h> 260 261 #include <net/icmp.h> 262 #include <net/tcp.h> 263 #include <net/xfrm.h> 264 #include <net/ip.h> 265 266 267 #include <asm/uaccess.h> 268 #include <asm/ioctls.h> 269 270 int sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT; 271 272 DEFINE_SNMP_STAT(struct tcp_mib, tcp_statistics); 273 274 kmem_cache_t *tcp_bucket_cachep; 275 kmem_cache_t *tcp_timewait_cachep; 276 277 atomic_t tcp_orphan_count = ATOMIC_INIT(0); 278 279 int sysctl_tcp_mem[3]; 280 int sysctl_tcp_wmem[3] = { 4 * 1024, 16 * 1024, 128 * 1024 }; 281 int sysctl_tcp_rmem[3] = { 4 * 1024, 87380, 87380 * 2 }; 282 283 EXPORT_SYMBOL(sysctl_tcp_mem); 284 EXPORT_SYMBOL(sysctl_tcp_rmem); 285 EXPORT_SYMBOL(sysctl_tcp_wmem); 286 287 atomic_t tcp_memory_allocated; /* Current allocated memory. */ 288 atomic_t tcp_sockets_allocated; /* Current number of TCP sockets. */ 289 290 EXPORT_SYMBOL(tcp_memory_allocated); 291 EXPORT_SYMBOL(tcp_sockets_allocated); 292 293 /* 294 * Pressure flag: try to collapse. 295 * Technical note: it is used by multiple contexts non atomically. 296 * All the sk_stream_mem_schedule() is of this nature: accounting 297 * is strict, actions are advisory and have some latency. 298 */ 299 int tcp_memory_pressure; 300 301 EXPORT_SYMBOL(tcp_memory_pressure); 302 303 void tcp_enter_memory_pressure(void) 304 { 305 if (!tcp_memory_pressure) { 306 NET_INC_STATS(LINUX_MIB_TCPMEMORYPRESSURES); 307 tcp_memory_pressure = 1; 308 } 309 } 310 311 EXPORT_SYMBOL(tcp_enter_memory_pressure); 312 313 /* 314 * LISTEN is a special case for poll.. 315 */ 316 static __inline__ unsigned int tcp_listen_poll(struct sock *sk, 317 poll_table *wait) 318 { 319 return !reqsk_queue_empty(&tcp_sk(sk)->accept_queue) ? (POLLIN | POLLRDNORM) : 0; 320 } 321 322 /* 323 * Wait for a TCP event. 324 * 325 * Note that we don't need to lock the socket, as the upper poll layers 326 * take care of normal races (between the test and the event) and we don't 327 * go look at any of the socket buffers directly. 328 */ 329 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait) 330 { 331 unsigned int mask; 332 struct sock *sk = sock->sk; 333 struct tcp_sock *tp = tcp_sk(sk); 334 335 poll_wait(file, sk->sk_sleep, wait); 336 if (sk->sk_state == TCP_LISTEN) 337 return tcp_listen_poll(sk, wait); 338 339 /* Socket is not locked. We are protected from async events 340 by poll logic and correct handling of state changes 341 made by another threads is impossible in any case. 342 */ 343 344 mask = 0; 345 if (sk->sk_err) 346 mask = POLLERR; 347 348 /* 349 * POLLHUP is certainly not done right. But poll() doesn't 350 * have a notion of HUP in just one direction, and for a 351 * socket the read side is more interesting. 352 * 353 * Some poll() documentation says that POLLHUP is incompatible 354 * with the POLLOUT/POLLWR flags, so somebody should check this 355 * all. But careful, it tends to be safer to return too many 356 * bits than too few, and you can easily break real applications 357 * if you don't tell them that something has hung up! 358 * 359 * Check-me. 360 * 361 * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and 362 * our fs/select.c). It means that after we received EOF, 363 * poll always returns immediately, making impossible poll() on write() 364 * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP 365 * if and only if shutdown has been made in both directions. 366 * Actually, it is interesting to look how Solaris and DUX 367 * solve this dilemma. I would prefer, if PULLHUP were maskable, 368 * then we could set it on SND_SHUTDOWN. BTW examples given 369 * in Stevens' books assume exactly this behaviour, it explains 370 * why PULLHUP is incompatible with POLLOUT. --ANK 371 * 372 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent 373 * blocking on fresh not-connected or disconnected socket. --ANK 374 */ 375 if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE) 376 mask |= POLLHUP; 377 if (sk->sk_shutdown & RCV_SHUTDOWN) 378 mask |= POLLIN | POLLRDNORM; 379 380 /* Connected? */ 381 if ((1 << sk->sk_state) & ~(TCPF_SYN_SENT | TCPF_SYN_RECV)) { 382 /* Potential race condition. If read of tp below will 383 * escape above sk->sk_state, we can be illegally awaken 384 * in SYN_* states. */ 385 if ((tp->rcv_nxt != tp->copied_seq) && 386 (tp->urg_seq != tp->copied_seq || 387 tp->rcv_nxt != tp->copied_seq + 1 || 388 sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data)) 389 mask |= POLLIN | POLLRDNORM; 390 391 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) { 392 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) { 393 mask |= POLLOUT | POLLWRNORM; 394 } else { /* send SIGIO later */ 395 set_bit(SOCK_ASYNC_NOSPACE, 396 &sk->sk_socket->flags); 397 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 398 399 /* Race breaker. If space is freed after 400 * wspace test but before the flags are set, 401 * IO signal will be lost. 402 */ 403 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) 404 mask |= POLLOUT | POLLWRNORM; 405 } 406 } 407 408 if (tp->urg_data & TCP_URG_VALID) 409 mask |= POLLPRI; 410 } 411 return mask; 412 } 413 414 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg) 415 { 416 struct tcp_sock *tp = tcp_sk(sk); 417 int answ; 418 419 switch (cmd) { 420 case SIOCINQ: 421 if (sk->sk_state == TCP_LISTEN) 422 return -EINVAL; 423 424 lock_sock(sk); 425 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 426 answ = 0; 427 else if (sock_flag(sk, SOCK_URGINLINE) || 428 !tp->urg_data || 429 before(tp->urg_seq, tp->copied_seq) || 430 !before(tp->urg_seq, tp->rcv_nxt)) { 431 answ = tp->rcv_nxt - tp->copied_seq; 432 433 /* Subtract 1, if FIN is in queue. */ 434 if (answ && !skb_queue_empty(&sk->sk_receive_queue)) 435 answ -= 436 ((struct sk_buff *)sk->sk_receive_queue.prev)->h.th->fin; 437 } else 438 answ = tp->urg_seq - tp->copied_seq; 439 release_sock(sk); 440 break; 441 case SIOCATMARK: 442 answ = tp->urg_data && tp->urg_seq == tp->copied_seq; 443 break; 444 case SIOCOUTQ: 445 if (sk->sk_state == TCP_LISTEN) 446 return -EINVAL; 447 448 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 449 answ = 0; 450 else 451 answ = tp->write_seq - tp->snd_una; 452 break; 453 default: 454 return -ENOIOCTLCMD; 455 }; 456 457 return put_user(answ, (int __user *)arg); 458 } 459 460 461 int tcp_listen_start(struct sock *sk) 462 { 463 struct inet_sock *inet = inet_sk(sk); 464 struct tcp_sock *tp = tcp_sk(sk); 465 int rc = reqsk_queue_alloc(&tp->accept_queue, TCP_SYNQ_HSIZE); 466 467 if (rc != 0) 468 return rc; 469 470 sk->sk_max_ack_backlog = 0; 471 sk->sk_ack_backlog = 0; 472 tcp_delack_init(tp); 473 474 /* There is race window here: we announce ourselves listening, 475 * but this transition is still not validated by get_port(). 476 * It is OK, because this socket enters to hash table only 477 * after validation is complete. 478 */ 479 sk->sk_state = TCP_LISTEN; 480 if (!sk->sk_prot->get_port(sk, inet->num)) { 481 inet->sport = htons(inet->num); 482 483 sk_dst_reset(sk); 484 sk->sk_prot->hash(sk); 485 486 return 0; 487 } 488 489 sk->sk_state = TCP_CLOSE; 490 reqsk_queue_destroy(&tp->accept_queue); 491 return -EADDRINUSE; 492 } 493 494 /* 495 * This routine closes sockets which have been at least partially 496 * opened, but not yet accepted. 497 */ 498 499 static void tcp_listen_stop (struct sock *sk) 500 { 501 struct tcp_sock *tp = tcp_sk(sk); 502 struct listen_sock *lopt; 503 struct request_sock *acc_req; 504 struct request_sock *req; 505 int i; 506 507 tcp_delete_keepalive_timer(sk); 508 509 /* make all the listen_opt local to us */ 510 lopt = reqsk_queue_yank_listen_sk(&tp->accept_queue); 511 acc_req = reqsk_queue_yank_acceptq(&tp->accept_queue); 512 513 if (lopt->qlen) { 514 for (i = 0; i < TCP_SYNQ_HSIZE; i++) { 515 while ((req = lopt->syn_table[i]) != NULL) { 516 lopt->syn_table[i] = req->dl_next; 517 lopt->qlen--; 518 reqsk_free(req); 519 520 /* Following specs, it would be better either to send FIN 521 * (and enter FIN-WAIT-1, it is normal close) 522 * or to send active reset (abort). 523 * Certainly, it is pretty dangerous while synflood, but it is 524 * bad justification for our negligence 8) 525 * To be honest, we are not able to make either 526 * of the variants now. --ANK 527 */ 528 } 529 } 530 } 531 BUG_TRAP(!lopt->qlen); 532 533 kfree(lopt); 534 535 while ((req = acc_req) != NULL) { 536 struct sock *child = req->sk; 537 538 acc_req = req->dl_next; 539 540 local_bh_disable(); 541 bh_lock_sock(child); 542 BUG_TRAP(!sock_owned_by_user(child)); 543 sock_hold(child); 544 545 tcp_disconnect(child, O_NONBLOCK); 546 547 sock_orphan(child); 548 549 atomic_inc(&tcp_orphan_count); 550 551 tcp_destroy_sock(child); 552 553 bh_unlock_sock(child); 554 local_bh_enable(); 555 sock_put(child); 556 557 sk_acceptq_removed(sk); 558 __reqsk_free(req); 559 } 560 BUG_TRAP(!sk->sk_ack_backlog); 561 } 562 563 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb) 564 { 565 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH; 566 tp->pushed_seq = tp->write_seq; 567 } 568 569 static inline int forced_push(struct tcp_sock *tp) 570 { 571 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1)); 572 } 573 574 static inline void skb_entail(struct sock *sk, struct tcp_sock *tp, 575 struct sk_buff *skb) 576 { 577 skb->csum = 0; 578 TCP_SKB_CB(skb)->seq = tp->write_seq; 579 TCP_SKB_CB(skb)->end_seq = tp->write_seq; 580 TCP_SKB_CB(skb)->flags = TCPCB_FLAG_ACK; 581 TCP_SKB_CB(skb)->sacked = 0; 582 skb_header_release(skb); 583 __skb_queue_tail(&sk->sk_write_queue, skb); 584 sk_charge_skb(sk, skb); 585 if (!sk->sk_send_head) 586 sk->sk_send_head = skb; 587 else if (tp->nonagle&TCP_NAGLE_PUSH) 588 tp->nonagle &= ~TCP_NAGLE_PUSH; 589 } 590 591 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags, 592 struct sk_buff *skb) 593 { 594 if (flags & MSG_OOB) { 595 tp->urg_mode = 1; 596 tp->snd_up = tp->write_seq; 597 TCP_SKB_CB(skb)->sacked |= TCPCB_URG; 598 } 599 } 600 601 static inline void tcp_push(struct sock *sk, struct tcp_sock *tp, int flags, 602 int mss_now, int nonagle) 603 { 604 if (sk->sk_send_head) { 605 struct sk_buff *skb = sk->sk_write_queue.prev; 606 if (!(flags & MSG_MORE) || forced_push(tp)) 607 tcp_mark_push(tp, skb); 608 tcp_mark_urg(tp, flags, skb); 609 __tcp_push_pending_frames(sk, tp, mss_now, 610 (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle); 611 } 612 } 613 614 static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset, 615 size_t psize, int flags) 616 { 617 struct tcp_sock *tp = tcp_sk(sk); 618 int mss_now; 619 int err; 620 ssize_t copied; 621 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 622 623 /* Wait for a connection to finish. */ 624 if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) 625 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0) 626 goto out_err; 627 628 clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags); 629 630 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); 631 copied = 0; 632 633 err = -EPIPE; 634 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 635 goto do_error; 636 637 while (psize > 0) { 638 struct sk_buff *skb = sk->sk_write_queue.prev; 639 struct page *page = pages[poffset / PAGE_SIZE]; 640 int copy, i, can_coalesce; 641 int offset = poffset % PAGE_SIZE; 642 int size = min_t(size_t, psize, PAGE_SIZE - offset); 643 644 if (!sk->sk_send_head || (copy = mss_now - skb->len) <= 0) { 645 new_segment: 646 if (!sk_stream_memory_free(sk)) 647 goto wait_for_sndbuf; 648 649 skb = sk_stream_alloc_pskb(sk, 0, 0, 650 sk->sk_allocation); 651 if (!skb) 652 goto wait_for_memory; 653 654 skb_entail(sk, tp, skb); 655 copy = mss_now; 656 } 657 658 if (copy > size) 659 copy = size; 660 661 i = skb_shinfo(skb)->nr_frags; 662 can_coalesce = skb_can_coalesce(skb, i, page, offset); 663 if (!can_coalesce && i >= MAX_SKB_FRAGS) { 664 tcp_mark_push(tp, skb); 665 goto new_segment; 666 } 667 if (sk->sk_forward_alloc < copy && 668 !sk_stream_mem_schedule(sk, copy, 0)) 669 goto wait_for_memory; 670 671 if (can_coalesce) { 672 skb_shinfo(skb)->frags[i - 1].size += copy; 673 } else { 674 get_page(page); 675 skb_fill_page_desc(skb, i, page, offset, copy); 676 } 677 678 skb->len += copy; 679 skb->data_len += copy; 680 skb->truesize += copy; 681 sk->sk_wmem_queued += copy; 682 sk->sk_forward_alloc -= copy; 683 skb->ip_summed = CHECKSUM_HW; 684 tp->write_seq += copy; 685 TCP_SKB_CB(skb)->end_seq += copy; 686 skb_shinfo(skb)->tso_segs = 0; 687 688 if (!copied) 689 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH; 690 691 copied += copy; 692 poffset += copy; 693 if (!(psize -= copy)) 694 goto out; 695 696 if (skb->len != mss_now || (flags & MSG_OOB)) 697 continue; 698 699 if (forced_push(tp)) { 700 tcp_mark_push(tp, skb); 701 __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH); 702 } else if (skb == sk->sk_send_head) 703 tcp_push_one(sk, mss_now); 704 continue; 705 706 wait_for_sndbuf: 707 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 708 wait_for_memory: 709 if (copied) 710 tcp_push(sk, tp, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH); 711 712 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0) 713 goto do_error; 714 715 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); 716 } 717 718 out: 719 if (copied) 720 tcp_push(sk, tp, flags, mss_now, tp->nonagle); 721 return copied; 722 723 do_error: 724 if (copied) 725 goto out; 726 out_err: 727 return sk_stream_error(sk, flags, err); 728 } 729 730 ssize_t tcp_sendpage(struct socket *sock, struct page *page, int offset, 731 size_t size, int flags) 732 { 733 ssize_t res; 734 struct sock *sk = sock->sk; 735 736 #define TCP_ZC_CSUM_FLAGS (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | NETIF_F_HW_CSUM) 737 738 if (!(sk->sk_route_caps & NETIF_F_SG) || 739 !(sk->sk_route_caps & TCP_ZC_CSUM_FLAGS)) 740 return sock_no_sendpage(sock, page, offset, size, flags); 741 742 #undef TCP_ZC_CSUM_FLAGS 743 744 lock_sock(sk); 745 TCP_CHECK_TIMER(sk); 746 res = do_tcp_sendpages(sk, &page, offset, size, flags); 747 TCP_CHECK_TIMER(sk); 748 release_sock(sk); 749 return res; 750 } 751 752 #define TCP_PAGE(sk) (sk->sk_sndmsg_page) 753 #define TCP_OFF(sk) (sk->sk_sndmsg_off) 754 755 static inline int select_size(struct sock *sk, struct tcp_sock *tp) 756 { 757 int tmp = tp->mss_cache_std; 758 759 if (sk->sk_route_caps & NETIF_F_SG) { 760 int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER); 761 762 if (tmp >= pgbreak && 763 tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE) 764 tmp = pgbreak; 765 } 766 return tmp; 767 } 768 769 int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, 770 size_t size) 771 { 772 struct iovec *iov; 773 struct tcp_sock *tp = tcp_sk(sk); 774 struct sk_buff *skb; 775 int iovlen, flags; 776 int mss_now; 777 int err, copied; 778 long timeo; 779 780 lock_sock(sk); 781 TCP_CHECK_TIMER(sk); 782 783 flags = msg->msg_flags; 784 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 785 786 /* Wait for a connection to finish. */ 787 if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) 788 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0) 789 goto out_err; 790 791 /* This should be in poll */ 792 clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags); 793 794 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); 795 796 /* Ok commence sending. */ 797 iovlen = msg->msg_iovlen; 798 iov = msg->msg_iov; 799 copied = 0; 800 801 err = -EPIPE; 802 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 803 goto do_error; 804 805 while (--iovlen >= 0) { 806 int seglen = iov->iov_len; 807 unsigned char __user *from = iov->iov_base; 808 809 iov++; 810 811 while (seglen > 0) { 812 int copy; 813 814 skb = sk->sk_write_queue.prev; 815 816 if (!sk->sk_send_head || 817 (copy = mss_now - skb->len) <= 0) { 818 819 new_segment: 820 /* Allocate new segment. If the interface is SG, 821 * allocate skb fitting to single page. 822 */ 823 if (!sk_stream_memory_free(sk)) 824 goto wait_for_sndbuf; 825 826 skb = sk_stream_alloc_pskb(sk, select_size(sk, tp), 827 0, sk->sk_allocation); 828 if (!skb) 829 goto wait_for_memory; 830 831 /* 832 * Check whether we can use HW checksum. 833 */ 834 if (sk->sk_route_caps & 835 (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | 836 NETIF_F_HW_CSUM)) 837 skb->ip_summed = CHECKSUM_HW; 838 839 skb_entail(sk, tp, skb); 840 copy = mss_now; 841 } 842 843 /* Try to append data to the end of skb. */ 844 if (copy > seglen) 845 copy = seglen; 846 847 /* Where to copy to? */ 848 if (skb_tailroom(skb) > 0) { 849 /* We have some space in skb head. Superb! */ 850 if (copy > skb_tailroom(skb)) 851 copy = skb_tailroom(skb); 852 if ((err = skb_add_data(skb, from, copy)) != 0) 853 goto do_fault; 854 } else { 855 int merge = 0; 856 int i = skb_shinfo(skb)->nr_frags; 857 struct page *page = TCP_PAGE(sk); 858 int off = TCP_OFF(sk); 859 860 if (skb_can_coalesce(skb, i, page, off) && 861 off != PAGE_SIZE) { 862 /* We can extend the last page 863 * fragment. */ 864 merge = 1; 865 } else if (i == MAX_SKB_FRAGS || 866 (!i && 867 !(sk->sk_route_caps & NETIF_F_SG))) { 868 /* Need to add new fragment and cannot 869 * do this because interface is non-SG, 870 * or because all the page slots are 871 * busy. */ 872 tcp_mark_push(tp, skb); 873 goto new_segment; 874 } else if (page) { 875 /* If page is cached, align 876 * offset to L1 cache boundary 877 */ 878 off = (off + L1_CACHE_BYTES - 1) & 879 ~(L1_CACHE_BYTES - 1); 880 if (off == PAGE_SIZE) { 881 put_page(page); 882 TCP_PAGE(sk) = page = NULL; 883 } 884 } 885 886 if (!page) { 887 /* Allocate new cache page. */ 888 if (!(page = sk_stream_alloc_page(sk))) 889 goto wait_for_memory; 890 off = 0; 891 } 892 893 if (copy > PAGE_SIZE - off) 894 copy = PAGE_SIZE - off; 895 896 /* Time to copy data. We are close to 897 * the end! */ 898 err = skb_copy_to_page(sk, from, skb, page, 899 off, copy); 900 if (err) { 901 /* If this page was new, give it to the 902 * socket so it does not get leaked. 903 */ 904 if (!TCP_PAGE(sk)) { 905 TCP_PAGE(sk) = page; 906 TCP_OFF(sk) = 0; 907 } 908 goto do_error; 909 } 910 911 /* Update the skb. */ 912 if (merge) { 913 skb_shinfo(skb)->frags[i - 1].size += 914 copy; 915 } else { 916 skb_fill_page_desc(skb, i, page, off, copy); 917 if (TCP_PAGE(sk)) { 918 get_page(page); 919 } else if (off + copy < PAGE_SIZE) { 920 get_page(page); 921 TCP_PAGE(sk) = page; 922 } 923 } 924 925 TCP_OFF(sk) = off + copy; 926 } 927 928 if (!copied) 929 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH; 930 931 tp->write_seq += copy; 932 TCP_SKB_CB(skb)->end_seq += copy; 933 skb_shinfo(skb)->tso_segs = 0; 934 935 from += copy; 936 copied += copy; 937 if ((seglen -= copy) == 0 && iovlen == 0) 938 goto out; 939 940 if (skb->len != mss_now || (flags & MSG_OOB)) 941 continue; 942 943 if (forced_push(tp)) { 944 tcp_mark_push(tp, skb); 945 __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH); 946 } else if (skb == sk->sk_send_head) 947 tcp_push_one(sk, mss_now); 948 continue; 949 950 wait_for_sndbuf: 951 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 952 wait_for_memory: 953 if (copied) 954 tcp_push(sk, tp, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH); 955 956 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0) 957 goto do_error; 958 959 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); 960 } 961 } 962 963 out: 964 if (copied) 965 tcp_push(sk, tp, flags, mss_now, tp->nonagle); 966 TCP_CHECK_TIMER(sk); 967 release_sock(sk); 968 return copied; 969 970 do_fault: 971 if (!skb->len) { 972 if (sk->sk_send_head == skb) 973 sk->sk_send_head = NULL; 974 __skb_unlink(skb, skb->list); 975 sk_stream_free_skb(sk, skb); 976 } 977 978 do_error: 979 if (copied) 980 goto out; 981 out_err: 982 err = sk_stream_error(sk, flags, err); 983 TCP_CHECK_TIMER(sk); 984 release_sock(sk); 985 return err; 986 } 987 988 /* 989 * Handle reading urgent data. BSD has very simple semantics for 990 * this, no blocking and very strange errors 8) 991 */ 992 993 static int tcp_recv_urg(struct sock *sk, long timeo, 994 struct msghdr *msg, int len, int flags, 995 int *addr_len) 996 { 997 struct tcp_sock *tp = tcp_sk(sk); 998 999 /* No URG data to read. */ 1000 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data || 1001 tp->urg_data == TCP_URG_READ) 1002 return -EINVAL; /* Yes this is right ! */ 1003 1004 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE)) 1005 return -ENOTCONN; 1006 1007 if (tp->urg_data & TCP_URG_VALID) { 1008 int err = 0; 1009 char c = tp->urg_data; 1010 1011 if (!(flags & MSG_PEEK)) 1012 tp->urg_data = TCP_URG_READ; 1013 1014 /* Read urgent data. */ 1015 msg->msg_flags |= MSG_OOB; 1016 1017 if (len > 0) { 1018 if (!(flags & MSG_TRUNC)) 1019 err = memcpy_toiovec(msg->msg_iov, &c, 1); 1020 len = 1; 1021 } else 1022 msg->msg_flags |= MSG_TRUNC; 1023 1024 return err ? -EFAULT : len; 1025 } 1026 1027 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 1028 return 0; 1029 1030 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and 1031 * the available implementations agree in this case: 1032 * this call should never block, independent of the 1033 * blocking state of the socket. 1034 * Mike <pall@rz.uni-karlsruhe.de> 1035 */ 1036 return -EAGAIN; 1037 } 1038 1039 /* Clean up the receive buffer for full frames taken by the user, 1040 * then send an ACK if necessary. COPIED is the number of bytes 1041 * tcp_recvmsg has given to the user so far, it speeds up the 1042 * calculation of whether or not we must ACK for the sake of 1043 * a window update. 1044 */ 1045 static void cleanup_rbuf(struct sock *sk, int copied) 1046 { 1047 struct tcp_sock *tp = tcp_sk(sk); 1048 int time_to_ack = 0; 1049 1050 #if TCP_DEBUG 1051 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 1052 1053 BUG_TRAP(!skb || before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq)); 1054 #endif 1055 1056 if (tcp_ack_scheduled(tp)) { 1057 /* Delayed ACKs frequently hit locked sockets during bulk 1058 * receive. */ 1059 if (tp->ack.blocked || 1060 /* Once-per-two-segments ACK was not sent by tcp_input.c */ 1061 tp->rcv_nxt - tp->rcv_wup > tp->ack.rcv_mss || 1062 /* 1063 * If this read emptied read buffer, we send ACK, if 1064 * connection is not bidirectional, user drained 1065 * receive buffer and there was a small segment 1066 * in queue. 1067 */ 1068 (copied > 0 && (tp->ack.pending & TCP_ACK_PUSHED) && 1069 !tp->ack.pingpong && !atomic_read(&sk->sk_rmem_alloc))) 1070 time_to_ack = 1; 1071 } 1072 1073 /* We send an ACK if we can now advertise a non-zero window 1074 * which has been raised "significantly". 1075 * 1076 * Even if window raised up to infinity, do not send window open ACK 1077 * in states, where we will not receive more. It is useless. 1078 */ 1079 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 1080 __u32 rcv_window_now = tcp_receive_window(tp); 1081 1082 /* Optimize, __tcp_select_window() is not cheap. */ 1083 if (2*rcv_window_now <= tp->window_clamp) { 1084 __u32 new_window = __tcp_select_window(sk); 1085 1086 /* Send ACK now, if this read freed lots of space 1087 * in our buffer. Certainly, new_window is new window. 1088 * We can advertise it now, if it is not less than current one. 1089 * "Lots" means "at least twice" here. 1090 */ 1091 if (new_window && new_window >= 2 * rcv_window_now) 1092 time_to_ack = 1; 1093 } 1094 } 1095 if (time_to_ack) 1096 tcp_send_ack(sk); 1097 } 1098 1099 static void tcp_prequeue_process(struct sock *sk) 1100 { 1101 struct sk_buff *skb; 1102 struct tcp_sock *tp = tcp_sk(sk); 1103 1104 NET_ADD_STATS_USER(LINUX_MIB_TCPPREQUEUED, skb_queue_len(&tp->ucopy.prequeue)); 1105 1106 /* RX process wants to run with disabled BHs, though it is not 1107 * necessary */ 1108 local_bh_disable(); 1109 while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL) 1110 sk->sk_backlog_rcv(sk, skb); 1111 local_bh_enable(); 1112 1113 /* Clear memory counter. */ 1114 tp->ucopy.memory = 0; 1115 } 1116 1117 static inline struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) 1118 { 1119 struct sk_buff *skb; 1120 u32 offset; 1121 1122 skb_queue_walk(&sk->sk_receive_queue, skb) { 1123 offset = seq - TCP_SKB_CB(skb)->seq; 1124 if (skb->h.th->syn) 1125 offset--; 1126 if (offset < skb->len || skb->h.th->fin) { 1127 *off = offset; 1128 return skb; 1129 } 1130 } 1131 return NULL; 1132 } 1133 1134 /* 1135 * This routine provides an alternative to tcp_recvmsg() for routines 1136 * that would like to handle copying from skbuffs directly in 'sendfile' 1137 * fashion. 1138 * Note: 1139 * - It is assumed that the socket was locked by the caller. 1140 * - The routine does not block. 1141 * - At present, there is no support for reading OOB data 1142 * or for 'peeking' the socket using this routine 1143 * (although both would be easy to implement). 1144 */ 1145 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 1146 sk_read_actor_t recv_actor) 1147 { 1148 struct sk_buff *skb; 1149 struct tcp_sock *tp = tcp_sk(sk); 1150 u32 seq = tp->copied_seq; 1151 u32 offset; 1152 int copied = 0; 1153 1154 if (sk->sk_state == TCP_LISTEN) 1155 return -ENOTCONN; 1156 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { 1157 if (offset < skb->len) { 1158 size_t used, len; 1159 1160 len = skb->len - offset; 1161 /* Stop reading if we hit a patch of urgent data */ 1162 if (tp->urg_data) { 1163 u32 urg_offset = tp->urg_seq - seq; 1164 if (urg_offset < len) 1165 len = urg_offset; 1166 if (!len) 1167 break; 1168 } 1169 used = recv_actor(desc, skb, offset, len); 1170 if (used <= len) { 1171 seq += used; 1172 copied += used; 1173 offset += used; 1174 } 1175 if (offset != skb->len) 1176 break; 1177 } 1178 if (skb->h.th->fin) { 1179 sk_eat_skb(sk, skb); 1180 ++seq; 1181 break; 1182 } 1183 sk_eat_skb(sk, skb); 1184 if (!desc->count) 1185 break; 1186 } 1187 tp->copied_seq = seq; 1188 1189 tcp_rcv_space_adjust(sk); 1190 1191 /* Clean up data we have read: This will do ACK frames. */ 1192 if (copied) 1193 cleanup_rbuf(sk, copied); 1194 return copied; 1195 } 1196 1197 /* 1198 * This routine copies from a sock struct into the user buffer. 1199 * 1200 * Technical note: in 2.3 we work on _locked_ socket, so that 1201 * tricks with *seq access order and skb->users are not required. 1202 * Probably, code can be easily improved even more. 1203 */ 1204 1205 int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, 1206 size_t len, int nonblock, int flags, int *addr_len) 1207 { 1208 struct tcp_sock *tp = tcp_sk(sk); 1209 int copied = 0; 1210 u32 peek_seq; 1211 u32 *seq; 1212 unsigned long used; 1213 int err; 1214 int target; /* Read at least this many bytes */ 1215 long timeo; 1216 struct task_struct *user_recv = NULL; 1217 1218 lock_sock(sk); 1219 1220 TCP_CHECK_TIMER(sk); 1221 1222 err = -ENOTCONN; 1223 if (sk->sk_state == TCP_LISTEN) 1224 goto out; 1225 1226 timeo = sock_rcvtimeo(sk, nonblock); 1227 1228 /* Urgent data needs to be handled specially. */ 1229 if (flags & MSG_OOB) 1230 goto recv_urg; 1231 1232 seq = &tp->copied_seq; 1233 if (flags & MSG_PEEK) { 1234 peek_seq = tp->copied_seq; 1235 seq = &peek_seq; 1236 } 1237 1238 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 1239 1240 do { 1241 struct sk_buff *skb; 1242 u32 offset; 1243 1244 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */ 1245 if (tp->urg_data && tp->urg_seq == *seq) { 1246 if (copied) 1247 break; 1248 if (signal_pending(current)) { 1249 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN; 1250 break; 1251 } 1252 } 1253 1254 /* Next get a buffer. */ 1255 1256 skb = skb_peek(&sk->sk_receive_queue); 1257 do { 1258 if (!skb) 1259 break; 1260 1261 /* Now that we have two receive queues this 1262 * shouldn't happen. 1263 */ 1264 if (before(*seq, TCP_SKB_CB(skb)->seq)) { 1265 printk(KERN_INFO "recvmsg bug: copied %X " 1266 "seq %X\n", *seq, TCP_SKB_CB(skb)->seq); 1267 break; 1268 } 1269 offset = *seq - TCP_SKB_CB(skb)->seq; 1270 if (skb->h.th->syn) 1271 offset--; 1272 if (offset < skb->len) 1273 goto found_ok_skb; 1274 if (skb->h.th->fin) 1275 goto found_fin_ok; 1276 BUG_TRAP(flags & MSG_PEEK); 1277 skb = skb->next; 1278 } while (skb != (struct sk_buff *)&sk->sk_receive_queue); 1279 1280 /* Well, if we have backlog, try to process it now yet. */ 1281 1282 if (copied >= target && !sk->sk_backlog.tail) 1283 break; 1284 1285 if (copied) { 1286 if (sk->sk_err || 1287 sk->sk_state == TCP_CLOSE || 1288 (sk->sk_shutdown & RCV_SHUTDOWN) || 1289 !timeo || 1290 signal_pending(current) || 1291 (flags & MSG_PEEK)) 1292 break; 1293 } else { 1294 if (sock_flag(sk, SOCK_DONE)) 1295 break; 1296 1297 if (sk->sk_err) { 1298 copied = sock_error(sk); 1299 break; 1300 } 1301 1302 if (sk->sk_shutdown & RCV_SHUTDOWN) 1303 break; 1304 1305 if (sk->sk_state == TCP_CLOSE) { 1306 if (!sock_flag(sk, SOCK_DONE)) { 1307 /* This occurs when user tries to read 1308 * from never connected socket. 1309 */ 1310 copied = -ENOTCONN; 1311 break; 1312 } 1313 break; 1314 } 1315 1316 if (!timeo) { 1317 copied = -EAGAIN; 1318 break; 1319 } 1320 1321 if (signal_pending(current)) { 1322 copied = sock_intr_errno(timeo); 1323 break; 1324 } 1325 } 1326 1327 cleanup_rbuf(sk, copied); 1328 1329 if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) { 1330 /* Install new reader */ 1331 if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) { 1332 user_recv = current; 1333 tp->ucopy.task = user_recv; 1334 tp->ucopy.iov = msg->msg_iov; 1335 } 1336 1337 tp->ucopy.len = len; 1338 1339 BUG_TRAP(tp->copied_seq == tp->rcv_nxt || 1340 (flags & (MSG_PEEK | MSG_TRUNC))); 1341 1342 /* Ugly... If prequeue is not empty, we have to 1343 * process it before releasing socket, otherwise 1344 * order will be broken at second iteration. 1345 * More elegant solution is required!!! 1346 * 1347 * Look: we have the following (pseudo)queues: 1348 * 1349 * 1. packets in flight 1350 * 2. backlog 1351 * 3. prequeue 1352 * 4. receive_queue 1353 * 1354 * Each queue can be processed only if the next ones 1355 * are empty. At this point we have empty receive_queue. 1356 * But prequeue _can_ be not empty after 2nd iteration, 1357 * when we jumped to start of loop because backlog 1358 * processing added something to receive_queue. 1359 * We cannot release_sock(), because backlog contains 1360 * packets arrived _after_ prequeued ones. 1361 * 1362 * Shortly, algorithm is clear --- to process all 1363 * the queues in order. We could make it more directly, 1364 * requeueing packets from backlog to prequeue, if 1365 * is not empty. It is more elegant, but eats cycles, 1366 * unfortunately. 1367 */ 1368 if (skb_queue_len(&tp->ucopy.prequeue)) 1369 goto do_prequeue; 1370 1371 /* __ Set realtime policy in scheduler __ */ 1372 } 1373 1374 if (copied >= target) { 1375 /* Do not sleep, just process backlog. */ 1376 release_sock(sk); 1377 lock_sock(sk); 1378 } else 1379 sk_wait_data(sk, &timeo); 1380 1381 if (user_recv) { 1382 int chunk; 1383 1384 /* __ Restore normal policy in scheduler __ */ 1385 1386 if ((chunk = len - tp->ucopy.len) != 0) { 1387 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk); 1388 len -= chunk; 1389 copied += chunk; 1390 } 1391 1392 if (tp->rcv_nxt == tp->copied_seq && 1393 skb_queue_len(&tp->ucopy.prequeue)) { 1394 do_prequeue: 1395 tcp_prequeue_process(sk); 1396 1397 if ((chunk = len - tp->ucopy.len) != 0) { 1398 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk); 1399 len -= chunk; 1400 copied += chunk; 1401 } 1402 } 1403 } 1404 if ((flags & MSG_PEEK) && peek_seq != tp->copied_seq) { 1405 if (net_ratelimit()) 1406 printk(KERN_DEBUG "TCP(%s:%d): Application bug, race in MSG_PEEK.\n", 1407 current->comm, current->pid); 1408 peek_seq = tp->copied_seq; 1409 } 1410 continue; 1411 1412 found_ok_skb: 1413 /* Ok so how much can we use? */ 1414 used = skb->len - offset; 1415 if (len < used) 1416 used = len; 1417 1418 /* Do we have urgent data here? */ 1419 if (tp->urg_data) { 1420 u32 urg_offset = tp->urg_seq - *seq; 1421 if (urg_offset < used) { 1422 if (!urg_offset) { 1423 if (!sock_flag(sk, SOCK_URGINLINE)) { 1424 ++*seq; 1425 offset++; 1426 used--; 1427 if (!used) 1428 goto skip_copy; 1429 } 1430 } else 1431 used = urg_offset; 1432 } 1433 } 1434 1435 if (!(flags & MSG_TRUNC)) { 1436 err = skb_copy_datagram_iovec(skb, offset, 1437 msg->msg_iov, used); 1438 if (err) { 1439 /* Exception. Bailout! */ 1440 if (!copied) 1441 copied = -EFAULT; 1442 break; 1443 } 1444 } 1445 1446 *seq += used; 1447 copied += used; 1448 len -= used; 1449 1450 tcp_rcv_space_adjust(sk); 1451 1452 skip_copy: 1453 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) { 1454 tp->urg_data = 0; 1455 tcp_fast_path_check(sk, tp); 1456 } 1457 if (used + offset < skb->len) 1458 continue; 1459 1460 if (skb->h.th->fin) 1461 goto found_fin_ok; 1462 if (!(flags & MSG_PEEK)) 1463 sk_eat_skb(sk, skb); 1464 continue; 1465 1466 found_fin_ok: 1467 /* Process the FIN. */ 1468 ++*seq; 1469 if (!(flags & MSG_PEEK)) 1470 sk_eat_skb(sk, skb); 1471 break; 1472 } while (len > 0); 1473 1474 if (user_recv) { 1475 if (skb_queue_len(&tp->ucopy.prequeue)) { 1476 int chunk; 1477 1478 tp->ucopy.len = copied > 0 ? len : 0; 1479 1480 tcp_prequeue_process(sk); 1481 1482 if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) { 1483 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk); 1484 len -= chunk; 1485 copied += chunk; 1486 } 1487 } 1488 1489 tp->ucopy.task = NULL; 1490 tp->ucopy.len = 0; 1491 } 1492 1493 /* According to UNIX98, msg_name/msg_namelen are ignored 1494 * on connected socket. I was just happy when found this 8) --ANK 1495 */ 1496 1497 /* Clean up data we have read: This will do ACK frames. */ 1498 cleanup_rbuf(sk, copied); 1499 1500 TCP_CHECK_TIMER(sk); 1501 release_sock(sk); 1502 return copied; 1503 1504 out: 1505 TCP_CHECK_TIMER(sk); 1506 release_sock(sk); 1507 return err; 1508 1509 recv_urg: 1510 err = tcp_recv_urg(sk, timeo, msg, len, flags, addr_len); 1511 goto out; 1512 } 1513 1514 /* 1515 * State processing on a close. This implements the state shift for 1516 * sending our FIN frame. Note that we only send a FIN for some 1517 * states. A shutdown() may have already sent the FIN, or we may be 1518 * closed. 1519 */ 1520 1521 static unsigned char new_state[16] = { 1522 /* current state: new state: action: */ 1523 /* (Invalid) */ TCP_CLOSE, 1524 /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, 1525 /* TCP_SYN_SENT */ TCP_CLOSE, 1526 /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, 1527 /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1, 1528 /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2, 1529 /* TCP_TIME_WAIT */ TCP_CLOSE, 1530 /* TCP_CLOSE */ TCP_CLOSE, 1531 /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN, 1532 /* TCP_LAST_ACK */ TCP_LAST_ACK, 1533 /* TCP_LISTEN */ TCP_CLOSE, 1534 /* TCP_CLOSING */ TCP_CLOSING, 1535 }; 1536 1537 static int tcp_close_state(struct sock *sk) 1538 { 1539 int next = (int)new_state[sk->sk_state]; 1540 int ns = next & TCP_STATE_MASK; 1541 1542 tcp_set_state(sk, ns); 1543 1544 return next & TCP_ACTION_FIN; 1545 } 1546 1547 /* 1548 * Shutdown the sending side of a connection. Much like close except 1549 * that we don't receive shut down or set_sock_flag(sk, SOCK_DEAD). 1550 */ 1551 1552 void tcp_shutdown(struct sock *sk, int how) 1553 { 1554 /* We need to grab some memory, and put together a FIN, 1555 * and then put it into the queue to be sent. 1556 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92. 1557 */ 1558 if (!(how & SEND_SHUTDOWN)) 1559 return; 1560 1561 /* If we've already sent a FIN, or it's a closed state, skip this. */ 1562 if ((1 << sk->sk_state) & 1563 (TCPF_ESTABLISHED | TCPF_SYN_SENT | 1564 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) { 1565 /* Clear out any half completed packets. FIN if needed. */ 1566 if (tcp_close_state(sk)) 1567 tcp_send_fin(sk); 1568 } 1569 } 1570 1571 /* 1572 * At this point, there should be no process reference to this 1573 * socket, and thus no user references at all. Therefore we 1574 * can assume the socket waitqueue is inactive and nobody will 1575 * try to jump onto it. 1576 */ 1577 void tcp_destroy_sock(struct sock *sk) 1578 { 1579 BUG_TRAP(sk->sk_state == TCP_CLOSE); 1580 BUG_TRAP(sock_flag(sk, SOCK_DEAD)); 1581 1582 /* It cannot be in hash table! */ 1583 BUG_TRAP(sk_unhashed(sk)); 1584 1585 /* If it has not 0 inet_sk(sk)->num, it must be bound */ 1586 BUG_TRAP(!inet_sk(sk)->num || tcp_sk(sk)->bind_hash); 1587 1588 sk->sk_prot->destroy(sk); 1589 1590 sk_stream_kill_queues(sk); 1591 1592 xfrm_sk_free_policy(sk); 1593 1594 #ifdef INET_REFCNT_DEBUG 1595 if (atomic_read(&sk->sk_refcnt) != 1) { 1596 printk(KERN_DEBUG "Destruction TCP %p delayed, c=%d\n", 1597 sk, atomic_read(&sk->sk_refcnt)); 1598 } 1599 #endif 1600 1601 atomic_dec(&tcp_orphan_count); 1602 sock_put(sk); 1603 } 1604 1605 void tcp_close(struct sock *sk, long timeout) 1606 { 1607 struct sk_buff *skb; 1608 int data_was_unread = 0; 1609 1610 lock_sock(sk); 1611 sk->sk_shutdown = SHUTDOWN_MASK; 1612 1613 if (sk->sk_state == TCP_LISTEN) { 1614 tcp_set_state(sk, TCP_CLOSE); 1615 1616 /* Special case. */ 1617 tcp_listen_stop(sk); 1618 1619 goto adjudge_to_death; 1620 } 1621 1622 /* We need to flush the recv. buffs. We do this only on the 1623 * descriptor close, not protocol-sourced closes, because the 1624 * reader process may not have drained the data yet! 1625 */ 1626 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { 1627 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq - 1628 skb->h.th->fin; 1629 data_was_unread += len; 1630 __kfree_skb(skb); 1631 } 1632 1633 sk_stream_mem_reclaim(sk); 1634 1635 /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section 1636 * 3.10, we send a RST here because data was lost. To 1637 * witness the awful effects of the old behavior of always 1638 * doing a FIN, run an older 2.1.x kernel or 2.0.x, start 1639 * a bulk GET in an FTP client, suspend the process, wait 1640 * for the client to advertise a zero window, then kill -9 1641 * the FTP client, wheee... Note: timeout is always zero 1642 * in such a case. 1643 */ 1644 if (data_was_unread) { 1645 /* Unread data was tossed, zap the connection. */ 1646 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONCLOSE); 1647 tcp_set_state(sk, TCP_CLOSE); 1648 tcp_send_active_reset(sk, GFP_KERNEL); 1649 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { 1650 /* Check zero linger _after_ checking for unread data. */ 1651 sk->sk_prot->disconnect(sk, 0); 1652 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONDATA); 1653 } else if (tcp_close_state(sk)) { 1654 /* We FIN if the application ate all the data before 1655 * zapping the connection. 1656 */ 1657 1658 /* RED-PEN. Formally speaking, we have broken TCP state 1659 * machine. State transitions: 1660 * 1661 * TCP_ESTABLISHED -> TCP_FIN_WAIT1 1662 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible) 1663 * TCP_CLOSE_WAIT -> TCP_LAST_ACK 1664 * 1665 * are legal only when FIN has been sent (i.e. in window), 1666 * rather than queued out of window. Purists blame. 1667 * 1668 * F.e. "RFC state" is ESTABLISHED, 1669 * if Linux state is FIN-WAIT-1, but FIN is still not sent. 1670 * 1671 * The visible declinations are that sometimes 1672 * we enter time-wait state, when it is not required really 1673 * (harmless), do not send active resets, when they are 1674 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when 1675 * they look as CLOSING or LAST_ACK for Linux) 1676 * Probably, I missed some more holelets. 1677 * --ANK 1678 */ 1679 tcp_send_fin(sk); 1680 } 1681 1682 sk_stream_wait_close(sk, timeout); 1683 1684 adjudge_to_death: 1685 /* It is the last release_sock in its life. It will remove backlog. */ 1686 release_sock(sk); 1687 1688 1689 /* Now socket is owned by kernel and we acquire BH lock 1690 to finish close. No need to check for user refs. 1691 */ 1692 local_bh_disable(); 1693 bh_lock_sock(sk); 1694 BUG_TRAP(!sock_owned_by_user(sk)); 1695 1696 sock_hold(sk); 1697 sock_orphan(sk); 1698 1699 /* This is a (useful) BSD violating of the RFC. There is a 1700 * problem with TCP as specified in that the other end could 1701 * keep a socket open forever with no application left this end. 1702 * We use a 3 minute timeout (about the same as BSD) then kill 1703 * our end. If they send after that then tough - BUT: long enough 1704 * that we won't make the old 4*rto = almost no time - whoops 1705 * reset mistake. 1706 * 1707 * Nope, it was not mistake. It is really desired behaviour 1708 * f.e. on http servers, when such sockets are useless, but 1709 * consume significant resources. Let's do it with special 1710 * linger2 option. --ANK 1711 */ 1712 1713 if (sk->sk_state == TCP_FIN_WAIT2) { 1714 struct tcp_sock *tp = tcp_sk(sk); 1715 if (tp->linger2 < 0) { 1716 tcp_set_state(sk, TCP_CLOSE); 1717 tcp_send_active_reset(sk, GFP_ATOMIC); 1718 NET_INC_STATS_BH(LINUX_MIB_TCPABORTONLINGER); 1719 } else { 1720 int tmo = tcp_fin_time(tp); 1721 1722 if (tmo > TCP_TIMEWAIT_LEN) { 1723 tcp_reset_keepalive_timer(sk, tcp_fin_time(tp)); 1724 } else { 1725 atomic_inc(&tcp_orphan_count); 1726 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo); 1727 goto out; 1728 } 1729 } 1730 } 1731 if (sk->sk_state != TCP_CLOSE) { 1732 sk_stream_mem_reclaim(sk); 1733 if (atomic_read(&tcp_orphan_count) > sysctl_tcp_max_orphans || 1734 (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 1735 atomic_read(&tcp_memory_allocated) > sysctl_tcp_mem[2])) { 1736 if (net_ratelimit()) 1737 printk(KERN_INFO "TCP: too many of orphaned " 1738 "sockets\n"); 1739 tcp_set_state(sk, TCP_CLOSE); 1740 tcp_send_active_reset(sk, GFP_ATOMIC); 1741 NET_INC_STATS_BH(LINUX_MIB_TCPABORTONMEMORY); 1742 } 1743 } 1744 atomic_inc(&tcp_orphan_count); 1745 1746 if (sk->sk_state == TCP_CLOSE) 1747 tcp_destroy_sock(sk); 1748 /* Otherwise, socket is reprieved until protocol close. */ 1749 1750 out: 1751 bh_unlock_sock(sk); 1752 local_bh_enable(); 1753 sock_put(sk); 1754 } 1755 1756 /* These states need RST on ABORT according to RFC793 */ 1757 1758 static inline int tcp_need_reset(int state) 1759 { 1760 return (1 << state) & 1761 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 | 1762 TCPF_FIN_WAIT2 | TCPF_SYN_RECV); 1763 } 1764 1765 int tcp_disconnect(struct sock *sk, int flags) 1766 { 1767 struct inet_sock *inet = inet_sk(sk); 1768 struct tcp_sock *tp = tcp_sk(sk); 1769 int err = 0; 1770 int old_state = sk->sk_state; 1771 1772 if (old_state != TCP_CLOSE) 1773 tcp_set_state(sk, TCP_CLOSE); 1774 1775 /* ABORT function of RFC793 */ 1776 if (old_state == TCP_LISTEN) { 1777 tcp_listen_stop(sk); 1778 } else if (tcp_need_reset(old_state) || 1779 (tp->snd_nxt != tp->write_seq && 1780 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) { 1781 /* The last check adjusts for discrepance of Linux wrt. RFC 1782 * states 1783 */ 1784 tcp_send_active_reset(sk, gfp_any()); 1785 sk->sk_err = ECONNRESET; 1786 } else if (old_state == TCP_SYN_SENT) 1787 sk->sk_err = ECONNRESET; 1788 1789 tcp_clear_xmit_timers(sk); 1790 __skb_queue_purge(&sk->sk_receive_queue); 1791 sk_stream_writequeue_purge(sk); 1792 __skb_queue_purge(&tp->out_of_order_queue); 1793 1794 inet->dport = 0; 1795 1796 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) 1797 inet_reset_saddr(sk); 1798 1799 sk->sk_shutdown = 0; 1800 sock_reset_flag(sk, SOCK_DONE); 1801 tp->srtt = 0; 1802 if ((tp->write_seq += tp->max_window + 2) == 0) 1803 tp->write_seq = 1; 1804 tp->backoff = 0; 1805 tp->snd_cwnd = 2; 1806 tp->probes_out = 0; 1807 tp->packets_out = 0; 1808 tp->snd_ssthresh = 0x7fffffff; 1809 tp->snd_cwnd_cnt = 0; 1810 tcp_set_ca_state(tp, TCP_CA_Open); 1811 tcp_clear_retrans(tp); 1812 tcp_delack_init(tp); 1813 sk->sk_send_head = NULL; 1814 tp->rx_opt.saw_tstamp = 0; 1815 tcp_sack_reset(&tp->rx_opt); 1816 __sk_dst_reset(sk); 1817 1818 BUG_TRAP(!inet->num || tp->bind_hash); 1819 1820 sk->sk_error_report(sk); 1821 return err; 1822 } 1823 1824 /* 1825 * Wait for an incoming connection, avoid race 1826 * conditions. This must be called with the socket locked. 1827 */ 1828 static int wait_for_connect(struct sock *sk, long timeo) 1829 { 1830 struct tcp_sock *tp = tcp_sk(sk); 1831 DEFINE_WAIT(wait); 1832 int err; 1833 1834 /* 1835 * True wake-one mechanism for incoming connections: only 1836 * one process gets woken up, not the 'whole herd'. 1837 * Since we do not 'race & poll' for established sockets 1838 * anymore, the common case will execute the loop only once. 1839 * 1840 * Subtle issue: "add_wait_queue_exclusive()" will be added 1841 * after any current non-exclusive waiters, and we know that 1842 * it will always _stay_ after any new non-exclusive waiters 1843 * because all non-exclusive waiters are added at the 1844 * beginning of the wait-queue. As such, it's ok to "drop" 1845 * our exclusiveness temporarily when we get woken up without 1846 * having to remove and re-insert us on the wait queue. 1847 */ 1848 for (;;) { 1849 prepare_to_wait_exclusive(sk->sk_sleep, &wait, 1850 TASK_INTERRUPTIBLE); 1851 release_sock(sk); 1852 if (reqsk_queue_empty(&tp->accept_queue)) 1853 timeo = schedule_timeout(timeo); 1854 lock_sock(sk); 1855 err = 0; 1856 if (!reqsk_queue_empty(&tp->accept_queue)) 1857 break; 1858 err = -EINVAL; 1859 if (sk->sk_state != TCP_LISTEN) 1860 break; 1861 err = sock_intr_errno(timeo); 1862 if (signal_pending(current)) 1863 break; 1864 err = -EAGAIN; 1865 if (!timeo) 1866 break; 1867 } 1868 finish_wait(sk->sk_sleep, &wait); 1869 return err; 1870 } 1871 1872 /* 1873 * This will accept the next outstanding connection. 1874 */ 1875 1876 struct sock *tcp_accept(struct sock *sk, int flags, int *err) 1877 { 1878 struct tcp_sock *tp = tcp_sk(sk); 1879 struct sock *newsk; 1880 int error; 1881 1882 lock_sock(sk); 1883 1884 /* We need to make sure that this socket is listening, 1885 * and that it has something pending. 1886 */ 1887 error = -EINVAL; 1888 if (sk->sk_state != TCP_LISTEN) 1889 goto out_err; 1890 1891 /* Find already established connection */ 1892 if (reqsk_queue_empty(&tp->accept_queue)) { 1893 long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); 1894 1895 /* If this is a non blocking socket don't sleep */ 1896 error = -EAGAIN; 1897 if (!timeo) 1898 goto out_err; 1899 1900 error = wait_for_connect(sk, timeo); 1901 if (error) 1902 goto out_err; 1903 } 1904 1905 newsk = reqsk_queue_get_child(&tp->accept_queue, sk); 1906 BUG_TRAP(newsk->sk_state != TCP_SYN_RECV); 1907 out: 1908 release_sock(sk); 1909 return newsk; 1910 out_err: 1911 newsk = NULL; 1912 *err = error; 1913 goto out; 1914 } 1915 1916 /* 1917 * Socket option code for TCP. 1918 */ 1919 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval, 1920 int optlen) 1921 { 1922 struct tcp_sock *tp = tcp_sk(sk); 1923 int val; 1924 int err = 0; 1925 1926 if (level != SOL_TCP) 1927 return tp->af_specific->setsockopt(sk, level, optname, 1928 optval, optlen); 1929 1930 if (optlen < sizeof(int)) 1931 return -EINVAL; 1932 1933 if (get_user(val, (int __user *)optval)) 1934 return -EFAULT; 1935 1936 lock_sock(sk); 1937 1938 switch (optname) { 1939 case TCP_MAXSEG: 1940 /* Values greater than interface MTU won't take effect. However 1941 * at the point when this call is done we typically don't yet 1942 * know which interface is going to be used */ 1943 if (val < 8 || val > MAX_TCP_WINDOW) { 1944 err = -EINVAL; 1945 break; 1946 } 1947 tp->rx_opt.user_mss = val; 1948 break; 1949 1950 case TCP_NODELAY: 1951 if (val) { 1952 /* TCP_NODELAY is weaker than TCP_CORK, so that 1953 * this option on corked socket is remembered, but 1954 * it is not activated until cork is cleared. 1955 * 1956 * However, when TCP_NODELAY is set we make 1957 * an explicit push, which overrides even TCP_CORK 1958 * for currently queued segments. 1959 */ 1960 tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; 1961 tcp_push_pending_frames(sk, tp); 1962 } else { 1963 tp->nonagle &= ~TCP_NAGLE_OFF; 1964 } 1965 break; 1966 1967 case TCP_CORK: 1968 /* When set indicates to always queue non-full frames. 1969 * Later the user clears this option and we transmit 1970 * any pending partial frames in the queue. This is 1971 * meant to be used alongside sendfile() to get properly 1972 * filled frames when the user (for example) must write 1973 * out headers with a write() call first and then use 1974 * sendfile to send out the data parts. 1975 * 1976 * TCP_CORK can be set together with TCP_NODELAY and it is 1977 * stronger than TCP_NODELAY. 1978 */ 1979 if (val) { 1980 tp->nonagle |= TCP_NAGLE_CORK; 1981 } else { 1982 tp->nonagle &= ~TCP_NAGLE_CORK; 1983 if (tp->nonagle&TCP_NAGLE_OFF) 1984 tp->nonagle |= TCP_NAGLE_PUSH; 1985 tcp_push_pending_frames(sk, tp); 1986 } 1987 break; 1988 1989 case TCP_KEEPIDLE: 1990 if (val < 1 || val > MAX_TCP_KEEPIDLE) 1991 err = -EINVAL; 1992 else { 1993 tp->keepalive_time = val * HZ; 1994 if (sock_flag(sk, SOCK_KEEPOPEN) && 1995 !((1 << sk->sk_state) & 1996 (TCPF_CLOSE | TCPF_LISTEN))) { 1997 __u32 elapsed = tcp_time_stamp - tp->rcv_tstamp; 1998 if (tp->keepalive_time > elapsed) 1999 elapsed = tp->keepalive_time - elapsed; 2000 else 2001 elapsed = 0; 2002 tcp_reset_keepalive_timer(sk, elapsed); 2003 } 2004 } 2005 break; 2006 case TCP_KEEPINTVL: 2007 if (val < 1 || val > MAX_TCP_KEEPINTVL) 2008 err = -EINVAL; 2009 else 2010 tp->keepalive_intvl = val * HZ; 2011 break; 2012 case TCP_KEEPCNT: 2013 if (val < 1 || val > MAX_TCP_KEEPCNT) 2014 err = -EINVAL; 2015 else 2016 tp->keepalive_probes = val; 2017 break; 2018 case TCP_SYNCNT: 2019 if (val < 1 || val > MAX_TCP_SYNCNT) 2020 err = -EINVAL; 2021 else 2022 tp->syn_retries = val; 2023 break; 2024 2025 case TCP_LINGER2: 2026 if (val < 0) 2027 tp->linger2 = -1; 2028 else if (val > sysctl_tcp_fin_timeout / HZ) 2029 tp->linger2 = 0; 2030 else 2031 tp->linger2 = val * HZ; 2032 break; 2033 2034 case TCP_DEFER_ACCEPT: 2035 tp->defer_accept = 0; 2036 if (val > 0) { 2037 /* Translate value in seconds to number of 2038 * retransmits */ 2039 while (tp->defer_accept < 32 && 2040 val > ((TCP_TIMEOUT_INIT / HZ) << 2041 tp->defer_accept)) 2042 tp->defer_accept++; 2043 tp->defer_accept++; 2044 } 2045 break; 2046 2047 case TCP_WINDOW_CLAMP: 2048 if (!val) { 2049 if (sk->sk_state != TCP_CLOSE) { 2050 err = -EINVAL; 2051 break; 2052 } 2053 tp->window_clamp = 0; 2054 } else 2055 tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ? 2056 SOCK_MIN_RCVBUF / 2 : val; 2057 break; 2058 2059 case TCP_QUICKACK: 2060 if (!val) { 2061 tp->ack.pingpong = 1; 2062 } else { 2063 tp->ack.pingpong = 0; 2064 if ((1 << sk->sk_state) & 2065 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && 2066 tcp_ack_scheduled(tp)) { 2067 tp->ack.pending |= TCP_ACK_PUSHED; 2068 cleanup_rbuf(sk, 1); 2069 if (!(val & 1)) 2070 tp->ack.pingpong = 1; 2071 } 2072 } 2073 break; 2074 2075 default: 2076 err = -ENOPROTOOPT; 2077 break; 2078 }; 2079 release_sock(sk); 2080 return err; 2081 } 2082 2083 /* Return information about state of tcp endpoint in API format. */ 2084 void tcp_get_info(struct sock *sk, struct tcp_info *info) 2085 { 2086 struct tcp_sock *tp = tcp_sk(sk); 2087 u32 now = tcp_time_stamp; 2088 2089 memset(info, 0, sizeof(*info)); 2090 2091 info->tcpi_state = sk->sk_state; 2092 info->tcpi_ca_state = tp->ca_state; 2093 info->tcpi_retransmits = tp->retransmits; 2094 info->tcpi_probes = tp->probes_out; 2095 info->tcpi_backoff = tp->backoff; 2096 2097 if (tp->rx_opt.tstamp_ok) 2098 info->tcpi_options |= TCPI_OPT_TIMESTAMPS; 2099 if (tp->rx_opt.sack_ok) 2100 info->tcpi_options |= TCPI_OPT_SACK; 2101 if (tp->rx_opt.wscale_ok) { 2102 info->tcpi_options |= TCPI_OPT_WSCALE; 2103 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale; 2104 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale; 2105 } 2106 2107 if (tp->ecn_flags&TCP_ECN_OK) 2108 info->tcpi_options |= TCPI_OPT_ECN; 2109 2110 info->tcpi_rto = jiffies_to_usecs(tp->rto); 2111 info->tcpi_ato = jiffies_to_usecs(tp->ack.ato); 2112 info->tcpi_snd_mss = tp->mss_cache_std; 2113 info->tcpi_rcv_mss = tp->ack.rcv_mss; 2114 2115 info->tcpi_unacked = tp->packets_out; 2116 info->tcpi_sacked = tp->sacked_out; 2117 info->tcpi_lost = tp->lost_out; 2118 info->tcpi_retrans = tp->retrans_out; 2119 info->tcpi_fackets = tp->fackets_out; 2120 2121 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime); 2122 info->tcpi_last_data_recv = jiffies_to_msecs(now - tp->ack.lrcvtime); 2123 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp); 2124 2125 info->tcpi_pmtu = tp->pmtu_cookie; 2126 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh; 2127 info->tcpi_rtt = jiffies_to_usecs(tp->srtt)>>3; 2128 info->tcpi_rttvar = jiffies_to_usecs(tp->mdev)>>2; 2129 info->tcpi_snd_ssthresh = tp->snd_ssthresh; 2130 info->tcpi_snd_cwnd = tp->snd_cwnd; 2131 info->tcpi_advmss = tp->advmss; 2132 info->tcpi_reordering = tp->reordering; 2133 2134 info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3; 2135 info->tcpi_rcv_space = tp->rcvq_space.space; 2136 2137 info->tcpi_total_retrans = tp->total_retrans; 2138 } 2139 2140 EXPORT_SYMBOL_GPL(tcp_get_info); 2141 2142 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, 2143 int __user *optlen) 2144 { 2145 struct tcp_sock *tp = tcp_sk(sk); 2146 int val, len; 2147 2148 if (level != SOL_TCP) 2149 return tp->af_specific->getsockopt(sk, level, optname, 2150 optval, optlen); 2151 2152 if (get_user(len, optlen)) 2153 return -EFAULT; 2154 2155 len = min_t(unsigned int, len, sizeof(int)); 2156 2157 if (len < 0) 2158 return -EINVAL; 2159 2160 switch (optname) { 2161 case TCP_MAXSEG: 2162 val = tp->mss_cache_std; 2163 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 2164 val = tp->rx_opt.user_mss; 2165 break; 2166 case TCP_NODELAY: 2167 val = !!(tp->nonagle&TCP_NAGLE_OFF); 2168 break; 2169 case TCP_CORK: 2170 val = !!(tp->nonagle&TCP_NAGLE_CORK); 2171 break; 2172 case TCP_KEEPIDLE: 2173 val = (tp->keepalive_time ? : sysctl_tcp_keepalive_time) / HZ; 2174 break; 2175 case TCP_KEEPINTVL: 2176 val = (tp->keepalive_intvl ? : sysctl_tcp_keepalive_intvl) / HZ; 2177 break; 2178 case TCP_KEEPCNT: 2179 val = tp->keepalive_probes ? : sysctl_tcp_keepalive_probes; 2180 break; 2181 case TCP_SYNCNT: 2182 val = tp->syn_retries ? : sysctl_tcp_syn_retries; 2183 break; 2184 case TCP_LINGER2: 2185 val = tp->linger2; 2186 if (val >= 0) 2187 val = (val ? : sysctl_tcp_fin_timeout) / HZ; 2188 break; 2189 case TCP_DEFER_ACCEPT: 2190 val = !tp->defer_accept ? 0 : ((TCP_TIMEOUT_INIT / HZ) << 2191 (tp->defer_accept - 1)); 2192 break; 2193 case TCP_WINDOW_CLAMP: 2194 val = tp->window_clamp; 2195 break; 2196 case TCP_INFO: { 2197 struct tcp_info info; 2198 2199 if (get_user(len, optlen)) 2200 return -EFAULT; 2201 2202 tcp_get_info(sk, &info); 2203 2204 len = min_t(unsigned int, len, sizeof(info)); 2205 if (put_user(len, optlen)) 2206 return -EFAULT; 2207 if (copy_to_user(optval, &info, len)) 2208 return -EFAULT; 2209 return 0; 2210 } 2211 case TCP_QUICKACK: 2212 val = !tp->ack.pingpong; 2213 break; 2214 default: 2215 return -ENOPROTOOPT; 2216 }; 2217 2218 if (put_user(len, optlen)) 2219 return -EFAULT; 2220 if (copy_to_user(optval, &val, len)) 2221 return -EFAULT; 2222 return 0; 2223 } 2224 2225 2226 extern void __skb_cb_too_small_for_tcp(int, int); 2227 extern void tcpdiag_init(void); 2228 2229 static __initdata unsigned long thash_entries; 2230 static int __init set_thash_entries(char *str) 2231 { 2232 if (!str) 2233 return 0; 2234 thash_entries = simple_strtoul(str, &str, 0); 2235 return 1; 2236 } 2237 __setup("thash_entries=", set_thash_entries); 2238 2239 void __init tcp_init(void) 2240 { 2241 struct sk_buff *skb = NULL; 2242 int order, i; 2243 2244 if (sizeof(struct tcp_skb_cb) > sizeof(skb->cb)) 2245 __skb_cb_too_small_for_tcp(sizeof(struct tcp_skb_cb), 2246 sizeof(skb->cb)); 2247 2248 tcp_bucket_cachep = kmem_cache_create("tcp_bind_bucket", 2249 sizeof(struct tcp_bind_bucket), 2250 0, SLAB_HWCACHE_ALIGN, 2251 NULL, NULL); 2252 if (!tcp_bucket_cachep) 2253 panic("tcp_init: Cannot alloc tcp_bind_bucket cache."); 2254 2255 tcp_timewait_cachep = kmem_cache_create("tcp_tw_bucket", 2256 sizeof(struct tcp_tw_bucket), 2257 0, SLAB_HWCACHE_ALIGN, 2258 NULL, NULL); 2259 if (!tcp_timewait_cachep) 2260 panic("tcp_init: Cannot alloc tcp_tw_bucket cache."); 2261 2262 /* Size and allocate the main established and bind bucket 2263 * hash tables. 2264 * 2265 * The methodology is similar to that of the buffer cache. 2266 */ 2267 tcp_ehash = (struct tcp_ehash_bucket *) 2268 alloc_large_system_hash("TCP established", 2269 sizeof(struct tcp_ehash_bucket), 2270 thash_entries, 2271 (num_physpages >= 128 * 1024) ? 2272 (25 - PAGE_SHIFT) : 2273 (27 - PAGE_SHIFT), 2274 HASH_HIGHMEM, 2275 &tcp_ehash_size, 2276 NULL, 2277 0); 2278 tcp_ehash_size = (1 << tcp_ehash_size) >> 1; 2279 for (i = 0; i < (tcp_ehash_size << 1); i++) { 2280 rwlock_init(&tcp_ehash[i].lock); 2281 INIT_HLIST_HEAD(&tcp_ehash[i].chain); 2282 } 2283 2284 tcp_bhash = (struct tcp_bind_hashbucket *) 2285 alloc_large_system_hash("TCP bind", 2286 sizeof(struct tcp_bind_hashbucket), 2287 tcp_ehash_size, 2288 (num_physpages >= 128 * 1024) ? 2289 (25 - PAGE_SHIFT) : 2290 (27 - PAGE_SHIFT), 2291 HASH_HIGHMEM, 2292 &tcp_bhash_size, 2293 NULL, 2294 64 * 1024); 2295 tcp_bhash_size = 1 << tcp_bhash_size; 2296 for (i = 0; i < tcp_bhash_size; i++) { 2297 spin_lock_init(&tcp_bhash[i].lock); 2298 INIT_HLIST_HEAD(&tcp_bhash[i].chain); 2299 } 2300 2301 /* Try to be a bit smarter and adjust defaults depending 2302 * on available memory. 2303 */ 2304 for (order = 0; ((1 << order) << PAGE_SHIFT) < 2305 (tcp_bhash_size * sizeof(struct tcp_bind_hashbucket)); 2306 order++) 2307 ; 2308 if (order >= 4) { 2309 sysctl_local_port_range[0] = 32768; 2310 sysctl_local_port_range[1] = 61000; 2311 sysctl_tcp_max_tw_buckets = 180000; 2312 sysctl_tcp_max_orphans = 4096 << (order - 4); 2313 sysctl_max_syn_backlog = 1024; 2314 } else if (order < 3) { 2315 sysctl_local_port_range[0] = 1024 * (3 - order); 2316 sysctl_tcp_max_tw_buckets >>= (3 - order); 2317 sysctl_tcp_max_orphans >>= (3 - order); 2318 sysctl_max_syn_backlog = 128; 2319 } 2320 tcp_port_rover = sysctl_local_port_range[0] - 1; 2321 2322 sysctl_tcp_mem[0] = 768 << order; 2323 sysctl_tcp_mem[1] = 1024 << order; 2324 sysctl_tcp_mem[2] = 1536 << order; 2325 2326 if (order < 3) { 2327 sysctl_tcp_wmem[2] = 64 * 1024; 2328 sysctl_tcp_rmem[0] = PAGE_SIZE; 2329 sysctl_tcp_rmem[1] = 43689; 2330 sysctl_tcp_rmem[2] = 2 * 43689; 2331 } 2332 2333 printk(KERN_INFO "TCP: Hash tables configured " 2334 "(established %d bind %d)\n", 2335 tcp_ehash_size << 1, tcp_bhash_size); 2336 } 2337 2338 EXPORT_SYMBOL(tcp_accept); 2339 EXPORT_SYMBOL(tcp_close); 2340 EXPORT_SYMBOL(tcp_destroy_sock); 2341 EXPORT_SYMBOL(tcp_disconnect); 2342 EXPORT_SYMBOL(tcp_getsockopt); 2343 EXPORT_SYMBOL(tcp_ioctl); 2344 EXPORT_SYMBOL(tcp_poll); 2345 EXPORT_SYMBOL(tcp_read_sock); 2346 EXPORT_SYMBOL(tcp_recvmsg); 2347 EXPORT_SYMBOL(tcp_sendmsg); 2348 EXPORT_SYMBOL(tcp_sendpage); 2349 EXPORT_SYMBOL(tcp_setsockopt); 2350 EXPORT_SYMBOL(tcp_shutdown); 2351 EXPORT_SYMBOL(tcp_statistics); 2352 EXPORT_SYMBOL(tcp_timewait_cachep); 2353