1 /* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * Implementation of the Transmission Control Protocol(TCP). 7 * 8 * Version: $Id: tcp.c,v 1.216 2002/02/01 22:01:04 davem Exp $ 9 * 10 * Authors: Ross Biro 11 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 12 * Mark Evans, <evansmp@uhura.aston.ac.uk> 13 * Corey Minyard <wf-rch!minyard@relay.EU.net> 14 * Florian La Roche, <flla@stud.uni-sb.de> 15 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu> 16 * Linus Torvalds, <torvalds@cs.helsinki.fi> 17 * Alan Cox, <gw4pts@gw4pts.ampr.org> 18 * Matthew Dillon, <dillon@apollo.west.oic.com> 19 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 20 * Jorge Cwik, <jorge@laser.satlink.net> 21 * 22 * Fixes: 23 * Alan Cox : Numerous verify_area() calls 24 * Alan Cox : Set the ACK bit on a reset 25 * Alan Cox : Stopped it crashing if it closed while 26 * sk->inuse=1 and was trying to connect 27 * (tcp_err()). 28 * Alan Cox : All icmp error handling was broken 29 * pointers passed where wrong and the 30 * socket was looked up backwards. Nobody 31 * tested any icmp error code obviously. 32 * Alan Cox : tcp_err() now handled properly. It 33 * wakes people on errors. poll 34 * behaves and the icmp error race 35 * has gone by moving it into sock.c 36 * Alan Cox : tcp_send_reset() fixed to work for 37 * everything not just packets for 38 * unknown sockets. 39 * Alan Cox : tcp option processing. 40 * Alan Cox : Reset tweaked (still not 100%) [Had 41 * syn rule wrong] 42 * Herp Rosmanith : More reset fixes 43 * Alan Cox : No longer acks invalid rst frames. 44 * Acking any kind of RST is right out. 45 * Alan Cox : Sets an ignore me flag on an rst 46 * receive otherwise odd bits of prattle 47 * escape still 48 * Alan Cox : Fixed another acking RST frame bug. 49 * Should stop LAN workplace lockups. 50 * Alan Cox : Some tidyups using the new skb list 51 * facilities 52 * Alan Cox : sk->keepopen now seems to work 53 * Alan Cox : Pulls options out correctly on accepts 54 * Alan Cox : Fixed assorted sk->rqueue->next errors 55 * Alan Cox : PSH doesn't end a TCP read. Switched a 56 * bit to skb ops. 57 * Alan Cox : Tidied tcp_data to avoid a potential 58 * nasty. 59 * Alan Cox : Added some better commenting, as the 60 * tcp is hard to follow 61 * Alan Cox : Removed incorrect check for 20 * psh 62 * Michael O'Reilly : ack < copied bug fix. 63 * Johannes Stille : Misc tcp fixes (not all in yet). 64 * Alan Cox : FIN with no memory -> CRASH 65 * Alan Cox : Added socket option proto entries. 66 * Also added awareness of them to accept. 67 * Alan Cox : Added TCP options (SOL_TCP) 68 * Alan Cox : Switched wakeup calls to callbacks, 69 * so the kernel can layer network 70 * sockets. 71 * Alan Cox : Use ip_tos/ip_ttl settings. 72 * Alan Cox : Handle FIN (more) properly (we hope). 73 * Alan Cox : RST frames sent on unsynchronised 74 * state ack error. 75 * Alan Cox : Put in missing check for SYN bit. 76 * Alan Cox : Added tcp_select_window() aka NET2E 77 * window non shrink trick. 78 * Alan Cox : Added a couple of small NET2E timer 79 * fixes 80 * Charles Hedrick : TCP fixes 81 * Toomas Tamm : TCP window fixes 82 * Alan Cox : Small URG fix to rlogin ^C ack fight 83 * Charles Hedrick : Rewrote most of it to actually work 84 * Linus : Rewrote tcp_read() and URG handling 85 * completely 86 * Gerhard Koerting: Fixed some missing timer handling 87 * Matthew Dillon : Reworked TCP machine states as per RFC 88 * Gerhard Koerting: PC/TCP workarounds 89 * Adam Caldwell : Assorted timer/timing errors 90 * Matthew Dillon : Fixed another RST bug 91 * Alan Cox : Move to kernel side addressing changes. 92 * Alan Cox : Beginning work on TCP fastpathing 93 * (not yet usable) 94 * Arnt Gulbrandsen: Turbocharged tcp_check() routine. 95 * Alan Cox : TCP fast path debugging 96 * Alan Cox : Window clamping 97 * Michael Riepe : Bug in tcp_check() 98 * Matt Dillon : More TCP improvements and RST bug fixes 99 * Matt Dillon : Yet more small nasties remove from the 100 * TCP code (Be very nice to this man if 101 * tcp finally works 100%) 8) 102 * Alan Cox : BSD accept semantics. 103 * Alan Cox : Reset on closedown bug. 104 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto(). 105 * Michael Pall : Handle poll() after URG properly in 106 * all cases. 107 * Michael Pall : Undo the last fix in tcp_read_urg() 108 * (multi URG PUSH broke rlogin). 109 * Michael Pall : Fix the multi URG PUSH problem in 110 * tcp_readable(), poll() after URG 111 * works now. 112 * Michael Pall : recv(...,MSG_OOB) never blocks in the 113 * BSD api. 114 * Alan Cox : Changed the semantics of sk->socket to 115 * fix a race and a signal problem with 116 * accept() and async I/O. 117 * Alan Cox : Relaxed the rules on tcp_sendto(). 118 * Yury Shevchuk : Really fixed accept() blocking problem. 119 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for 120 * clients/servers which listen in on 121 * fixed ports. 122 * Alan Cox : Cleaned the above up and shrank it to 123 * a sensible code size. 124 * Alan Cox : Self connect lockup fix. 125 * Alan Cox : No connect to multicast. 126 * Ross Biro : Close unaccepted children on master 127 * socket close. 128 * Alan Cox : Reset tracing code. 129 * Alan Cox : Spurious resets on shutdown. 130 * Alan Cox : Giant 15 minute/60 second timer error 131 * Alan Cox : Small whoops in polling before an 132 * accept. 133 * Alan Cox : Kept the state trace facility since 134 * it's handy for debugging. 135 * Alan Cox : More reset handler fixes. 136 * Alan Cox : Started rewriting the code based on 137 * the RFC's for other useful protocol 138 * references see: Comer, KA9Q NOS, and 139 * for a reference on the difference 140 * between specifications and how BSD 141 * works see the 4.4lite source. 142 * A.N.Kuznetsov : Don't time wait on completion of tidy 143 * close. 144 * Linus Torvalds : Fin/Shutdown & copied_seq changes. 145 * Linus Torvalds : Fixed BSD port reuse to work first syn 146 * Alan Cox : Reimplemented timers as per the RFC 147 * and using multiple timers for sanity. 148 * Alan Cox : Small bug fixes, and a lot of new 149 * comments. 150 * Alan Cox : Fixed dual reader crash by locking 151 * the buffers (much like datagram.c) 152 * Alan Cox : Fixed stuck sockets in probe. A probe 153 * now gets fed up of retrying without 154 * (even a no space) answer. 155 * Alan Cox : Extracted closing code better 156 * Alan Cox : Fixed the closing state machine to 157 * resemble the RFC. 158 * Alan Cox : More 'per spec' fixes. 159 * Jorge Cwik : Even faster checksumming. 160 * Alan Cox : tcp_data() doesn't ack illegal PSH 161 * only frames. At least one pc tcp stack 162 * generates them. 163 * Alan Cox : Cache last socket. 164 * Alan Cox : Per route irtt. 165 * Matt Day : poll()->select() match BSD precisely on error 166 * Alan Cox : New buffers 167 * Marc Tamsky : Various sk->prot->retransmits and 168 * sk->retransmits misupdating fixed. 169 * Fixed tcp_write_timeout: stuck close, 170 * and TCP syn retries gets used now. 171 * Mark Yarvis : In tcp_read_wakeup(), don't send an 172 * ack if state is TCP_CLOSED. 173 * Alan Cox : Look up device on a retransmit - routes may 174 * change. Doesn't yet cope with MSS shrink right 175 * but it's a start! 176 * Marc Tamsky : Closing in closing fixes. 177 * Mike Shaver : RFC1122 verifications. 178 * Alan Cox : rcv_saddr errors. 179 * Alan Cox : Block double connect(). 180 * Alan Cox : Small hooks for enSKIP. 181 * Alexey Kuznetsov: Path MTU discovery. 182 * Alan Cox : Support soft errors. 183 * Alan Cox : Fix MTU discovery pathological case 184 * when the remote claims no mtu! 185 * Marc Tamsky : TCP_CLOSE fix. 186 * Colin (G3TNE) : Send a reset on syn ack replies in 187 * window but wrong (fixes NT lpd problems) 188 * Pedro Roque : Better TCP window handling, delayed ack. 189 * Joerg Reuter : No modification of locked buffers in 190 * tcp_do_retransmit() 191 * Eric Schenk : Changed receiver side silly window 192 * avoidance algorithm to BSD style 193 * algorithm. This doubles throughput 194 * against machines running Solaris, 195 * and seems to result in general 196 * improvement. 197 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD 198 * Willy Konynenberg : Transparent proxying support. 199 * Mike McLagan : Routing by source 200 * Keith Owens : Do proper merging with partial SKB's in 201 * tcp_do_sendmsg to avoid burstiness. 202 * Eric Schenk : Fix fast close down bug with 203 * shutdown() followed by close(). 204 * Andi Kleen : Make poll agree with SIGIO 205 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and 206 * lingertime == 0 (RFC 793 ABORT Call) 207 * Hirokazu Takahashi : Use copy_from_user() instead of 208 * csum_and_copy_from_user() if possible. 209 * 210 * This program is free software; you can redistribute it and/or 211 * modify it under the terms of the GNU General Public License 212 * as published by the Free Software Foundation; either version 213 * 2 of the License, or(at your option) any later version. 214 * 215 * Description of States: 216 * 217 * TCP_SYN_SENT sent a connection request, waiting for ack 218 * 219 * TCP_SYN_RECV received a connection request, sent ack, 220 * waiting for final ack in three-way handshake. 221 * 222 * TCP_ESTABLISHED connection established 223 * 224 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete 225 * transmission of remaining buffered data 226 * 227 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote 228 * to shutdown 229 * 230 * TCP_CLOSING both sides have shutdown but we still have 231 * data we have to finish sending 232 * 233 * TCP_TIME_WAIT timeout to catch resent junk before entering 234 * closed, can only be entered from FIN_WAIT2 235 * or CLOSING. Required because the other end 236 * may not have gotten our last ACK causing it 237 * to retransmit the data packet (which we ignore) 238 * 239 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for 240 * us to finish writing our data and to shutdown 241 * (we have to close() to move on to LAST_ACK) 242 * 243 * TCP_LAST_ACK out side has shutdown after remote has 244 * shutdown. There may still be data in our 245 * buffer that we have to finish sending 246 * 247 * TCP_CLOSE socket is finished 248 */ 249 250 #include <linux/config.h> 251 #include <linux/module.h> 252 #include <linux/types.h> 253 #include <linux/fcntl.h> 254 #include <linux/poll.h> 255 #include <linux/init.h> 256 #include <linux/smp_lock.h> 257 #include <linux/fs.h> 258 #include <linux/random.h> 259 #include <linux/bootmem.h> 260 261 #include <net/icmp.h> 262 #include <net/tcp.h> 263 #include <net/xfrm.h> 264 #include <net/ip.h> 265 266 267 #include <asm/uaccess.h> 268 #include <asm/ioctls.h> 269 270 int sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT; 271 272 DEFINE_SNMP_STAT(struct tcp_mib, tcp_statistics); 273 274 kmem_cache_t *tcp_bucket_cachep; 275 kmem_cache_t *tcp_timewait_cachep; 276 277 atomic_t tcp_orphan_count = ATOMIC_INIT(0); 278 279 int sysctl_tcp_mem[3]; 280 int sysctl_tcp_wmem[3] = { 4 * 1024, 16 * 1024, 128 * 1024 }; 281 int sysctl_tcp_rmem[3] = { 4 * 1024, 87380, 87380 * 2 }; 282 283 EXPORT_SYMBOL(sysctl_tcp_mem); 284 EXPORT_SYMBOL(sysctl_tcp_rmem); 285 EXPORT_SYMBOL(sysctl_tcp_wmem); 286 287 atomic_t tcp_memory_allocated; /* Current allocated memory. */ 288 atomic_t tcp_sockets_allocated; /* Current number of TCP sockets. */ 289 290 EXPORT_SYMBOL(tcp_memory_allocated); 291 EXPORT_SYMBOL(tcp_sockets_allocated); 292 293 /* 294 * Pressure flag: try to collapse. 295 * Technical note: it is used by multiple contexts non atomically. 296 * All the sk_stream_mem_schedule() is of this nature: accounting 297 * is strict, actions are advisory and have some latency. 298 */ 299 int tcp_memory_pressure; 300 301 EXPORT_SYMBOL(tcp_memory_pressure); 302 303 void tcp_enter_memory_pressure(void) 304 { 305 if (!tcp_memory_pressure) { 306 NET_INC_STATS(LINUX_MIB_TCPMEMORYPRESSURES); 307 tcp_memory_pressure = 1; 308 } 309 } 310 311 EXPORT_SYMBOL(tcp_enter_memory_pressure); 312 313 /* 314 * LISTEN is a special case for poll.. 315 */ 316 static __inline__ unsigned int tcp_listen_poll(struct sock *sk, 317 poll_table *wait) 318 { 319 return !reqsk_queue_empty(&tcp_sk(sk)->accept_queue) ? (POLLIN | POLLRDNORM) : 0; 320 } 321 322 /* 323 * Wait for a TCP event. 324 * 325 * Note that we don't need to lock the socket, as the upper poll layers 326 * take care of normal races (between the test and the event) and we don't 327 * go look at any of the socket buffers directly. 328 */ 329 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait) 330 { 331 unsigned int mask; 332 struct sock *sk = sock->sk; 333 struct tcp_sock *tp = tcp_sk(sk); 334 335 poll_wait(file, sk->sk_sleep, wait); 336 if (sk->sk_state == TCP_LISTEN) 337 return tcp_listen_poll(sk, wait); 338 339 /* Socket is not locked. We are protected from async events 340 by poll logic and correct handling of state changes 341 made by another threads is impossible in any case. 342 */ 343 344 mask = 0; 345 if (sk->sk_err) 346 mask = POLLERR; 347 348 /* 349 * POLLHUP is certainly not done right. But poll() doesn't 350 * have a notion of HUP in just one direction, and for a 351 * socket the read side is more interesting. 352 * 353 * Some poll() documentation says that POLLHUP is incompatible 354 * with the POLLOUT/POLLWR flags, so somebody should check this 355 * all. But careful, it tends to be safer to return too many 356 * bits than too few, and you can easily break real applications 357 * if you don't tell them that something has hung up! 358 * 359 * Check-me. 360 * 361 * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and 362 * our fs/select.c). It means that after we received EOF, 363 * poll always returns immediately, making impossible poll() on write() 364 * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP 365 * if and only if shutdown has been made in both directions. 366 * Actually, it is interesting to look how Solaris and DUX 367 * solve this dilemma. I would prefer, if PULLHUP were maskable, 368 * then we could set it on SND_SHUTDOWN. BTW examples given 369 * in Stevens' books assume exactly this behaviour, it explains 370 * why PULLHUP is incompatible with POLLOUT. --ANK 371 * 372 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent 373 * blocking on fresh not-connected or disconnected socket. --ANK 374 */ 375 if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE) 376 mask |= POLLHUP; 377 if (sk->sk_shutdown & RCV_SHUTDOWN) 378 mask |= POLLIN | POLLRDNORM; 379 380 /* Connected? */ 381 if ((1 << sk->sk_state) & ~(TCPF_SYN_SENT | TCPF_SYN_RECV)) { 382 /* Potential race condition. If read of tp below will 383 * escape above sk->sk_state, we can be illegally awaken 384 * in SYN_* states. */ 385 if ((tp->rcv_nxt != tp->copied_seq) && 386 (tp->urg_seq != tp->copied_seq || 387 tp->rcv_nxt != tp->copied_seq + 1 || 388 sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data)) 389 mask |= POLLIN | POLLRDNORM; 390 391 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) { 392 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) { 393 mask |= POLLOUT | POLLWRNORM; 394 } else { /* send SIGIO later */ 395 set_bit(SOCK_ASYNC_NOSPACE, 396 &sk->sk_socket->flags); 397 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 398 399 /* Race breaker. If space is freed after 400 * wspace test but before the flags are set, 401 * IO signal will be lost. 402 */ 403 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) 404 mask |= POLLOUT | POLLWRNORM; 405 } 406 } 407 408 if (tp->urg_data & TCP_URG_VALID) 409 mask |= POLLPRI; 410 } 411 return mask; 412 } 413 414 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg) 415 { 416 struct tcp_sock *tp = tcp_sk(sk); 417 int answ; 418 419 switch (cmd) { 420 case SIOCINQ: 421 if (sk->sk_state == TCP_LISTEN) 422 return -EINVAL; 423 424 lock_sock(sk); 425 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 426 answ = 0; 427 else if (sock_flag(sk, SOCK_URGINLINE) || 428 !tp->urg_data || 429 before(tp->urg_seq, tp->copied_seq) || 430 !before(tp->urg_seq, tp->rcv_nxt)) { 431 answ = tp->rcv_nxt - tp->copied_seq; 432 433 /* Subtract 1, if FIN is in queue. */ 434 if (answ && !skb_queue_empty(&sk->sk_receive_queue)) 435 answ -= 436 ((struct sk_buff *)sk->sk_receive_queue.prev)->h.th->fin; 437 } else 438 answ = tp->urg_seq - tp->copied_seq; 439 release_sock(sk); 440 break; 441 case SIOCATMARK: 442 answ = tp->urg_data && tp->urg_seq == tp->copied_seq; 443 break; 444 case SIOCOUTQ: 445 if (sk->sk_state == TCP_LISTEN) 446 return -EINVAL; 447 448 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 449 answ = 0; 450 else 451 answ = tp->write_seq - tp->snd_una; 452 break; 453 default: 454 return -ENOIOCTLCMD; 455 }; 456 457 return put_user(answ, (int __user *)arg); 458 } 459 460 461 int tcp_listen_start(struct sock *sk) 462 { 463 struct inet_sock *inet = inet_sk(sk); 464 struct tcp_sock *tp = tcp_sk(sk); 465 int rc = reqsk_queue_alloc(&tp->accept_queue, TCP_SYNQ_HSIZE); 466 467 if (rc != 0) 468 return rc; 469 470 sk->sk_max_ack_backlog = 0; 471 sk->sk_ack_backlog = 0; 472 tcp_delack_init(tp); 473 474 /* There is race window here: we announce ourselves listening, 475 * but this transition is still not validated by get_port(). 476 * It is OK, because this socket enters to hash table only 477 * after validation is complete. 478 */ 479 sk->sk_state = TCP_LISTEN; 480 if (!sk->sk_prot->get_port(sk, inet->num)) { 481 inet->sport = htons(inet->num); 482 483 sk_dst_reset(sk); 484 sk->sk_prot->hash(sk); 485 486 return 0; 487 } 488 489 sk->sk_state = TCP_CLOSE; 490 reqsk_queue_destroy(&tp->accept_queue); 491 return -EADDRINUSE; 492 } 493 494 /* 495 * This routine closes sockets which have been at least partially 496 * opened, but not yet accepted. 497 */ 498 499 static void tcp_listen_stop (struct sock *sk) 500 { 501 struct tcp_sock *tp = tcp_sk(sk); 502 struct listen_sock *lopt; 503 struct request_sock *acc_req; 504 struct request_sock *req; 505 int i; 506 507 tcp_delete_keepalive_timer(sk); 508 509 /* make all the listen_opt local to us */ 510 lopt = reqsk_queue_yank_listen_sk(&tp->accept_queue); 511 acc_req = reqsk_queue_yank_acceptq(&tp->accept_queue); 512 513 if (lopt->qlen) { 514 for (i = 0; i < TCP_SYNQ_HSIZE; i++) { 515 while ((req = lopt->syn_table[i]) != NULL) { 516 lopt->syn_table[i] = req->dl_next; 517 lopt->qlen--; 518 reqsk_free(req); 519 520 /* Following specs, it would be better either to send FIN 521 * (and enter FIN-WAIT-1, it is normal close) 522 * or to send active reset (abort). 523 * Certainly, it is pretty dangerous while synflood, but it is 524 * bad justification for our negligence 8) 525 * To be honest, we are not able to make either 526 * of the variants now. --ANK 527 */ 528 } 529 } 530 } 531 BUG_TRAP(!lopt->qlen); 532 533 kfree(lopt); 534 535 while ((req = acc_req) != NULL) { 536 struct sock *child = req->sk; 537 538 acc_req = req->dl_next; 539 540 local_bh_disable(); 541 bh_lock_sock(child); 542 BUG_TRAP(!sock_owned_by_user(child)); 543 sock_hold(child); 544 545 tcp_disconnect(child, O_NONBLOCK); 546 547 sock_orphan(child); 548 549 atomic_inc(&tcp_orphan_count); 550 551 tcp_destroy_sock(child); 552 553 bh_unlock_sock(child); 554 local_bh_enable(); 555 sock_put(child); 556 557 sk_acceptq_removed(sk); 558 __reqsk_free(req); 559 } 560 BUG_TRAP(!sk->sk_ack_backlog); 561 } 562 563 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb) 564 { 565 TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH; 566 tp->pushed_seq = tp->write_seq; 567 } 568 569 static inline int forced_push(struct tcp_sock *tp) 570 { 571 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1)); 572 } 573 574 static inline void skb_entail(struct sock *sk, struct tcp_sock *tp, 575 struct sk_buff *skb) 576 { 577 skb->csum = 0; 578 TCP_SKB_CB(skb)->seq = tp->write_seq; 579 TCP_SKB_CB(skb)->end_seq = tp->write_seq; 580 TCP_SKB_CB(skb)->flags = TCPCB_FLAG_ACK; 581 TCP_SKB_CB(skb)->sacked = 0; 582 skb_header_release(skb); 583 __skb_queue_tail(&sk->sk_write_queue, skb); 584 sk_charge_skb(sk, skb); 585 if (!sk->sk_send_head) 586 sk->sk_send_head = skb; 587 else if (tp->nonagle&TCP_NAGLE_PUSH) 588 tp->nonagle &= ~TCP_NAGLE_PUSH; 589 } 590 591 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags, 592 struct sk_buff *skb) 593 { 594 if (flags & MSG_OOB) { 595 tp->urg_mode = 1; 596 tp->snd_up = tp->write_seq; 597 TCP_SKB_CB(skb)->sacked |= TCPCB_URG; 598 } 599 } 600 601 static inline void tcp_push(struct sock *sk, struct tcp_sock *tp, int flags, 602 int mss_now, int nonagle) 603 { 604 if (sk->sk_send_head) { 605 struct sk_buff *skb = sk->sk_write_queue.prev; 606 if (!(flags & MSG_MORE) || forced_push(tp)) 607 tcp_mark_push(tp, skb); 608 tcp_mark_urg(tp, flags, skb); 609 __tcp_push_pending_frames(sk, tp, mss_now, 610 (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle); 611 } 612 } 613 614 static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset, 615 size_t psize, int flags) 616 { 617 struct tcp_sock *tp = tcp_sk(sk); 618 int mss_now, size_goal; 619 int err; 620 ssize_t copied; 621 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 622 623 /* Wait for a connection to finish. */ 624 if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) 625 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0) 626 goto out_err; 627 628 clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags); 629 630 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); 631 size_goal = tp->xmit_size_goal; 632 copied = 0; 633 634 err = -EPIPE; 635 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 636 goto do_error; 637 638 while (psize > 0) { 639 struct sk_buff *skb = sk->sk_write_queue.prev; 640 struct page *page = pages[poffset / PAGE_SIZE]; 641 int copy, i, can_coalesce; 642 int offset = poffset % PAGE_SIZE; 643 int size = min_t(size_t, psize, PAGE_SIZE - offset); 644 645 if (!sk->sk_send_head || (copy = size_goal - skb->len) <= 0) { 646 new_segment: 647 if (!sk_stream_memory_free(sk)) 648 goto wait_for_sndbuf; 649 650 skb = sk_stream_alloc_pskb(sk, 0, 0, 651 sk->sk_allocation); 652 if (!skb) 653 goto wait_for_memory; 654 655 skb_entail(sk, tp, skb); 656 copy = size_goal; 657 } 658 659 if (copy > size) 660 copy = size; 661 662 i = skb_shinfo(skb)->nr_frags; 663 can_coalesce = skb_can_coalesce(skb, i, page, offset); 664 if (!can_coalesce && i >= MAX_SKB_FRAGS) { 665 tcp_mark_push(tp, skb); 666 goto new_segment; 667 } 668 if (sk->sk_forward_alloc < copy && 669 !sk_stream_mem_schedule(sk, copy, 0)) 670 goto wait_for_memory; 671 672 if (can_coalesce) { 673 skb_shinfo(skb)->frags[i - 1].size += copy; 674 } else { 675 get_page(page); 676 skb_fill_page_desc(skb, i, page, offset, copy); 677 } 678 679 skb->len += copy; 680 skb->data_len += copy; 681 skb->truesize += copy; 682 sk->sk_wmem_queued += copy; 683 sk->sk_forward_alloc -= copy; 684 skb->ip_summed = CHECKSUM_HW; 685 tp->write_seq += copy; 686 TCP_SKB_CB(skb)->end_seq += copy; 687 skb_shinfo(skb)->tso_segs = 0; 688 689 if (!copied) 690 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH; 691 692 copied += copy; 693 poffset += copy; 694 if (!(psize -= copy)) 695 goto out; 696 697 if (skb->len < mss_now || (flags & MSG_OOB)) 698 continue; 699 700 if (forced_push(tp)) { 701 tcp_mark_push(tp, skb); 702 __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH); 703 } else if (skb == sk->sk_send_head) 704 tcp_push_one(sk, mss_now); 705 continue; 706 707 wait_for_sndbuf: 708 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 709 wait_for_memory: 710 if (copied) 711 tcp_push(sk, tp, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH); 712 713 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0) 714 goto do_error; 715 716 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); 717 size_goal = tp->xmit_size_goal; 718 } 719 720 out: 721 if (copied) 722 tcp_push(sk, tp, flags, mss_now, tp->nonagle); 723 return copied; 724 725 do_error: 726 if (copied) 727 goto out; 728 out_err: 729 return sk_stream_error(sk, flags, err); 730 } 731 732 ssize_t tcp_sendpage(struct socket *sock, struct page *page, int offset, 733 size_t size, int flags) 734 { 735 ssize_t res; 736 struct sock *sk = sock->sk; 737 738 #define TCP_ZC_CSUM_FLAGS (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | NETIF_F_HW_CSUM) 739 740 if (!(sk->sk_route_caps & NETIF_F_SG) || 741 !(sk->sk_route_caps & TCP_ZC_CSUM_FLAGS)) 742 return sock_no_sendpage(sock, page, offset, size, flags); 743 744 #undef TCP_ZC_CSUM_FLAGS 745 746 lock_sock(sk); 747 TCP_CHECK_TIMER(sk); 748 res = do_tcp_sendpages(sk, &page, offset, size, flags); 749 TCP_CHECK_TIMER(sk); 750 release_sock(sk); 751 return res; 752 } 753 754 #define TCP_PAGE(sk) (sk->sk_sndmsg_page) 755 #define TCP_OFF(sk) (sk->sk_sndmsg_off) 756 757 static inline int select_size(struct sock *sk, struct tcp_sock *tp) 758 { 759 int tmp = tp->mss_cache; 760 761 if (sk->sk_route_caps & NETIF_F_SG) { 762 if (sk->sk_route_caps & NETIF_F_TSO) 763 tmp = 0; 764 else { 765 int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER); 766 767 if (tmp >= pgbreak && 768 tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE) 769 tmp = pgbreak; 770 } 771 } 772 773 return tmp; 774 } 775 776 int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, 777 size_t size) 778 { 779 struct iovec *iov; 780 struct tcp_sock *tp = tcp_sk(sk); 781 struct sk_buff *skb; 782 int iovlen, flags; 783 int mss_now, size_goal; 784 int err, copied; 785 long timeo; 786 787 lock_sock(sk); 788 TCP_CHECK_TIMER(sk); 789 790 flags = msg->msg_flags; 791 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); 792 793 /* Wait for a connection to finish. */ 794 if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) 795 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0) 796 goto out_err; 797 798 /* This should be in poll */ 799 clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags); 800 801 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); 802 size_goal = tp->xmit_size_goal; 803 804 /* Ok commence sending. */ 805 iovlen = msg->msg_iovlen; 806 iov = msg->msg_iov; 807 copied = 0; 808 809 err = -EPIPE; 810 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) 811 goto do_error; 812 813 while (--iovlen >= 0) { 814 int seglen = iov->iov_len; 815 unsigned char __user *from = iov->iov_base; 816 817 iov++; 818 819 while (seglen > 0) { 820 int copy; 821 822 skb = sk->sk_write_queue.prev; 823 824 if (!sk->sk_send_head || 825 (copy = size_goal - skb->len) <= 0) { 826 827 new_segment: 828 /* Allocate new segment. If the interface is SG, 829 * allocate skb fitting to single page. 830 */ 831 if (!sk_stream_memory_free(sk)) 832 goto wait_for_sndbuf; 833 834 skb = sk_stream_alloc_pskb(sk, select_size(sk, tp), 835 0, sk->sk_allocation); 836 if (!skb) 837 goto wait_for_memory; 838 839 /* 840 * Check whether we can use HW checksum. 841 */ 842 if (sk->sk_route_caps & 843 (NETIF_F_IP_CSUM | NETIF_F_NO_CSUM | 844 NETIF_F_HW_CSUM)) 845 skb->ip_summed = CHECKSUM_HW; 846 847 skb_entail(sk, tp, skb); 848 copy = size_goal; 849 } 850 851 /* Try to append data to the end of skb. */ 852 if (copy > seglen) 853 copy = seglen; 854 855 /* Where to copy to? */ 856 if (skb_tailroom(skb) > 0) { 857 /* We have some space in skb head. Superb! */ 858 if (copy > skb_tailroom(skb)) 859 copy = skb_tailroom(skb); 860 if ((err = skb_add_data(skb, from, copy)) != 0) 861 goto do_fault; 862 } else { 863 int merge = 0; 864 int i = skb_shinfo(skb)->nr_frags; 865 struct page *page = TCP_PAGE(sk); 866 int off = TCP_OFF(sk); 867 868 if (skb_can_coalesce(skb, i, page, off) && 869 off != PAGE_SIZE) { 870 /* We can extend the last page 871 * fragment. */ 872 merge = 1; 873 } else if (i == MAX_SKB_FRAGS || 874 (!i && 875 !(sk->sk_route_caps & NETIF_F_SG))) { 876 /* Need to add new fragment and cannot 877 * do this because interface is non-SG, 878 * or because all the page slots are 879 * busy. */ 880 tcp_mark_push(tp, skb); 881 goto new_segment; 882 } else if (page) { 883 if (off == PAGE_SIZE) { 884 put_page(page); 885 TCP_PAGE(sk) = page = NULL; 886 } 887 } 888 889 if (!page) { 890 /* Allocate new cache page. */ 891 if (!(page = sk_stream_alloc_page(sk))) 892 goto wait_for_memory; 893 off = 0; 894 } 895 896 if (copy > PAGE_SIZE - off) 897 copy = PAGE_SIZE - off; 898 899 /* Time to copy data. We are close to 900 * the end! */ 901 err = skb_copy_to_page(sk, from, skb, page, 902 off, copy); 903 if (err) { 904 /* If this page was new, give it to the 905 * socket so it does not get leaked. 906 */ 907 if (!TCP_PAGE(sk)) { 908 TCP_PAGE(sk) = page; 909 TCP_OFF(sk) = 0; 910 } 911 goto do_error; 912 } 913 914 /* Update the skb. */ 915 if (merge) { 916 skb_shinfo(skb)->frags[i - 1].size += 917 copy; 918 } else { 919 skb_fill_page_desc(skb, i, page, off, copy); 920 if (TCP_PAGE(sk)) { 921 get_page(page); 922 } else if (off + copy < PAGE_SIZE) { 923 get_page(page); 924 TCP_PAGE(sk) = page; 925 } 926 } 927 928 TCP_OFF(sk) = off + copy; 929 } 930 931 if (!copied) 932 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH; 933 934 tp->write_seq += copy; 935 TCP_SKB_CB(skb)->end_seq += copy; 936 skb_shinfo(skb)->tso_segs = 0; 937 938 from += copy; 939 copied += copy; 940 if ((seglen -= copy) == 0 && iovlen == 0) 941 goto out; 942 943 if (skb->len < mss_now || (flags & MSG_OOB)) 944 continue; 945 946 if (forced_push(tp)) { 947 tcp_mark_push(tp, skb); 948 __tcp_push_pending_frames(sk, tp, mss_now, TCP_NAGLE_PUSH); 949 } else if (skb == sk->sk_send_head) 950 tcp_push_one(sk, mss_now); 951 continue; 952 953 wait_for_sndbuf: 954 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 955 wait_for_memory: 956 if (copied) 957 tcp_push(sk, tp, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH); 958 959 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0) 960 goto do_error; 961 962 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); 963 size_goal = tp->xmit_size_goal; 964 } 965 } 966 967 out: 968 if (copied) 969 tcp_push(sk, tp, flags, mss_now, tp->nonagle); 970 TCP_CHECK_TIMER(sk); 971 release_sock(sk); 972 return copied; 973 974 do_fault: 975 if (!skb->len) { 976 if (sk->sk_send_head == skb) 977 sk->sk_send_head = NULL; 978 __skb_unlink(skb, skb->list); 979 sk_stream_free_skb(sk, skb); 980 } 981 982 do_error: 983 if (copied) 984 goto out; 985 out_err: 986 err = sk_stream_error(sk, flags, err); 987 TCP_CHECK_TIMER(sk); 988 release_sock(sk); 989 return err; 990 } 991 992 /* 993 * Handle reading urgent data. BSD has very simple semantics for 994 * this, no blocking and very strange errors 8) 995 */ 996 997 static int tcp_recv_urg(struct sock *sk, long timeo, 998 struct msghdr *msg, int len, int flags, 999 int *addr_len) 1000 { 1001 struct tcp_sock *tp = tcp_sk(sk); 1002 1003 /* No URG data to read. */ 1004 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data || 1005 tp->urg_data == TCP_URG_READ) 1006 return -EINVAL; /* Yes this is right ! */ 1007 1008 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE)) 1009 return -ENOTCONN; 1010 1011 if (tp->urg_data & TCP_URG_VALID) { 1012 int err = 0; 1013 char c = tp->urg_data; 1014 1015 if (!(flags & MSG_PEEK)) 1016 tp->urg_data = TCP_URG_READ; 1017 1018 /* Read urgent data. */ 1019 msg->msg_flags |= MSG_OOB; 1020 1021 if (len > 0) { 1022 if (!(flags & MSG_TRUNC)) 1023 err = memcpy_toiovec(msg->msg_iov, &c, 1); 1024 len = 1; 1025 } else 1026 msg->msg_flags |= MSG_TRUNC; 1027 1028 return err ? -EFAULT : len; 1029 } 1030 1031 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 1032 return 0; 1033 1034 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and 1035 * the available implementations agree in this case: 1036 * this call should never block, independent of the 1037 * blocking state of the socket. 1038 * Mike <pall@rz.uni-karlsruhe.de> 1039 */ 1040 return -EAGAIN; 1041 } 1042 1043 /* Clean up the receive buffer for full frames taken by the user, 1044 * then send an ACK if necessary. COPIED is the number of bytes 1045 * tcp_recvmsg has given to the user so far, it speeds up the 1046 * calculation of whether or not we must ACK for the sake of 1047 * a window update. 1048 */ 1049 static void cleanup_rbuf(struct sock *sk, int copied) 1050 { 1051 struct tcp_sock *tp = tcp_sk(sk); 1052 int time_to_ack = 0; 1053 1054 #if TCP_DEBUG 1055 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 1056 1057 BUG_TRAP(!skb || before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq)); 1058 #endif 1059 1060 if (tcp_ack_scheduled(tp)) { 1061 /* Delayed ACKs frequently hit locked sockets during bulk 1062 * receive. */ 1063 if (tp->ack.blocked || 1064 /* Once-per-two-segments ACK was not sent by tcp_input.c */ 1065 tp->rcv_nxt - tp->rcv_wup > tp->ack.rcv_mss || 1066 /* 1067 * If this read emptied read buffer, we send ACK, if 1068 * connection is not bidirectional, user drained 1069 * receive buffer and there was a small segment 1070 * in queue. 1071 */ 1072 (copied > 0 && (tp->ack.pending & TCP_ACK_PUSHED) && 1073 !tp->ack.pingpong && !atomic_read(&sk->sk_rmem_alloc))) 1074 time_to_ack = 1; 1075 } 1076 1077 /* We send an ACK if we can now advertise a non-zero window 1078 * which has been raised "significantly". 1079 * 1080 * Even if window raised up to infinity, do not send window open ACK 1081 * in states, where we will not receive more. It is useless. 1082 */ 1083 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 1084 __u32 rcv_window_now = tcp_receive_window(tp); 1085 1086 /* Optimize, __tcp_select_window() is not cheap. */ 1087 if (2*rcv_window_now <= tp->window_clamp) { 1088 __u32 new_window = __tcp_select_window(sk); 1089 1090 /* Send ACK now, if this read freed lots of space 1091 * in our buffer. Certainly, new_window is new window. 1092 * We can advertise it now, if it is not less than current one. 1093 * "Lots" means "at least twice" here. 1094 */ 1095 if (new_window && new_window >= 2 * rcv_window_now) 1096 time_to_ack = 1; 1097 } 1098 } 1099 if (time_to_ack) 1100 tcp_send_ack(sk); 1101 } 1102 1103 static void tcp_prequeue_process(struct sock *sk) 1104 { 1105 struct sk_buff *skb; 1106 struct tcp_sock *tp = tcp_sk(sk); 1107 1108 NET_INC_STATS_USER(LINUX_MIB_TCPPREQUEUED); 1109 1110 /* RX process wants to run with disabled BHs, though it is not 1111 * necessary */ 1112 local_bh_disable(); 1113 while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL) 1114 sk->sk_backlog_rcv(sk, skb); 1115 local_bh_enable(); 1116 1117 /* Clear memory counter. */ 1118 tp->ucopy.memory = 0; 1119 } 1120 1121 static inline struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) 1122 { 1123 struct sk_buff *skb; 1124 u32 offset; 1125 1126 skb_queue_walk(&sk->sk_receive_queue, skb) { 1127 offset = seq - TCP_SKB_CB(skb)->seq; 1128 if (skb->h.th->syn) 1129 offset--; 1130 if (offset < skb->len || skb->h.th->fin) { 1131 *off = offset; 1132 return skb; 1133 } 1134 } 1135 return NULL; 1136 } 1137 1138 /* 1139 * This routine provides an alternative to tcp_recvmsg() for routines 1140 * that would like to handle copying from skbuffs directly in 'sendfile' 1141 * fashion. 1142 * Note: 1143 * - It is assumed that the socket was locked by the caller. 1144 * - The routine does not block. 1145 * - At present, there is no support for reading OOB data 1146 * or for 'peeking' the socket using this routine 1147 * (although both would be easy to implement). 1148 */ 1149 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, 1150 sk_read_actor_t recv_actor) 1151 { 1152 struct sk_buff *skb; 1153 struct tcp_sock *tp = tcp_sk(sk); 1154 u32 seq = tp->copied_seq; 1155 u32 offset; 1156 int copied = 0; 1157 1158 if (sk->sk_state == TCP_LISTEN) 1159 return -ENOTCONN; 1160 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) { 1161 if (offset < skb->len) { 1162 size_t used, len; 1163 1164 len = skb->len - offset; 1165 /* Stop reading if we hit a patch of urgent data */ 1166 if (tp->urg_data) { 1167 u32 urg_offset = tp->urg_seq - seq; 1168 if (urg_offset < len) 1169 len = urg_offset; 1170 if (!len) 1171 break; 1172 } 1173 used = recv_actor(desc, skb, offset, len); 1174 if (used <= len) { 1175 seq += used; 1176 copied += used; 1177 offset += used; 1178 } 1179 if (offset != skb->len) 1180 break; 1181 } 1182 if (skb->h.th->fin) { 1183 sk_eat_skb(sk, skb); 1184 ++seq; 1185 break; 1186 } 1187 sk_eat_skb(sk, skb); 1188 if (!desc->count) 1189 break; 1190 } 1191 tp->copied_seq = seq; 1192 1193 tcp_rcv_space_adjust(sk); 1194 1195 /* Clean up data we have read: This will do ACK frames. */ 1196 if (copied) 1197 cleanup_rbuf(sk, copied); 1198 return copied; 1199 } 1200 1201 /* 1202 * This routine copies from a sock struct into the user buffer. 1203 * 1204 * Technical note: in 2.3 we work on _locked_ socket, so that 1205 * tricks with *seq access order and skb->users are not required. 1206 * Probably, code can be easily improved even more. 1207 */ 1208 1209 int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, 1210 size_t len, int nonblock, int flags, int *addr_len) 1211 { 1212 struct tcp_sock *tp = tcp_sk(sk); 1213 int copied = 0; 1214 u32 peek_seq; 1215 u32 *seq; 1216 unsigned long used; 1217 int err; 1218 int target; /* Read at least this many bytes */ 1219 long timeo; 1220 struct task_struct *user_recv = NULL; 1221 1222 lock_sock(sk); 1223 1224 TCP_CHECK_TIMER(sk); 1225 1226 err = -ENOTCONN; 1227 if (sk->sk_state == TCP_LISTEN) 1228 goto out; 1229 1230 timeo = sock_rcvtimeo(sk, nonblock); 1231 1232 /* Urgent data needs to be handled specially. */ 1233 if (flags & MSG_OOB) 1234 goto recv_urg; 1235 1236 seq = &tp->copied_seq; 1237 if (flags & MSG_PEEK) { 1238 peek_seq = tp->copied_seq; 1239 seq = &peek_seq; 1240 } 1241 1242 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 1243 1244 do { 1245 struct sk_buff *skb; 1246 u32 offset; 1247 1248 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */ 1249 if (tp->urg_data && tp->urg_seq == *seq) { 1250 if (copied) 1251 break; 1252 if (signal_pending(current)) { 1253 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN; 1254 break; 1255 } 1256 } 1257 1258 /* Next get a buffer. */ 1259 1260 skb = skb_peek(&sk->sk_receive_queue); 1261 do { 1262 if (!skb) 1263 break; 1264 1265 /* Now that we have two receive queues this 1266 * shouldn't happen. 1267 */ 1268 if (before(*seq, TCP_SKB_CB(skb)->seq)) { 1269 printk(KERN_INFO "recvmsg bug: copied %X " 1270 "seq %X\n", *seq, TCP_SKB_CB(skb)->seq); 1271 break; 1272 } 1273 offset = *seq - TCP_SKB_CB(skb)->seq; 1274 if (skb->h.th->syn) 1275 offset--; 1276 if (offset < skb->len) 1277 goto found_ok_skb; 1278 if (skb->h.th->fin) 1279 goto found_fin_ok; 1280 BUG_TRAP(flags & MSG_PEEK); 1281 skb = skb->next; 1282 } while (skb != (struct sk_buff *)&sk->sk_receive_queue); 1283 1284 /* Well, if we have backlog, try to process it now yet. */ 1285 1286 if (copied >= target && !sk->sk_backlog.tail) 1287 break; 1288 1289 if (copied) { 1290 if (sk->sk_err || 1291 sk->sk_state == TCP_CLOSE || 1292 (sk->sk_shutdown & RCV_SHUTDOWN) || 1293 !timeo || 1294 signal_pending(current) || 1295 (flags & MSG_PEEK)) 1296 break; 1297 } else { 1298 if (sock_flag(sk, SOCK_DONE)) 1299 break; 1300 1301 if (sk->sk_err) { 1302 copied = sock_error(sk); 1303 break; 1304 } 1305 1306 if (sk->sk_shutdown & RCV_SHUTDOWN) 1307 break; 1308 1309 if (sk->sk_state == TCP_CLOSE) { 1310 if (!sock_flag(sk, SOCK_DONE)) { 1311 /* This occurs when user tries to read 1312 * from never connected socket. 1313 */ 1314 copied = -ENOTCONN; 1315 break; 1316 } 1317 break; 1318 } 1319 1320 if (!timeo) { 1321 copied = -EAGAIN; 1322 break; 1323 } 1324 1325 if (signal_pending(current)) { 1326 copied = sock_intr_errno(timeo); 1327 break; 1328 } 1329 } 1330 1331 cleanup_rbuf(sk, copied); 1332 1333 if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) { 1334 /* Install new reader */ 1335 if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) { 1336 user_recv = current; 1337 tp->ucopy.task = user_recv; 1338 tp->ucopy.iov = msg->msg_iov; 1339 } 1340 1341 tp->ucopy.len = len; 1342 1343 BUG_TRAP(tp->copied_seq == tp->rcv_nxt || 1344 (flags & (MSG_PEEK | MSG_TRUNC))); 1345 1346 /* Ugly... If prequeue is not empty, we have to 1347 * process it before releasing socket, otherwise 1348 * order will be broken at second iteration. 1349 * More elegant solution is required!!! 1350 * 1351 * Look: we have the following (pseudo)queues: 1352 * 1353 * 1. packets in flight 1354 * 2. backlog 1355 * 3. prequeue 1356 * 4. receive_queue 1357 * 1358 * Each queue can be processed only if the next ones 1359 * are empty. At this point we have empty receive_queue. 1360 * But prequeue _can_ be not empty after 2nd iteration, 1361 * when we jumped to start of loop because backlog 1362 * processing added something to receive_queue. 1363 * We cannot release_sock(), because backlog contains 1364 * packets arrived _after_ prequeued ones. 1365 * 1366 * Shortly, algorithm is clear --- to process all 1367 * the queues in order. We could make it more directly, 1368 * requeueing packets from backlog to prequeue, if 1369 * is not empty. It is more elegant, but eats cycles, 1370 * unfortunately. 1371 */ 1372 if (!skb_queue_empty(&tp->ucopy.prequeue)) 1373 goto do_prequeue; 1374 1375 /* __ Set realtime policy in scheduler __ */ 1376 } 1377 1378 if (copied >= target) { 1379 /* Do not sleep, just process backlog. */ 1380 release_sock(sk); 1381 lock_sock(sk); 1382 } else 1383 sk_wait_data(sk, &timeo); 1384 1385 if (user_recv) { 1386 int chunk; 1387 1388 /* __ Restore normal policy in scheduler __ */ 1389 1390 if ((chunk = len - tp->ucopy.len) != 0) { 1391 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk); 1392 len -= chunk; 1393 copied += chunk; 1394 } 1395 1396 if (tp->rcv_nxt == tp->copied_seq && 1397 !skb_queue_empty(&tp->ucopy.prequeue)) { 1398 do_prequeue: 1399 tcp_prequeue_process(sk); 1400 1401 if ((chunk = len - tp->ucopy.len) != 0) { 1402 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk); 1403 len -= chunk; 1404 copied += chunk; 1405 } 1406 } 1407 } 1408 if ((flags & MSG_PEEK) && peek_seq != tp->copied_seq) { 1409 if (net_ratelimit()) 1410 printk(KERN_DEBUG "TCP(%s:%d): Application bug, race in MSG_PEEK.\n", 1411 current->comm, current->pid); 1412 peek_seq = tp->copied_seq; 1413 } 1414 continue; 1415 1416 found_ok_skb: 1417 /* Ok so how much can we use? */ 1418 used = skb->len - offset; 1419 if (len < used) 1420 used = len; 1421 1422 /* Do we have urgent data here? */ 1423 if (tp->urg_data) { 1424 u32 urg_offset = tp->urg_seq - *seq; 1425 if (urg_offset < used) { 1426 if (!urg_offset) { 1427 if (!sock_flag(sk, SOCK_URGINLINE)) { 1428 ++*seq; 1429 offset++; 1430 used--; 1431 if (!used) 1432 goto skip_copy; 1433 } 1434 } else 1435 used = urg_offset; 1436 } 1437 } 1438 1439 if (!(flags & MSG_TRUNC)) { 1440 err = skb_copy_datagram_iovec(skb, offset, 1441 msg->msg_iov, used); 1442 if (err) { 1443 /* Exception. Bailout! */ 1444 if (!copied) 1445 copied = -EFAULT; 1446 break; 1447 } 1448 } 1449 1450 *seq += used; 1451 copied += used; 1452 len -= used; 1453 1454 tcp_rcv_space_adjust(sk); 1455 1456 skip_copy: 1457 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) { 1458 tp->urg_data = 0; 1459 tcp_fast_path_check(sk, tp); 1460 } 1461 if (used + offset < skb->len) 1462 continue; 1463 1464 if (skb->h.th->fin) 1465 goto found_fin_ok; 1466 if (!(flags & MSG_PEEK)) 1467 sk_eat_skb(sk, skb); 1468 continue; 1469 1470 found_fin_ok: 1471 /* Process the FIN. */ 1472 ++*seq; 1473 if (!(flags & MSG_PEEK)) 1474 sk_eat_skb(sk, skb); 1475 break; 1476 } while (len > 0); 1477 1478 if (user_recv) { 1479 if (!skb_queue_empty(&tp->ucopy.prequeue)) { 1480 int chunk; 1481 1482 tp->ucopy.len = copied > 0 ? len : 0; 1483 1484 tcp_prequeue_process(sk); 1485 1486 if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) { 1487 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk); 1488 len -= chunk; 1489 copied += chunk; 1490 } 1491 } 1492 1493 tp->ucopy.task = NULL; 1494 tp->ucopy.len = 0; 1495 } 1496 1497 /* According to UNIX98, msg_name/msg_namelen are ignored 1498 * on connected socket. I was just happy when found this 8) --ANK 1499 */ 1500 1501 /* Clean up data we have read: This will do ACK frames. */ 1502 cleanup_rbuf(sk, copied); 1503 1504 TCP_CHECK_TIMER(sk); 1505 release_sock(sk); 1506 return copied; 1507 1508 out: 1509 TCP_CHECK_TIMER(sk); 1510 release_sock(sk); 1511 return err; 1512 1513 recv_urg: 1514 err = tcp_recv_urg(sk, timeo, msg, len, flags, addr_len); 1515 goto out; 1516 } 1517 1518 /* 1519 * State processing on a close. This implements the state shift for 1520 * sending our FIN frame. Note that we only send a FIN for some 1521 * states. A shutdown() may have already sent the FIN, or we may be 1522 * closed. 1523 */ 1524 1525 static unsigned char new_state[16] = { 1526 /* current state: new state: action: */ 1527 /* (Invalid) */ TCP_CLOSE, 1528 /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, 1529 /* TCP_SYN_SENT */ TCP_CLOSE, 1530 /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN, 1531 /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1, 1532 /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2, 1533 /* TCP_TIME_WAIT */ TCP_CLOSE, 1534 /* TCP_CLOSE */ TCP_CLOSE, 1535 /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN, 1536 /* TCP_LAST_ACK */ TCP_LAST_ACK, 1537 /* TCP_LISTEN */ TCP_CLOSE, 1538 /* TCP_CLOSING */ TCP_CLOSING, 1539 }; 1540 1541 static int tcp_close_state(struct sock *sk) 1542 { 1543 int next = (int)new_state[sk->sk_state]; 1544 int ns = next & TCP_STATE_MASK; 1545 1546 tcp_set_state(sk, ns); 1547 1548 return next & TCP_ACTION_FIN; 1549 } 1550 1551 /* 1552 * Shutdown the sending side of a connection. Much like close except 1553 * that we don't receive shut down or set_sock_flag(sk, SOCK_DEAD). 1554 */ 1555 1556 void tcp_shutdown(struct sock *sk, int how) 1557 { 1558 /* We need to grab some memory, and put together a FIN, 1559 * and then put it into the queue to be sent. 1560 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92. 1561 */ 1562 if (!(how & SEND_SHUTDOWN)) 1563 return; 1564 1565 /* If we've already sent a FIN, or it's a closed state, skip this. */ 1566 if ((1 << sk->sk_state) & 1567 (TCPF_ESTABLISHED | TCPF_SYN_SENT | 1568 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) { 1569 /* Clear out any half completed packets. FIN if needed. */ 1570 if (tcp_close_state(sk)) 1571 tcp_send_fin(sk); 1572 } 1573 } 1574 1575 /* 1576 * At this point, there should be no process reference to this 1577 * socket, and thus no user references at all. Therefore we 1578 * can assume the socket waitqueue is inactive and nobody will 1579 * try to jump onto it. 1580 */ 1581 void tcp_destroy_sock(struct sock *sk) 1582 { 1583 BUG_TRAP(sk->sk_state == TCP_CLOSE); 1584 BUG_TRAP(sock_flag(sk, SOCK_DEAD)); 1585 1586 /* It cannot be in hash table! */ 1587 BUG_TRAP(sk_unhashed(sk)); 1588 1589 /* If it has not 0 inet_sk(sk)->num, it must be bound */ 1590 BUG_TRAP(!inet_sk(sk)->num || tcp_sk(sk)->bind_hash); 1591 1592 sk->sk_prot->destroy(sk); 1593 1594 sk_stream_kill_queues(sk); 1595 1596 xfrm_sk_free_policy(sk); 1597 1598 #ifdef INET_REFCNT_DEBUG 1599 if (atomic_read(&sk->sk_refcnt) != 1) { 1600 printk(KERN_DEBUG "Destruction TCP %p delayed, c=%d\n", 1601 sk, atomic_read(&sk->sk_refcnt)); 1602 } 1603 #endif 1604 1605 atomic_dec(&tcp_orphan_count); 1606 sock_put(sk); 1607 } 1608 1609 void tcp_close(struct sock *sk, long timeout) 1610 { 1611 struct sk_buff *skb; 1612 int data_was_unread = 0; 1613 1614 lock_sock(sk); 1615 sk->sk_shutdown = SHUTDOWN_MASK; 1616 1617 if (sk->sk_state == TCP_LISTEN) { 1618 tcp_set_state(sk, TCP_CLOSE); 1619 1620 /* Special case. */ 1621 tcp_listen_stop(sk); 1622 1623 goto adjudge_to_death; 1624 } 1625 1626 /* We need to flush the recv. buffs. We do this only on the 1627 * descriptor close, not protocol-sourced closes, because the 1628 * reader process may not have drained the data yet! 1629 */ 1630 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { 1631 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq - 1632 skb->h.th->fin; 1633 data_was_unread += len; 1634 __kfree_skb(skb); 1635 } 1636 1637 sk_stream_mem_reclaim(sk); 1638 1639 /* As outlined in draft-ietf-tcpimpl-prob-03.txt, section 1640 * 3.10, we send a RST here because data was lost. To 1641 * witness the awful effects of the old behavior of always 1642 * doing a FIN, run an older 2.1.x kernel or 2.0.x, start 1643 * a bulk GET in an FTP client, suspend the process, wait 1644 * for the client to advertise a zero window, then kill -9 1645 * the FTP client, wheee... Note: timeout is always zero 1646 * in such a case. 1647 */ 1648 if (data_was_unread) { 1649 /* Unread data was tossed, zap the connection. */ 1650 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONCLOSE); 1651 tcp_set_state(sk, TCP_CLOSE); 1652 tcp_send_active_reset(sk, GFP_KERNEL); 1653 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { 1654 /* Check zero linger _after_ checking for unread data. */ 1655 sk->sk_prot->disconnect(sk, 0); 1656 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONDATA); 1657 } else if (tcp_close_state(sk)) { 1658 /* We FIN if the application ate all the data before 1659 * zapping the connection. 1660 */ 1661 1662 /* RED-PEN. Formally speaking, we have broken TCP state 1663 * machine. State transitions: 1664 * 1665 * TCP_ESTABLISHED -> TCP_FIN_WAIT1 1666 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible) 1667 * TCP_CLOSE_WAIT -> TCP_LAST_ACK 1668 * 1669 * are legal only when FIN has been sent (i.e. in window), 1670 * rather than queued out of window. Purists blame. 1671 * 1672 * F.e. "RFC state" is ESTABLISHED, 1673 * if Linux state is FIN-WAIT-1, but FIN is still not sent. 1674 * 1675 * The visible declinations are that sometimes 1676 * we enter time-wait state, when it is not required really 1677 * (harmless), do not send active resets, when they are 1678 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when 1679 * they look as CLOSING or LAST_ACK for Linux) 1680 * Probably, I missed some more holelets. 1681 * --ANK 1682 */ 1683 tcp_send_fin(sk); 1684 } 1685 1686 sk_stream_wait_close(sk, timeout); 1687 1688 adjudge_to_death: 1689 /* It is the last release_sock in its life. It will remove backlog. */ 1690 release_sock(sk); 1691 1692 1693 /* Now socket is owned by kernel and we acquire BH lock 1694 to finish close. No need to check for user refs. 1695 */ 1696 local_bh_disable(); 1697 bh_lock_sock(sk); 1698 BUG_TRAP(!sock_owned_by_user(sk)); 1699 1700 sock_hold(sk); 1701 sock_orphan(sk); 1702 1703 /* This is a (useful) BSD violating of the RFC. There is a 1704 * problem with TCP as specified in that the other end could 1705 * keep a socket open forever with no application left this end. 1706 * We use a 3 minute timeout (about the same as BSD) then kill 1707 * our end. If they send after that then tough - BUT: long enough 1708 * that we won't make the old 4*rto = almost no time - whoops 1709 * reset mistake. 1710 * 1711 * Nope, it was not mistake. It is really desired behaviour 1712 * f.e. on http servers, when such sockets are useless, but 1713 * consume significant resources. Let's do it with special 1714 * linger2 option. --ANK 1715 */ 1716 1717 if (sk->sk_state == TCP_FIN_WAIT2) { 1718 struct tcp_sock *tp = tcp_sk(sk); 1719 if (tp->linger2 < 0) { 1720 tcp_set_state(sk, TCP_CLOSE); 1721 tcp_send_active_reset(sk, GFP_ATOMIC); 1722 NET_INC_STATS_BH(LINUX_MIB_TCPABORTONLINGER); 1723 } else { 1724 int tmo = tcp_fin_time(tp); 1725 1726 if (tmo > TCP_TIMEWAIT_LEN) { 1727 tcp_reset_keepalive_timer(sk, tcp_fin_time(tp)); 1728 } else { 1729 atomic_inc(&tcp_orphan_count); 1730 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo); 1731 goto out; 1732 } 1733 } 1734 } 1735 if (sk->sk_state != TCP_CLOSE) { 1736 sk_stream_mem_reclaim(sk); 1737 if (atomic_read(&tcp_orphan_count) > sysctl_tcp_max_orphans || 1738 (sk->sk_wmem_queued > SOCK_MIN_SNDBUF && 1739 atomic_read(&tcp_memory_allocated) > sysctl_tcp_mem[2])) { 1740 if (net_ratelimit()) 1741 printk(KERN_INFO "TCP: too many of orphaned " 1742 "sockets\n"); 1743 tcp_set_state(sk, TCP_CLOSE); 1744 tcp_send_active_reset(sk, GFP_ATOMIC); 1745 NET_INC_STATS_BH(LINUX_MIB_TCPABORTONMEMORY); 1746 } 1747 } 1748 atomic_inc(&tcp_orphan_count); 1749 1750 if (sk->sk_state == TCP_CLOSE) 1751 tcp_destroy_sock(sk); 1752 /* Otherwise, socket is reprieved until protocol close. */ 1753 1754 out: 1755 bh_unlock_sock(sk); 1756 local_bh_enable(); 1757 sock_put(sk); 1758 } 1759 1760 /* These states need RST on ABORT according to RFC793 */ 1761 1762 static inline int tcp_need_reset(int state) 1763 { 1764 return (1 << state) & 1765 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 | 1766 TCPF_FIN_WAIT2 | TCPF_SYN_RECV); 1767 } 1768 1769 int tcp_disconnect(struct sock *sk, int flags) 1770 { 1771 struct inet_sock *inet = inet_sk(sk); 1772 struct tcp_sock *tp = tcp_sk(sk); 1773 int err = 0; 1774 int old_state = sk->sk_state; 1775 1776 if (old_state != TCP_CLOSE) 1777 tcp_set_state(sk, TCP_CLOSE); 1778 1779 /* ABORT function of RFC793 */ 1780 if (old_state == TCP_LISTEN) { 1781 tcp_listen_stop(sk); 1782 } else if (tcp_need_reset(old_state) || 1783 (tp->snd_nxt != tp->write_seq && 1784 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) { 1785 /* The last check adjusts for discrepance of Linux wrt. RFC 1786 * states 1787 */ 1788 tcp_send_active_reset(sk, gfp_any()); 1789 sk->sk_err = ECONNRESET; 1790 } else if (old_state == TCP_SYN_SENT) 1791 sk->sk_err = ECONNRESET; 1792 1793 tcp_clear_xmit_timers(sk); 1794 __skb_queue_purge(&sk->sk_receive_queue); 1795 sk_stream_writequeue_purge(sk); 1796 __skb_queue_purge(&tp->out_of_order_queue); 1797 1798 inet->dport = 0; 1799 1800 if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) 1801 inet_reset_saddr(sk); 1802 1803 sk->sk_shutdown = 0; 1804 sock_reset_flag(sk, SOCK_DONE); 1805 tp->srtt = 0; 1806 if ((tp->write_seq += tp->max_window + 2) == 0) 1807 tp->write_seq = 1; 1808 tp->backoff = 0; 1809 tp->snd_cwnd = 2; 1810 tp->probes_out = 0; 1811 tp->packets_out = 0; 1812 tp->snd_ssthresh = 0x7fffffff; 1813 tp->snd_cwnd_cnt = 0; 1814 tcp_set_ca_state(tp, TCP_CA_Open); 1815 tcp_clear_retrans(tp); 1816 tcp_delack_init(tp); 1817 sk->sk_send_head = NULL; 1818 tp->rx_opt.saw_tstamp = 0; 1819 tcp_sack_reset(&tp->rx_opt); 1820 __sk_dst_reset(sk); 1821 1822 BUG_TRAP(!inet->num || tp->bind_hash); 1823 1824 sk->sk_error_report(sk); 1825 return err; 1826 } 1827 1828 /* 1829 * Wait for an incoming connection, avoid race 1830 * conditions. This must be called with the socket locked. 1831 */ 1832 static int wait_for_connect(struct sock *sk, long timeo) 1833 { 1834 struct tcp_sock *tp = tcp_sk(sk); 1835 DEFINE_WAIT(wait); 1836 int err; 1837 1838 /* 1839 * True wake-one mechanism for incoming connections: only 1840 * one process gets woken up, not the 'whole herd'. 1841 * Since we do not 'race & poll' for established sockets 1842 * anymore, the common case will execute the loop only once. 1843 * 1844 * Subtle issue: "add_wait_queue_exclusive()" will be added 1845 * after any current non-exclusive waiters, and we know that 1846 * it will always _stay_ after any new non-exclusive waiters 1847 * because all non-exclusive waiters are added at the 1848 * beginning of the wait-queue. As such, it's ok to "drop" 1849 * our exclusiveness temporarily when we get woken up without 1850 * having to remove and re-insert us on the wait queue. 1851 */ 1852 for (;;) { 1853 prepare_to_wait_exclusive(sk->sk_sleep, &wait, 1854 TASK_INTERRUPTIBLE); 1855 release_sock(sk); 1856 if (reqsk_queue_empty(&tp->accept_queue)) 1857 timeo = schedule_timeout(timeo); 1858 lock_sock(sk); 1859 err = 0; 1860 if (!reqsk_queue_empty(&tp->accept_queue)) 1861 break; 1862 err = -EINVAL; 1863 if (sk->sk_state != TCP_LISTEN) 1864 break; 1865 err = sock_intr_errno(timeo); 1866 if (signal_pending(current)) 1867 break; 1868 err = -EAGAIN; 1869 if (!timeo) 1870 break; 1871 } 1872 finish_wait(sk->sk_sleep, &wait); 1873 return err; 1874 } 1875 1876 /* 1877 * This will accept the next outstanding connection. 1878 */ 1879 1880 struct sock *tcp_accept(struct sock *sk, int flags, int *err) 1881 { 1882 struct tcp_sock *tp = tcp_sk(sk); 1883 struct sock *newsk; 1884 int error; 1885 1886 lock_sock(sk); 1887 1888 /* We need to make sure that this socket is listening, 1889 * and that it has something pending. 1890 */ 1891 error = -EINVAL; 1892 if (sk->sk_state != TCP_LISTEN) 1893 goto out_err; 1894 1895 /* Find already established connection */ 1896 if (reqsk_queue_empty(&tp->accept_queue)) { 1897 long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK); 1898 1899 /* If this is a non blocking socket don't sleep */ 1900 error = -EAGAIN; 1901 if (!timeo) 1902 goto out_err; 1903 1904 error = wait_for_connect(sk, timeo); 1905 if (error) 1906 goto out_err; 1907 } 1908 1909 newsk = reqsk_queue_get_child(&tp->accept_queue, sk); 1910 BUG_TRAP(newsk->sk_state != TCP_SYN_RECV); 1911 out: 1912 release_sock(sk); 1913 return newsk; 1914 out_err: 1915 newsk = NULL; 1916 *err = error; 1917 goto out; 1918 } 1919 1920 /* 1921 * Socket option code for TCP. 1922 */ 1923 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval, 1924 int optlen) 1925 { 1926 struct tcp_sock *tp = tcp_sk(sk); 1927 int val; 1928 int err = 0; 1929 1930 if (level != SOL_TCP) 1931 return tp->af_specific->setsockopt(sk, level, optname, 1932 optval, optlen); 1933 1934 /* This is a string value all the others are int's */ 1935 if (optname == TCP_CONGESTION) { 1936 char name[TCP_CA_NAME_MAX]; 1937 1938 if (optlen < 1) 1939 return -EINVAL; 1940 1941 val = strncpy_from_user(name, optval, 1942 min(TCP_CA_NAME_MAX-1, optlen)); 1943 if (val < 0) 1944 return -EFAULT; 1945 name[val] = 0; 1946 1947 lock_sock(sk); 1948 err = tcp_set_congestion_control(tp, name); 1949 release_sock(sk); 1950 return err; 1951 } 1952 1953 if (optlen < sizeof(int)) 1954 return -EINVAL; 1955 1956 if (get_user(val, (int __user *)optval)) 1957 return -EFAULT; 1958 1959 lock_sock(sk); 1960 1961 switch (optname) { 1962 case TCP_MAXSEG: 1963 /* Values greater than interface MTU won't take effect. However 1964 * at the point when this call is done we typically don't yet 1965 * know which interface is going to be used */ 1966 if (val < 8 || val > MAX_TCP_WINDOW) { 1967 err = -EINVAL; 1968 break; 1969 } 1970 tp->rx_opt.user_mss = val; 1971 break; 1972 1973 case TCP_NODELAY: 1974 if (val) { 1975 /* TCP_NODELAY is weaker than TCP_CORK, so that 1976 * this option on corked socket is remembered, but 1977 * it is not activated until cork is cleared. 1978 * 1979 * However, when TCP_NODELAY is set we make 1980 * an explicit push, which overrides even TCP_CORK 1981 * for currently queued segments. 1982 */ 1983 tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; 1984 tcp_push_pending_frames(sk, tp); 1985 } else { 1986 tp->nonagle &= ~TCP_NAGLE_OFF; 1987 } 1988 break; 1989 1990 case TCP_CORK: 1991 /* When set indicates to always queue non-full frames. 1992 * Later the user clears this option and we transmit 1993 * any pending partial frames in the queue. This is 1994 * meant to be used alongside sendfile() to get properly 1995 * filled frames when the user (for example) must write 1996 * out headers with a write() call first and then use 1997 * sendfile to send out the data parts. 1998 * 1999 * TCP_CORK can be set together with TCP_NODELAY and it is 2000 * stronger than TCP_NODELAY. 2001 */ 2002 if (val) { 2003 tp->nonagle |= TCP_NAGLE_CORK; 2004 } else { 2005 tp->nonagle &= ~TCP_NAGLE_CORK; 2006 if (tp->nonagle&TCP_NAGLE_OFF) 2007 tp->nonagle |= TCP_NAGLE_PUSH; 2008 tcp_push_pending_frames(sk, tp); 2009 } 2010 break; 2011 2012 case TCP_KEEPIDLE: 2013 if (val < 1 || val > MAX_TCP_KEEPIDLE) 2014 err = -EINVAL; 2015 else { 2016 tp->keepalive_time = val * HZ; 2017 if (sock_flag(sk, SOCK_KEEPOPEN) && 2018 !((1 << sk->sk_state) & 2019 (TCPF_CLOSE | TCPF_LISTEN))) { 2020 __u32 elapsed = tcp_time_stamp - tp->rcv_tstamp; 2021 if (tp->keepalive_time > elapsed) 2022 elapsed = tp->keepalive_time - elapsed; 2023 else 2024 elapsed = 0; 2025 tcp_reset_keepalive_timer(sk, elapsed); 2026 } 2027 } 2028 break; 2029 case TCP_KEEPINTVL: 2030 if (val < 1 || val > MAX_TCP_KEEPINTVL) 2031 err = -EINVAL; 2032 else 2033 tp->keepalive_intvl = val * HZ; 2034 break; 2035 case TCP_KEEPCNT: 2036 if (val < 1 || val > MAX_TCP_KEEPCNT) 2037 err = -EINVAL; 2038 else 2039 tp->keepalive_probes = val; 2040 break; 2041 case TCP_SYNCNT: 2042 if (val < 1 || val > MAX_TCP_SYNCNT) 2043 err = -EINVAL; 2044 else 2045 tp->syn_retries = val; 2046 break; 2047 2048 case TCP_LINGER2: 2049 if (val < 0) 2050 tp->linger2 = -1; 2051 else if (val > sysctl_tcp_fin_timeout / HZ) 2052 tp->linger2 = 0; 2053 else 2054 tp->linger2 = val * HZ; 2055 break; 2056 2057 case TCP_DEFER_ACCEPT: 2058 tp->defer_accept = 0; 2059 if (val > 0) { 2060 /* Translate value in seconds to number of 2061 * retransmits */ 2062 while (tp->defer_accept < 32 && 2063 val > ((TCP_TIMEOUT_INIT / HZ) << 2064 tp->defer_accept)) 2065 tp->defer_accept++; 2066 tp->defer_accept++; 2067 } 2068 break; 2069 2070 case TCP_WINDOW_CLAMP: 2071 if (!val) { 2072 if (sk->sk_state != TCP_CLOSE) { 2073 err = -EINVAL; 2074 break; 2075 } 2076 tp->window_clamp = 0; 2077 } else 2078 tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ? 2079 SOCK_MIN_RCVBUF / 2 : val; 2080 break; 2081 2082 case TCP_QUICKACK: 2083 if (!val) { 2084 tp->ack.pingpong = 1; 2085 } else { 2086 tp->ack.pingpong = 0; 2087 if ((1 << sk->sk_state) & 2088 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) && 2089 tcp_ack_scheduled(tp)) { 2090 tp->ack.pending |= TCP_ACK_PUSHED; 2091 cleanup_rbuf(sk, 1); 2092 if (!(val & 1)) 2093 tp->ack.pingpong = 1; 2094 } 2095 } 2096 break; 2097 2098 default: 2099 err = -ENOPROTOOPT; 2100 break; 2101 }; 2102 release_sock(sk); 2103 return err; 2104 } 2105 2106 /* Return information about state of tcp endpoint in API format. */ 2107 void tcp_get_info(struct sock *sk, struct tcp_info *info) 2108 { 2109 struct tcp_sock *tp = tcp_sk(sk); 2110 u32 now = tcp_time_stamp; 2111 2112 memset(info, 0, sizeof(*info)); 2113 2114 info->tcpi_state = sk->sk_state; 2115 info->tcpi_ca_state = tp->ca_state; 2116 info->tcpi_retransmits = tp->retransmits; 2117 info->tcpi_probes = tp->probes_out; 2118 info->tcpi_backoff = tp->backoff; 2119 2120 if (tp->rx_opt.tstamp_ok) 2121 info->tcpi_options |= TCPI_OPT_TIMESTAMPS; 2122 if (tp->rx_opt.sack_ok) 2123 info->tcpi_options |= TCPI_OPT_SACK; 2124 if (tp->rx_opt.wscale_ok) { 2125 info->tcpi_options |= TCPI_OPT_WSCALE; 2126 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale; 2127 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale; 2128 } 2129 2130 if (tp->ecn_flags&TCP_ECN_OK) 2131 info->tcpi_options |= TCPI_OPT_ECN; 2132 2133 info->tcpi_rto = jiffies_to_usecs(tp->rto); 2134 info->tcpi_ato = jiffies_to_usecs(tp->ack.ato); 2135 info->tcpi_snd_mss = tp->mss_cache; 2136 info->tcpi_rcv_mss = tp->ack.rcv_mss; 2137 2138 info->tcpi_unacked = tp->packets_out; 2139 info->tcpi_sacked = tp->sacked_out; 2140 info->tcpi_lost = tp->lost_out; 2141 info->tcpi_retrans = tp->retrans_out; 2142 info->tcpi_fackets = tp->fackets_out; 2143 2144 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime); 2145 info->tcpi_last_data_recv = jiffies_to_msecs(now - tp->ack.lrcvtime); 2146 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp); 2147 2148 info->tcpi_pmtu = tp->pmtu_cookie; 2149 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh; 2150 info->tcpi_rtt = jiffies_to_usecs(tp->srtt)>>3; 2151 info->tcpi_rttvar = jiffies_to_usecs(tp->mdev)>>2; 2152 info->tcpi_snd_ssthresh = tp->snd_ssthresh; 2153 info->tcpi_snd_cwnd = tp->snd_cwnd; 2154 info->tcpi_advmss = tp->advmss; 2155 info->tcpi_reordering = tp->reordering; 2156 2157 info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3; 2158 info->tcpi_rcv_space = tp->rcvq_space.space; 2159 2160 info->tcpi_total_retrans = tp->total_retrans; 2161 } 2162 2163 EXPORT_SYMBOL_GPL(tcp_get_info); 2164 2165 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, 2166 int __user *optlen) 2167 { 2168 struct tcp_sock *tp = tcp_sk(sk); 2169 int val, len; 2170 2171 if (level != SOL_TCP) 2172 return tp->af_specific->getsockopt(sk, level, optname, 2173 optval, optlen); 2174 2175 if (get_user(len, optlen)) 2176 return -EFAULT; 2177 2178 len = min_t(unsigned int, len, sizeof(int)); 2179 2180 if (len < 0) 2181 return -EINVAL; 2182 2183 switch (optname) { 2184 case TCP_MAXSEG: 2185 val = tp->mss_cache; 2186 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 2187 val = tp->rx_opt.user_mss; 2188 break; 2189 case TCP_NODELAY: 2190 val = !!(tp->nonagle&TCP_NAGLE_OFF); 2191 break; 2192 case TCP_CORK: 2193 val = !!(tp->nonagle&TCP_NAGLE_CORK); 2194 break; 2195 case TCP_KEEPIDLE: 2196 val = (tp->keepalive_time ? : sysctl_tcp_keepalive_time) / HZ; 2197 break; 2198 case TCP_KEEPINTVL: 2199 val = (tp->keepalive_intvl ? : sysctl_tcp_keepalive_intvl) / HZ; 2200 break; 2201 case TCP_KEEPCNT: 2202 val = tp->keepalive_probes ? : sysctl_tcp_keepalive_probes; 2203 break; 2204 case TCP_SYNCNT: 2205 val = tp->syn_retries ? : sysctl_tcp_syn_retries; 2206 break; 2207 case TCP_LINGER2: 2208 val = tp->linger2; 2209 if (val >= 0) 2210 val = (val ? : sysctl_tcp_fin_timeout) / HZ; 2211 break; 2212 case TCP_DEFER_ACCEPT: 2213 val = !tp->defer_accept ? 0 : ((TCP_TIMEOUT_INIT / HZ) << 2214 (tp->defer_accept - 1)); 2215 break; 2216 case TCP_WINDOW_CLAMP: 2217 val = tp->window_clamp; 2218 break; 2219 case TCP_INFO: { 2220 struct tcp_info info; 2221 2222 if (get_user(len, optlen)) 2223 return -EFAULT; 2224 2225 tcp_get_info(sk, &info); 2226 2227 len = min_t(unsigned int, len, sizeof(info)); 2228 if (put_user(len, optlen)) 2229 return -EFAULT; 2230 if (copy_to_user(optval, &info, len)) 2231 return -EFAULT; 2232 return 0; 2233 } 2234 case TCP_QUICKACK: 2235 val = !tp->ack.pingpong; 2236 break; 2237 2238 case TCP_CONGESTION: 2239 if (get_user(len, optlen)) 2240 return -EFAULT; 2241 len = min_t(unsigned int, len, TCP_CA_NAME_MAX); 2242 if (put_user(len, optlen)) 2243 return -EFAULT; 2244 if (copy_to_user(optval, tp->ca_ops->name, len)) 2245 return -EFAULT; 2246 return 0; 2247 default: 2248 return -ENOPROTOOPT; 2249 }; 2250 2251 if (put_user(len, optlen)) 2252 return -EFAULT; 2253 if (copy_to_user(optval, &val, len)) 2254 return -EFAULT; 2255 return 0; 2256 } 2257 2258 2259 extern void __skb_cb_too_small_for_tcp(int, int); 2260 extern struct tcp_congestion_ops tcp_reno; 2261 2262 static __initdata unsigned long thash_entries; 2263 static int __init set_thash_entries(char *str) 2264 { 2265 if (!str) 2266 return 0; 2267 thash_entries = simple_strtoul(str, &str, 0); 2268 return 1; 2269 } 2270 __setup("thash_entries=", set_thash_entries); 2271 2272 void __init tcp_init(void) 2273 { 2274 struct sk_buff *skb = NULL; 2275 int order, i; 2276 2277 if (sizeof(struct tcp_skb_cb) > sizeof(skb->cb)) 2278 __skb_cb_too_small_for_tcp(sizeof(struct tcp_skb_cb), 2279 sizeof(skb->cb)); 2280 2281 tcp_bucket_cachep = kmem_cache_create("tcp_bind_bucket", 2282 sizeof(struct tcp_bind_bucket), 2283 0, SLAB_HWCACHE_ALIGN, 2284 NULL, NULL); 2285 if (!tcp_bucket_cachep) 2286 panic("tcp_init: Cannot alloc tcp_bind_bucket cache."); 2287 2288 tcp_timewait_cachep = kmem_cache_create("tcp_tw_bucket", 2289 sizeof(struct tcp_tw_bucket), 2290 0, SLAB_HWCACHE_ALIGN, 2291 NULL, NULL); 2292 if (!tcp_timewait_cachep) 2293 panic("tcp_init: Cannot alloc tcp_tw_bucket cache."); 2294 2295 /* Size and allocate the main established and bind bucket 2296 * hash tables. 2297 * 2298 * The methodology is similar to that of the buffer cache. 2299 */ 2300 tcp_ehash = (struct tcp_ehash_bucket *) 2301 alloc_large_system_hash("TCP established", 2302 sizeof(struct tcp_ehash_bucket), 2303 thash_entries, 2304 (num_physpages >= 128 * 1024) ? 2305 (25 - PAGE_SHIFT) : 2306 (27 - PAGE_SHIFT), 2307 HASH_HIGHMEM, 2308 &tcp_ehash_size, 2309 NULL, 2310 0); 2311 tcp_ehash_size = (1 << tcp_ehash_size) >> 1; 2312 for (i = 0; i < (tcp_ehash_size << 1); i++) { 2313 rwlock_init(&tcp_ehash[i].lock); 2314 INIT_HLIST_HEAD(&tcp_ehash[i].chain); 2315 } 2316 2317 tcp_bhash = (struct tcp_bind_hashbucket *) 2318 alloc_large_system_hash("TCP bind", 2319 sizeof(struct tcp_bind_hashbucket), 2320 tcp_ehash_size, 2321 (num_physpages >= 128 * 1024) ? 2322 (25 - PAGE_SHIFT) : 2323 (27 - PAGE_SHIFT), 2324 HASH_HIGHMEM, 2325 &tcp_bhash_size, 2326 NULL, 2327 64 * 1024); 2328 tcp_bhash_size = 1 << tcp_bhash_size; 2329 for (i = 0; i < tcp_bhash_size; i++) { 2330 spin_lock_init(&tcp_bhash[i].lock); 2331 INIT_HLIST_HEAD(&tcp_bhash[i].chain); 2332 } 2333 2334 /* Try to be a bit smarter and adjust defaults depending 2335 * on available memory. 2336 */ 2337 for (order = 0; ((1 << order) << PAGE_SHIFT) < 2338 (tcp_bhash_size * sizeof(struct tcp_bind_hashbucket)); 2339 order++) 2340 ; 2341 if (order >= 4) { 2342 sysctl_local_port_range[0] = 32768; 2343 sysctl_local_port_range[1] = 61000; 2344 sysctl_tcp_max_tw_buckets = 180000; 2345 sysctl_tcp_max_orphans = 4096 << (order - 4); 2346 sysctl_max_syn_backlog = 1024; 2347 } else if (order < 3) { 2348 sysctl_local_port_range[0] = 1024 * (3 - order); 2349 sysctl_tcp_max_tw_buckets >>= (3 - order); 2350 sysctl_tcp_max_orphans >>= (3 - order); 2351 sysctl_max_syn_backlog = 128; 2352 } 2353 tcp_port_rover = sysctl_local_port_range[0] - 1; 2354 2355 sysctl_tcp_mem[0] = 768 << order; 2356 sysctl_tcp_mem[1] = 1024 << order; 2357 sysctl_tcp_mem[2] = 1536 << order; 2358 2359 if (order < 3) { 2360 sysctl_tcp_wmem[2] = 64 * 1024; 2361 sysctl_tcp_rmem[0] = PAGE_SIZE; 2362 sysctl_tcp_rmem[1] = 43689; 2363 sysctl_tcp_rmem[2] = 2 * 43689; 2364 } 2365 2366 printk(KERN_INFO "TCP: Hash tables configured " 2367 "(established %d bind %d)\n", 2368 tcp_ehash_size << 1, tcp_bhash_size); 2369 2370 tcp_register_congestion_control(&tcp_reno); 2371 } 2372 2373 EXPORT_SYMBOL(tcp_accept); 2374 EXPORT_SYMBOL(tcp_close); 2375 EXPORT_SYMBOL(tcp_destroy_sock); 2376 EXPORT_SYMBOL(tcp_disconnect); 2377 EXPORT_SYMBOL(tcp_getsockopt); 2378 EXPORT_SYMBOL(tcp_ioctl); 2379 EXPORT_SYMBOL(tcp_poll); 2380 EXPORT_SYMBOL(tcp_read_sock); 2381 EXPORT_SYMBOL(tcp_recvmsg); 2382 EXPORT_SYMBOL(tcp_sendmsg); 2383 EXPORT_SYMBOL(tcp_sendpage); 2384 EXPORT_SYMBOL(tcp_setsockopt); 2385 EXPORT_SYMBOL(tcp_shutdown); 2386 EXPORT_SYMBOL(tcp_statistics); 2387 EXPORT_SYMBOL(tcp_timewait_cachep); 2388