1 /* 2 * linux/net/sunrpc/svcsock.c 3 * 4 * These are the RPC server socket internals. 5 * 6 * The server scheduling algorithm does not always distribute the load 7 * evenly when servicing a single client. May need to modify the 8 * svc_sock_enqueue procedure... 9 * 10 * TCP support is largely untested and may be a little slow. The problem 11 * is that we currently do two separate recvfrom's, one for the 4-byte 12 * record length, and the second for the actual record. This could possibly 13 * be improved by always reading a minimum size of around 100 bytes and 14 * tucking any superfluous bytes away in a temporary store. Still, that 15 * leaves write requests out in the rain. An alternative may be to peek at 16 * the first skb in the queue, and if it matches the next TCP sequence 17 * number, to extract the record marker. Yuck. 18 * 19 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de> 20 */ 21 22 #include <linux/sched.h> 23 #include <linux/errno.h> 24 #include <linux/fcntl.h> 25 #include <linux/net.h> 26 #include <linux/in.h> 27 #include <linux/inet.h> 28 #include <linux/udp.h> 29 #include <linux/tcp.h> 30 #include <linux/unistd.h> 31 #include <linux/slab.h> 32 #include <linux/netdevice.h> 33 #include <linux/skbuff.h> 34 #include <linux/file.h> 35 #include <linux/freezer.h> 36 #include <net/sock.h> 37 #include <net/checksum.h> 38 #include <net/ip.h> 39 #include <net/ipv6.h> 40 #include <net/tcp_states.h> 41 #include <asm/uaccess.h> 42 #include <asm/ioctls.h> 43 44 #include <linux/sunrpc/types.h> 45 #include <linux/sunrpc/clnt.h> 46 #include <linux/sunrpc/xdr.h> 47 #include <linux/sunrpc/svcsock.h> 48 #include <linux/sunrpc/stats.h> 49 50 /* SMP locking strategy: 51 * 52 * svc_pool->sp_lock protects most of the fields of that pool. 53 * svc_serv->sv_lock protects sv_tempsocks, sv_permsocks, sv_tmpcnt. 54 * when both need to be taken (rare), svc_serv->sv_lock is first. 55 * BKL protects svc_serv->sv_nrthread. 56 * svc_sock->sk_defer_lock protects the svc_sock->sk_deferred list 57 * svc_sock->sk_flags.SK_BUSY prevents a svc_sock being enqueued multiply. 58 * 59 * Some flags can be set to certain values at any time 60 * providing that certain rules are followed: 61 * 62 * SK_CONN, SK_DATA, can be set or cleared at any time. 63 * after a set, svc_sock_enqueue must be called. 64 * after a clear, the socket must be read/accepted 65 * if this succeeds, it must be set again. 66 * SK_CLOSE can set at any time. It is never cleared. 67 * sk_inuse contains a bias of '1' until SK_DEAD is set. 68 * so when sk_inuse hits zero, we know the socket is dead 69 * and no-one is using it. 70 * SK_DEAD can only be set while SK_BUSY is held which ensures 71 * no other thread will be using the socket or will try to 72 * set SK_DEAD. 73 * 74 */ 75 76 #define RPCDBG_FACILITY RPCDBG_SVCSOCK 77 78 79 static struct svc_sock *svc_setup_socket(struct svc_serv *, struct socket *, 80 int *errp, int flags); 81 static void svc_delete_socket(struct svc_sock *svsk); 82 static void svc_udp_data_ready(struct sock *, int); 83 static int svc_udp_recvfrom(struct svc_rqst *); 84 static int svc_udp_sendto(struct svc_rqst *); 85 static void svc_close_socket(struct svc_sock *svsk); 86 87 static struct svc_deferred_req *svc_deferred_dequeue(struct svc_sock *svsk); 88 static int svc_deferred_recv(struct svc_rqst *rqstp); 89 static struct cache_deferred_req *svc_defer(struct cache_req *req); 90 91 /* apparently the "standard" is that clients close 92 * idle connections after 5 minutes, servers after 93 * 6 minutes 94 * http://www.connectathon.org/talks96/nfstcp.pdf 95 */ 96 static int svc_conn_age_period = 6*60; 97 98 #ifdef CONFIG_DEBUG_LOCK_ALLOC 99 static struct lock_class_key svc_key[2]; 100 static struct lock_class_key svc_slock_key[2]; 101 102 static inline void svc_reclassify_socket(struct socket *sock) 103 { 104 struct sock *sk = sock->sk; 105 BUG_ON(sk->sk_lock.owner != NULL); 106 switch (sk->sk_family) { 107 case AF_INET: 108 sock_lock_init_class_and_name(sk, "slock-AF_INET-NFSD", 109 &svc_slock_key[0], "sk_lock-AF_INET-NFSD", &svc_key[0]); 110 break; 111 112 case AF_INET6: 113 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NFSD", 114 &svc_slock_key[1], "sk_lock-AF_INET6-NFSD", &svc_key[1]); 115 break; 116 117 default: 118 BUG(); 119 } 120 } 121 #else 122 static inline void svc_reclassify_socket(struct socket *sock) 123 { 124 } 125 #endif 126 127 static char *__svc_print_addr(struct sockaddr *addr, char *buf, size_t len) 128 { 129 switch (addr->sa_family) { 130 case AF_INET: 131 snprintf(buf, len, "%u.%u.%u.%u, port=%u", 132 NIPQUAD(((struct sockaddr_in *) addr)->sin_addr), 133 htons(((struct sockaddr_in *) addr)->sin_port)); 134 break; 135 136 case AF_INET6: 137 snprintf(buf, len, "%x:%x:%x:%x:%x:%x:%x:%x, port=%u", 138 NIP6(((struct sockaddr_in6 *) addr)->sin6_addr), 139 htons(((struct sockaddr_in6 *) addr)->sin6_port)); 140 break; 141 142 default: 143 snprintf(buf, len, "unknown address type: %d", addr->sa_family); 144 break; 145 } 146 return buf; 147 } 148 149 /** 150 * svc_print_addr - Format rq_addr field for printing 151 * @rqstp: svc_rqst struct containing address to print 152 * @buf: target buffer for formatted address 153 * @len: length of target buffer 154 * 155 */ 156 char *svc_print_addr(struct svc_rqst *rqstp, char *buf, size_t len) 157 { 158 return __svc_print_addr(svc_addr(rqstp), buf, len); 159 } 160 EXPORT_SYMBOL_GPL(svc_print_addr); 161 162 /* 163 * Queue up an idle server thread. Must have pool->sp_lock held. 164 * Note: this is really a stack rather than a queue, so that we only 165 * use as many different threads as we need, and the rest don't pollute 166 * the cache. 167 */ 168 static inline void 169 svc_thread_enqueue(struct svc_pool *pool, struct svc_rqst *rqstp) 170 { 171 list_add(&rqstp->rq_list, &pool->sp_threads); 172 } 173 174 /* 175 * Dequeue an nfsd thread. Must have pool->sp_lock held. 176 */ 177 static inline void 178 svc_thread_dequeue(struct svc_pool *pool, struct svc_rqst *rqstp) 179 { 180 list_del(&rqstp->rq_list); 181 } 182 183 /* 184 * Release an skbuff after use 185 */ 186 static inline void 187 svc_release_skb(struct svc_rqst *rqstp) 188 { 189 struct sk_buff *skb = rqstp->rq_skbuff; 190 struct svc_deferred_req *dr = rqstp->rq_deferred; 191 192 if (skb) { 193 rqstp->rq_skbuff = NULL; 194 195 dprintk("svc: service %p, releasing skb %p\n", rqstp, skb); 196 skb_free_datagram(rqstp->rq_sock->sk_sk, skb); 197 } 198 if (dr) { 199 rqstp->rq_deferred = NULL; 200 kfree(dr); 201 } 202 } 203 204 /* 205 * Any space to write? 206 */ 207 static inline unsigned long 208 svc_sock_wspace(struct svc_sock *svsk) 209 { 210 int wspace; 211 212 if (svsk->sk_sock->type == SOCK_STREAM) 213 wspace = sk_stream_wspace(svsk->sk_sk); 214 else 215 wspace = sock_wspace(svsk->sk_sk); 216 217 return wspace; 218 } 219 220 /* 221 * Queue up a socket with data pending. If there are idle nfsd 222 * processes, wake 'em up. 223 * 224 */ 225 static void 226 svc_sock_enqueue(struct svc_sock *svsk) 227 { 228 struct svc_serv *serv = svsk->sk_server; 229 struct svc_pool *pool; 230 struct svc_rqst *rqstp; 231 int cpu; 232 233 if (!(svsk->sk_flags & 234 ( (1<<SK_CONN)|(1<<SK_DATA)|(1<<SK_CLOSE)|(1<<SK_DEFERRED)) )) 235 return; 236 if (test_bit(SK_DEAD, &svsk->sk_flags)) 237 return; 238 239 cpu = get_cpu(); 240 pool = svc_pool_for_cpu(svsk->sk_server, cpu); 241 put_cpu(); 242 243 spin_lock_bh(&pool->sp_lock); 244 245 if (!list_empty(&pool->sp_threads) && 246 !list_empty(&pool->sp_sockets)) 247 printk(KERN_ERR 248 "svc_sock_enqueue: threads and sockets both waiting??\n"); 249 250 if (test_bit(SK_DEAD, &svsk->sk_flags)) { 251 /* Don't enqueue dead sockets */ 252 dprintk("svc: socket %p is dead, not enqueued\n", svsk->sk_sk); 253 goto out_unlock; 254 } 255 256 /* Mark socket as busy. It will remain in this state until the 257 * server has processed all pending data and put the socket back 258 * on the idle list. We update SK_BUSY atomically because 259 * it also guards against trying to enqueue the svc_sock twice. 260 */ 261 if (test_and_set_bit(SK_BUSY, &svsk->sk_flags)) { 262 /* Don't enqueue socket while already enqueued */ 263 dprintk("svc: socket %p busy, not enqueued\n", svsk->sk_sk); 264 goto out_unlock; 265 } 266 BUG_ON(svsk->sk_pool != NULL); 267 svsk->sk_pool = pool; 268 269 set_bit(SOCK_NOSPACE, &svsk->sk_sock->flags); 270 if (((atomic_read(&svsk->sk_reserved) + serv->sv_max_mesg)*2 271 > svc_sock_wspace(svsk)) 272 && !test_bit(SK_CLOSE, &svsk->sk_flags) 273 && !test_bit(SK_CONN, &svsk->sk_flags)) { 274 /* Don't enqueue while not enough space for reply */ 275 dprintk("svc: socket %p no space, %d*2 > %ld, not enqueued\n", 276 svsk->sk_sk, atomic_read(&svsk->sk_reserved)+serv->sv_max_mesg, 277 svc_sock_wspace(svsk)); 278 svsk->sk_pool = NULL; 279 clear_bit(SK_BUSY, &svsk->sk_flags); 280 goto out_unlock; 281 } 282 clear_bit(SOCK_NOSPACE, &svsk->sk_sock->flags); 283 284 285 if (!list_empty(&pool->sp_threads)) { 286 rqstp = list_entry(pool->sp_threads.next, 287 struct svc_rqst, 288 rq_list); 289 dprintk("svc: socket %p served by daemon %p\n", 290 svsk->sk_sk, rqstp); 291 svc_thread_dequeue(pool, rqstp); 292 if (rqstp->rq_sock) 293 printk(KERN_ERR 294 "svc_sock_enqueue: server %p, rq_sock=%p!\n", 295 rqstp, rqstp->rq_sock); 296 rqstp->rq_sock = svsk; 297 atomic_inc(&svsk->sk_inuse); 298 rqstp->rq_reserved = serv->sv_max_mesg; 299 atomic_add(rqstp->rq_reserved, &svsk->sk_reserved); 300 BUG_ON(svsk->sk_pool != pool); 301 wake_up(&rqstp->rq_wait); 302 } else { 303 dprintk("svc: socket %p put into queue\n", svsk->sk_sk); 304 list_add_tail(&svsk->sk_ready, &pool->sp_sockets); 305 BUG_ON(svsk->sk_pool != pool); 306 } 307 308 out_unlock: 309 spin_unlock_bh(&pool->sp_lock); 310 } 311 312 /* 313 * Dequeue the first socket. Must be called with the pool->sp_lock held. 314 */ 315 static inline struct svc_sock * 316 svc_sock_dequeue(struct svc_pool *pool) 317 { 318 struct svc_sock *svsk; 319 320 if (list_empty(&pool->sp_sockets)) 321 return NULL; 322 323 svsk = list_entry(pool->sp_sockets.next, 324 struct svc_sock, sk_ready); 325 list_del_init(&svsk->sk_ready); 326 327 dprintk("svc: socket %p dequeued, inuse=%d\n", 328 svsk->sk_sk, atomic_read(&svsk->sk_inuse)); 329 330 return svsk; 331 } 332 333 /* 334 * Having read something from a socket, check whether it 335 * needs to be re-enqueued. 336 * Note: SK_DATA only gets cleared when a read-attempt finds 337 * no (or insufficient) data. 338 */ 339 static inline void 340 svc_sock_received(struct svc_sock *svsk) 341 { 342 svsk->sk_pool = NULL; 343 clear_bit(SK_BUSY, &svsk->sk_flags); 344 svc_sock_enqueue(svsk); 345 } 346 347 348 /** 349 * svc_reserve - change the space reserved for the reply to a request. 350 * @rqstp: The request in question 351 * @space: new max space to reserve 352 * 353 * Each request reserves some space on the output queue of the socket 354 * to make sure the reply fits. This function reduces that reserved 355 * space to be the amount of space used already, plus @space. 356 * 357 */ 358 void svc_reserve(struct svc_rqst *rqstp, int space) 359 { 360 space += rqstp->rq_res.head[0].iov_len; 361 362 if (space < rqstp->rq_reserved) { 363 struct svc_sock *svsk = rqstp->rq_sock; 364 atomic_sub((rqstp->rq_reserved - space), &svsk->sk_reserved); 365 rqstp->rq_reserved = space; 366 367 svc_sock_enqueue(svsk); 368 } 369 } 370 371 /* 372 * Release a socket after use. 373 */ 374 static inline void 375 svc_sock_put(struct svc_sock *svsk) 376 { 377 if (atomic_dec_and_test(&svsk->sk_inuse)) { 378 BUG_ON(! test_bit(SK_DEAD, &svsk->sk_flags)); 379 380 dprintk("svc: releasing dead socket\n"); 381 if (svsk->sk_sock->file) 382 sockfd_put(svsk->sk_sock); 383 else 384 sock_release(svsk->sk_sock); 385 if (svsk->sk_info_authunix != NULL) 386 svcauth_unix_info_release(svsk->sk_info_authunix); 387 kfree(svsk); 388 } 389 } 390 391 static void 392 svc_sock_release(struct svc_rqst *rqstp) 393 { 394 struct svc_sock *svsk = rqstp->rq_sock; 395 396 svc_release_skb(rqstp); 397 398 svc_free_res_pages(rqstp); 399 rqstp->rq_res.page_len = 0; 400 rqstp->rq_res.page_base = 0; 401 402 403 /* Reset response buffer and release 404 * the reservation. 405 * But first, check that enough space was reserved 406 * for the reply, otherwise we have a bug! 407 */ 408 if ((rqstp->rq_res.len) > rqstp->rq_reserved) 409 printk(KERN_ERR "RPC request reserved %d but used %d\n", 410 rqstp->rq_reserved, 411 rqstp->rq_res.len); 412 413 rqstp->rq_res.head[0].iov_len = 0; 414 svc_reserve(rqstp, 0); 415 rqstp->rq_sock = NULL; 416 417 svc_sock_put(svsk); 418 } 419 420 /* 421 * External function to wake up a server waiting for data 422 * This really only makes sense for services like lockd 423 * which have exactly one thread anyway. 424 */ 425 void 426 svc_wake_up(struct svc_serv *serv) 427 { 428 struct svc_rqst *rqstp; 429 unsigned int i; 430 struct svc_pool *pool; 431 432 for (i = 0; i < serv->sv_nrpools; i++) { 433 pool = &serv->sv_pools[i]; 434 435 spin_lock_bh(&pool->sp_lock); 436 if (!list_empty(&pool->sp_threads)) { 437 rqstp = list_entry(pool->sp_threads.next, 438 struct svc_rqst, 439 rq_list); 440 dprintk("svc: daemon %p woken up.\n", rqstp); 441 /* 442 svc_thread_dequeue(pool, rqstp); 443 rqstp->rq_sock = NULL; 444 */ 445 wake_up(&rqstp->rq_wait); 446 } 447 spin_unlock_bh(&pool->sp_lock); 448 } 449 } 450 451 union svc_pktinfo_u { 452 struct in_pktinfo pkti; 453 struct in6_pktinfo pkti6; 454 }; 455 #define SVC_PKTINFO_SPACE \ 456 CMSG_SPACE(sizeof(union svc_pktinfo_u)) 457 458 static void svc_set_cmsg_data(struct svc_rqst *rqstp, struct cmsghdr *cmh) 459 { 460 switch (rqstp->rq_sock->sk_sk->sk_family) { 461 case AF_INET: { 462 struct in_pktinfo *pki = CMSG_DATA(cmh); 463 464 cmh->cmsg_level = SOL_IP; 465 cmh->cmsg_type = IP_PKTINFO; 466 pki->ipi_ifindex = 0; 467 pki->ipi_spec_dst.s_addr = rqstp->rq_daddr.addr.s_addr; 468 cmh->cmsg_len = CMSG_LEN(sizeof(*pki)); 469 } 470 break; 471 472 case AF_INET6: { 473 struct in6_pktinfo *pki = CMSG_DATA(cmh); 474 475 cmh->cmsg_level = SOL_IPV6; 476 cmh->cmsg_type = IPV6_PKTINFO; 477 pki->ipi6_ifindex = 0; 478 ipv6_addr_copy(&pki->ipi6_addr, 479 &rqstp->rq_daddr.addr6); 480 cmh->cmsg_len = CMSG_LEN(sizeof(*pki)); 481 } 482 break; 483 } 484 return; 485 } 486 487 /* 488 * Generic sendto routine 489 */ 490 static int 491 svc_sendto(struct svc_rqst *rqstp, struct xdr_buf *xdr) 492 { 493 struct svc_sock *svsk = rqstp->rq_sock; 494 struct socket *sock = svsk->sk_sock; 495 int slen; 496 union { 497 struct cmsghdr hdr; 498 long all[SVC_PKTINFO_SPACE / sizeof(long)]; 499 } buffer; 500 struct cmsghdr *cmh = &buffer.hdr; 501 int len = 0; 502 int result; 503 int size; 504 struct page **ppage = xdr->pages; 505 size_t base = xdr->page_base; 506 unsigned int pglen = xdr->page_len; 507 unsigned int flags = MSG_MORE; 508 char buf[RPC_MAX_ADDRBUFLEN]; 509 510 slen = xdr->len; 511 512 if (rqstp->rq_prot == IPPROTO_UDP) { 513 struct msghdr msg = { 514 .msg_name = &rqstp->rq_addr, 515 .msg_namelen = rqstp->rq_addrlen, 516 .msg_control = cmh, 517 .msg_controllen = sizeof(buffer), 518 .msg_flags = MSG_MORE, 519 }; 520 521 svc_set_cmsg_data(rqstp, cmh); 522 523 if (sock_sendmsg(sock, &msg, 0) < 0) 524 goto out; 525 } 526 527 /* send head */ 528 if (slen == xdr->head[0].iov_len) 529 flags = 0; 530 len = kernel_sendpage(sock, rqstp->rq_respages[0], 0, 531 xdr->head[0].iov_len, flags); 532 if (len != xdr->head[0].iov_len) 533 goto out; 534 slen -= xdr->head[0].iov_len; 535 if (slen == 0) 536 goto out; 537 538 /* send page data */ 539 size = PAGE_SIZE - base < pglen ? PAGE_SIZE - base : pglen; 540 while (pglen > 0) { 541 if (slen == size) 542 flags = 0; 543 result = kernel_sendpage(sock, *ppage, base, size, flags); 544 if (result > 0) 545 len += result; 546 if (result != size) 547 goto out; 548 slen -= size; 549 pglen -= size; 550 size = PAGE_SIZE < pglen ? PAGE_SIZE : pglen; 551 base = 0; 552 ppage++; 553 } 554 /* send tail */ 555 if (xdr->tail[0].iov_len) { 556 result = kernel_sendpage(sock, rqstp->rq_respages[0], 557 ((unsigned long)xdr->tail[0].iov_base) 558 & (PAGE_SIZE-1), 559 xdr->tail[0].iov_len, 0); 560 561 if (result > 0) 562 len += result; 563 } 564 out: 565 dprintk("svc: socket %p sendto([%p %Zu... ], %d) = %d (addr %s)\n", 566 rqstp->rq_sock, xdr->head[0].iov_base, xdr->head[0].iov_len, 567 xdr->len, len, svc_print_addr(rqstp, buf, sizeof(buf))); 568 569 return len; 570 } 571 572 /* 573 * Report socket names for nfsdfs 574 */ 575 static int one_sock_name(char *buf, struct svc_sock *svsk) 576 { 577 int len; 578 579 switch(svsk->sk_sk->sk_family) { 580 case AF_INET: 581 len = sprintf(buf, "ipv4 %s %u.%u.%u.%u %d\n", 582 svsk->sk_sk->sk_protocol==IPPROTO_UDP? 583 "udp" : "tcp", 584 NIPQUAD(inet_sk(svsk->sk_sk)->rcv_saddr), 585 inet_sk(svsk->sk_sk)->num); 586 break; 587 default: 588 len = sprintf(buf, "*unknown-%d*\n", 589 svsk->sk_sk->sk_family); 590 } 591 return len; 592 } 593 594 int 595 svc_sock_names(char *buf, struct svc_serv *serv, char *toclose) 596 { 597 struct svc_sock *svsk, *closesk = NULL; 598 int len = 0; 599 600 if (!serv) 601 return 0; 602 spin_lock_bh(&serv->sv_lock); 603 list_for_each_entry(svsk, &serv->sv_permsocks, sk_list) { 604 int onelen = one_sock_name(buf+len, svsk); 605 if (toclose && strcmp(toclose, buf+len) == 0) 606 closesk = svsk; 607 else 608 len += onelen; 609 } 610 spin_unlock_bh(&serv->sv_lock); 611 if (closesk) 612 /* Should unregister with portmap, but you cannot 613 * unregister just one protocol... 614 */ 615 svc_close_socket(closesk); 616 else if (toclose) 617 return -ENOENT; 618 return len; 619 } 620 EXPORT_SYMBOL(svc_sock_names); 621 622 /* 623 * Check input queue length 624 */ 625 static int 626 svc_recv_available(struct svc_sock *svsk) 627 { 628 struct socket *sock = svsk->sk_sock; 629 int avail, err; 630 631 err = kernel_sock_ioctl(sock, TIOCINQ, (unsigned long) &avail); 632 633 return (err >= 0)? avail : err; 634 } 635 636 /* 637 * Generic recvfrom routine. 638 */ 639 static int 640 svc_recvfrom(struct svc_rqst *rqstp, struct kvec *iov, int nr, int buflen) 641 { 642 struct svc_sock *svsk = rqstp->rq_sock; 643 struct msghdr msg = { 644 .msg_flags = MSG_DONTWAIT, 645 }; 646 int len; 647 648 len = kernel_recvmsg(svsk->sk_sock, &msg, iov, nr, buflen, 649 msg.msg_flags); 650 651 /* sock_recvmsg doesn't fill in the name/namelen, so we must.. 652 */ 653 memcpy(&rqstp->rq_addr, &svsk->sk_remote, svsk->sk_remotelen); 654 rqstp->rq_addrlen = svsk->sk_remotelen; 655 656 dprintk("svc: socket %p recvfrom(%p, %Zu) = %d\n", 657 svsk, iov[0].iov_base, iov[0].iov_len, len); 658 659 return len; 660 } 661 662 /* 663 * Set socket snd and rcv buffer lengths 664 */ 665 static inline void 666 svc_sock_setbufsize(struct socket *sock, unsigned int snd, unsigned int rcv) 667 { 668 #if 0 669 mm_segment_t oldfs; 670 oldfs = get_fs(); set_fs(KERNEL_DS); 671 sock_setsockopt(sock, SOL_SOCKET, SO_SNDBUF, 672 (char*)&snd, sizeof(snd)); 673 sock_setsockopt(sock, SOL_SOCKET, SO_RCVBUF, 674 (char*)&rcv, sizeof(rcv)); 675 #else 676 /* sock_setsockopt limits use to sysctl_?mem_max, 677 * which isn't acceptable. Until that is made conditional 678 * on not having CAP_SYS_RESOURCE or similar, we go direct... 679 * DaveM said I could! 680 */ 681 lock_sock(sock->sk); 682 sock->sk->sk_sndbuf = snd * 2; 683 sock->sk->sk_rcvbuf = rcv * 2; 684 sock->sk->sk_userlocks |= SOCK_SNDBUF_LOCK|SOCK_RCVBUF_LOCK; 685 release_sock(sock->sk); 686 #endif 687 } 688 /* 689 * INET callback when data has been received on the socket. 690 */ 691 static void 692 svc_udp_data_ready(struct sock *sk, int count) 693 { 694 struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; 695 696 if (svsk) { 697 dprintk("svc: socket %p(inet %p), count=%d, busy=%d\n", 698 svsk, sk, count, test_bit(SK_BUSY, &svsk->sk_flags)); 699 set_bit(SK_DATA, &svsk->sk_flags); 700 svc_sock_enqueue(svsk); 701 } 702 if (sk->sk_sleep && waitqueue_active(sk->sk_sleep)) 703 wake_up_interruptible(sk->sk_sleep); 704 } 705 706 /* 707 * INET callback when space is newly available on the socket. 708 */ 709 static void 710 svc_write_space(struct sock *sk) 711 { 712 struct svc_sock *svsk = (struct svc_sock *)(sk->sk_user_data); 713 714 if (svsk) { 715 dprintk("svc: socket %p(inet %p), write_space busy=%d\n", 716 svsk, sk, test_bit(SK_BUSY, &svsk->sk_flags)); 717 svc_sock_enqueue(svsk); 718 } 719 720 if (sk->sk_sleep && waitqueue_active(sk->sk_sleep)) { 721 dprintk("RPC svc_write_space: someone sleeping on %p\n", 722 svsk); 723 wake_up_interruptible(sk->sk_sleep); 724 } 725 } 726 727 static inline void svc_udp_get_dest_address(struct svc_rqst *rqstp, 728 struct cmsghdr *cmh) 729 { 730 switch (rqstp->rq_sock->sk_sk->sk_family) { 731 case AF_INET: { 732 struct in_pktinfo *pki = CMSG_DATA(cmh); 733 rqstp->rq_daddr.addr.s_addr = pki->ipi_spec_dst.s_addr; 734 break; 735 } 736 case AF_INET6: { 737 struct in6_pktinfo *pki = CMSG_DATA(cmh); 738 ipv6_addr_copy(&rqstp->rq_daddr.addr6, &pki->ipi6_addr); 739 break; 740 } 741 } 742 } 743 744 /* 745 * Receive a datagram from a UDP socket. 746 */ 747 static int 748 svc_udp_recvfrom(struct svc_rqst *rqstp) 749 { 750 struct svc_sock *svsk = rqstp->rq_sock; 751 struct svc_serv *serv = svsk->sk_server; 752 struct sk_buff *skb; 753 union { 754 struct cmsghdr hdr; 755 long all[SVC_PKTINFO_SPACE / sizeof(long)]; 756 } buffer; 757 struct cmsghdr *cmh = &buffer.hdr; 758 int err, len; 759 struct msghdr msg = { 760 .msg_name = svc_addr(rqstp), 761 .msg_control = cmh, 762 .msg_controllen = sizeof(buffer), 763 .msg_flags = MSG_DONTWAIT, 764 }; 765 766 if (test_and_clear_bit(SK_CHNGBUF, &svsk->sk_flags)) 767 /* udp sockets need large rcvbuf as all pending 768 * requests are still in that buffer. sndbuf must 769 * also be large enough that there is enough space 770 * for one reply per thread. We count all threads 771 * rather than threads in a particular pool, which 772 * provides an upper bound on the number of threads 773 * which will access the socket. 774 */ 775 svc_sock_setbufsize(svsk->sk_sock, 776 (serv->sv_nrthreads+3) * serv->sv_max_mesg, 777 (serv->sv_nrthreads+3) * serv->sv_max_mesg); 778 779 if ((rqstp->rq_deferred = svc_deferred_dequeue(svsk))) { 780 svc_sock_received(svsk); 781 return svc_deferred_recv(rqstp); 782 } 783 784 if (test_bit(SK_CLOSE, &svsk->sk_flags)) { 785 svc_delete_socket(svsk); 786 return 0; 787 } 788 789 clear_bit(SK_DATA, &svsk->sk_flags); 790 while ((err = kernel_recvmsg(svsk->sk_sock, &msg, NULL, 791 0, 0, MSG_PEEK | MSG_DONTWAIT)) < 0 || 792 (skb = skb_recv_datagram(svsk->sk_sk, 0, 1, &err)) == NULL) { 793 if (err == -EAGAIN) { 794 svc_sock_received(svsk); 795 return err; 796 } 797 /* possibly an icmp error */ 798 dprintk("svc: recvfrom returned error %d\n", -err); 799 } 800 rqstp->rq_addrlen = sizeof(rqstp->rq_addr); 801 if (skb->tstamp.tv64 == 0) { 802 skb->tstamp = ktime_get_real(); 803 /* Don't enable netstamp, sunrpc doesn't 804 need that much accuracy */ 805 } 806 svsk->sk_sk->sk_stamp = skb->tstamp; 807 set_bit(SK_DATA, &svsk->sk_flags); /* there may be more data... */ 808 809 /* 810 * Maybe more packets - kick another thread ASAP. 811 */ 812 svc_sock_received(svsk); 813 814 len = skb->len - sizeof(struct udphdr); 815 rqstp->rq_arg.len = len; 816 817 rqstp->rq_prot = IPPROTO_UDP; 818 819 if (cmh->cmsg_level != IPPROTO_IP || 820 cmh->cmsg_type != IP_PKTINFO) { 821 if (net_ratelimit()) 822 printk("rpcsvc: received unknown control message:" 823 "%d/%d\n", 824 cmh->cmsg_level, cmh->cmsg_type); 825 skb_free_datagram(svsk->sk_sk, skb); 826 return 0; 827 } 828 svc_udp_get_dest_address(rqstp, cmh); 829 830 if (skb_is_nonlinear(skb)) { 831 /* we have to copy */ 832 local_bh_disable(); 833 if (csum_partial_copy_to_xdr(&rqstp->rq_arg, skb)) { 834 local_bh_enable(); 835 /* checksum error */ 836 skb_free_datagram(svsk->sk_sk, skb); 837 return 0; 838 } 839 local_bh_enable(); 840 skb_free_datagram(svsk->sk_sk, skb); 841 } else { 842 /* we can use it in-place */ 843 rqstp->rq_arg.head[0].iov_base = skb->data + sizeof(struct udphdr); 844 rqstp->rq_arg.head[0].iov_len = len; 845 if (skb_checksum_complete(skb)) { 846 skb_free_datagram(svsk->sk_sk, skb); 847 return 0; 848 } 849 rqstp->rq_skbuff = skb; 850 } 851 852 rqstp->rq_arg.page_base = 0; 853 if (len <= rqstp->rq_arg.head[0].iov_len) { 854 rqstp->rq_arg.head[0].iov_len = len; 855 rqstp->rq_arg.page_len = 0; 856 rqstp->rq_respages = rqstp->rq_pages+1; 857 } else { 858 rqstp->rq_arg.page_len = len - rqstp->rq_arg.head[0].iov_len; 859 rqstp->rq_respages = rqstp->rq_pages + 1 + 860 (rqstp->rq_arg.page_len + PAGE_SIZE - 1)/ PAGE_SIZE; 861 } 862 863 if (serv->sv_stats) 864 serv->sv_stats->netudpcnt++; 865 866 return len; 867 } 868 869 static int 870 svc_udp_sendto(struct svc_rqst *rqstp) 871 { 872 int error; 873 874 error = svc_sendto(rqstp, &rqstp->rq_res); 875 if (error == -ECONNREFUSED) 876 /* ICMP error on earlier request. */ 877 error = svc_sendto(rqstp, &rqstp->rq_res); 878 879 return error; 880 } 881 882 static void 883 svc_udp_init(struct svc_sock *svsk) 884 { 885 int one = 1; 886 mm_segment_t oldfs; 887 888 svsk->sk_sk->sk_data_ready = svc_udp_data_ready; 889 svsk->sk_sk->sk_write_space = svc_write_space; 890 svsk->sk_recvfrom = svc_udp_recvfrom; 891 svsk->sk_sendto = svc_udp_sendto; 892 893 /* initialise setting must have enough space to 894 * receive and respond to one request. 895 * svc_udp_recvfrom will re-adjust if necessary 896 */ 897 svc_sock_setbufsize(svsk->sk_sock, 898 3 * svsk->sk_server->sv_max_mesg, 899 3 * svsk->sk_server->sv_max_mesg); 900 901 set_bit(SK_DATA, &svsk->sk_flags); /* might have come in before data_ready set up */ 902 set_bit(SK_CHNGBUF, &svsk->sk_flags); 903 904 oldfs = get_fs(); 905 set_fs(KERNEL_DS); 906 /* make sure we get destination address info */ 907 svsk->sk_sock->ops->setsockopt(svsk->sk_sock, IPPROTO_IP, IP_PKTINFO, 908 (char __user *)&one, sizeof(one)); 909 set_fs(oldfs); 910 } 911 912 /* 913 * A data_ready event on a listening socket means there's a connection 914 * pending. Do not use state_change as a substitute for it. 915 */ 916 static void 917 svc_tcp_listen_data_ready(struct sock *sk, int count_unused) 918 { 919 struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; 920 921 dprintk("svc: socket %p TCP (listen) state change %d\n", 922 sk, sk->sk_state); 923 924 /* 925 * This callback may called twice when a new connection 926 * is established as a child socket inherits everything 927 * from a parent LISTEN socket. 928 * 1) data_ready method of the parent socket will be called 929 * when one of child sockets become ESTABLISHED. 930 * 2) data_ready method of the child socket may be called 931 * when it receives data before the socket is accepted. 932 * In case of 2, we should ignore it silently. 933 */ 934 if (sk->sk_state == TCP_LISTEN) { 935 if (svsk) { 936 set_bit(SK_CONN, &svsk->sk_flags); 937 svc_sock_enqueue(svsk); 938 } else 939 printk("svc: socket %p: no user data\n", sk); 940 } 941 942 if (sk->sk_sleep && waitqueue_active(sk->sk_sleep)) 943 wake_up_interruptible_all(sk->sk_sleep); 944 } 945 946 /* 947 * A state change on a connected socket means it's dying or dead. 948 */ 949 static void 950 svc_tcp_state_change(struct sock *sk) 951 { 952 struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; 953 954 dprintk("svc: socket %p TCP (connected) state change %d (svsk %p)\n", 955 sk, sk->sk_state, sk->sk_user_data); 956 957 if (!svsk) 958 printk("svc: socket %p: no user data\n", sk); 959 else { 960 set_bit(SK_CLOSE, &svsk->sk_flags); 961 svc_sock_enqueue(svsk); 962 } 963 if (sk->sk_sleep && waitqueue_active(sk->sk_sleep)) 964 wake_up_interruptible_all(sk->sk_sleep); 965 } 966 967 static void 968 svc_tcp_data_ready(struct sock *sk, int count) 969 { 970 struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; 971 972 dprintk("svc: socket %p TCP data ready (svsk %p)\n", 973 sk, sk->sk_user_data); 974 if (svsk) { 975 set_bit(SK_DATA, &svsk->sk_flags); 976 svc_sock_enqueue(svsk); 977 } 978 if (sk->sk_sleep && waitqueue_active(sk->sk_sleep)) 979 wake_up_interruptible(sk->sk_sleep); 980 } 981 982 static inline int svc_port_is_privileged(struct sockaddr *sin) 983 { 984 switch (sin->sa_family) { 985 case AF_INET: 986 return ntohs(((struct sockaddr_in *)sin)->sin_port) 987 < PROT_SOCK; 988 case AF_INET6: 989 return ntohs(((struct sockaddr_in6 *)sin)->sin6_port) 990 < PROT_SOCK; 991 default: 992 return 0; 993 } 994 } 995 996 /* 997 * Accept a TCP connection 998 */ 999 static void 1000 svc_tcp_accept(struct svc_sock *svsk) 1001 { 1002 struct sockaddr_storage addr; 1003 struct sockaddr *sin = (struct sockaddr *) &addr; 1004 struct svc_serv *serv = svsk->sk_server; 1005 struct socket *sock = svsk->sk_sock; 1006 struct socket *newsock; 1007 struct svc_sock *newsvsk; 1008 int err, slen; 1009 char buf[RPC_MAX_ADDRBUFLEN]; 1010 1011 dprintk("svc: tcp_accept %p sock %p\n", svsk, sock); 1012 if (!sock) 1013 return; 1014 1015 clear_bit(SK_CONN, &svsk->sk_flags); 1016 err = kernel_accept(sock, &newsock, O_NONBLOCK); 1017 if (err < 0) { 1018 if (err == -ENOMEM) 1019 printk(KERN_WARNING "%s: no more sockets!\n", 1020 serv->sv_name); 1021 else if (err != -EAGAIN && net_ratelimit()) 1022 printk(KERN_WARNING "%s: accept failed (err %d)!\n", 1023 serv->sv_name, -err); 1024 return; 1025 } 1026 1027 set_bit(SK_CONN, &svsk->sk_flags); 1028 svc_sock_enqueue(svsk); 1029 1030 err = kernel_getpeername(newsock, sin, &slen); 1031 if (err < 0) { 1032 if (net_ratelimit()) 1033 printk(KERN_WARNING "%s: peername failed (err %d)!\n", 1034 serv->sv_name, -err); 1035 goto failed; /* aborted connection or whatever */ 1036 } 1037 1038 /* Ideally, we would want to reject connections from unauthorized 1039 * hosts here, but when we get encryption, the IP of the host won't 1040 * tell us anything. For now just warn about unpriv connections. 1041 */ 1042 if (!svc_port_is_privileged(sin)) { 1043 dprintk(KERN_WARNING 1044 "%s: connect from unprivileged port: %s\n", 1045 serv->sv_name, 1046 __svc_print_addr(sin, buf, sizeof(buf))); 1047 } 1048 dprintk("%s: connect from %s\n", serv->sv_name, 1049 __svc_print_addr(sin, buf, sizeof(buf))); 1050 1051 /* make sure that a write doesn't block forever when 1052 * low on memory 1053 */ 1054 newsock->sk->sk_sndtimeo = HZ*30; 1055 1056 if (!(newsvsk = svc_setup_socket(serv, newsock, &err, 1057 (SVC_SOCK_ANONYMOUS | SVC_SOCK_TEMPORARY)))) 1058 goto failed; 1059 memcpy(&newsvsk->sk_remote, sin, slen); 1060 newsvsk->sk_remotelen = slen; 1061 1062 svc_sock_received(newsvsk); 1063 1064 /* make sure that we don't have too many active connections. 1065 * If we have, something must be dropped. 1066 * 1067 * There's no point in trying to do random drop here for 1068 * DoS prevention. The NFS clients does 1 reconnect in 15 1069 * seconds. An attacker can easily beat that. 1070 * 1071 * The only somewhat efficient mechanism would be if drop 1072 * old connections from the same IP first. But right now 1073 * we don't even record the client IP in svc_sock. 1074 */ 1075 if (serv->sv_tmpcnt > (serv->sv_nrthreads+3)*20) { 1076 struct svc_sock *svsk = NULL; 1077 spin_lock_bh(&serv->sv_lock); 1078 if (!list_empty(&serv->sv_tempsocks)) { 1079 if (net_ratelimit()) { 1080 /* Try to help the admin */ 1081 printk(KERN_NOTICE "%s: too many open TCP " 1082 "sockets, consider increasing the " 1083 "number of nfsd threads\n", 1084 serv->sv_name); 1085 printk(KERN_NOTICE 1086 "%s: last TCP connect from %s\n", 1087 serv->sv_name, buf); 1088 } 1089 /* 1090 * Always select the oldest socket. It's not fair, 1091 * but so is life 1092 */ 1093 svsk = list_entry(serv->sv_tempsocks.prev, 1094 struct svc_sock, 1095 sk_list); 1096 set_bit(SK_CLOSE, &svsk->sk_flags); 1097 atomic_inc(&svsk->sk_inuse); 1098 } 1099 spin_unlock_bh(&serv->sv_lock); 1100 1101 if (svsk) { 1102 svc_sock_enqueue(svsk); 1103 svc_sock_put(svsk); 1104 } 1105 1106 } 1107 1108 if (serv->sv_stats) 1109 serv->sv_stats->nettcpconn++; 1110 1111 return; 1112 1113 failed: 1114 sock_release(newsock); 1115 return; 1116 } 1117 1118 /* 1119 * Receive data from a TCP socket. 1120 */ 1121 static int 1122 svc_tcp_recvfrom(struct svc_rqst *rqstp) 1123 { 1124 struct svc_sock *svsk = rqstp->rq_sock; 1125 struct svc_serv *serv = svsk->sk_server; 1126 int len; 1127 struct kvec *vec; 1128 int pnum, vlen; 1129 1130 dprintk("svc: tcp_recv %p data %d conn %d close %d\n", 1131 svsk, test_bit(SK_DATA, &svsk->sk_flags), 1132 test_bit(SK_CONN, &svsk->sk_flags), 1133 test_bit(SK_CLOSE, &svsk->sk_flags)); 1134 1135 if ((rqstp->rq_deferred = svc_deferred_dequeue(svsk))) { 1136 svc_sock_received(svsk); 1137 return svc_deferred_recv(rqstp); 1138 } 1139 1140 if (test_bit(SK_CLOSE, &svsk->sk_flags)) { 1141 svc_delete_socket(svsk); 1142 return 0; 1143 } 1144 1145 if (svsk->sk_sk->sk_state == TCP_LISTEN) { 1146 svc_tcp_accept(svsk); 1147 svc_sock_received(svsk); 1148 return 0; 1149 } 1150 1151 if (test_and_clear_bit(SK_CHNGBUF, &svsk->sk_flags)) 1152 /* sndbuf needs to have room for one request 1153 * per thread, otherwise we can stall even when the 1154 * network isn't a bottleneck. 1155 * 1156 * We count all threads rather than threads in a 1157 * particular pool, which provides an upper bound 1158 * on the number of threads which will access the socket. 1159 * 1160 * rcvbuf just needs to be able to hold a few requests. 1161 * Normally they will be removed from the queue 1162 * as soon a a complete request arrives. 1163 */ 1164 svc_sock_setbufsize(svsk->sk_sock, 1165 (serv->sv_nrthreads+3) * serv->sv_max_mesg, 1166 3 * serv->sv_max_mesg); 1167 1168 clear_bit(SK_DATA, &svsk->sk_flags); 1169 1170 /* Receive data. If we haven't got the record length yet, get 1171 * the next four bytes. Otherwise try to gobble up as much as 1172 * possible up to the complete record length. 1173 */ 1174 if (svsk->sk_tcplen < 4) { 1175 unsigned long want = 4 - svsk->sk_tcplen; 1176 struct kvec iov; 1177 1178 iov.iov_base = ((char *) &svsk->sk_reclen) + svsk->sk_tcplen; 1179 iov.iov_len = want; 1180 if ((len = svc_recvfrom(rqstp, &iov, 1, want)) < 0) 1181 goto error; 1182 svsk->sk_tcplen += len; 1183 1184 if (len < want) { 1185 dprintk("svc: short recvfrom while reading record length (%d of %lu)\n", 1186 len, want); 1187 svc_sock_received(svsk); 1188 return -EAGAIN; /* record header not complete */ 1189 } 1190 1191 svsk->sk_reclen = ntohl(svsk->sk_reclen); 1192 if (!(svsk->sk_reclen & 0x80000000)) { 1193 /* FIXME: technically, a record can be fragmented, 1194 * and non-terminal fragments will not have the top 1195 * bit set in the fragment length header. 1196 * But apparently no known nfs clients send fragmented 1197 * records. */ 1198 if (net_ratelimit()) 1199 printk(KERN_NOTICE "RPC: bad TCP reclen 0x%08lx" 1200 " (non-terminal)\n", 1201 (unsigned long) svsk->sk_reclen); 1202 goto err_delete; 1203 } 1204 svsk->sk_reclen &= 0x7fffffff; 1205 dprintk("svc: TCP record, %d bytes\n", svsk->sk_reclen); 1206 if (svsk->sk_reclen > serv->sv_max_mesg) { 1207 if (net_ratelimit()) 1208 printk(KERN_NOTICE "RPC: bad TCP reclen 0x%08lx" 1209 " (large)\n", 1210 (unsigned long) svsk->sk_reclen); 1211 goto err_delete; 1212 } 1213 } 1214 1215 /* Check whether enough data is available */ 1216 len = svc_recv_available(svsk); 1217 if (len < 0) 1218 goto error; 1219 1220 if (len < svsk->sk_reclen) { 1221 dprintk("svc: incomplete TCP record (%d of %d)\n", 1222 len, svsk->sk_reclen); 1223 svc_sock_received(svsk); 1224 return -EAGAIN; /* record not complete */ 1225 } 1226 len = svsk->sk_reclen; 1227 set_bit(SK_DATA, &svsk->sk_flags); 1228 1229 vec = rqstp->rq_vec; 1230 vec[0] = rqstp->rq_arg.head[0]; 1231 vlen = PAGE_SIZE; 1232 pnum = 1; 1233 while (vlen < len) { 1234 vec[pnum].iov_base = page_address(rqstp->rq_pages[pnum]); 1235 vec[pnum].iov_len = PAGE_SIZE; 1236 pnum++; 1237 vlen += PAGE_SIZE; 1238 } 1239 rqstp->rq_respages = &rqstp->rq_pages[pnum]; 1240 1241 /* Now receive data */ 1242 len = svc_recvfrom(rqstp, vec, pnum, len); 1243 if (len < 0) 1244 goto error; 1245 1246 dprintk("svc: TCP complete record (%d bytes)\n", len); 1247 rqstp->rq_arg.len = len; 1248 rqstp->rq_arg.page_base = 0; 1249 if (len <= rqstp->rq_arg.head[0].iov_len) { 1250 rqstp->rq_arg.head[0].iov_len = len; 1251 rqstp->rq_arg.page_len = 0; 1252 } else { 1253 rqstp->rq_arg.page_len = len - rqstp->rq_arg.head[0].iov_len; 1254 } 1255 1256 rqstp->rq_skbuff = NULL; 1257 rqstp->rq_prot = IPPROTO_TCP; 1258 1259 /* Reset TCP read info */ 1260 svsk->sk_reclen = 0; 1261 svsk->sk_tcplen = 0; 1262 1263 svc_sock_received(svsk); 1264 if (serv->sv_stats) 1265 serv->sv_stats->nettcpcnt++; 1266 1267 return len; 1268 1269 err_delete: 1270 svc_delete_socket(svsk); 1271 return -EAGAIN; 1272 1273 error: 1274 if (len == -EAGAIN) { 1275 dprintk("RPC: TCP recvfrom got EAGAIN\n"); 1276 svc_sock_received(svsk); 1277 } else { 1278 printk(KERN_NOTICE "%s: recvfrom returned errno %d\n", 1279 svsk->sk_server->sv_name, -len); 1280 goto err_delete; 1281 } 1282 1283 return len; 1284 } 1285 1286 /* 1287 * Send out data on TCP socket. 1288 */ 1289 static int 1290 svc_tcp_sendto(struct svc_rqst *rqstp) 1291 { 1292 struct xdr_buf *xbufp = &rqstp->rq_res; 1293 int sent; 1294 __be32 reclen; 1295 1296 /* Set up the first element of the reply kvec. 1297 * Any other kvecs that may be in use have been taken 1298 * care of by the server implementation itself. 1299 */ 1300 reclen = htonl(0x80000000|((xbufp->len ) - 4)); 1301 memcpy(xbufp->head[0].iov_base, &reclen, 4); 1302 1303 if (test_bit(SK_DEAD, &rqstp->rq_sock->sk_flags)) 1304 return -ENOTCONN; 1305 1306 sent = svc_sendto(rqstp, &rqstp->rq_res); 1307 if (sent != xbufp->len) { 1308 printk(KERN_NOTICE "rpc-srv/tcp: %s: %s %d when sending %d bytes - shutting down socket\n", 1309 rqstp->rq_sock->sk_server->sv_name, 1310 (sent<0)?"got error":"sent only", 1311 sent, xbufp->len); 1312 set_bit(SK_CLOSE, &rqstp->rq_sock->sk_flags); 1313 svc_sock_enqueue(rqstp->rq_sock); 1314 sent = -EAGAIN; 1315 } 1316 return sent; 1317 } 1318 1319 static void 1320 svc_tcp_init(struct svc_sock *svsk) 1321 { 1322 struct sock *sk = svsk->sk_sk; 1323 struct tcp_sock *tp = tcp_sk(sk); 1324 1325 svsk->sk_recvfrom = svc_tcp_recvfrom; 1326 svsk->sk_sendto = svc_tcp_sendto; 1327 1328 if (sk->sk_state == TCP_LISTEN) { 1329 dprintk("setting up TCP socket for listening\n"); 1330 sk->sk_data_ready = svc_tcp_listen_data_ready; 1331 set_bit(SK_CONN, &svsk->sk_flags); 1332 } else { 1333 dprintk("setting up TCP socket for reading\n"); 1334 sk->sk_state_change = svc_tcp_state_change; 1335 sk->sk_data_ready = svc_tcp_data_ready; 1336 sk->sk_write_space = svc_write_space; 1337 1338 svsk->sk_reclen = 0; 1339 svsk->sk_tcplen = 0; 1340 1341 tp->nonagle = 1; /* disable Nagle's algorithm */ 1342 1343 /* initialise setting must have enough space to 1344 * receive and respond to one request. 1345 * svc_tcp_recvfrom will re-adjust if necessary 1346 */ 1347 svc_sock_setbufsize(svsk->sk_sock, 1348 3 * svsk->sk_server->sv_max_mesg, 1349 3 * svsk->sk_server->sv_max_mesg); 1350 1351 set_bit(SK_CHNGBUF, &svsk->sk_flags); 1352 set_bit(SK_DATA, &svsk->sk_flags); 1353 if (sk->sk_state != TCP_ESTABLISHED) 1354 set_bit(SK_CLOSE, &svsk->sk_flags); 1355 } 1356 } 1357 1358 void 1359 svc_sock_update_bufs(struct svc_serv *serv) 1360 { 1361 /* 1362 * The number of server threads has changed. Update 1363 * rcvbuf and sndbuf accordingly on all sockets 1364 */ 1365 struct list_head *le; 1366 1367 spin_lock_bh(&serv->sv_lock); 1368 list_for_each(le, &serv->sv_permsocks) { 1369 struct svc_sock *svsk = 1370 list_entry(le, struct svc_sock, sk_list); 1371 set_bit(SK_CHNGBUF, &svsk->sk_flags); 1372 } 1373 list_for_each(le, &serv->sv_tempsocks) { 1374 struct svc_sock *svsk = 1375 list_entry(le, struct svc_sock, sk_list); 1376 set_bit(SK_CHNGBUF, &svsk->sk_flags); 1377 } 1378 spin_unlock_bh(&serv->sv_lock); 1379 } 1380 1381 /* 1382 * Receive the next request on any socket. This code is carefully 1383 * organised not to touch any cachelines in the shared svc_serv 1384 * structure, only cachelines in the local svc_pool. 1385 */ 1386 int 1387 svc_recv(struct svc_rqst *rqstp, long timeout) 1388 { 1389 struct svc_sock *svsk = NULL; 1390 struct svc_serv *serv = rqstp->rq_server; 1391 struct svc_pool *pool = rqstp->rq_pool; 1392 int len, i; 1393 int pages; 1394 struct xdr_buf *arg; 1395 DECLARE_WAITQUEUE(wait, current); 1396 1397 dprintk("svc: server %p waiting for data (to = %ld)\n", 1398 rqstp, timeout); 1399 1400 if (rqstp->rq_sock) 1401 printk(KERN_ERR 1402 "svc_recv: service %p, socket not NULL!\n", 1403 rqstp); 1404 if (waitqueue_active(&rqstp->rq_wait)) 1405 printk(KERN_ERR 1406 "svc_recv: service %p, wait queue active!\n", 1407 rqstp); 1408 1409 1410 /* now allocate needed pages. If we get a failure, sleep briefly */ 1411 pages = (serv->sv_max_mesg + PAGE_SIZE) / PAGE_SIZE; 1412 for (i=0; i < pages ; i++) 1413 while (rqstp->rq_pages[i] == NULL) { 1414 struct page *p = alloc_page(GFP_KERNEL); 1415 if (!p) 1416 schedule_timeout_uninterruptible(msecs_to_jiffies(500)); 1417 rqstp->rq_pages[i] = p; 1418 } 1419 rqstp->rq_pages[i++] = NULL; /* this might be seen in nfs_read_actor */ 1420 BUG_ON(pages >= RPCSVC_MAXPAGES); 1421 1422 /* Make arg->head point to first page and arg->pages point to rest */ 1423 arg = &rqstp->rq_arg; 1424 arg->head[0].iov_base = page_address(rqstp->rq_pages[0]); 1425 arg->head[0].iov_len = PAGE_SIZE; 1426 arg->pages = rqstp->rq_pages + 1; 1427 arg->page_base = 0; 1428 /* save at least one page for response */ 1429 arg->page_len = (pages-2)*PAGE_SIZE; 1430 arg->len = (pages-1)*PAGE_SIZE; 1431 arg->tail[0].iov_len = 0; 1432 1433 try_to_freeze(); 1434 cond_resched(); 1435 if (signalled()) 1436 return -EINTR; 1437 1438 spin_lock_bh(&pool->sp_lock); 1439 if ((svsk = svc_sock_dequeue(pool)) != NULL) { 1440 rqstp->rq_sock = svsk; 1441 atomic_inc(&svsk->sk_inuse); 1442 rqstp->rq_reserved = serv->sv_max_mesg; 1443 atomic_add(rqstp->rq_reserved, &svsk->sk_reserved); 1444 } else { 1445 /* No data pending. Go to sleep */ 1446 svc_thread_enqueue(pool, rqstp); 1447 1448 /* 1449 * We have to be able to interrupt this wait 1450 * to bring down the daemons ... 1451 */ 1452 set_current_state(TASK_INTERRUPTIBLE); 1453 add_wait_queue(&rqstp->rq_wait, &wait); 1454 spin_unlock_bh(&pool->sp_lock); 1455 1456 schedule_timeout(timeout); 1457 1458 try_to_freeze(); 1459 1460 spin_lock_bh(&pool->sp_lock); 1461 remove_wait_queue(&rqstp->rq_wait, &wait); 1462 1463 if (!(svsk = rqstp->rq_sock)) { 1464 svc_thread_dequeue(pool, rqstp); 1465 spin_unlock_bh(&pool->sp_lock); 1466 dprintk("svc: server %p, no data yet\n", rqstp); 1467 return signalled()? -EINTR : -EAGAIN; 1468 } 1469 } 1470 spin_unlock_bh(&pool->sp_lock); 1471 1472 dprintk("svc: server %p, pool %u, socket %p, inuse=%d\n", 1473 rqstp, pool->sp_id, svsk, atomic_read(&svsk->sk_inuse)); 1474 len = svsk->sk_recvfrom(rqstp); 1475 dprintk("svc: got len=%d\n", len); 1476 1477 /* No data, incomplete (TCP) read, or accept() */ 1478 if (len == 0 || len == -EAGAIN) { 1479 rqstp->rq_res.len = 0; 1480 svc_sock_release(rqstp); 1481 return -EAGAIN; 1482 } 1483 svsk->sk_lastrecv = get_seconds(); 1484 clear_bit(SK_OLD, &svsk->sk_flags); 1485 1486 rqstp->rq_secure = svc_port_is_privileged(svc_addr(rqstp)); 1487 rqstp->rq_chandle.defer = svc_defer; 1488 1489 if (serv->sv_stats) 1490 serv->sv_stats->netcnt++; 1491 return len; 1492 } 1493 1494 /* 1495 * Drop request 1496 */ 1497 void 1498 svc_drop(struct svc_rqst *rqstp) 1499 { 1500 dprintk("svc: socket %p dropped request\n", rqstp->rq_sock); 1501 svc_sock_release(rqstp); 1502 } 1503 1504 /* 1505 * Return reply to client. 1506 */ 1507 int 1508 svc_send(struct svc_rqst *rqstp) 1509 { 1510 struct svc_sock *svsk; 1511 int len; 1512 struct xdr_buf *xb; 1513 1514 if ((svsk = rqstp->rq_sock) == NULL) { 1515 printk(KERN_WARNING "NULL socket pointer in %s:%d\n", 1516 __FILE__, __LINE__); 1517 return -EFAULT; 1518 } 1519 1520 /* release the receive skb before sending the reply */ 1521 svc_release_skb(rqstp); 1522 1523 /* calculate over-all length */ 1524 xb = & rqstp->rq_res; 1525 xb->len = xb->head[0].iov_len + 1526 xb->page_len + 1527 xb->tail[0].iov_len; 1528 1529 /* Grab svsk->sk_mutex to serialize outgoing data. */ 1530 mutex_lock(&svsk->sk_mutex); 1531 if (test_bit(SK_DEAD, &svsk->sk_flags)) 1532 len = -ENOTCONN; 1533 else 1534 len = svsk->sk_sendto(rqstp); 1535 mutex_unlock(&svsk->sk_mutex); 1536 svc_sock_release(rqstp); 1537 1538 if (len == -ECONNREFUSED || len == -ENOTCONN || len == -EAGAIN) 1539 return 0; 1540 return len; 1541 } 1542 1543 /* 1544 * Timer function to close old temporary sockets, using 1545 * a mark-and-sweep algorithm. 1546 */ 1547 static void 1548 svc_age_temp_sockets(unsigned long closure) 1549 { 1550 struct svc_serv *serv = (struct svc_serv *)closure; 1551 struct svc_sock *svsk; 1552 struct list_head *le, *next; 1553 LIST_HEAD(to_be_aged); 1554 1555 dprintk("svc_age_temp_sockets\n"); 1556 1557 if (!spin_trylock_bh(&serv->sv_lock)) { 1558 /* busy, try again 1 sec later */ 1559 dprintk("svc_age_temp_sockets: busy\n"); 1560 mod_timer(&serv->sv_temptimer, jiffies + HZ); 1561 return; 1562 } 1563 1564 list_for_each_safe(le, next, &serv->sv_tempsocks) { 1565 svsk = list_entry(le, struct svc_sock, sk_list); 1566 1567 if (!test_and_set_bit(SK_OLD, &svsk->sk_flags)) 1568 continue; 1569 if (atomic_read(&svsk->sk_inuse) || test_bit(SK_BUSY, &svsk->sk_flags)) 1570 continue; 1571 atomic_inc(&svsk->sk_inuse); 1572 list_move(le, &to_be_aged); 1573 set_bit(SK_CLOSE, &svsk->sk_flags); 1574 set_bit(SK_DETACHED, &svsk->sk_flags); 1575 } 1576 spin_unlock_bh(&serv->sv_lock); 1577 1578 while (!list_empty(&to_be_aged)) { 1579 le = to_be_aged.next; 1580 /* fiddling the sk_list node is safe 'cos we're SK_DETACHED */ 1581 list_del_init(le); 1582 svsk = list_entry(le, struct svc_sock, sk_list); 1583 1584 dprintk("queuing svsk %p for closing, %lu seconds old\n", 1585 svsk, get_seconds() - svsk->sk_lastrecv); 1586 1587 /* a thread will dequeue and close it soon */ 1588 svc_sock_enqueue(svsk); 1589 svc_sock_put(svsk); 1590 } 1591 1592 mod_timer(&serv->sv_temptimer, jiffies + svc_conn_age_period * HZ); 1593 } 1594 1595 /* 1596 * Initialize socket for RPC use and create svc_sock struct 1597 * XXX: May want to setsockopt SO_SNDBUF and SO_RCVBUF. 1598 */ 1599 static struct svc_sock *svc_setup_socket(struct svc_serv *serv, 1600 struct socket *sock, 1601 int *errp, int flags) 1602 { 1603 struct svc_sock *svsk; 1604 struct sock *inet; 1605 int pmap_register = !(flags & SVC_SOCK_ANONYMOUS); 1606 int is_temporary = flags & SVC_SOCK_TEMPORARY; 1607 1608 dprintk("svc: svc_setup_socket %p\n", sock); 1609 if (!(svsk = kzalloc(sizeof(*svsk), GFP_KERNEL))) { 1610 *errp = -ENOMEM; 1611 return NULL; 1612 } 1613 1614 inet = sock->sk; 1615 1616 /* Register socket with portmapper */ 1617 if (*errp >= 0 && pmap_register) 1618 *errp = svc_register(serv, inet->sk_protocol, 1619 ntohs(inet_sk(inet)->sport)); 1620 1621 if (*errp < 0) { 1622 kfree(svsk); 1623 return NULL; 1624 } 1625 1626 set_bit(SK_BUSY, &svsk->sk_flags); 1627 inet->sk_user_data = svsk; 1628 svsk->sk_sock = sock; 1629 svsk->sk_sk = inet; 1630 svsk->sk_ostate = inet->sk_state_change; 1631 svsk->sk_odata = inet->sk_data_ready; 1632 svsk->sk_owspace = inet->sk_write_space; 1633 svsk->sk_server = serv; 1634 atomic_set(&svsk->sk_inuse, 1); 1635 svsk->sk_lastrecv = get_seconds(); 1636 spin_lock_init(&svsk->sk_defer_lock); 1637 INIT_LIST_HEAD(&svsk->sk_deferred); 1638 INIT_LIST_HEAD(&svsk->sk_ready); 1639 mutex_init(&svsk->sk_mutex); 1640 1641 /* Initialize the socket */ 1642 if (sock->type == SOCK_DGRAM) 1643 svc_udp_init(svsk); 1644 else 1645 svc_tcp_init(svsk); 1646 1647 spin_lock_bh(&serv->sv_lock); 1648 if (is_temporary) { 1649 set_bit(SK_TEMP, &svsk->sk_flags); 1650 list_add(&svsk->sk_list, &serv->sv_tempsocks); 1651 serv->sv_tmpcnt++; 1652 if (serv->sv_temptimer.function == NULL) { 1653 /* setup timer to age temp sockets */ 1654 setup_timer(&serv->sv_temptimer, svc_age_temp_sockets, 1655 (unsigned long)serv); 1656 mod_timer(&serv->sv_temptimer, 1657 jiffies + svc_conn_age_period * HZ); 1658 } 1659 } else { 1660 clear_bit(SK_TEMP, &svsk->sk_flags); 1661 list_add(&svsk->sk_list, &serv->sv_permsocks); 1662 } 1663 spin_unlock_bh(&serv->sv_lock); 1664 1665 dprintk("svc: svc_setup_socket created %p (inet %p)\n", 1666 svsk, svsk->sk_sk); 1667 1668 return svsk; 1669 } 1670 1671 int svc_addsock(struct svc_serv *serv, 1672 int fd, 1673 char *name_return, 1674 int *proto) 1675 { 1676 int err = 0; 1677 struct socket *so = sockfd_lookup(fd, &err); 1678 struct svc_sock *svsk = NULL; 1679 1680 if (!so) 1681 return err; 1682 if (so->sk->sk_family != AF_INET) 1683 err = -EAFNOSUPPORT; 1684 else if (so->sk->sk_protocol != IPPROTO_TCP && 1685 so->sk->sk_protocol != IPPROTO_UDP) 1686 err = -EPROTONOSUPPORT; 1687 else if (so->state > SS_UNCONNECTED) 1688 err = -EISCONN; 1689 else { 1690 svsk = svc_setup_socket(serv, so, &err, SVC_SOCK_DEFAULTS); 1691 if (svsk) { 1692 svc_sock_received(svsk); 1693 err = 0; 1694 } 1695 } 1696 if (err) { 1697 sockfd_put(so); 1698 return err; 1699 } 1700 if (proto) *proto = so->sk->sk_protocol; 1701 return one_sock_name(name_return, svsk); 1702 } 1703 EXPORT_SYMBOL_GPL(svc_addsock); 1704 1705 /* 1706 * Create socket for RPC service. 1707 */ 1708 static int svc_create_socket(struct svc_serv *serv, int protocol, 1709 struct sockaddr *sin, int len, int flags) 1710 { 1711 struct svc_sock *svsk; 1712 struct socket *sock; 1713 int error; 1714 int type; 1715 char buf[RPC_MAX_ADDRBUFLEN]; 1716 1717 dprintk("svc: svc_create_socket(%s, %d, %s)\n", 1718 serv->sv_program->pg_name, protocol, 1719 __svc_print_addr(sin, buf, sizeof(buf))); 1720 1721 if (protocol != IPPROTO_UDP && protocol != IPPROTO_TCP) { 1722 printk(KERN_WARNING "svc: only UDP and TCP " 1723 "sockets supported\n"); 1724 return -EINVAL; 1725 } 1726 type = (protocol == IPPROTO_UDP)? SOCK_DGRAM : SOCK_STREAM; 1727 1728 error = sock_create_kern(sin->sa_family, type, protocol, &sock); 1729 if (error < 0) 1730 return error; 1731 1732 svc_reclassify_socket(sock); 1733 1734 if (type == SOCK_STREAM) 1735 sock->sk->sk_reuse = 1; /* allow address reuse */ 1736 error = kernel_bind(sock, sin, len); 1737 if (error < 0) 1738 goto bummer; 1739 1740 if (protocol == IPPROTO_TCP) { 1741 if ((error = kernel_listen(sock, 64)) < 0) 1742 goto bummer; 1743 } 1744 1745 if ((svsk = svc_setup_socket(serv, sock, &error, flags)) != NULL) { 1746 svc_sock_received(svsk); 1747 return ntohs(inet_sk(svsk->sk_sk)->sport); 1748 } 1749 1750 bummer: 1751 dprintk("svc: svc_create_socket error = %d\n", -error); 1752 sock_release(sock); 1753 return error; 1754 } 1755 1756 /* 1757 * Remove a dead socket 1758 */ 1759 static void 1760 svc_delete_socket(struct svc_sock *svsk) 1761 { 1762 struct svc_serv *serv; 1763 struct sock *sk; 1764 1765 dprintk("svc: svc_delete_socket(%p)\n", svsk); 1766 1767 serv = svsk->sk_server; 1768 sk = svsk->sk_sk; 1769 1770 sk->sk_state_change = svsk->sk_ostate; 1771 sk->sk_data_ready = svsk->sk_odata; 1772 sk->sk_write_space = svsk->sk_owspace; 1773 1774 spin_lock_bh(&serv->sv_lock); 1775 1776 if (!test_and_set_bit(SK_DETACHED, &svsk->sk_flags)) 1777 list_del_init(&svsk->sk_list); 1778 /* 1779 * We used to delete the svc_sock from whichever list 1780 * it's sk_ready node was on, but we don't actually 1781 * need to. This is because the only time we're called 1782 * while still attached to a queue, the queue itself 1783 * is about to be destroyed (in svc_destroy). 1784 */ 1785 if (!test_and_set_bit(SK_DEAD, &svsk->sk_flags)) { 1786 BUG_ON(atomic_read(&svsk->sk_inuse)<2); 1787 atomic_dec(&svsk->sk_inuse); 1788 if (test_bit(SK_TEMP, &svsk->sk_flags)) 1789 serv->sv_tmpcnt--; 1790 } 1791 1792 spin_unlock_bh(&serv->sv_lock); 1793 } 1794 1795 static void svc_close_socket(struct svc_sock *svsk) 1796 { 1797 set_bit(SK_CLOSE, &svsk->sk_flags); 1798 if (test_and_set_bit(SK_BUSY, &svsk->sk_flags)) 1799 /* someone else will have to effect the close */ 1800 return; 1801 1802 atomic_inc(&svsk->sk_inuse); 1803 svc_delete_socket(svsk); 1804 clear_bit(SK_BUSY, &svsk->sk_flags); 1805 svc_sock_put(svsk); 1806 } 1807 1808 void svc_force_close_socket(struct svc_sock *svsk) 1809 { 1810 set_bit(SK_CLOSE, &svsk->sk_flags); 1811 if (test_bit(SK_BUSY, &svsk->sk_flags)) { 1812 /* Waiting to be processed, but no threads left, 1813 * So just remove it from the waiting list 1814 */ 1815 list_del_init(&svsk->sk_ready); 1816 clear_bit(SK_BUSY, &svsk->sk_flags); 1817 } 1818 svc_close_socket(svsk); 1819 } 1820 1821 /** 1822 * svc_makesock - Make a socket for nfsd and lockd 1823 * @serv: RPC server structure 1824 * @protocol: transport protocol to use 1825 * @port: port to use 1826 * @flags: requested socket characteristics 1827 * 1828 */ 1829 int svc_makesock(struct svc_serv *serv, int protocol, unsigned short port, 1830 int flags) 1831 { 1832 struct sockaddr_in sin = { 1833 .sin_family = AF_INET, 1834 .sin_addr.s_addr = INADDR_ANY, 1835 .sin_port = htons(port), 1836 }; 1837 1838 dprintk("svc: creating socket proto = %d\n", protocol); 1839 return svc_create_socket(serv, protocol, (struct sockaddr *) &sin, 1840 sizeof(sin), flags); 1841 } 1842 1843 /* 1844 * Handle defer and revisit of requests 1845 */ 1846 1847 static void svc_revisit(struct cache_deferred_req *dreq, int too_many) 1848 { 1849 struct svc_deferred_req *dr = container_of(dreq, struct svc_deferred_req, handle); 1850 struct svc_sock *svsk; 1851 1852 if (too_many) { 1853 svc_sock_put(dr->svsk); 1854 kfree(dr); 1855 return; 1856 } 1857 dprintk("revisit queued\n"); 1858 svsk = dr->svsk; 1859 dr->svsk = NULL; 1860 spin_lock_bh(&svsk->sk_defer_lock); 1861 list_add(&dr->handle.recent, &svsk->sk_deferred); 1862 spin_unlock_bh(&svsk->sk_defer_lock); 1863 set_bit(SK_DEFERRED, &svsk->sk_flags); 1864 svc_sock_enqueue(svsk); 1865 svc_sock_put(svsk); 1866 } 1867 1868 static struct cache_deferred_req * 1869 svc_defer(struct cache_req *req) 1870 { 1871 struct svc_rqst *rqstp = container_of(req, struct svc_rqst, rq_chandle); 1872 int size = sizeof(struct svc_deferred_req) + (rqstp->rq_arg.len); 1873 struct svc_deferred_req *dr; 1874 1875 if (rqstp->rq_arg.page_len) 1876 return NULL; /* if more than a page, give up FIXME */ 1877 if (rqstp->rq_deferred) { 1878 dr = rqstp->rq_deferred; 1879 rqstp->rq_deferred = NULL; 1880 } else { 1881 int skip = rqstp->rq_arg.len - rqstp->rq_arg.head[0].iov_len; 1882 /* FIXME maybe discard if size too large */ 1883 dr = kmalloc(size, GFP_KERNEL); 1884 if (dr == NULL) 1885 return NULL; 1886 1887 dr->handle.owner = rqstp->rq_server; 1888 dr->prot = rqstp->rq_prot; 1889 memcpy(&dr->addr, &rqstp->rq_addr, rqstp->rq_addrlen); 1890 dr->addrlen = rqstp->rq_addrlen; 1891 dr->daddr = rqstp->rq_daddr; 1892 dr->argslen = rqstp->rq_arg.len >> 2; 1893 memcpy(dr->args, rqstp->rq_arg.head[0].iov_base-skip, dr->argslen<<2); 1894 } 1895 atomic_inc(&rqstp->rq_sock->sk_inuse); 1896 dr->svsk = rqstp->rq_sock; 1897 1898 dr->handle.revisit = svc_revisit; 1899 return &dr->handle; 1900 } 1901 1902 /* 1903 * recv data from a deferred request into an active one 1904 */ 1905 static int svc_deferred_recv(struct svc_rqst *rqstp) 1906 { 1907 struct svc_deferred_req *dr = rqstp->rq_deferred; 1908 1909 rqstp->rq_arg.head[0].iov_base = dr->args; 1910 rqstp->rq_arg.head[0].iov_len = dr->argslen<<2; 1911 rqstp->rq_arg.page_len = 0; 1912 rqstp->rq_arg.len = dr->argslen<<2; 1913 rqstp->rq_prot = dr->prot; 1914 memcpy(&rqstp->rq_addr, &dr->addr, dr->addrlen); 1915 rqstp->rq_addrlen = dr->addrlen; 1916 rqstp->rq_daddr = dr->daddr; 1917 rqstp->rq_respages = rqstp->rq_pages; 1918 return dr->argslen<<2; 1919 } 1920 1921 1922 static struct svc_deferred_req *svc_deferred_dequeue(struct svc_sock *svsk) 1923 { 1924 struct svc_deferred_req *dr = NULL; 1925 1926 if (!test_bit(SK_DEFERRED, &svsk->sk_flags)) 1927 return NULL; 1928 spin_lock_bh(&svsk->sk_defer_lock); 1929 clear_bit(SK_DEFERRED, &svsk->sk_flags); 1930 if (!list_empty(&svsk->sk_deferred)) { 1931 dr = list_entry(svsk->sk_deferred.next, 1932 struct svc_deferred_req, 1933 handle.recent); 1934 list_del_init(&dr->handle.recent); 1935 set_bit(SK_DEFERRED, &svsk->sk_flags); 1936 } 1937 spin_unlock_bh(&svsk->sk_defer_lock); 1938 return dr; 1939 } 1940