1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include <sys/types.h> 28 #include <sys/param.h> 29 #include <sys/signal.h> 30 #include <sys/cmn_err.h> 31 32 #include <sys/stropts.h> 33 #include <sys/socket.h> 34 #include <sys/socketvar.h> 35 #include <sys/sockio.h> 36 #include <sys/strsubr.h> 37 #include <sys/strsun.h> 38 #include <sys/atomic.h> 39 #include <sys/tihdr.h> 40 41 #include <fs/sockfs/sockcommon.h> 42 #include <fs/sockfs/socktpi.h> 43 #include <fs/sockfs/sodirect.h> 44 #include <sys/ddi.h> 45 #include <inet/ip.h> 46 #include <sys/time.h> 47 #include <sys/cmn_err.h> 48 49 #ifdef SOCK_TEST 50 extern int do_useracc; 51 extern clock_t sock_test_timelimit; 52 #endif /* SOCK_TEST */ 53 54 #define MBLK_PULL_LEN 64 55 uint32_t so_mblk_pull_len = MBLK_PULL_LEN; 56 57 #ifdef DEBUG 58 boolean_t so_debug_length = B_FALSE; 59 static boolean_t so_check_length(sonode_t *so); 60 #endif 61 62 int 63 so_acceptq_enqueue_locked(struct sonode *so, struct sonode *nso) 64 { 65 ASSERT(MUTEX_HELD(&so->so_acceptq_lock)); 66 ASSERT(nso->so_acceptq_next == NULL); 67 68 *so->so_acceptq_tail = nso; 69 so->so_acceptq_tail = &nso->so_acceptq_next; 70 so->so_acceptq_len++; 71 72 /* Wakeup a single consumer */ 73 cv_signal(&so->so_acceptq_cv); 74 75 return (so->so_acceptq_len); 76 } 77 78 /* 79 * int so_acceptq_enqueue(struct sonode *so, struct sonode *nso) 80 * 81 * Enqueue an incoming connection on a listening socket. 82 * 83 * Arguments: 84 * so - listening socket 85 * nso - new connection 86 * 87 * Returns: 88 * Number of queued connections, including the new connection 89 */ 90 int 91 so_acceptq_enqueue(struct sonode *so, struct sonode *nso) 92 { 93 int conns; 94 95 mutex_enter(&so->so_acceptq_lock); 96 conns = so_acceptq_enqueue_locked(so, nso); 97 mutex_exit(&so->so_acceptq_lock); 98 99 return (conns); 100 } 101 102 static int 103 so_acceptq_dequeue_locked(struct sonode *so, boolean_t dontblock, 104 struct sonode **nsop) 105 { 106 struct sonode *nso = NULL; 107 108 *nsop = NULL; 109 ASSERT(MUTEX_HELD(&so->so_acceptq_lock)); 110 while ((nso = so->so_acceptq_head) == NULL) { 111 /* 112 * No need to check so_error here, because it is not 113 * possible for a listening socket to be reset or otherwise 114 * disconnected. 115 * 116 * So now we just need check if it's ok to wait. 117 */ 118 if (dontblock) 119 return (EWOULDBLOCK); 120 if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING)) 121 return (EINTR); 122 123 if (cv_wait_sig_swap(&so->so_acceptq_cv, 124 &so->so_acceptq_lock) == 0) 125 return (EINTR); 126 } 127 128 ASSERT(nso != NULL); 129 so->so_acceptq_head = nso->so_acceptq_next; 130 nso->so_acceptq_next = NULL; 131 132 if (so->so_acceptq_head == NULL) { 133 ASSERT(so->so_acceptq_tail == &nso->so_acceptq_next); 134 so->so_acceptq_tail = &so->so_acceptq_head; 135 } 136 ASSERT(so->so_acceptq_len > 0); 137 --so->so_acceptq_len; 138 139 *nsop = nso; 140 141 return (0); 142 } 143 144 /* 145 * int so_acceptq_dequeue(struct sonode *, boolean_t, struct sonode **) 146 * 147 * Pulls a connection off of the accept queue. 148 * 149 * Arguments: 150 * so - listening socket 151 * dontblock - indicate whether it's ok to sleep if there are no 152 * connections on the queue 153 * nsop - Value-return argument 154 * 155 * Return values: 156 * 0 when a connection is successfully dequeued, in which case nsop 157 * is set to point to the new connection. Upon failure a non-zero 158 * value is returned, and the value of nsop is set to NULL. 159 * 160 * Note: 161 * so_acceptq_dequeue() may return prematurly if the socket is falling 162 * back to TPI. 163 */ 164 int 165 so_acceptq_dequeue(struct sonode *so, boolean_t dontblock, 166 struct sonode **nsop) 167 { 168 int error; 169 170 mutex_enter(&so->so_acceptq_lock); 171 error = so_acceptq_dequeue_locked(so, dontblock, nsop); 172 mutex_exit(&so->so_acceptq_lock); 173 174 return (error); 175 } 176 177 /* 178 * void so_acceptq_flush(struct sonode *so, boolean_t doclose) 179 * 180 * Removes all pending connections from a listening socket, and 181 * frees the associated resources. 182 * 183 * Arguments 184 * so - listening socket 185 * doclose - make a close downcall for each socket on the accept queue 186 * (Note, only SCTP and SDP sockets rely on this) 187 * 188 * Return values: 189 * None. 190 * 191 * Note: 192 * The caller has to ensure that no calls to so_acceptq_enqueue() or 193 * so_acceptq_dequeue() occur while the accept queue is being flushed. 194 * So either the socket needs to be in a state where no operations 195 * would come in, or so_lock needs to be obtained. 196 */ 197 void 198 so_acceptq_flush(struct sonode *so, boolean_t doclose) 199 { 200 struct sonode *nso; 201 202 while ((nso = so->so_acceptq_head) != NULL) { 203 so->so_acceptq_head = nso->so_acceptq_next; 204 nso->so_acceptq_next = NULL; 205 206 if (doclose) { 207 (void) socket_close(nso, 0, CRED()); 208 } else { 209 /* 210 * Since the socket is on the accept queue, there can 211 * only be one reference. We drop the reference and 212 * just blow off the socket. 213 */ 214 ASSERT(nso->so_count == 1); 215 nso->so_count--; 216 } 217 socket_destroy(nso); 218 } 219 220 so->so_acceptq_head = NULL; 221 so->so_acceptq_tail = &so->so_acceptq_head; 222 so->so_acceptq_len = 0; 223 } 224 225 int 226 so_wait_connected_locked(struct sonode *so, boolean_t nonblock, 227 sock_connid_t id) 228 { 229 ASSERT(MUTEX_HELD(&so->so_lock)); 230 231 /* 232 * The protocol has notified us that a connection attempt is being 233 * made, so before we wait for a notification to arrive we must 234 * clear out any errors associated with earlier connection attempts. 235 */ 236 if (so->so_error != 0 && SOCK_CONNID_LT(so->so_proto_connid, id)) 237 so->so_error = 0; 238 239 while (SOCK_CONNID_LT(so->so_proto_connid, id)) { 240 if (nonblock) 241 return (EINPROGRESS); 242 243 if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING)) 244 return (EINTR); 245 246 if (cv_wait_sig_swap(&so->so_state_cv, &so->so_lock) == 0) 247 return (EINTR); 248 } 249 250 if (so->so_error != 0) 251 return (sogeterr(so, B_TRUE)); 252 /* 253 * Under normal circumstances, so_error should contain an error 254 * in case the connect failed. However, it is possible for another 255 * thread to come in a consume the error, so generate a sensible 256 * error in that case. 257 */ 258 if ((so->so_state & SS_ISCONNECTED) == 0) 259 return (ECONNREFUSED); 260 261 return (0); 262 } 263 264 /* 265 * int so_wait_connected(struct sonode *so, boolean_t nonblock, 266 * sock_connid_t id) 267 * 268 * Wait until the socket is connected or an error has occured. 269 * 270 * Arguments: 271 * so - socket 272 * nonblock - indicate whether it's ok to sleep if the connection has 273 * not yet been established 274 * gen - generation number that was returned by the protocol 275 * when the operation was started 276 * 277 * Returns: 278 * 0 if the connection attempt was successful, or an error indicating why 279 * the connection attempt failed. 280 */ 281 int 282 so_wait_connected(struct sonode *so, boolean_t nonblock, sock_connid_t id) 283 { 284 int error; 285 286 mutex_enter(&so->so_lock); 287 error = so_wait_connected_locked(so, nonblock, id); 288 mutex_exit(&so->so_lock); 289 290 return (error); 291 } 292 293 int 294 so_snd_wait_qnotfull_locked(struct sonode *so, boolean_t dontblock) 295 { 296 int error; 297 298 ASSERT(MUTEX_HELD(&so->so_lock)); 299 while (so->so_snd_qfull) { 300 if (so->so_state & SS_CANTSENDMORE) 301 return (EPIPE); 302 if (dontblock) 303 return (EWOULDBLOCK); 304 305 if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING)) 306 return (EINTR); 307 308 if (so->so_sndtimeo == 0) { 309 /* 310 * Zero means disable timeout. 311 */ 312 error = cv_wait_sig(&so->so_snd_cv, &so->so_lock); 313 } else { 314 clock_t now; 315 316 time_to_wait(&now, so->so_sndtimeo); 317 error = cv_timedwait_sig(&so->so_snd_cv, &so->so_lock, 318 now); 319 } 320 if (error == 0) 321 return (EINTR); 322 else if (error == -1) 323 return (EAGAIN); 324 } 325 return (0); 326 } 327 328 /* 329 * int so_wait_sendbuf(struct sonode *so, boolean_t dontblock) 330 * 331 * Wait for the transport to notify us about send buffers becoming 332 * available. 333 */ 334 int 335 so_snd_wait_qnotfull(struct sonode *so, boolean_t dontblock) 336 { 337 int error = 0; 338 339 mutex_enter(&so->so_lock); 340 if (so->so_snd_qfull) { 341 so->so_snd_wakeup = B_TRUE; 342 error = so_snd_wait_qnotfull_locked(so, dontblock); 343 so->so_snd_wakeup = B_FALSE; 344 } 345 mutex_exit(&so->so_lock); 346 347 return (error); 348 } 349 350 void 351 so_snd_qfull(struct sonode *so) 352 { 353 mutex_enter(&so->so_lock); 354 so->so_snd_qfull = B_TRUE; 355 mutex_exit(&so->so_lock); 356 } 357 358 void 359 so_snd_qnotfull(struct sonode *so) 360 { 361 mutex_enter(&so->so_lock); 362 so->so_snd_qfull = B_FALSE; 363 /* wake up everyone waiting for buffers */ 364 cv_broadcast(&so->so_snd_cv); 365 mutex_exit(&so->so_lock); 366 } 367 368 /* 369 * Change the process/process group to which SIGIO is sent. 370 */ 371 int 372 socket_chgpgrp(struct sonode *so, pid_t pid) 373 { 374 int error; 375 376 ASSERT(MUTEX_HELD(&so->so_lock)); 377 if (pid != 0) { 378 /* 379 * Permissions check by sending signal 0. 380 * Note that when kill fails it does a 381 * set_errno causing the system call to fail. 382 */ 383 error = kill(pid, 0); 384 if (error != 0) { 385 return (error); 386 } 387 } 388 so->so_pgrp = pid; 389 return (0); 390 } 391 392 393 /* 394 * Generate a SIGIO, for 'writable' events include siginfo structure, 395 * for read events just send the signal. 396 */ 397 /*ARGSUSED*/ 398 static void 399 socket_sigproc(proc_t *proc, int event) 400 { 401 k_siginfo_t info; 402 403 ASSERT(event & (SOCKETSIG_WRITE | SOCKETSIG_READ | SOCKETSIG_URG)); 404 405 if (event & SOCKETSIG_WRITE) { 406 info.si_signo = SIGPOLL; 407 info.si_code = POLL_OUT; 408 info.si_errno = 0; 409 info.si_fd = 0; 410 info.si_band = 0; 411 sigaddq(proc, NULL, &info, KM_NOSLEEP); 412 } 413 if (event & SOCKETSIG_READ) { 414 sigtoproc(proc, NULL, SIGPOLL); 415 } 416 if (event & SOCKETSIG_URG) { 417 sigtoproc(proc, NULL, SIGURG); 418 } 419 } 420 421 void 422 socket_sendsig(struct sonode *so, int event) 423 { 424 proc_t *proc; 425 426 ASSERT(MUTEX_HELD(&so->so_lock)); 427 428 if (so->so_pgrp == 0 || (!(so->so_state & SS_ASYNC) && 429 event != SOCKETSIG_URG)) { 430 return; 431 } 432 433 dprint(3, ("sending sig %d to %d\n", event, so->so_pgrp)); 434 435 if (so->so_pgrp > 0) { 436 /* 437 * XXX This unfortunately still generates 438 * a signal when a fd is closed but 439 * the proc is active. 440 */ 441 mutex_enter(&pidlock); 442 proc = prfind(so->so_pgrp); 443 if (proc == NULL) { 444 mutex_exit(&pidlock); 445 return; 446 } 447 mutex_enter(&proc->p_lock); 448 mutex_exit(&pidlock); 449 socket_sigproc(proc, event); 450 mutex_exit(&proc->p_lock); 451 } else { 452 /* 453 * Send to process group. Hold pidlock across 454 * calls to socket_sigproc(). 455 */ 456 pid_t pgrp = -so->so_pgrp; 457 458 mutex_enter(&pidlock); 459 proc = pgfind(pgrp); 460 while (proc != NULL) { 461 mutex_enter(&proc->p_lock); 462 socket_sigproc(proc, event); 463 mutex_exit(&proc->p_lock); 464 proc = proc->p_pglink; 465 } 466 mutex_exit(&pidlock); 467 } 468 } 469 470 #define MIN(a, b) ((a) < (b) ? (a) : (b)) 471 /* Copy userdata into a new mblk_t */ 472 mblk_t * 473 socopyinuio(uio_t *uiop, ssize_t iosize, size_t wroff, ssize_t maxblk, 474 size_t tail_len, int *errorp) 475 { 476 mblk_t *head = NULL, **tail = &head; 477 478 ASSERT(iosize == INFPSZ || iosize > 0); 479 480 if (iosize == INFPSZ || iosize > uiop->uio_resid) 481 iosize = uiop->uio_resid; 482 483 if (maxblk == INFPSZ) 484 maxblk = iosize; 485 486 /* Nothing to do in these cases, so we're done */ 487 if (iosize < 0 || maxblk < 0 || (maxblk == 0 && iosize > 0)) 488 goto done; 489 490 /* 491 * We will enter the loop below if iosize is 0; it will allocate an 492 * empty message block and call uiomove(9F) which will just return. 493 * We could avoid that with an extra check but would only slow 494 * down the much more likely case where iosize is larger than 0. 495 */ 496 do { 497 ssize_t blocksize; 498 mblk_t *mp; 499 500 blocksize = MIN(iosize, maxblk); 501 ASSERT(blocksize >= 0); 502 mp = allocb(wroff + blocksize + tail_len, BPRI_MED); 503 if (mp == NULL) { 504 *errorp = ENOMEM; 505 return (head); 506 } 507 mp->b_rptr += wroff; 508 mp->b_wptr = mp->b_rptr + blocksize; 509 510 *tail = mp; 511 tail = &mp->b_cont; 512 513 /* uiomove(9F) either returns 0 or EFAULT */ 514 if ((*errorp = uiomove(mp->b_rptr, (size_t)blocksize, 515 UIO_WRITE, uiop)) != 0) { 516 ASSERT(*errorp != ENOMEM); 517 freemsg(head); 518 return (NULL); 519 } 520 521 iosize -= blocksize; 522 } while (iosize > 0); 523 524 done: 525 *errorp = 0; 526 return (head); 527 } 528 529 mblk_t * 530 socopyoutuio(mblk_t *mp, struct uio *uiop, ssize_t max_read, int *errorp) 531 { 532 int error; 533 ptrdiff_t n; 534 mblk_t *nmp; 535 536 ASSERT(mp->b_wptr >= mp->b_rptr); 537 538 /* 539 * max_read is the offset of the oobmark and read can not go pass 540 * the oobmark. 541 */ 542 if (max_read == INFPSZ || max_read > uiop->uio_resid) 543 max_read = uiop->uio_resid; 544 545 do { 546 if ((n = MIN(max_read, MBLKL(mp))) != 0) { 547 ASSERT(n > 0); 548 549 error = uiomove(mp->b_rptr, n, UIO_READ, uiop); 550 if (error != 0) { 551 freemsg(mp); 552 *errorp = error; 553 return (NULL); 554 } 555 } 556 557 mp->b_rptr += n; 558 max_read -= n; 559 while (mp != NULL && (mp->b_rptr >= mp->b_wptr)) { 560 /* 561 * get rid of zero length mblks 562 */ 563 nmp = mp; 564 mp = mp->b_cont; 565 freeb(nmp); 566 } 567 } while (mp != NULL && max_read > 0); 568 569 *errorp = 0; 570 return (mp); 571 } 572 573 static void 574 so_prepend_msg(struct sonode *so, mblk_t *mp, mblk_t *last_tail) 575 { 576 ASSERT(last_tail != NULL); 577 mp->b_next = so->so_rcv_q_head; 578 mp->b_prev = last_tail; 579 ASSERT(!(DB_FLAGS(mp) & DBLK_UIOA)); 580 581 if (so->so_rcv_q_head == NULL) { 582 ASSERT(so->so_rcv_q_last_head == NULL); 583 so->so_rcv_q_last_head = mp; 584 #ifdef DEBUG 585 } else { 586 ASSERT(!(DB_FLAGS(so->so_rcv_q_head) & DBLK_UIOA)); 587 #endif 588 } 589 so->so_rcv_q_head = mp; 590 591 #ifdef DEBUG 592 if (so_debug_length) { 593 mutex_enter(&so->so_lock); 594 ASSERT(so_check_length(so)); 595 mutex_exit(&so->so_lock); 596 } 597 #endif 598 } 599 600 /* 601 * Move a mblk chain (mp_head, mp_last_head) to the sonode's rcv queue so it 602 * can be processed by so_dequeue_msg(). 603 */ 604 void 605 so_process_new_message(struct sonode *so, mblk_t *mp_head, mblk_t *mp_last_head) 606 { 607 ASSERT(mp_head->b_prev != NULL); 608 if (so->so_rcv_q_head == NULL) { 609 so->so_rcv_q_head = mp_head; 610 so->so_rcv_q_last_head = mp_last_head; 611 ASSERT(so->so_rcv_q_last_head->b_prev != NULL); 612 } else { 613 boolean_t flag_equal = ((DB_FLAGS(mp_head) & DBLK_UIOA) == 614 (DB_FLAGS(so->so_rcv_q_last_head) & DBLK_UIOA)); 615 616 if (mp_head->b_next == NULL && 617 DB_TYPE(mp_head) == M_DATA && 618 DB_TYPE(so->so_rcv_q_last_head) == M_DATA && flag_equal) { 619 so->so_rcv_q_last_head->b_prev->b_cont = mp_head; 620 so->so_rcv_q_last_head->b_prev = mp_head->b_prev; 621 mp_head->b_prev = NULL; 622 } else if (flag_equal && (DB_FLAGS(mp_head) & DBLK_UIOA)) { 623 /* 624 * Append to last_head if more than one mblks, and both 625 * mp_head and last_head are I/OAT mblks. 626 */ 627 ASSERT(mp_head->b_next != NULL); 628 so->so_rcv_q_last_head->b_prev->b_cont = mp_head; 629 so->so_rcv_q_last_head->b_prev = mp_head->b_prev; 630 mp_head->b_prev = NULL; 631 632 so->so_rcv_q_last_head->b_next = mp_head->b_next; 633 mp_head->b_next = NULL; 634 so->so_rcv_q_last_head = mp_last_head; 635 } else { 636 #ifdef DEBUG 637 { 638 mblk_t *tmp_mblk; 639 tmp_mblk = mp_head; 640 while (tmp_mblk != NULL) { 641 ASSERT(tmp_mblk->b_prev != NULL); 642 tmp_mblk = tmp_mblk->b_next; 643 } 644 } 645 #endif 646 so->so_rcv_q_last_head->b_next = mp_head; 647 so->so_rcv_q_last_head = mp_last_head; 648 } 649 } 650 } 651 652 /* 653 * Check flow control on a given sonode. Must have so_lock held, and 654 * this function will release the hold. 655 */ 656 657 static void 658 so_check_flow_control(struct sonode *so) 659 { 660 ASSERT(MUTEX_HELD(&so->so_lock)); 661 662 if (so->so_flowctrld && so->so_rcv_queued < so->so_rcvlowat) { 663 so->so_flowctrld = B_FALSE; 664 mutex_exit(&so->so_lock); 665 /* 666 * Open up flow control. SCTP does not have any downcalls, and 667 * it will clr flow ctrl in sosctp_recvmsg(). 668 */ 669 if (so->so_downcalls != NULL && 670 so->so_downcalls->sd_clr_flowctrl != NULL) { 671 (*so->so_downcalls->sd_clr_flowctrl) 672 (so->so_proto_handle); 673 } 674 } else { 675 mutex_exit(&so->so_lock); 676 } 677 } 678 679 int 680 so_dequeue_msg(struct sonode *so, mblk_t **mctlp, struct uio *uiop, 681 rval_t *rvalp, int flags) 682 { 683 mblk_t *mp, *nmp; 684 mblk_t *savemp, *savemptail; 685 mblk_t *new_msg_head; 686 mblk_t *new_msg_last_head; 687 mblk_t *last_tail; 688 boolean_t partial_read; 689 boolean_t reset_atmark = B_FALSE; 690 int more = 0; 691 int error; 692 ssize_t oobmark; 693 sodirect_t *sodp = so->so_direct; 694 695 partial_read = B_FALSE; 696 *mctlp = NULL; 697 again: 698 mutex_enter(&so->so_lock); 699 again1: 700 #ifdef DEBUG 701 if (so_debug_length) { 702 ASSERT(so_check_length(so)); 703 } 704 #endif 705 if (so->so_state & SS_RCVATMARK) { 706 /* Check whether the caller is OK to read past the mark */ 707 if (flags & MSG_NOMARK) { 708 mutex_exit(&so->so_lock); 709 return (EWOULDBLOCK); 710 } 711 reset_atmark = B_TRUE; 712 } 713 /* 714 * First move messages from the dump area to processing area 715 */ 716 if (sodp != NULL) { 717 if (sodp->sod_enabled) { 718 if (sodp->sod_uioa.uioa_state & UIOA_ALLOC) { 719 /* nothing to uioamove */ 720 sodp = NULL; 721 } else if (sodp->sod_uioa.uioa_state & UIOA_INIT) { 722 sodp->sod_uioa.uioa_state &= UIOA_CLR; 723 sodp->sod_uioa.uioa_state |= UIOA_ENABLED; 724 /* 725 * try to uioamove() the data that 726 * has already queued. 727 */ 728 sod_uioa_so_init(so, sodp, uiop); 729 } 730 } else { 731 sodp = NULL; 732 } 733 } 734 new_msg_head = so->so_rcv_head; 735 new_msg_last_head = so->so_rcv_last_head; 736 so->so_rcv_head = NULL; 737 so->so_rcv_last_head = NULL; 738 oobmark = so->so_oobmark; 739 /* 740 * We can release the lock as there can only be one reader 741 */ 742 mutex_exit(&so->so_lock); 743 744 if (new_msg_head != NULL) { 745 so_process_new_message(so, new_msg_head, new_msg_last_head); 746 } 747 savemp = savemptail = NULL; 748 rvalp->r_val1 = 0; 749 error = 0; 750 mp = so->so_rcv_q_head; 751 752 if (mp != NULL && 753 (so->so_rcv_timer_tid == 0 || 754 so->so_rcv_queued >= so->so_rcv_thresh)) { 755 partial_read = B_FALSE; 756 757 if (flags & MSG_PEEK) { 758 if ((nmp = dupmsg(mp)) == NULL && 759 (nmp = copymsg(mp)) == NULL) { 760 size_t size = msgsize(mp); 761 762 error = strwaitbuf(size, BPRI_HI); 763 if (error) { 764 return (error); 765 } 766 goto again; 767 } 768 mp = nmp; 769 } else { 770 ASSERT(mp->b_prev != NULL); 771 last_tail = mp->b_prev; 772 mp->b_prev = NULL; 773 so->so_rcv_q_head = mp->b_next; 774 if (so->so_rcv_q_head == NULL) { 775 so->so_rcv_q_last_head = NULL; 776 } 777 mp->b_next = NULL; 778 } 779 780 ASSERT(mctlp != NULL); 781 /* 782 * First process PROTO or PCPROTO blocks, if any. 783 */ 784 if (DB_TYPE(mp) != M_DATA) { 785 *mctlp = mp; 786 savemp = mp; 787 savemptail = mp; 788 ASSERT(DB_TYPE(mp) == M_PROTO || 789 DB_TYPE(mp) == M_PCPROTO); 790 while (mp->b_cont != NULL && 791 DB_TYPE(mp->b_cont) != M_DATA) { 792 ASSERT(DB_TYPE(mp->b_cont) == M_PROTO || 793 DB_TYPE(mp->b_cont) == M_PCPROTO); 794 mp = mp->b_cont; 795 savemptail = mp; 796 } 797 mp = savemptail->b_cont; 798 savemptail->b_cont = NULL; 799 } 800 801 ASSERT(DB_TYPE(mp) == M_DATA); 802 /* 803 * Now process DATA blocks, if any. Note that for sodirect 804 * enabled socket, uio_resid can be 0. 805 */ 806 if (uiop->uio_resid >= 0) { 807 ssize_t copied = 0; 808 809 if (sodp != NULL && (DB_FLAGS(mp) & DBLK_UIOA)) { 810 mutex_enter(&so->so_lock); 811 ASSERT(uiop == (uio_t *)&sodp->sod_uioa); 812 copied = sod_uioa_mblk(so, mp); 813 if (copied > 0) 814 partial_read = B_TRUE; 815 mutex_exit(&so->so_lock); 816 /* mark this mblk as processed */ 817 mp = NULL; 818 } else { 819 ssize_t oldresid = uiop->uio_resid; 820 821 if (MBLKL(mp) < so_mblk_pull_len) { 822 if (pullupmsg(mp, -1) == 1) { 823 last_tail = mp; 824 } 825 } 826 /* 827 * Can not read beyond the oobmark 828 */ 829 mp = socopyoutuio(mp, uiop, 830 oobmark == 0 ? INFPSZ : oobmark, &error); 831 if (error != 0) { 832 freemsg(*mctlp); 833 *mctlp = NULL; 834 more = 0; 835 goto done; 836 } 837 ASSERT(oldresid >= uiop->uio_resid); 838 copied = oldresid - uiop->uio_resid; 839 if (oldresid > uiop->uio_resid) 840 partial_read = B_TRUE; 841 } 842 ASSERT(copied >= 0); 843 if (copied > 0 && !(flags & MSG_PEEK)) { 844 mutex_enter(&so->so_lock); 845 so->so_rcv_queued -= copied; 846 ASSERT(so->so_oobmark >= 0); 847 if (so->so_oobmark > 0) { 848 so->so_oobmark -= copied; 849 ASSERT(so->so_oobmark >= 0); 850 if (so->so_oobmark == 0) { 851 ASSERT(so->so_state & 852 SS_OOBPEND); 853 so->so_oobmark = 0; 854 so->so_state |= SS_RCVATMARK; 855 } 856 } 857 /* 858 * so_check_flow_control() will drop 859 * so->so_lock. 860 */ 861 so_check_flow_control(so); 862 } 863 } 864 if (mp != NULL) { /* more data blocks in msg */ 865 more |= MOREDATA; 866 if ((flags & (MSG_PEEK|MSG_TRUNC))) { 867 if (flags & MSG_PEEK) { 868 freemsg(mp); 869 } else { 870 unsigned int msize = msgdsize(mp); 871 872 freemsg(mp); 873 mutex_enter(&so->so_lock); 874 so->so_rcv_queued -= msize; 875 /* 876 * so_check_flow_control() will drop 877 * so->so_lock. 878 */ 879 so_check_flow_control(so); 880 } 881 } else if (partial_read && !somsghasdata(mp)) { 882 /* 883 * Avoid queuing a zero-length tail part of 884 * a message. partial_read == 1 indicates that 885 * we read some of the message. 886 */ 887 freemsg(mp); 888 more &= ~MOREDATA; 889 } else { 890 if (savemp != NULL && 891 (flags & MSG_DUPCTRL)) { 892 mblk_t *nmp; 893 /* 894 * There should only be non data mblks 895 */ 896 ASSERT(DB_TYPE(savemp) != M_DATA && 897 DB_TYPE(savemptail) != M_DATA); 898 try_again: 899 if ((nmp = dupmsg(savemp)) == NULL && 900 (nmp = copymsg(savemp)) == NULL) { 901 902 size_t size = msgsize(savemp); 903 904 error = strwaitbuf(size, 905 BPRI_HI); 906 if (error != 0) { 907 /* 908 * In case we 909 * cannot copy 910 * control data 911 * free the remaining 912 * data. 913 */ 914 freemsg(mp); 915 goto done; 916 } 917 goto try_again; 918 } 919 920 ASSERT(nmp != NULL); 921 ASSERT(DB_TYPE(nmp) != M_DATA); 922 savemptail->b_cont = mp; 923 *mctlp = nmp; 924 mp = savemp; 925 } 926 /* 927 * putback mp 928 */ 929 so_prepend_msg(so, mp, last_tail); 930 } 931 } 932 933 /* fast check so_rcv_head if there is more data */ 934 if (partial_read && !(so->so_state & SS_RCVATMARK) && 935 *mctlp == NULL && uiop->uio_resid > 0 && 936 !(flags & MSG_PEEK) && so->so_rcv_head != NULL) { 937 goto again; 938 } 939 } else if (!partial_read) { 940 mutex_enter(&so->so_lock); 941 if (so->so_error != 0) { 942 error = sogeterr(so, !(flags & MSG_PEEK)); 943 mutex_exit(&so->so_lock); 944 return (error); 945 } 946 /* 947 * No pending data. Return right away for nonblocking 948 * socket, otherwise sleep waiting for data. 949 */ 950 if (!(so->so_state & SS_CANTRCVMORE) && uiop->uio_resid > 0) { 951 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 952 (flags & MSG_DONTWAIT)) { 953 error = EWOULDBLOCK; 954 } else { 955 if (so->so_state & (SS_CLOSING | 956 SS_FALLBACK_PENDING)) { 957 mutex_exit(&so->so_lock); 958 error = EINTR; 959 goto done; 960 } 961 962 if (so->so_rcv_head != NULL) { 963 goto again1; 964 } 965 so->so_rcv_wakeup = B_TRUE; 966 so->so_rcv_wanted = uiop->uio_resid; 967 if (so->so_rcvtimeo == 0) { 968 /* 969 * Zero means disable timeout. 970 */ 971 error = cv_wait_sig(&so->so_rcv_cv, 972 &so->so_lock); 973 } else { 974 clock_t now; 975 time_to_wait(&now, so->so_rcvtimeo); 976 error = cv_timedwait_sig(&so->so_rcv_cv, 977 &so->so_lock, now); 978 } 979 so->so_rcv_wakeup = B_FALSE; 980 so->so_rcv_wanted = 0; 981 982 if (error == 0) { 983 error = EINTR; 984 } else if (error == -1) { 985 error = EAGAIN; 986 } else { 987 goto again1; 988 } 989 } 990 } 991 mutex_exit(&so->so_lock); 992 } 993 if (reset_atmark && partial_read && !(flags & MSG_PEEK)) { 994 /* 995 * We are passed the mark, update state 996 * 4.3BSD and 4.4BSD clears the mark when peeking across it. 997 * The draft Posix socket spec states that the mark should 998 * not be cleared when peeking. We follow the latter. 999 */ 1000 mutex_enter(&so->so_lock); 1001 ASSERT(so_verify_oobstate(so)); 1002 so->so_state &= ~(SS_OOBPEND|SS_HAVEOOBDATA|SS_RCVATMARK); 1003 freemsg(so->so_oobmsg); 1004 so->so_oobmsg = NULL; 1005 ASSERT(so_verify_oobstate(so)); 1006 mutex_exit(&so->so_lock); 1007 } 1008 ASSERT(so->so_rcv_wakeup == B_FALSE); 1009 done: 1010 if (sodp != NULL) { 1011 mutex_enter(&so->so_lock); 1012 if (sodp->sod_enabled && 1013 (sodp->sod_uioa.uioa_state & UIOA_ENABLED)) { 1014 SOD_UIOAFINI(sodp); 1015 if (sodp->sod_uioa.uioa_mbytes > 0) { 1016 ASSERT(so->so_rcv_q_head != NULL || 1017 so->so_rcv_head != NULL); 1018 so->so_rcv_queued -= sod_uioa_mblk(so, NULL); 1019 if (error == EWOULDBLOCK) 1020 error = 0; 1021 } 1022 } 1023 mutex_exit(&so->so_lock); 1024 } 1025 #ifdef DEBUG 1026 if (so_debug_length) { 1027 mutex_enter(&so->so_lock); 1028 ASSERT(so_check_length(so)); 1029 mutex_exit(&so->so_lock); 1030 } 1031 #endif 1032 rvalp->r_val1 = more; 1033 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 1034 return (error); 1035 } 1036 1037 /* 1038 * Enqueue data from the protocol on the socket's rcv queue. 1039 * 1040 * We try to hook new M_DATA mblks onto an existing chain, however, 1041 * that cannot be done if the existing chain has already been 1042 * processed by I/OAT. Non-M_DATA mblks are just linked together via 1043 * b_next. In all cases the b_prev of the enqueued mblk is set to 1044 * point to the last mblk in its b_cont chain. 1045 */ 1046 void 1047 so_enqueue_msg(struct sonode *so, mblk_t *mp, size_t msg_size) 1048 { 1049 ASSERT(MUTEX_HELD(&so->so_lock)); 1050 1051 #ifdef DEBUG 1052 if (so_debug_length) { 1053 ASSERT(so_check_length(so)); 1054 } 1055 #endif 1056 so->so_rcv_queued += msg_size; 1057 1058 if (so->so_rcv_head == NULL) { 1059 ASSERT(so->so_rcv_last_head == NULL); 1060 so->so_rcv_head = mp; 1061 so->so_rcv_last_head = mp; 1062 } else if ((DB_TYPE(mp) == M_DATA && 1063 DB_TYPE(so->so_rcv_last_head) == M_DATA) && 1064 ((DB_FLAGS(mp) & DBLK_UIOA) == 1065 (DB_FLAGS(so->so_rcv_last_head) & DBLK_UIOA))) { 1066 /* Added to the end */ 1067 ASSERT(so->so_rcv_last_head != NULL); 1068 ASSERT(so->so_rcv_last_head->b_prev != NULL); 1069 so->so_rcv_last_head->b_prev->b_cont = mp; 1070 } else { 1071 /* Start a new end */ 1072 so->so_rcv_last_head->b_next = mp; 1073 so->so_rcv_last_head = mp; 1074 } 1075 while (mp->b_cont != NULL) 1076 mp = mp->b_cont; 1077 1078 so->so_rcv_last_head->b_prev = mp; 1079 #ifdef DEBUG 1080 if (so_debug_length) { 1081 ASSERT(so_check_length(so)); 1082 } 1083 #endif 1084 } 1085 1086 /* 1087 * Return B_TRUE if there is data in the message, B_FALSE otherwise. 1088 */ 1089 boolean_t 1090 somsghasdata(mblk_t *mp) 1091 { 1092 for (; mp; mp = mp->b_cont) 1093 if (mp->b_datap->db_type == M_DATA) { 1094 ASSERT(mp->b_wptr >= mp->b_rptr); 1095 if (mp->b_wptr > mp->b_rptr) 1096 return (B_TRUE); 1097 } 1098 return (B_FALSE); 1099 } 1100 1101 /* 1102 * Flush the read side of sockfs. 1103 * 1104 * The caller must be sure that a reader is not already active when the 1105 * buffer is being flushed. 1106 */ 1107 void 1108 so_rcv_flush(struct sonode *so) 1109 { 1110 mblk_t *mp; 1111 1112 ASSERT(MUTEX_HELD(&so->so_lock)); 1113 1114 if (so->so_oobmsg != NULL) { 1115 freemsg(so->so_oobmsg); 1116 so->so_oobmsg = NULL; 1117 so->so_oobmark = 0; 1118 so->so_state &= 1119 ~(SS_OOBPEND|SS_HAVEOOBDATA|SS_HADOOBDATA|SS_RCVATMARK); 1120 } 1121 1122 /* 1123 * Free messages sitting in the send and recv queue 1124 */ 1125 while (so->so_rcv_q_head != NULL) { 1126 mp = so->so_rcv_q_head; 1127 so->so_rcv_q_head = mp->b_next; 1128 mp->b_next = mp->b_prev = NULL; 1129 freemsg(mp); 1130 } 1131 while (so->so_rcv_head != NULL) { 1132 mp = so->so_rcv_head; 1133 so->so_rcv_head = mp->b_next; 1134 mp->b_next = mp->b_prev = NULL; 1135 freemsg(mp); 1136 } 1137 so->so_rcv_queued = 0; 1138 so->so_rcv_q_head = NULL; 1139 so->so_rcv_q_last_head = NULL; 1140 so->so_rcv_head = NULL; 1141 so->so_rcv_last_head = NULL; 1142 } 1143 1144 /* 1145 * Handle recv* calls that set MSG_OOB or MSG_OOB together with MSG_PEEK. 1146 */ 1147 int 1148 sorecvoob(struct sonode *so, struct nmsghdr *msg, struct uio *uiop, int flags, 1149 boolean_t oob_inline) 1150 { 1151 mblk_t *mp, *nmp; 1152 int error; 1153 1154 dprintso(so, 1, ("sorecvoob(%p, %p, 0x%x)\n", (void *)so, (void *)msg, 1155 flags)); 1156 1157 if (msg != NULL) { 1158 /* 1159 * There is never any oob data with addresses or control since 1160 * the T_EXDATA_IND does not carry any options. 1161 */ 1162 msg->msg_controllen = 0; 1163 msg->msg_namelen = 0; 1164 msg->msg_flags = 0; 1165 } 1166 1167 mutex_enter(&so->so_lock); 1168 ASSERT(so_verify_oobstate(so)); 1169 if (oob_inline || 1170 (so->so_state & (SS_OOBPEND|SS_HADOOBDATA)) != SS_OOBPEND) { 1171 dprintso(so, 1, ("sorecvoob: inline or data consumed\n")); 1172 mutex_exit(&so->so_lock); 1173 return (EINVAL); 1174 } 1175 if (!(so->so_state & SS_HAVEOOBDATA)) { 1176 dprintso(so, 1, ("sorecvoob: no data yet\n")); 1177 mutex_exit(&so->so_lock); 1178 return (EWOULDBLOCK); 1179 } 1180 ASSERT(so->so_oobmsg != NULL); 1181 mp = so->so_oobmsg; 1182 if (flags & MSG_PEEK) { 1183 /* 1184 * Since recv* can not return ENOBUFS we can not use dupmsg. 1185 * Instead we revert to the consolidation private 1186 * allocb_wait plus bcopy. 1187 */ 1188 mblk_t *mp1; 1189 1190 mp1 = allocb_wait(msgdsize(mp), BPRI_MED, STR_NOSIG, NULL); 1191 ASSERT(mp1); 1192 1193 while (mp != NULL) { 1194 ssize_t size; 1195 1196 size = MBLKL(mp); 1197 bcopy(mp->b_rptr, mp1->b_wptr, size); 1198 mp1->b_wptr += size; 1199 ASSERT(mp1->b_wptr <= mp1->b_datap->db_lim); 1200 mp = mp->b_cont; 1201 } 1202 mp = mp1; 1203 } else { 1204 /* 1205 * Update the state indicating that the data has been consumed. 1206 * Keep SS_OOBPEND set until data is consumed past the mark. 1207 */ 1208 so->so_oobmsg = NULL; 1209 so->so_state ^= SS_HAVEOOBDATA|SS_HADOOBDATA; 1210 } 1211 ASSERT(so_verify_oobstate(so)); 1212 mutex_exit(&so->so_lock); 1213 1214 error = 0; 1215 nmp = mp; 1216 while (nmp != NULL && uiop->uio_resid > 0) { 1217 ssize_t n = MBLKL(nmp); 1218 1219 n = MIN(n, uiop->uio_resid); 1220 if (n > 0) 1221 error = uiomove(nmp->b_rptr, n, 1222 UIO_READ, uiop); 1223 if (error) 1224 break; 1225 nmp = nmp->b_cont; 1226 } 1227 ASSERT(mp->b_next == NULL && mp->b_prev == NULL); 1228 freemsg(mp); 1229 return (error); 1230 } 1231 1232 /* 1233 * Allocate and initializ sonode 1234 */ 1235 /* ARGSUSED */ 1236 struct sonode * 1237 socket_sonode_create(struct sockparams *sp, int family, int type, 1238 int protocol, int version, int sflags, int *errorp, struct cred *cr) 1239 { 1240 sonode_t *so; 1241 int kmflags; 1242 1243 /* 1244 * Choose the right set of sonodeops based on the upcall and 1245 * down call version that the protocol has provided 1246 */ 1247 if (SOCK_UC_VERSION != sp->sp_smod_info->smod_uc_version || 1248 SOCK_DC_VERSION != sp->sp_smod_info->smod_dc_version) { 1249 /* 1250 * mismatch 1251 */ 1252 #ifdef DEBUG 1253 cmn_err(CE_CONT, "protocol and socket module version mismatch"); 1254 #endif 1255 *errorp = EINVAL; 1256 return (NULL); 1257 } 1258 1259 kmflags = (sflags & SOCKET_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 1260 1261 so = kmem_cache_alloc(socket_cache, kmflags); 1262 if (so == NULL) { 1263 *errorp = ENOMEM; 1264 return (NULL); 1265 } 1266 1267 sonode_init(so, sp, family, type, protocol, &so_sonodeops); 1268 1269 if (version == SOV_DEFAULT) 1270 version = so_default_version; 1271 1272 so->so_version = (short)version; 1273 1274 /* 1275 * set the default values to be INFPSZ 1276 * if a protocol desires it can change the value later 1277 */ 1278 so->so_proto_props.sopp_rxhiwat = SOCKET_RECVHIWATER; 1279 so->so_proto_props.sopp_rxlowat = SOCKET_RECVLOWATER; 1280 so->so_proto_props.sopp_maxpsz = INFPSZ; 1281 so->so_proto_props.sopp_maxblk = INFPSZ; 1282 1283 return (so); 1284 } 1285 1286 int 1287 socket_init_common(struct sonode *so, struct sonode *pso, int flags, cred_t *cr) 1288 { 1289 int error = 0; 1290 1291 if (pso != NULL) { 1292 /* 1293 * We have a passive open, so inherit basic state from 1294 * the parent (listener). 1295 * 1296 * No need to grab the new sonode's lock, since there is no 1297 * one that can have a reference to it. 1298 */ 1299 mutex_enter(&pso->so_lock); 1300 1301 so->so_state |= SS_ISCONNECTED | (pso->so_state & SS_ASYNC); 1302 so->so_pgrp = pso->so_pgrp; 1303 so->so_rcvtimeo = pso->so_rcvtimeo; 1304 so->so_sndtimeo = pso->so_sndtimeo; 1305 so->so_xpg_rcvbuf = pso->so_xpg_rcvbuf; 1306 /* 1307 * Make note of the socket level options. TCP and IP level 1308 * options are already inherited. We could do all this after 1309 * accept is successful but doing it here simplifies code and 1310 * no harm done for error case. 1311 */ 1312 so->so_options = pso->so_options & (SO_DEBUG|SO_REUSEADDR| 1313 SO_KEEPALIVE|SO_DONTROUTE|SO_BROADCAST|SO_USELOOPBACK| 1314 SO_OOBINLINE|SO_DGRAM_ERRIND|SO_LINGER); 1315 so->so_proto_props = pso->so_proto_props; 1316 so->so_mode = pso->so_mode; 1317 so->so_pollev = pso->so_pollev & SO_POLLEV_ALWAYS; 1318 1319 mutex_exit(&pso->so_lock); 1320 } else { 1321 struct sockparams *sp = so->so_sockparams; 1322 sock_upcalls_t *upcalls_to_use; 1323 1324 /* 1325 * Based on the version number select the right upcalls to 1326 * pass down. Currently we only have one version so choose 1327 * default 1328 */ 1329 upcalls_to_use = &so_upcalls; 1330 1331 /* active open, so create a lower handle */ 1332 so->so_proto_handle = 1333 sp->sp_smod_info->smod_proto_create_func(so->so_family, 1334 so->so_type, so->so_protocol, &so->so_downcalls, 1335 &so->so_mode, &error, flags, cr); 1336 1337 if (so->so_proto_handle == NULL) { 1338 ASSERT(error != 0); 1339 /* 1340 * To be safe; if a lower handle cannot be created, and 1341 * the proto does not give a reason why, assume there 1342 * was a lack of memory. 1343 */ 1344 return ((error == 0) ? ENOMEM : error); 1345 } 1346 ASSERT(so->so_downcalls != NULL); 1347 ASSERT(so->so_downcalls->sd_send != NULL || 1348 so->so_downcalls->sd_send_uio != NULL); 1349 if (so->so_downcalls->sd_recv_uio != NULL) { 1350 ASSERT(so->so_downcalls->sd_poll != NULL); 1351 so->so_pollev |= SO_POLLEV_ALWAYS; 1352 } 1353 1354 (*so->so_downcalls->sd_activate)(so->so_proto_handle, 1355 (sock_upper_handle_t)so, upcalls_to_use, 0, cr); 1356 1357 /* Wildcard */ 1358 1359 /* 1360 * FIXME No need for this, the protocol can deal with it in 1361 * sd_create(). Should update ICMP. 1362 */ 1363 if (so->so_protocol != so->so_sockparams->sp_protocol) { 1364 int protocol = so->so_protocol; 1365 int error; 1366 /* 1367 * Issue SO_PROTOTYPE setsockopt. 1368 */ 1369 error = socket_setsockopt(so, SOL_SOCKET, SO_PROTOTYPE, 1370 &protocol, (t_uscalar_t)sizeof (protocol), cr); 1371 if (error) { 1372 (void) (*so->so_downcalls->sd_close) 1373 (so->so_proto_handle, 0, cr); 1374 1375 mutex_enter(&so->so_lock); 1376 so_rcv_flush(so); 1377 mutex_exit(&so->so_lock); 1378 /* 1379 * Setsockopt often fails with ENOPROTOOPT but 1380 * socket() should fail with 1381 * EPROTONOSUPPORT/EPROTOTYPE. 1382 */ 1383 return (EPROTONOSUPPORT); 1384 } 1385 } 1386 } 1387 1388 if (uioasync.enabled) 1389 sod_sock_init(so); 1390 1391 return (0); 1392 } 1393 1394 /* 1395 * int socket_ioctl_common(struct sonode *so, int cmd, intptr_t arg, int mode, 1396 * struct cred *cr, int32_t *rvalp) 1397 * 1398 * Handle ioctls that manipulate basic socket state; non-blocking, 1399 * async, etc. 1400 * 1401 * Returns: 1402 * < 0 - ioctl was not handle 1403 * >= 0 - ioctl was handled, if > 0, then it is an errno 1404 * 1405 * Notes: 1406 * Assumes the standard receive buffer is used to obtain info for 1407 * NREAD. 1408 */ 1409 /* ARGSUSED */ 1410 int 1411 socket_ioctl_common(struct sonode *so, int cmd, intptr_t arg, int mode, 1412 struct cred *cr, int32_t *rvalp) 1413 { 1414 switch (cmd) { 1415 case SIOCSQPTR: 1416 /* 1417 * SIOCSQPTR is valid only when helper stream is created 1418 * by the protocol. 1419 */ 1420 1421 return (EOPNOTSUPP); 1422 case FIONBIO: { 1423 int32_t value; 1424 1425 if (so_copyin((void *)arg, &value, sizeof (int32_t), 1426 (mode & (int)FKIOCTL))) 1427 return (EFAULT); 1428 1429 mutex_enter(&so->so_lock); 1430 if (value) { 1431 so->so_state |= SS_NDELAY; 1432 } else { 1433 so->so_state &= ~SS_NDELAY; 1434 } 1435 mutex_exit(&so->so_lock); 1436 return (0); 1437 } 1438 case FIOASYNC: { 1439 int32_t value; 1440 1441 if (so_copyin((void *)arg, &value, sizeof (int32_t), 1442 (mode & (int)FKIOCTL))) 1443 return (EFAULT); 1444 1445 mutex_enter(&so->so_lock); 1446 1447 if (value) { 1448 /* Turn on SIGIO */ 1449 so->so_state |= SS_ASYNC; 1450 } else { 1451 /* Turn off SIGIO */ 1452 so->so_state &= ~SS_ASYNC; 1453 } 1454 mutex_exit(&so->so_lock); 1455 1456 return (0); 1457 } 1458 1459 case SIOCSPGRP: 1460 case FIOSETOWN: { 1461 int error; 1462 pid_t pid; 1463 1464 if (so_copyin((void *)arg, &pid, sizeof (pid_t), 1465 (mode & (int)FKIOCTL))) 1466 return (EFAULT); 1467 1468 mutex_enter(&so->so_lock); 1469 error = (pid != so->so_pgrp) ? socket_chgpgrp(so, pid) : 0; 1470 mutex_exit(&so->so_lock); 1471 return (error); 1472 } 1473 case SIOCGPGRP: 1474 case FIOGETOWN: 1475 if (so_copyout(&so->so_pgrp, (void *)arg, 1476 sizeof (pid_t), (mode & (int)FKIOCTL))) 1477 return (EFAULT); 1478 1479 return (0); 1480 case SIOCATMARK: { 1481 int retval; 1482 1483 /* 1484 * Only protocols that support urgent data can handle ATMARK. 1485 */ 1486 if ((so->so_mode & SM_EXDATA) == 0) 1487 return (EINVAL); 1488 1489 /* 1490 * If the protocol is maintaining its own buffer, then the 1491 * request must be passed down. 1492 */ 1493 if (so->so_downcalls->sd_recv_uio != NULL) 1494 return (-1); 1495 1496 retval = (so->so_state & SS_RCVATMARK) != 0; 1497 1498 if (so_copyout(&retval, (void *)arg, sizeof (int), 1499 (mode & (int)FKIOCTL))) { 1500 return (EFAULT); 1501 } 1502 return (0); 1503 } 1504 1505 case FIONREAD: { 1506 int retval; 1507 1508 /* 1509 * If the protocol is maintaining its own buffer, then the 1510 * request must be passed down. 1511 */ 1512 if (so->so_downcalls->sd_recv_uio != NULL) 1513 return (-1); 1514 1515 retval = MIN(so->so_rcv_queued, INT_MAX); 1516 1517 if (so_copyout(&retval, (void *)arg, 1518 sizeof (retval), (mode & (int)FKIOCTL))) { 1519 return (EFAULT); 1520 } 1521 return (0); 1522 } 1523 1524 case _I_GETPEERCRED: { 1525 int error = 0; 1526 1527 if ((mode & FKIOCTL) == 0) 1528 return (EINVAL); 1529 1530 mutex_enter(&so->so_lock); 1531 if ((so->so_mode & SM_CONNREQUIRED) == 0) { 1532 error = ENOTSUP; 1533 } else if ((so->so_state & SS_ISCONNECTED) == 0) { 1534 error = ENOTCONN; 1535 } else if (so->so_peercred != NULL) { 1536 k_peercred_t *kp = (k_peercred_t *)arg; 1537 kp->pc_cr = so->so_peercred; 1538 kp->pc_cpid = so->so_cpid; 1539 crhold(so->so_peercred); 1540 } else { 1541 error = EINVAL; 1542 } 1543 mutex_exit(&so->so_lock); 1544 return (error); 1545 } 1546 default: 1547 return (-1); 1548 } 1549 } 1550 1551 /* 1552 * Handle the I_NREAD STREAM ioctl. 1553 */ 1554 static int 1555 so_strioc_nread(struct sonode *so, intptr_t arg, int mode, int32_t *rvalp) 1556 { 1557 size_t size = 0; 1558 int retval; 1559 int count = 0; 1560 mblk_t *mp; 1561 1562 if (so->so_downcalls == NULL || 1563 so->so_downcalls->sd_recv_uio != NULL) 1564 return (EINVAL); 1565 1566 mutex_enter(&so->so_lock); 1567 /* Wait for reader to get out of the way. */ 1568 while (so->so_flag & SOREADLOCKED) { 1569 /* 1570 * If reader is waiting for data, then there should be nothing 1571 * on the rcv queue. 1572 */ 1573 if (so->so_rcv_wakeup) 1574 goto out; 1575 1576 so->so_flag |= SOWANT; 1577 /* Do a timed sleep, in case the reader goes to sleep. */ 1578 (void) cv_timedwait(&so->so_state_cv, &so->so_lock, 1579 lbolt + drv_usectohz(10)); 1580 } 1581 1582 /* 1583 * Since we are holding so_lock no new reader will come in, and the 1584 * protocol will not be able to enqueue data. So it's safe to walk 1585 * both rcv queues. 1586 */ 1587 mp = so->so_rcv_q_head; 1588 if (mp != NULL) { 1589 size = msgdsize(so->so_rcv_q_head); 1590 for (; mp != NULL; mp = mp->b_next) 1591 count++; 1592 } else { 1593 /* 1594 * In case the processing list was empty, get the size of the 1595 * next msg in line. 1596 */ 1597 size = msgdsize(so->so_rcv_head); 1598 } 1599 1600 for (mp = so->so_rcv_head; mp != NULL; mp = mp->b_next) 1601 count++; 1602 out: 1603 mutex_exit(&so->so_lock); 1604 1605 /* 1606 * Drop down from size_t to the "int" required by the 1607 * interface. Cap at INT_MAX. 1608 */ 1609 retval = MIN(size, INT_MAX); 1610 if (so_copyout(&retval, (void *)arg, sizeof (retval), 1611 (mode & (int)FKIOCTL))) { 1612 return (EFAULT); 1613 } else { 1614 *rvalp = count; 1615 return (0); 1616 } 1617 } 1618 1619 /* 1620 * Process STREAM ioctls. 1621 * 1622 * Returns: 1623 * < 0 - ioctl was not handle 1624 * >= 0 - ioctl was handled, if > 0, then it is an errno 1625 */ 1626 int 1627 socket_strioc_common(struct sonode *so, int cmd, intptr_t arg, int mode, 1628 struct cred *cr, int32_t *rvalp) 1629 { 1630 int retval; 1631 1632 /* Only STREAM iotcls are handled here */ 1633 if ((cmd & 0xffffff00U) != STR) 1634 return (-1); 1635 1636 switch (cmd) { 1637 case I_CANPUT: 1638 /* 1639 * We return an error for I_CANPUT so that isastream(3C) will 1640 * not report the socket as being a STREAM. 1641 */ 1642 return (EOPNOTSUPP); 1643 case I_NREAD: 1644 /* Avoid doing a fallback for I_NREAD. */ 1645 return (so_strioc_nread(so, arg, mode, rvalp)); 1646 case I_LOOK: 1647 /* Avoid doing a fallback for I_LOOK. */ 1648 if (so_copyout("sockmod", (void *)arg, strlen("sockmod") + 1, 1649 (mode & (int)FKIOCTL))) { 1650 return (EFAULT); 1651 } 1652 return (0); 1653 default: 1654 break; 1655 } 1656 1657 /* 1658 * Try to fall back to TPI, and if successful, reissue the ioctl. 1659 */ 1660 if ((retval = so_tpi_fallback(so, cr)) == 0) { 1661 /* Reissue the ioctl */ 1662 ASSERT(so->so_rcv_q_head == NULL); 1663 return (SOP_IOCTL(so, cmd, arg, mode, cr, rvalp)); 1664 } else { 1665 return (retval); 1666 } 1667 } 1668 1669 /* 1670 * This is called for all socket types to verify that the buffer size is large 1671 * enough for the option, and if we can, handle the request as well. Most 1672 * options will be forwarded to the protocol. 1673 */ 1674 int 1675 socket_getopt_common(struct sonode *so, int level, int option_name, 1676 void *optval, socklen_t *optlenp, int flags) 1677 { 1678 if (level != SOL_SOCKET) 1679 return (-1); 1680 1681 switch (option_name) { 1682 case SO_ERROR: 1683 case SO_DOMAIN: 1684 case SO_TYPE: 1685 case SO_ACCEPTCONN: { 1686 int32_t value; 1687 socklen_t optlen = *optlenp; 1688 1689 if (optlen < (t_uscalar_t)sizeof (int32_t)) { 1690 return (EINVAL); 1691 } 1692 1693 switch (option_name) { 1694 case SO_ERROR: 1695 mutex_enter(&so->so_lock); 1696 value = sogeterr(so, B_TRUE); 1697 mutex_exit(&so->so_lock); 1698 break; 1699 case SO_DOMAIN: 1700 value = so->so_family; 1701 break; 1702 case SO_TYPE: 1703 value = so->so_type; 1704 break; 1705 case SO_ACCEPTCONN: 1706 if (so->so_state & SS_ACCEPTCONN) 1707 value = SO_ACCEPTCONN; 1708 else 1709 value = 0; 1710 break; 1711 } 1712 1713 bcopy(&value, optval, sizeof (value)); 1714 *optlenp = sizeof (value); 1715 1716 return (0); 1717 } 1718 case SO_SNDTIMEO: 1719 case SO_RCVTIMEO: { 1720 clock_t value; 1721 socklen_t optlen = *optlenp; 1722 1723 if (get_udatamodel() == DATAMODEL_NONE || 1724 get_udatamodel() == DATAMODEL_NATIVE) { 1725 if (optlen < sizeof (struct timeval)) 1726 return (EINVAL); 1727 } else { 1728 if (optlen < sizeof (struct timeval32)) 1729 return (EINVAL); 1730 } 1731 if (option_name == SO_RCVTIMEO) 1732 value = drv_hztousec(so->so_rcvtimeo); 1733 else 1734 value = drv_hztousec(so->so_sndtimeo); 1735 1736 if (get_udatamodel() == DATAMODEL_NONE || 1737 get_udatamodel() == DATAMODEL_NATIVE) { 1738 ((struct timeval *)(optval))->tv_sec = 1739 value / (1000 * 1000); 1740 ((struct timeval *)(optval))->tv_usec = 1741 value % (1000 * 1000); 1742 *optlenp = sizeof (struct timeval); 1743 } else { 1744 ((struct timeval32 *)(optval))->tv_sec = 1745 value / (1000 * 1000); 1746 ((struct timeval32 *)(optval))->tv_usec = 1747 value % (1000 * 1000); 1748 *optlenp = sizeof (struct timeval32); 1749 } 1750 return (0); 1751 } 1752 case SO_DEBUG: 1753 case SO_REUSEADDR: 1754 case SO_KEEPALIVE: 1755 case SO_DONTROUTE: 1756 case SO_BROADCAST: 1757 case SO_USELOOPBACK: 1758 case SO_OOBINLINE: 1759 case SO_SNDBUF: 1760 #ifdef notyet 1761 case SO_SNDLOWAT: 1762 case SO_RCVLOWAT: 1763 #endif /* notyet */ 1764 case SO_DGRAM_ERRIND: { 1765 socklen_t optlen = *optlenp; 1766 1767 if (optlen < (t_uscalar_t)sizeof (int32_t)) 1768 return (EINVAL); 1769 break; 1770 } 1771 case SO_RCVBUF: { 1772 socklen_t optlen = *optlenp; 1773 1774 if (optlen < (t_uscalar_t)sizeof (int32_t)) 1775 return (EINVAL); 1776 1777 if ((flags & _SOGETSOCKOPT_XPG4_2) && so->so_xpg_rcvbuf != 0) { 1778 /* 1779 * XXX If SO_RCVBUF has been set and this is an 1780 * XPG 4.2 application then do not ask the transport 1781 * since the transport might adjust the value and not 1782 * return exactly what was set by the application. 1783 * For non-XPG 4.2 application we return the value 1784 * that the transport is actually using. 1785 */ 1786 *(int32_t *)optval = so->so_xpg_rcvbuf; 1787 *optlenp = sizeof (so->so_xpg_rcvbuf); 1788 return (0); 1789 } 1790 /* 1791 * If the option has not been set then get a default 1792 * value from the transport. 1793 */ 1794 break; 1795 } 1796 case SO_LINGER: { 1797 socklen_t optlen = *optlenp; 1798 1799 if (optlen < (t_uscalar_t)sizeof (struct linger)) 1800 return (EINVAL); 1801 break; 1802 } 1803 case SO_SND_BUFINFO: { 1804 socklen_t optlen = *optlenp; 1805 1806 if (optlen < (t_uscalar_t)sizeof (struct so_snd_bufinfo)) 1807 return (EINVAL); 1808 ((struct so_snd_bufinfo *)(optval))->sbi_wroff = 1809 (so->so_proto_props).sopp_wroff; 1810 ((struct so_snd_bufinfo *)(optval))->sbi_maxblk = 1811 (so->so_proto_props).sopp_maxblk; 1812 ((struct so_snd_bufinfo *)(optval))->sbi_maxpsz = 1813 (so->so_proto_props).sopp_maxpsz; 1814 ((struct so_snd_bufinfo *)(optval))->sbi_tail = 1815 (so->so_proto_props).sopp_tail; 1816 *optlenp = sizeof (struct so_snd_bufinfo); 1817 return (0); 1818 } 1819 default: 1820 break; 1821 } 1822 1823 /* Unknown Option */ 1824 return (-1); 1825 } 1826 1827 void 1828 socket_sonode_destroy(struct sonode *so) 1829 { 1830 sonode_fini(so); 1831 kmem_cache_free(socket_cache, so); 1832 } 1833 1834 int 1835 so_zcopy_wait(struct sonode *so) 1836 { 1837 int error = 0; 1838 1839 mutex_enter(&so->so_lock); 1840 while (!(so->so_copyflag & STZCNOTIFY)) { 1841 if (so->so_state & SS_CLOSING) { 1842 mutex_exit(&so->so_lock); 1843 return (EINTR); 1844 } 1845 if (cv_wait_sig(&so->so_copy_cv, &so->so_lock) == 0) { 1846 error = EINTR; 1847 break; 1848 } 1849 } 1850 so->so_copyflag &= ~STZCNOTIFY; 1851 mutex_exit(&so->so_lock); 1852 return (error); 1853 } 1854 1855 void 1856 so_timer_callback(void *arg) 1857 { 1858 struct sonode *so = (struct sonode *)arg; 1859 1860 mutex_enter(&so->so_lock); 1861 1862 so->so_rcv_timer_tid = 0; 1863 if (so->so_rcv_queued > 0) { 1864 so_notify_data(so, so->so_rcv_queued); 1865 } else { 1866 mutex_exit(&so->so_lock); 1867 } 1868 } 1869 1870 #ifdef DEBUG 1871 /* 1872 * Verify that the length stored in so_rcv_queued and the length of data blocks 1873 * queued is same. 1874 */ 1875 static boolean_t 1876 so_check_length(sonode_t *so) 1877 { 1878 mblk_t *mp = so->so_rcv_q_head; 1879 int len = 0; 1880 1881 ASSERT(MUTEX_HELD(&so->so_lock)); 1882 1883 if (mp != NULL) { 1884 len = msgdsize(mp); 1885 while ((mp = mp->b_next) != NULL) 1886 len += msgdsize(mp); 1887 } 1888 mp = so->so_rcv_head; 1889 if (mp != NULL) { 1890 len += msgdsize(mp); 1891 while ((mp = mp->b_next) != NULL) 1892 len += msgdsize(mp); 1893 } 1894 return ((len == so->so_rcv_queued) ? B_TRUE : B_FALSE); 1895 } 1896 #endif 1897 1898 int 1899 so_get_mod_version(struct sockparams *sp) 1900 { 1901 ASSERT(sp != NULL && sp->sp_smod_info != NULL); 1902 return (sp->sp_smod_info->smod_version); 1903 } 1904 1905 /* 1906 * so_start_fallback() 1907 * 1908 * Block new socket operations from coming in, and wait for active operations 1909 * to complete. Threads that are sleeping will be woken up so they can get 1910 * out of the way. 1911 * 1912 * The caller must be a reader on so_fallback_rwlock. 1913 */ 1914 static boolean_t 1915 so_start_fallback(struct sonode *so) 1916 { 1917 ASSERT(RW_READ_HELD(&so->so_fallback_rwlock)); 1918 1919 mutex_enter(&so->so_lock); 1920 if (so->so_state & SS_FALLBACK_PENDING) { 1921 mutex_exit(&so->so_lock); 1922 return (B_FALSE); 1923 } 1924 so->so_state |= SS_FALLBACK_PENDING; 1925 /* 1926 * Poke all threads that might be sleeping. Any operation that comes 1927 * in after the cv_broadcast will observe the fallback pending flag 1928 * which cause the call to return where it would normally sleep. 1929 */ 1930 cv_broadcast(&so->so_state_cv); /* threads in connect() */ 1931 cv_broadcast(&so->so_rcv_cv); /* threads in recvmsg() */ 1932 cv_broadcast(&so->so_snd_cv); /* threads in sendmsg() */ 1933 mutex_enter(&so->so_acceptq_lock); 1934 cv_broadcast(&so->so_acceptq_cv); /* threads in accept() */ 1935 mutex_exit(&so->so_acceptq_lock); 1936 mutex_exit(&so->so_lock); 1937 1938 /* 1939 * The main reason for the rw_tryupgrade call is to provide 1940 * observability during the fallback process. We want to 1941 * be able to see if there are pending operations. 1942 */ 1943 if (rw_tryupgrade(&so->so_fallback_rwlock) == 0) { 1944 /* 1945 * It is safe to drop and reaquire the fallback lock, because 1946 * we are guaranteed that another fallback cannot take place. 1947 */ 1948 rw_exit(&so->so_fallback_rwlock); 1949 DTRACE_PROBE1(pending__ops__wait, (struct sonode *), so); 1950 rw_enter(&so->so_fallback_rwlock, RW_WRITER); 1951 DTRACE_PROBE1(pending__ops__complete, (struct sonode *), so); 1952 } 1953 1954 return (B_TRUE); 1955 } 1956 1957 /* 1958 * so_end_fallback() 1959 * 1960 * Allow socket opertions back in. 1961 * 1962 * The caller must be a writer on so_fallback_rwlock. 1963 */ 1964 static void 1965 so_end_fallback(struct sonode *so) 1966 { 1967 ASSERT(RW_ISWRITER(&so->so_fallback_rwlock)); 1968 1969 mutex_enter(&so->so_lock); 1970 so->so_state &= ~(SS_FALLBACK_PENDING|SS_FALLBACK_DRAIN); 1971 mutex_exit(&so->so_lock); 1972 1973 rw_downgrade(&so->so_fallback_rwlock); 1974 } 1975 1976 /* 1977 * so_quiesced_cb() 1978 * 1979 * Callback passed to the protocol during fallback. It is called once 1980 * the endpoint is quiescent. 1981 * 1982 * No requests from the user, no notifications from the protocol, so it 1983 * is safe to synchronize the state. Data can also be moved without 1984 * risk for reordering. 1985 * 1986 * We do not need to hold so_lock, since there can be only one thread 1987 * operating on the sonode. 1988 */ 1989 static void 1990 so_quiesced_cb(sock_upper_handle_t sock_handle, queue_t *q, 1991 struct T_capability_ack *tcap, struct sockaddr *laddr, socklen_t laddrlen, 1992 struct sockaddr *faddr, socklen_t faddrlen, short opts) 1993 { 1994 struct sonode *so = (struct sonode *)sock_handle; 1995 boolean_t atmark; 1996 1997 sotpi_update_state(so, tcap, laddr, laddrlen, faddr, faddrlen, opts); 1998 1999 /* 2000 * Some protocols do not quiece the data path during fallback. Once 2001 * we set the SS_FALLBACK_DRAIN flag any attempt to queue data will 2002 * fail and the protocol is responsible for saving the data for later 2003 * delivery (i.e., once the fallback has completed). 2004 */ 2005 mutex_enter(&so->so_lock); 2006 so->so_state |= SS_FALLBACK_DRAIN; 2007 SOCKET_TIMER_CANCEL(so); 2008 mutex_exit(&so->so_lock); 2009 2010 if (so->so_rcv_head != NULL) { 2011 if (so->so_rcv_q_last_head == NULL) 2012 so->so_rcv_q_head = so->so_rcv_head; 2013 else 2014 so->so_rcv_q_last_head->b_next = so->so_rcv_head; 2015 so->so_rcv_q_last_head = so->so_rcv_last_head; 2016 } 2017 2018 atmark = (so->so_state & SS_RCVATMARK) != 0; 2019 /* 2020 * Clear any OOB state having to do with pending data. The TPI 2021 * code path will set the appropriate oob state when we move the 2022 * oob data to the STREAM head. We leave SS_HADOOBDATA since the oob 2023 * data has already been consumed. 2024 */ 2025 so->so_state &= ~(SS_RCVATMARK|SS_OOBPEND|SS_HAVEOOBDATA); 2026 2027 ASSERT(so->so_oobmsg != NULL || so->so_oobmark <= so->so_rcv_queued); 2028 2029 /* 2030 * Move data to the STREAM head. 2031 */ 2032 while (so->so_rcv_q_head != NULL) { 2033 mblk_t *mp = so->so_rcv_q_head; 2034 size_t mlen = msgdsize(mp); 2035 2036 so->so_rcv_q_head = mp->b_next; 2037 mp->b_next = NULL; 2038 mp->b_prev = NULL; 2039 2040 /* 2041 * Send T_EXDATA_IND if we are at the oob mark. 2042 */ 2043 if (atmark) { 2044 struct T_exdata_ind *tei; 2045 mblk_t *mp1 = SOTOTPI(so)->sti_exdata_mp; 2046 2047 SOTOTPI(so)->sti_exdata_mp = NULL; 2048 ASSERT(mp1 != NULL); 2049 mp1->b_datap->db_type = M_PROTO; 2050 tei = (struct T_exdata_ind *)mp1->b_rptr; 2051 tei->PRIM_type = T_EXDATA_IND; 2052 tei->MORE_flag = 0; 2053 mp1->b_wptr = (uchar_t *)&tei[1]; 2054 2055 if (IS_SO_OOB_INLINE(so)) { 2056 mp1->b_cont = mp; 2057 } else { 2058 ASSERT(so->so_oobmsg != NULL); 2059 mp1->b_cont = so->so_oobmsg; 2060 so->so_oobmsg = NULL; 2061 2062 /* process current mp next time around */ 2063 mp->b_next = so->so_rcv_q_head; 2064 so->so_rcv_q_head = mp; 2065 mlen = 0; 2066 } 2067 mp = mp1; 2068 2069 /* we have consumed the oob mark */ 2070 atmark = B_FALSE; 2071 } else if (so->so_oobmark > 0) { 2072 /* 2073 * Check if the OOB mark is within the current 2074 * mblk chain. In that case we have to split it up. 2075 */ 2076 if (so->so_oobmark < mlen) { 2077 mblk_t *urg_mp = mp; 2078 2079 atmark = B_TRUE; 2080 mp = NULL; 2081 mlen = so->so_oobmark; 2082 2083 /* 2084 * It is assumed that the OOB mark does 2085 * not land within a mblk. 2086 */ 2087 do { 2088 so->so_oobmark -= MBLKL(urg_mp); 2089 mp = urg_mp; 2090 urg_mp = urg_mp->b_cont; 2091 } while (so->so_oobmark > 0); 2092 mp->b_cont = NULL; 2093 if (urg_mp != NULL) { 2094 urg_mp->b_next = so->so_rcv_q_head; 2095 so->so_rcv_q_head = urg_mp; 2096 } 2097 } else { 2098 so->so_oobmark -= mlen; 2099 if (so->so_oobmark == 0) 2100 atmark = B_TRUE; 2101 } 2102 } 2103 2104 /* 2105 * Queue data on the STREAM head. 2106 */ 2107 so->so_rcv_queued -= mlen; 2108 putnext(q, mp); 2109 } 2110 so->so_rcv_head = NULL; 2111 so->so_rcv_last_head = NULL; 2112 so->so_rcv_q_head = NULL; 2113 so->so_rcv_q_last_head = NULL; 2114 2115 /* 2116 * Check if the oob byte is at the end of the data stream, or if the 2117 * oob byte has not yet arrived. In the latter case we have to send a 2118 * SIGURG and a mark indicator to the STREAM head. The mark indicator 2119 * is needed to guarantee correct behavior for SIOCATMARK. See block 2120 * comment in socktpi.h for more details. 2121 */ 2122 if (atmark || so->so_oobmark > 0) { 2123 mblk_t *mp; 2124 2125 if (atmark && so->so_oobmsg != NULL) { 2126 struct T_exdata_ind *tei; 2127 2128 mp = SOTOTPI(so)->sti_exdata_mp; 2129 SOTOTPI(so)->sti_exdata_mp = NULL; 2130 ASSERT(mp != NULL); 2131 mp->b_datap->db_type = M_PROTO; 2132 tei = (struct T_exdata_ind *)mp->b_rptr; 2133 tei->PRIM_type = T_EXDATA_IND; 2134 tei->MORE_flag = 0; 2135 mp->b_wptr = (uchar_t *)&tei[1]; 2136 2137 mp->b_cont = so->so_oobmsg; 2138 so->so_oobmsg = NULL; 2139 2140 putnext(q, mp); 2141 } else { 2142 /* Send up the signal */ 2143 mp = SOTOTPI(so)->sti_exdata_mp; 2144 SOTOTPI(so)->sti_exdata_mp = NULL; 2145 ASSERT(mp != NULL); 2146 DB_TYPE(mp) = M_PCSIG; 2147 *mp->b_wptr++ = (uchar_t)SIGURG; 2148 putnext(q, mp); 2149 2150 /* Send up the mark indicator */ 2151 mp = SOTOTPI(so)->sti_urgmark_mp; 2152 SOTOTPI(so)->sti_urgmark_mp = NULL; 2153 mp->b_flag = atmark ? MSGMARKNEXT : MSGNOTMARKNEXT; 2154 putnext(q, mp); 2155 2156 so->so_oobmark = 0; 2157 } 2158 } 2159 2160 if (SOTOTPI(so)->sti_exdata_mp != NULL) { 2161 freeb(SOTOTPI(so)->sti_exdata_mp); 2162 SOTOTPI(so)->sti_exdata_mp = NULL; 2163 } 2164 2165 if (SOTOTPI(so)->sti_urgmark_mp != NULL) { 2166 freeb(SOTOTPI(so)->sti_urgmark_mp); 2167 SOTOTPI(so)->sti_urgmark_mp = NULL; 2168 } 2169 2170 ASSERT(so->so_oobmark == 0); 2171 ASSERT(so->so_rcv_queued == 0); 2172 } 2173 2174 #ifdef DEBUG 2175 /* 2176 * Do an integrity check of the sonode. This should be done if a 2177 * fallback fails after sonode has initially been converted to use 2178 * TPI and subsequently have to be reverted. 2179 * 2180 * Failure to pass the integrity check will panic the system. 2181 */ 2182 void 2183 so_integrity_check(struct sonode *cur, struct sonode *orig) 2184 { 2185 VERIFY(cur->so_vnode == orig->so_vnode); 2186 VERIFY(cur->so_ops == orig->so_ops); 2187 /* 2188 * For so_state we can only VERIFY the state flags in CHECK_STATE. 2189 * The other state flags might be affected by a notification from the 2190 * protocol. 2191 */ 2192 #define CHECK_STATE (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_NDELAY|SS_NONBLOCK| \ 2193 SS_ASYNC|SS_ACCEPTCONN|SS_SAVEDEOR|SS_RCVATMARK|SS_OOBPEND| \ 2194 SS_HAVEOOBDATA|SS_HADOOBDATA|SS_SENTLASTREADSIG|SS_SENTLASTWRITESIG) 2195 VERIFY((cur->so_state & (orig->so_state & CHECK_STATE)) == 2196 (orig->so_state & CHECK_STATE)); 2197 VERIFY(cur->so_mode == orig->so_mode); 2198 VERIFY(cur->so_flag == orig->so_flag); 2199 VERIFY(cur->so_count == orig->so_count); 2200 /* Cannot VERIFY so_proto_connid; proto can update it */ 2201 VERIFY(cur->so_sockparams == orig->so_sockparams); 2202 /* an error might have been recorded, but it can not be lost */ 2203 VERIFY(cur->so_error != 0 || orig->so_error == 0); 2204 VERIFY(cur->so_family == orig->so_family); 2205 VERIFY(cur->so_type == orig->so_type); 2206 VERIFY(cur->so_protocol == orig->so_protocol); 2207 VERIFY(cur->so_version == orig->so_version); 2208 /* New conns might have arrived, but none should have been lost */ 2209 VERIFY(cur->so_acceptq_len >= orig->so_acceptq_len); 2210 VERIFY(cur->so_acceptq_head == orig->so_acceptq_head); 2211 VERIFY(cur->so_backlog == orig->so_backlog); 2212 /* New OOB migth have arrived, but mark should not have been lost */ 2213 VERIFY(cur->so_oobmark >= orig->so_oobmark); 2214 /* Cannot VERIFY so_oobmsg; the proto might have sent up a new one */ 2215 VERIFY(cur->so_pgrp == orig->so_pgrp); 2216 VERIFY(cur->so_peercred == orig->so_peercred); 2217 VERIFY(cur->so_cpid == orig->so_cpid); 2218 VERIFY(cur->so_zoneid == orig->so_zoneid); 2219 /* New data migth have arrived, but none should have been lost */ 2220 VERIFY(cur->so_rcv_queued >= orig->so_rcv_queued); 2221 VERIFY(cur->so_rcv_q_head == orig->so_rcv_q_head); 2222 VERIFY(cur->so_rcv_head == orig->so_rcv_head); 2223 VERIFY(cur->so_proto_handle == orig->so_proto_handle); 2224 VERIFY(cur->so_downcalls == orig->so_downcalls); 2225 /* Cannot VERIFY so_proto_props; they can be updated by proto */ 2226 } 2227 #endif 2228 2229 /* 2230 * so_tpi_fallback() 2231 * 2232 * This is the fallback initation routine; things start here. 2233 * 2234 * Basic strategy: 2235 * o Block new socket operations from coming in 2236 * o Allocate/initate info needed by TPI 2237 * o Quiesce the connection, at which point we sync 2238 * state and move data 2239 * o Change operations (sonodeops) associated with the socket 2240 * o Unblock threads waiting for the fallback to finish 2241 */ 2242 int 2243 so_tpi_fallback(struct sonode *so, struct cred *cr) 2244 { 2245 int error; 2246 queue_t *q; 2247 struct sockparams *sp; 2248 struct sockparams *newsp = NULL; 2249 so_proto_fallback_func_t fbfunc; 2250 boolean_t direct; 2251 struct sonode *nso; 2252 #ifdef DEBUG 2253 struct sonode origso; 2254 #endif 2255 error = 0; 2256 sp = so->so_sockparams; 2257 fbfunc = sp->sp_smod_info->smod_proto_fallback_func; 2258 2259 /* 2260 * Fallback can only happen if there is a device associated 2261 * with the sonode, and the socket module has a fallback function. 2262 */ 2263 if (!SOCKPARAMS_HAS_DEVICE(sp) || fbfunc == NULL) 2264 return (EINVAL); 2265 2266 /* 2267 * Initiate fallback; upon success we know that no new requests 2268 * will come in from the user. 2269 */ 2270 if (!so_start_fallback(so)) 2271 return (EAGAIN); 2272 #ifdef DEBUG 2273 /* 2274 * Make a copy of the sonode in case we need to make an integrity 2275 * check later on. 2276 */ 2277 bcopy(so, &origso, sizeof (*so)); 2278 #endif 2279 2280 sp->sp_stats.sps_nfallback.value.ui64++; 2281 2282 newsp = sockparams_hold_ephemeral_bydev(so->so_family, so->so_type, 2283 so->so_protocol, so->so_sockparams->sp_sdev_info.sd_devpath, 2284 KM_SLEEP, &error); 2285 if (error != 0) 2286 goto out; 2287 2288 if (so->so_direct != NULL) { 2289 sodirect_t *sodp = so->so_direct; 2290 mutex_enter(&so->so_lock); 2291 2292 so->so_direct->sod_enabled = B_FALSE; 2293 so->so_state &= ~SS_SODIRECT; 2294 ASSERT(sodp->sod_uioafh == NULL); 2295 mutex_exit(&so->so_lock); 2296 } 2297 2298 /* Turn sonode into a TPI socket */ 2299 error = sotpi_convert_sonode(so, newsp, &direct, &q, cr); 2300 if (error != 0) 2301 goto out; 2302 2303 2304 /* 2305 * Now tell the protocol to start using TPI. so_quiesced_cb be 2306 * called once it's safe to synchronize state. 2307 */ 2308 DTRACE_PROBE1(proto__fallback__begin, struct sonode *, so); 2309 error = (*fbfunc)(so->so_proto_handle, q, direct, so_quiesced_cb); 2310 DTRACE_PROBE1(proto__fallback__end, struct sonode *, so); 2311 2312 if (error != 0) { 2313 /* protocol was unable to do a fallback, revert the sonode */ 2314 sotpi_revert_sonode(so, cr); 2315 goto out; 2316 } 2317 2318 /* 2319 * Walk the accept queue and notify the proto that they should 2320 * fall back to TPI. The protocol will send up the T_CONN_IND. 2321 */ 2322 nso = so->so_acceptq_head; 2323 while (nso != NULL) { 2324 int rval; 2325 2326 DTRACE_PROBE1(proto__fallback__begin, struct sonode *, nso); 2327 rval = (*fbfunc)(nso->so_proto_handle, NULL, direct, NULL); 2328 DTRACE_PROBE1(proto__fallback__end, struct sonode *, nso); 2329 if (rval != 0) { 2330 zcmn_err(getzoneid(), CE_WARN, 2331 "Failed to convert socket in accept queue to TPI. " 2332 "Pid = %d\n", curproc->p_pid); 2333 } 2334 nso = nso->so_acceptq_next; 2335 } 2336 2337 /* 2338 * Now flush the acceptq, this will destroy all sockets. They will 2339 * be recreated in sotpi_accept(). 2340 */ 2341 so_acceptq_flush(so, B_FALSE); 2342 2343 mutex_enter(&so->so_lock); 2344 so->so_state |= SS_FALLBACK_COMP; 2345 mutex_exit(&so->so_lock); 2346 2347 /* 2348 * Swap the sonode ops. Socket opertations that come in once this 2349 * is done will proceed without blocking. 2350 */ 2351 so->so_ops = &sotpi_sonodeops; 2352 2353 /* 2354 * Wake up any threads stuck in poll. This is needed since the poll 2355 * head changes when the fallback happens (moves from the sonode to 2356 * the STREAMS head). 2357 */ 2358 pollwakeup(&so->so_poll_list, POLLERR); 2359 out: 2360 so_end_fallback(so); 2361 2362 if (error != 0) { 2363 #ifdef DEBUG 2364 so_integrity_check(so, &origso); 2365 #endif 2366 zcmn_err(getzoneid(), CE_WARN, 2367 "Failed to convert socket to TPI (err=%d). Pid = %d\n", 2368 error, curproc->p_pid); 2369 if (newsp != NULL) 2370 SOCKPARAMS_DEC_REF(newsp); 2371 } 2372 2373 return (error); 2374 } 2375