1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 /* 26 * Copyright 2014, OmniTI Computer Consulting, Inc. All rights reserved. 27 * Copyright 2019 Joyent, Inc. 28 */ 29 30 #include <sys/types.h> 31 #include <sys/param.h> 32 #include <sys/signal.h> 33 #include <sys/cmn_err.h> 34 35 #include <sys/stropts.h> 36 #include <sys/socket.h> 37 #include <sys/socketvar.h> 38 #include <sys/sockio.h> 39 #include <sys/strsubr.h> 40 #include <sys/strsun.h> 41 #include <sys/atomic.h> 42 #include <sys/tihdr.h> 43 44 #include <fs/sockfs/sockcommon.h> 45 #include <fs/sockfs/sockfilter_impl.h> 46 #include <fs/sockfs/socktpi.h> 47 #include <fs/sockfs/sodirect.h> 48 #include <sys/ddi.h> 49 #include <inet/ip.h> 50 #include <sys/time.h> 51 #include <sys/cmn_err.h> 52 53 #ifdef SOCK_TEST 54 extern int do_useracc; 55 extern clock_t sock_test_timelimit; 56 #endif /* SOCK_TEST */ 57 58 #define MBLK_PULL_LEN 64 59 uint32_t so_mblk_pull_len = MBLK_PULL_LEN; 60 61 #ifdef DEBUG 62 boolean_t so_debug_length = B_FALSE; 63 static boolean_t so_check_length(sonode_t *so); 64 #endif 65 66 static int 67 so_acceptq_dequeue_locked(struct sonode *so, boolean_t dontblock, 68 struct sonode **nsop) 69 { 70 struct sonode *nso = NULL; 71 72 *nsop = NULL; 73 ASSERT(MUTEX_HELD(&so->so_acceptq_lock)); 74 while ((nso = list_remove_head(&so->so_acceptq_list)) == NULL) { 75 /* 76 * No need to check so_error here, because it is not 77 * possible for a listening socket to be reset or otherwise 78 * disconnected. 79 * 80 * So now we just need check if it's ok to wait. 81 */ 82 if (dontblock) 83 return (EWOULDBLOCK); 84 if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING)) 85 return (EINTR); 86 87 if (cv_wait_sig_swap(&so->so_acceptq_cv, 88 &so->so_acceptq_lock) == 0) 89 return (EINTR); 90 } 91 92 ASSERT(nso != NULL); 93 ASSERT(so->so_acceptq_len > 0); 94 so->so_acceptq_len--; 95 nso->so_listener = NULL; 96 97 *nsop = nso; 98 99 return (0); 100 } 101 102 /* 103 * int so_acceptq_dequeue(struct sonode *, boolean_t, struct sonode **) 104 * 105 * Pulls a connection off of the accept queue. 106 * 107 * Arguments: 108 * so - listening socket 109 * dontblock - indicate whether it's ok to sleep if there are no 110 * connections on the queue 111 * nsop - Value-return argument 112 * 113 * Return values: 114 * 0 when a connection is successfully dequeued, in which case nsop 115 * is set to point to the new connection. Upon failure a non-zero 116 * value is returned, and the value of nsop is set to NULL. 117 * 118 * Note: 119 * so_acceptq_dequeue() may return prematurly if the socket is falling 120 * back to TPI. 121 */ 122 int 123 so_acceptq_dequeue(struct sonode *so, boolean_t dontblock, 124 struct sonode **nsop) 125 { 126 int error; 127 128 mutex_enter(&so->so_acceptq_lock); 129 error = so_acceptq_dequeue_locked(so, dontblock, nsop); 130 mutex_exit(&so->so_acceptq_lock); 131 132 return (error); 133 } 134 135 static void 136 so_acceptq_flush_impl(struct sonode *so, list_t *list, boolean_t doclose) 137 { 138 struct sonode *nso; 139 140 while ((nso = list_remove_head(list)) != NULL) { 141 nso->so_listener = NULL; 142 if (doclose) { 143 (void) socket_close(nso, 0, CRED()); 144 } else { 145 /* 146 * Only used for fallback - not possible when filters 147 * are present. 148 */ 149 ASSERT(so->so_filter_active == 0); 150 /* 151 * Since the socket is on the accept queue, there can 152 * only be one reference. We drop the reference and 153 * just blow off the socket. 154 */ 155 ASSERT(nso->so_count == 1); 156 nso->so_count--; 157 /* drop the proto ref */ 158 VN_RELE(SOTOV(nso)); 159 } 160 socket_destroy(nso); 161 } 162 } 163 /* 164 * void so_acceptq_flush(struct sonode *so) 165 * 166 * Removes all pending connections from a listening socket, and 167 * frees the associated resources. 168 * 169 * Arguments 170 * so - listening socket 171 * doclose - make a close downcall for each socket on the accept queue 172 * 173 * Return values: 174 * None. 175 * 176 * Note: 177 * The caller has to ensure that no calls to so_acceptq_enqueue() or 178 * so_acceptq_dequeue() occur while the accept queue is being flushed. 179 * So either the socket needs to be in a state where no operations 180 * would come in, or so_lock needs to be obtained. 181 */ 182 void 183 so_acceptq_flush(struct sonode *so, boolean_t doclose) 184 { 185 so_acceptq_flush_impl(so, &so->so_acceptq_list, doclose); 186 so_acceptq_flush_impl(so, &so->so_acceptq_defer, doclose); 187 188 so->so_acceptq_len = 0; 189 } 190 191 int 192 so_wait_connected_locked(struct sonode *so, boolean_t nonblock, 193 sock_connid_t id) 194 { 195 ASSERT(MUTEX_HELD(&so->so_lock)); 196 197 /* 198 * The protocol has notified us that a connection attempt is being 199 * made, so before we wait for a notification to arrive we must 200 * clear out any errors associated with earlier connection attempts. 201 */ 202 if (so->so_error != 0 && SOCK_CONNID_LT(so->so_proto_connid, id)) 203 so->so_error = 0; 204 205 while (SOCK_CONNID_LT(so->so_proto_connid, id)) { 206 if (nonblock) 207 return (EINPROGRESS); 208 209 if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING)) 210 return (EINTR); 211 212 if (cv_wait_sig_swap(&so->so_state_cv, &so->so_lock) == 0) 213 return (EINTR); 214 } 215 216 if (so->so_error != 0) 217 return (sogeterr(so, B_TRUE)); 218 /* 219 * Under normal circumstances, so_error should contain an error 220 * in case the connect failed. However, it is possible for another 221 * thread to come in a consume the error, so generate a sensible 222 * error in that case. 223 */ 224 if ((so->so_state & SS_ISCONNECTED) == 0) 225 return (ECONNREFUSED); 226 227 return (0); 228 } 229 230 /* 231 * int so_wait_connected(struct sonode *so, boolean_t nonblock, 232 * sock_connid_t id) 233 * 234 * Wait until the socket is connected or an error has occured. 235 * 236 * Arguments: 237 * so - socket 238 * nonblock - indicate whether it's ok to sleep if the connection has 239 * not yet been established 240 * gen - generation number that was returned by the protocol 241 * when the operation was started 242 * 243 * Returns: 244 * 0 if the connection attempt was successful, or an error indicating why 245 * the connection attempt failed. 246 */ 247 int 248 so_wait_connected(struct sonode *so, boolean_t nonblock, sock_connid_t id) 249 { 250 int error; 251 252 mutex_enter(&so->so_lock); 253 error = so_wait_connected_locked(so, nonblock, id); 254 mutex_exit(&so->so_lock); 255 256 return (error); 257 } 258 259 int 260 so_snd_wait_qnotfull_locked(struct sonode *so, boolean_t dontblock) 261 { 262 int error; 263 264 ASSERT(MUTEX_HELD(&so->so_lock)); 265 while (SO_SND_FLOWCTRLD(so)) { 266 if (so->so_state & SS_CANTSENDMORE) 267 return (EPIPE); 268 if (dontblock) 269 return (EWOULDBLOCK); 270 271 if (so->so_state & (SS_CLOSING | SS_FALLBACK_PENDING)) 272 return (EINTR); 273 274 if (so->so_sndtimeo == 0) { 275 /* 276 * Zero means disable timeout. 277 */ 278 error = cv_wait_sig(&so->so_snd_cv, &so->so_lock); 279 } else { 280 error = cv_reltimedwait_sig(&so->so_snd_cv, 281 &so->so_lock, so->so_sndtimeo, TR_CLOCK_TICK); 282 } 283 if (error == 0) 284 return (EINTR); 285 else if (error == -1) 286 return (EAGAIN); 287 } 288 return (0); 289 } 290 291 /* 292 * int so_wait_sendbuf(struct sonode *so, boolean_t dontblock) 293 * 294 * Wait for the transport to notify us about send buffers becoming 295 * available. 296 */ 297 int 298 so_snd_wait_qnotfull(struct sonode *so, boolean_t dontblock) 299 { 300 int error = 0; 301 302 mutex_enter(&so->so_lock); 303 so->so_snd_wakeup = B_TRUE; 304 error = so_snd_wait_qnotfull_locked(so, dontblock); 305 so->so_snd_wakeup = B_FALSE; 306 mutex_exit(&so->so_lock); 307 308 return (error); 309 } 310 311 void 312 so_snd_qfull(struct sonode *so) 313 { 314 mutex_enter(&so->so_lock); 315 so->so_snd_qfull = B_TRUE; 316 mutex_exit(&so->so_lock); 317 } 318 319 void 320 so_snd_qnotfull(struct sonode *so) 321 { 322 mutex_enter(&so->so_lock); 323 so->so_snd_qfull = B_FALSE; 324 /* wake up everyone waiting for buffers */ 325 cv_broadcast(&so->so_snd_cv); 326 mutex_exit(&so->so_lock); 327 } 328 329 /* 330 * Change the process/process group to which SIGIO is sent. 331 */ 332 int 333 socket_chgpgrp(struct sonode *so, pid_t pid) 334 { 335 int error; 336 337 ASSERT(MUTEX_HELD(&so->so_lock)); 338 if (pid != 0) { 339 /* 340 * Permissions check by sending signal 0. 341 * Note that when kill fails it does a 342 * set_errno causing the system call to fail. 343 */ 344 error = kill(pid, 0); 345 if (error != 0) { 346 return (error); 347 } 348 } 349 so->so_pgrp = pid; 350 return (0); 351 } 352 353 354 /* 355 * Generate a SIGIO, for 'writable' events include siginfo structure, 356 * for read events just send the signal. 357 */ 358 /*ARGSUSED*/ 359 static void 360 socket_sigproc(proc_t *proc, int event) 361 { 362 k_siginfo_t info; 363 364 ASSERT(event & (SOCKETSIG_WRITE | SOCKETSIG_READ | SOCKETSIG_URG)); 365 366 if (event & SOCKETSIG_WRITE) { 367 info.si_signo = SIGPOLL; 368 info.si_code = POLL_OUT; 369 info.si_errno = 0; 370 info.si_fd = 0; 371 info.si_band = 0; 372 sigaddq(proc, NULL, &info, KM_NOSLEEP); 373 } 374 if (event & SOCKETSIG_READ) { 375 sigtoproc(proc, NULL, SIGPOLL); 376 } 377 if (event & SOCKETSIG_URG) { 378 sigtoproc(proc, NULL, SIGURG); 379 } 380 } 381 382 void 383 socket_sendsig(struct sonode *so, int event) 384 { 385 proc_t *proc; 386 387 ASSERT(MUTEX_HELD(&so->so_lock)); 388 389 if (so->so_pgrp == 0 || (!(so->so_state & SS_ASYNC) && 390 event != SOCKETSIG_URG)) { 391 return; 392 } 393 394 dprint(3, ("sending sig %d to %d\n", event, so->so_pgrp)); 395 396 if (so->so_pgrp > 0) { 397 /* 398 * XXX This unfortunately still generates 399 * a signal when a fd is closed but 400 * the proc is active. 401 */ 402 mutex_enter(&pidlock); 403 /* 404 * Even if the thread started in another zone, we're receiving 405 * on behalf of this socket's zone, so find the proc using the 406 * socket's zone ID. 407 */ 408 proc = prfind_zone(so->so_pgrp, so->so_zoneid); 409 if (proc == NULL) { 410 mutex_exit(&pidlock); 411 return; 412 } 413 mutex_enter(&proc->p_lock); 414 mutex_exit(&pidlock); 415 socket_sigproc(proc, event); 416 mutex_exit(&proc->p_lock); 417 } else { 418 /* 419 * Send to process group. Hold pidlock across 420 * calls to socket_sigproc(). 421 */ 422 pid_t pgrp = -so->so_pgrp; 423 424 mutex_enter(&pidlock); 425 /* 426 * Even if the thread started in another zone, we're receiving 427 * on behalf of this socket's zone, so find the pgrp using the 428 * socket's zone ID. 429 */ 430 proc = pgfind_zone(pgrp, so->so_zoneid); 431 while (proc != NULL) { 432 mutex_enter(&proc->p_lock); 433 socket_sigproc(proc, event); 434 mutex_exit(&proc->p_lock); 435 proc = proc->p_pglink; 436 } 437 mutex_exit(&pidlock); 438 } 439 } 440 441 #define MIN(a, b) ((a) < (b) ? (a) : (b)) 442 /* Copy userdata into a new mblk_t */ 443 mblk_t * 444 socopyinuio(uio_t *uiop, ssize_t iosize, size_t wroff, ssize_t maxblk, 445 size_t tail_len, int *errorp) 446 { 447 mblk_t *head = NULL, **tail = &head; 448 449 ASSERT(iosize == INFPSZ || iosize > 0); 450 451 if (iosize == INFPSZ || iosize > uiop->uio_resid) 452 iosize = uiop->uio_resid; 453 454 if (maxblk == INFPSZ) 455 maxblk = iosize; 456 457 /* Nothing to do in these cases, so we're done */ 458 if (iosize < 0 || maxblk < 0 || (maxblk == 0 && iosize > 0)) 459 goto done; 460 461 /* 462 * We will enter the loop below if iosize is 0; it will allocate an 463 * empty message block and call uiomove(9F) which will just return. 464 * We could avoid that with an extra check but would only slow 465 * down the much more likely case where iosize is larger than 0. 466 */ 467 do { 468 ssize_t blocksize; 469 mblk_t *mp; 470 471 blocksize = MIN(iosize, maxblk); 472 ASSERT(blocksize >= 0); 473 mp = allocb(wroff + blocksize + tail_len, BPRI_MED); 474 if (mp == NULL) { 475 *errorp = ENOMEM; 476 return (head); 477 } 478 mp->b_rptr += wroff; 479 mp->b_wptr = mp->b_rptr + blocksize; 480 481 *tail = mp; 482 tail = &mp->b_cont; 483 484 /* uiomove(9F) either returns 0 or EFAULT */ 485 if ((*errorp = uiomove(mp->b_rptr, (size_t)blocksize, 486 UIO_WRITE, uiop)) != 0) { 487 ASSERT(*errorp != ENOMEM); 488 freemsg(head); 489 return (NULL); 490 } 491 492 iosize -= blocksize; 493 } while (iosize > 0); 494 495 done: 496 *errorp = 0; 497 return (head); 498 } 499 500 mblk_t * 501 socopyoutuio(mblk_t *mp, struct uio *uiop, ssize_t max_read, int *errorp) 502 { 503 int error; 504 ptrdiff_t n; 505 mblk_t *nmp; 506 507 ASSERT(mp->b_wptr >= mp->b_rptr); 508 509 /* 510 * max_read is the offset of the oobmark and read can not go pass 511 * the oobmark. 512 */ 513 if (max_read == INFPSZ || max_read > uiop->uio_resid) 514 max_read = uiop->uio_resid; 515 516 do { 517 if ((n = MIN(max_read, MBLKL(mp))) != 0) { 518 ASSERT(n > 0); 519 520 error = uiomove(mp->b_rptr, n, UIO_READ, uiop); 521 if (error != 0) { 522 freemsg(mp); 523 *errorp = error; 524 return (NULL); 525 } 526 } 527 528 mp->b_rptr += n; 529 max_read -= n; 530 while (mp != NULL && (mp->b_rptr >= mp->b_wptr)) { 531 /* 532 * get rid of zero length mblks 533 */ 534 nmp = mp; 535 mp = mp->b_cont; 536 freeb(nmp); 537 } 538 } while (mp != NULL && max_read > 0); 539 540 *errorp = 0; 541 return (mp); 542 } 543 544 static void 545 so_prepend_msg(struct sonode *so, mblk_t *mp, mblk_t *last_tail) 546 { 547 ASSERT(last_tail != NULL); 548 mp->b_next = so->so_rcv_q_head; 549 mp->b_prev = last_tail; 550 ASSERT(!(DB_FLAGS(mp) & DBLK_UIOA)); 551 552 if (so->so_rcv_q_head == NULL) { 553 ASSERT(so->so_rcv_q_last_head == NULL); 554 so->so_rcv_q_last_head = mp; 555 #ifdef DEBUG 556 } else { 557 ASSERT(!(DB_FLAGS(so->so_rcv_q_head) & DBLK_UIOA)); 558 #endif 559 } 560 so->so_rcv_q_head = mp; 561 562 #ifdef DEBUG 563 if (so_debug_length) { 564 mutex_enter(&so->so_lock); 565 ASSERT(so_check_length(so)); 566 mutex_exit(&so->so_lock); 567 } 568 #endif 569 } 570 571 /* 572 * Move a mblk chain (mp_head, mp_last_head) to the sonode's rcv queue so it 573 * can be processed by so_dequeue_msg(). 574 */ 575 void 576 so_process_new_message(struct sonode *so, mblk_t *mp_head, mblk_t *mp_last_head) 577 { 578 if (so->so_filter_active > 0 && 579 (mp_head = sof_filter_data_in_proc(so, mp_head, 580 &mp_last_head)) == NULL) 581 return; 582 583 ASSERT(mp_head->b_prev != NULL); 584 if (so->so_rcv_q_head == NULL) { 585 so->so_rcv_q_head = mp_head; 586 so->so_rcv_q_last_head = mp_last_head; 587 ASSERT(so->so_rcv_q_last_head->b_prev != NULL); 588 } else { 589 boolean_t flag_equal = ((DB_FLAGS(mp_head) & DBLK_UIOA) == 590 (DB_FLAGS(so->so_rcv_q_last_head) & DBLK_UIOA)); 591 592 if (mp_head->b_next == NULL && 593 DB_TYPE(mp_head) == M_DATA && 594 DB_TYPE(so->so_rcv_q_last_head) == M_DATA && flag_equal) { 595 so->so_rcv_q_last_head->b_prev->b_cont = mp_head; 596 so->so_rcv_q_last_head->b_prev = mp_head->b_prev; 597 mp_head->b_prev = NULL; 598 } else if (flag_equal && (DB_FLAGS(mp_head) & DBLK_UIOA)) { 599 /* 600 * Append to last_head if more than one mblks, and both 601 * mp_head and last_head are I/OAT mblks. 602 */ 603 ASSERT(mp_head->b_next != NULL); 604 so->so_rcv_q_last_head->b_prev->b_cont = mp_head; 605 so->so_rcv_q_last_head->b_prev = mp_head->b_prev; 606 mp_head->b_prev = NULL; 607 608 so->so_rcv_q_last_head->b_next = mp_head->b_next; 609 mp_head->b_next = NULL; 610 so->so_rcv_q_last_head = mp_last_head; 611 } else { 612 #ifdef DEBUG 613 { 614 mblk_t *tmp_mblk; 615 tmp_mblk = mp_head; 616 while (tmp_mblk != NULL) { 617 ASSERT(tmp_mblk->b_prev != NULL); 618 tmp_mblk = tmp_mblk->b_next; 619 } 620 } 621 #endif 622 so->so_rcv_q_last_head->b_next = mp_head; 623 so->so_rcv_q_last_head = mp_last_head; 624 } 625 } 626 } 627 628 /* 629 * Check flow control on a given sonode. Must have so_lock held, and 630 * this function will release the hold. Return true if flow control 631 * is cleared. 632 */ 633 boolean_t 634 so_check_flow_control(struct sonode *so) 635 { 636 ASSERT(MUTEX_HELD(&so->so_lock)); 637 638 if (so->so_flowctrld && (so->so_rcv_queued < so->so_rcvlowat && 639 !(so->so_state & SS_FIL_RCV_FLOWCTRL))) { 640 so->so_flowctrld = B_FALSE; 641 mutex_exit(&so->so_lock); 642 /* 643 * Open up flow control. SCTP does not have any downcalls, and 644 * it will clr flow ctrl in sosctp_recvmsg(). 645 */ 646 if (so->so_downcalls != NULL && 647 so->so_downcalls->sd_clr_flowctrl != NULL) { 648 (*so->so_downcalls->sd_clr_flowctrl) 649 (so->so_proto_handle); 650 } 651 /* filters can start injecting data */ 652 sof_sonode_notify_filters(so, SOF_EV_INJECT_DATA_IN_OK, 0); 653 return (B_TRUE); 654 } else { 655 mutex_exit(&so->so_lock); 656 return (B_FALSE); 657 } 658 } 659 660 int 661 so_dequeue_msg(struct sonode *so, mblk_t **mctlp, struct uio *uiop, 662 rval_t *rvalp, int flags) 663 { 664 mblk_t *mp, *nmp; 665 mblk_t *savemp, *savemptail; 666 mblk_t *new_msg_head; 667 mblk_t *new_msg_last_head; 668 mblk_t *last_tail = NULL; 669 boolean_t partial_read; 670 boolean_t reset_atmark = B_FALSE; 671 int more = 0; 672 int error; 673 ssize_t oobmark; 674 sodirect_t *sodp = so->so_direct; 675 676 partial_read = B_FALSE; 677 *mctlp = NULL; 678 again: 679 mutex_enter(&so->so_lock); 680 again1: 681 #ifdef DEBUG 682 if (so_debug_length) { 683 ASSERT(so_check_length(so)); 684 } 685 #endif 686 if (so->so_state & SS_RCVATMARK) { 687 /* Check whether the caller is OK to read past the mark */ 688 if (flags & MSG_NOMARK) { 689 mutex_exit(&so->so_lock); 690 return (EWOULDBLOCK); 691 } 692 reset_atmark = B_TRUE; 693 } 694 /* 695 * First move messages from the dump area to processing area 696 */ 697 if (sodp != NULL) { 698 if (sodp->sod_enabled) { 699 if (sodp->sod_uioa.uioa_state & UIOA_ALLOC) { 700 /* nothing to uioamove */ 701 sodp = NULL; 702 } else if (sodp->sod_uioa.uioa_state & UIOA_INIT) { 703 sodp->sod_uioa.uioa_state &= UIOA_CLR; 704 sodp->sod_uioa.uioa_state |= UIOA_ENABLED; 705 /* 706 * try to uioamove() the data that 707 * has already queued. 708 */ 709 sod_uioa_so_init(so, sodp, uiop); 710 } 711 } else { 712 sodp = NULL; 713 } 714 } 715 new_msg_head = so->so_rcv_head; 716 new_msg_last_head = so->so_rcv_last_head; 717 so->so_rcv_head = NULL; 718 so->so_rcv_last_head = NULL; 719 oobmark = so->so_oobmark; 720 /* 721 * We can release the lock as there can only be one reader 722 */ 723 mutex_exit(&so->so_lock); 724 725 if (new_msg_head != NULL) { 726 so_process_new_message(so, new_msg_head, new_msg_last_head); 727 } 728 savemp = savemptail = NULL; 729 rvalp->r_vals = 0; 730 error = 0; 731 mp = so->so_rcv_q_head; 732 733 if (mp != NULL && 734 (so->so_rcv_timer_tid == 0 || 735 so->so_rcv_queued >= so->so_rcv_thresh)) { 736 partial_read = B_FALSE; 737 738 if (flags & MSG_PEEK) { 739 if ((nmp = dupmsg(mp)) == NULL && 740 (nmp = copymsg(mp)) == NULL) { 741 size_t size = msgsize(mp); 742 743 error = strwaitbuf(size, BPRI_HI); 744 if (error) { 745 return (error); 746 } 747 goto again; 748 } 749 mp = nmp; 750 } else { 751 ASSERT(mp->b_prev != NULL); 752 last_tail = mp->b_prev; 753 mp->b_prev = NULL; 754 so->so_rcv_q_head = mp->b_next; 755 if (so->so_rcv_q_head == NULL) { 756 so->so_rcv_q_last_head = NULL; 757 } 758 mp->b_next = NULL; 759 } 760 761 ASSERT(mctlp != NULL); 762 /* 763 * First process PROTO or PCPROTO blocks, if any. 764 */ 765 if (DB_TYPE(mp) != M_DATA) { 766 *mctlp = mp; 767 savemp = mp; 768 savemptail = mp; 769 ASSERT(DB_TYPE(mp) == M_PROTO || 770 DB_TYPE(mp) == M_PCPROTO); 771 while (mp->b_cont != NULL && 772 DB_TYPE(mp->b_cont) != M_DATA) { 773 ASSERT(DB_TYPE(mp->b_cont) == M_PROTO || 774 DB_TYPE(mp->b_cont) == M_PCPROTO); 775 mp = mp->b_cont; 776 savemptail = mp; 777 } 778 mp = savemptail->b_cont; 779 savemptail->b_cont = NULL; 780 } 781 782 ASSERT(DB_TYPE(mp) == M_DATA); 783 /* 784 * Now process DATA blocks, if any. Note that for sodirect 785 * enabled socket, uio_resid can be 0. 786 */ 787 if (uiop->uio_resid >= 0) { 788 ssize_t copied = 0; 789 790 if (sodp != NULL && (DB_FLAGS(mp) & DBLK_UIOA)) { 791 mutex_enter(&so->so_lock); 792 ASSERT(uiop == (uio_t *)&sodp->sod_uioa); 793 copied = sod_uioa_mblk(so, mp); 794 if (copied > 0) 795 partial_read = B_TRUE; 796 mutex_exit(&so->so_lock); 797 /* mark this mblk as processed */ 798 mp = NULL; 799 } else { 800 ssize_t oldresid = uiop->uio_resid; 801 802 if (MBLKL(mp) < so_mblk_pull_len) { 803 if (pullupmsg(mp, -1) == 1) { 804 last_tail = mp; 805 } 806 } 807 /* 808 * Can not read beyond the oobmark 809 */ 810 mp = socopyoutuio(mp, uiop, 811 oobmark == 0 ? INFPSZ : oobmark, &error); 812 if (error != 0) { 813 freemsg(*mctlp); 814 *mctlp = NULL; 815 more = 0; 816 goto done; 817 } 818 ASSERT(oldresid >= uiop->uio_resid); 819 copied = oldresid - uiop->uio_resid; 820 if (oldresid > uiop->uio_resid) 821 partial_read = B_TRUE; 822 } 823 ASSERT(copied >= 0); 824 if (copied > 0 && !(flags & MSG_PEEK)) { 825 mutex_enter(&so->so_lock); 826 so->so_rcv_queued -= copied; 827 ASSERT(so->so_oobmark >= 0); 828 if (so->so_oobmark > 0) { 829 so->so_oobmark -= copied; 830 ASSERT(so->so_oobmark >= 0); 831 if (so->so_oobmark == 0) { 832 ASSERT(so->so_state & 833 SS_OOBPEND); 834 so->so_oobmark = 0; 835 so->so_state |= SS_RCVATMARK; 836 } 837 } 838 /* 839 * so_check_flow_control() will drop 840 * so->so_lock. 841 */ 842 rvalp->r_val2 = so_check_flow_control(so); 843 } 844 } 845 if (mp != NULL) { /* more data blocks in msg */ 846 more |= MOREDATA; 847 if ((flags & (MSG_PEEK|MSG_TRUNC))) { 848 if (flags & MSG_PEEK) { 849 freemsg(mp); 850 } else { 851 unsigned int msize = msgdsize(mp); 852 853 freemsg(mp); 854 mutex_enter(&so->so_lock); 855 so->so_rcv_queued -= msize; 856 /* 857 * so_check_flow_control() will drop 858 * so->so_lock. 859 */ 860 rvalp->r_val2 = 861 so_check_flow_control(so); 862 } 863 } else if (partial_read && !somsghasdata(mp)) { 864 /* 865 * Avoid queuing a zero-length tail part of 866 * a message. partial_read == 1 indicates that 867 * we read some of the message. 868 */ 869 freemsg(mp); 870 more &= ~MOREDATA; 871 } else { 872 if (savemp != NULL && 873 (flags & MSG_DUPCTRL)) { 874 mblk_t *nmp; 875 /* 876 * There should only be non data mblks 877 */ 878 ASSERT(DB_TYPE(savemp) != M_DATA && 879 DB_TYPE(savemptail) != M_DATA); 880 try_again: 881 if ((nmp = dupmsg(savemp)) == NULL && 882 (nmp = copymsg(savemp)) == NULL) { 883 884 size_t size = msgsize(savemp); 885 886 error = strwaitbuf(size, 887 BPRI_HI); 888 if (error != 0) { 889 /* 890 * In case we 891 * cannot copy 892 * control data 893 * free the remaining 894 * data. 895 */ 896 freemsg(mp); 897 goto done; 898 } 899 goto try_again; 900 } 901 902 ASSERT(nmp != NULL); 903 ASSERT(DB_TYPE(nmp) != M_DATA); 904 savemptail->b_cont = mp; 905 *mctlp = nmp; 906 mp = savemp; 907 } 908 /* 909 * putback mp 910 */ 911 so_prepend_msg(so, mp, last_tail); 912 } 913 } 914 915 /* fast check so_rcv_head if there is more data */ 916 if (partial_read && !(so->so_state & SS_RCVATMARK) && 917 *mctlp == NULL && uiop->uio_resid > 0 && 918 !(flags & MSG_PEEK) && so->so_rcv_head != NULL) { 919 goto again; 920 } 921 } else if (!partial_read) { 922 mutex_enter(&so->so_lock); 923 if (so->so_error != 0) { 924 error = sogeterr(so, !(flags & MSG_PEEK)); 925 mutex_exit(&so->so_lock); 926 return (error); 927 } 928 929 /* See if new data has arrived in the meantime */ 930 if (so->so_rcv_head != NULL) 931 goto again1; 932 933 /* 934 * No pending data. Return right away for nonblocking 935 * socket, otherwise sleep waiting for data. 936 */ 937 if (!(so->so_state & SS_CANTRCVMORE) && uiop->uio_resid > 0) { 938 if ((uiop->uio_fmode & (FNDELAY|FNONBLOCK)) || 939 (flags & MSG_DONTWAIT)) { 940 error = EWOULDBLOCK; 941 } else { 942 if (so->so_state & (SS_CLOSING | 943 SS_FALLBACK_PENDING)) { 944 mutex_exit(&so->so_lock); 945 error = EINTR; 946 goto done; 947 } 948 949 so->so_rcv_wakeup = B_TRUE; 950 so->so_rcv_wanted = uiop->uio_resid; 951 if (so->so_rcvtimeo == 0) { 952 /* 953 * Zero means disable timeout. 954 */ 955 error = cv_wait_sig(&so->so_rcv_cv, 956 &so->so_lock); 957 } else { 958 error = cv_reltimedwait_sig( 959 &so->so_rcv_cv, &so->so_lock, 960 so->so_rcvtimeo, TR_CLOCK_TICK); 961 } 962 so->so_rcv_wakeup = B_FALSE; 963 so->so_rcv_wanted = 0; 964 965 if (error == 0) { 966 error = EINTR; 967 } else if (error == -1) { 968 error = EAGAIN; 969 } else { 970 goto again1; 971 } 972 } 973 } 974 mutex_exit(&so->so_lock); 975 } 976 if (reset_atmark && partial_read && !(flags & MSG_PEEK)) { 977 /* 978 * We are passed the mark, update state 979 * 4.3BSD and 4.4BSD clears the mark when peeking across it. 980 * The draft Posix socket spec states that the mark should 981 * not be cleared when peeking. We follow the latter. 982 */ 983 mutex_enter(&so->so_lock); 984 ASSERT(so_verify_oobstate(so)); 985 so->so_state &= ~(SS_OOBPEND|SS_HAVEOOBDATA|SS_RCVATMARK); 986 freemsg(so->so_oobmsg); 987 so->so_oobmsg = NULL; 988 ASSERT(so_verify_oobstate(so)); 989 mutex_exit(&so->so_lock); 990 } 991 ASSERT(so->so_rcv_wakeup == B_FALSE); 992 done: 993 if (sodp != NULL) { 994 mutex_enter(&so->so_lock); 995 if (sodp->sod_enabled && 996 (sodp->sod_uioa.uioa_state & UIOA_ENABLED)) { 997 SOD_UIOAFINI(sodp); 998 if (sodp->sod_uioa.uioa_mbytes > 0) { 999 ASSERT(so->so_rcv_q_head != NULL || 1000 so->so_rcv_head != NULL); 1001 so->so_rcv_queued -= sod_uioa_mblk(so, NULL); 1002 if (error == EWOULDBLOCK) 1003 error = 0; 1004 } 1005 } 1006 mutex_exit(&so->so_lock); 1007 } 1008 #ifdef DEBUG 1009 if (so_debug_length) { 1010 mutex_enter(&so->so_lock); 1011 ASSERT(so_check_length(so)); 1012 mutex_exit(&so->so_lock); 1013 } 1014 #endif 1015 rvalp->r_val1 = more; 1016 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 1017 return (error); 1018 } 1019 1020 /* 1021 * Enqueue data from the protocol on the socket's rcv queue. 1022 * 1023 * We try to hook new M_DATA mblks onto an existing chain, however, 1024 * that cannot be done if the existing chain has already been 1025 * processed by I/OAT. Non-M_DATA mblks are just linked together via 1026 * b_next. In all cases the b_prev of the enqueued mblk is set to 1027 * point to the last mblk in its b_cont chain. 1028 */ 1029 void 1030 so_enqueue_msg(struct sonode *so, mblk_t *mp, size_t msg_size) 1031 { 1032 ASSERT(MUTEX_HELD(&so->so_lock)); 1033 1034 #ifdef DEBUG 1035 if (so_debug_length) { 1036 ASSERT(so_check_length(so)); 1037 } 1038 #endif 1039 so->so_rcv_queued += msg_size; 1040 1041 if (so->so_rcv_head == NULL) { 1042 ASSERT(so->so_rcv_last_head == NULL); 1043 so->so_rcv_head = mp; 1044 so->so_rcv_last_head = mp; 1045 } else if ((DB_TYPE(mp) == M_DATA && 1046 DB_TYPE(so->so_rcv_last_head) == M_DATA) && 1047 ((DB_FLAGS(mp) & DBLK_UIOA) == 1048 (DB_FLAGS(so->so_rcv_last_head) & DBLK_UIOA))) { 1049 /* Added to the end */ 1050 ASSERT(so->so_rcv_last_head != NULL); 1051 ASSERT(so->so_rcv_last_head->b_prev != NULL); 1052 so->so_rcv_last_head->b_prev->b_cont = mp; 1053 } else { 1054 /* Start a new end */ 1055 so->so_rcv_last_head->b_next = mp; 1056 so->so_rcv_last_head = mp; 1057 } 1058 while (mp->b_cont != NULL) 1059 mp = mp->b_cont; 1060 1061 so->so_rcv_last_head->b_prev = mp; 1062 #ifdef DEBUG 1063 if (so_debug_length) { 1064 ASSERT(so_check_length(so)); 1065 } 1066 #endif 1067 } 1068 1069 /* 1070 * Return B_TRUE if there is data in the message, B_FALSE otherwise. 1071 */ 1072 boolean_t 1073 somsghasdata(mblk_t *mp) 1074 { 1075 for (; mp; mp = mp->b_cont) 1076 if (mp->b_datap->db_type == M_DATA) { 1077 ASSERT(mp->b_wptr >= mp->b_rptr); 1078 if (mp->b_wptr > mp->b_rptr) 1079 return (B_TRUE); 1080 } 1081 return (B_FALSE); 1082 } 1083 1084 /* 1085 * Flush the read side of sockfs. 1086 * 1087 * The caller must be sure that a reader is not already active when the 1088 * buffer is being flushed. 1089 */ 1090 void 1091 so_rcv_flush(struct sonode *so) 1092 { 1093 mblk_t *mp; 1094 1095 ASSERT(MUTEX_HELD(&so->so_lock)); 1096 1097 if (so->so_oobmsg != NULL) { 1098 freemsg(so->so_oobmsg); 1099 so->so_oobmsg = NULL; 1100 so->so_oobmark = 0; 1101 so->so_state &= 1102 ~(SS_OOBPEND|SS_HAVEOOBDATA|SS_HADOOBDATA|SS_RCVATMARK); 1103 } 1104 1105 /* 1106 * Free messages sitting in the recv queues 1107 */ 1108 while (so->so_rcv_q_head != NULL) { 1109 mp = so->so_rcv_q_head; 1110 so->so_rcv_q_head = mp->b_next; 1111 mp->b_next = mp->b_prev = NULL; 1112 freemsg(mp); 1113 } 1114 while (so->so_rcv_head != NULL) { 1115 mp = so->so_rcv_head; 1116 so->so_rcv_head = mp->b_next; 1117 mp->b_next = mp->b_prev = NULL; 1118 freemsg(mp); 1119 } 1120 so->so_rcv_queued = 0; 1121 so->so_rcv_q_head = NULL; 1122 so->so_rcv_q_last_head = NULL; 1123 so->so_rcv_head = NULL; 1124 so->so_rcv_last_head = NULL; 1125 } 1126 1127 /* 1128 * Handle recv* calls that set MSG_OOB or MSG_OOB together with MSG_PEEK. 1129 */ 1130 int 1131 sorecvoob(struct sonode *so, struct nmsghdr *msg, struct uio *uiop, int flags, 1132 boolean_t oob_inline) 1133 { 1134 mblk_t *mp, *nmp; 1135 int error; 1136 1137 dprintso(so, 1, ("sorecvoob(%p, %p, 0x%x)\n", (void *)so, (void *)msg, 1138 flags)); 1139 1140 if (msg != NULL) { 1141 /* 1142 * There is never any oob data with addresses or control since 1143 * the T_EXDATA_IND does not carry any options. 1144 */ 1145 msg->msg_controllen = 0; 1146 msg->msg_namelen = 0; 1147 msg->msg_flags = 0; 1148 } 1149 1150 mutex_enter(&so->so_lock); 1151 ASSERT(so_verify_oobstate(so)); 1152 if (oob_inline || 1153 (so->so_state & (SS_OOBPEND|SS_HADOOBDATA)) != SS_OOBPEND) { 1154 dprintso(so, 1, ("sorecvoob: inline or data consumed\n")); 1155 mutex_exit(&so->so_lock); 1156 return (EINVAL); 1157 } 1158 if (!(so->so_state & SS_HAVEOOBDATA)) { 1159 dprintso(so, 1, ("sorecvoob: no data yet\n")); 1160 mutex_exit(&so->so_lock); 1161 return (EWOULDBLOCK); 1162 } 1163 ASSERT(so->so_oobmsg != NULL); 1164 mp = so->so_oobmsg; 1165 if (flags & MSG_PEEK) { 1166 /* 1167 * Since recv* can not return ENOBUFS we can not use dupmsg. 1168 * Instead we revert to the consolidation private 1169 * allocb_wait plus bcopy. 1170 */ 1171 mblk_t *mp1; 1172 1173 mp1 = allocb_wait(msgdsize(mp), BPRI_MED, STR_NOSIG, NULL); 1174 ASSERT(mp1); 1175 1176 while (mp != NULL) { 1177 ssize_t size; 1178 1179 size = MBLKL(mp); 1180 bcopy(mp->b_rptr, mp1->b_wptr, size); 1181 mp1->b_wptr += size; 1182 ASSERT(mp1->b_wptr <= mp1->b_datap->db_lim); 1183 mp = mp->b_cont; 1184 } 1185 mp = mp1; 1186 } else { 1187 /* 1188 * Update the state indicating that the data has been consumed. 1189 * Keep SS_OOBPEND set until data is consumed past the mark. 1190 */ 1191 so->so_oobmsg = NULL; 1192 so->so_state ^= SS_HAVEOOBDATA|SS_HADOOBDATA; 1193 } 1194 ASSERT(so_verify_oobstate(so)); 1195 mutex_exit(&so->so_lock); 1196 1197 error = 0; 1198 nmp = mp; 1199 while (nmp != NULL && uiop->uio_resid > 0) { 1200 ssize_t n = MBLKL(nmp); 1201 1202 n = MIN(n, uiop->uio_resid); 1203 if (n > 0) 1204 error = uiomove(nmp->b_rptr, n, 1205 UIO_READ, uiop); 1206 if (error) 1207 break; 1208 nmp = nmp->b_cont; 1209 } 1210 ASSERT(mp->b_next == NULL && mp->b_prev == NULL); 1211 freemsg(mp); 1212 return (error); 1213 } 1214 1215 /* 1216 * Allocate and initializ sonode 1217 */ 1218 /* ARGSUSED */ 1219 struct sonode * 1220 socket_sonode_create(struct sockparams *sp, int family, int type, 1221 int protocol, int version, int sflags, int *errorp, struct cred *cr) 1222 { 1223 sonode_t *so; 1224 int kmflags; 1225 1226 /* 1227 * Choose the right set of sonodeops based on the upcall and 1228 * down call version that the protocol has provided 1229 */ 1230 if (SOCK_UC_VERSION != sp->sp_smod_info->smod_uc_version || 1231 SOCK_DC_VERSION != sp->sp_smod_info->smod_dc_version) { 1232 /* 1233 * mismatch 1234 */ 1235 #ifdef DEBUG 1236 cmn_err(CE_CONT, "protocol and socket module version mismatch"); 1237 #endif 1238 *errorp = EINVAL; 1239 return (NULL); 1240 } 1241 1242 kmflags = (sflags & SOCKET_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; 1243 1244 so = kmem_cache_alloc(socket_cache, kmflags); 1245 if (so == NULL) { 1246 *errorp = ENOMEM; 1247 return (NULL); 1248 } 1249 1250 sonode_init(so, sp, family, type, protocol, &so_sonodeops); 1251 1252 if (version == SOV_DEFAULT) 1253 version = so_default_version; 1254 1255 so->so_version = (short)version; 1256 1257 /* 1258 * set the default values to be INFPSZ 1259 * if a protocol desires it can change the value later 1260 */ 1261 so->so_proto_props.sopp_rxhiwat = SOCKET_RECVHIWATER; 1262 so->so_proto_props.sopp_rxlowat = SOCKET_RECVLOWATER; 1263 so->so_proto_props.sopp_maxpsz = INFPSZ; 1264 so->so_proto_props.sopp_maxblk = INFPSZ; 1265 1266 return (so); 1267 } 1268 1269 int 1270 socket_init_common(struct sonode *so, struct sonode *pso, int flags, cred_t *cr) 1271 { 1272 int error = 0; 1273 1274 if (pso != NULL) { 1275 /* 1276 * We have a passive open, so inherit basic state from 1277 * the parent (listener). 1278 * 1279 * No need to grab the new sonode's lock, since there is no 1280 * one that can have a reference to it. 1281 */ 1282 mutex_enter(&pso->so_lock); 1283 1284 so->so_state |= SS_ISCONNECTED | (pso->so_state & SS_ASYNC); 1285 so->so_pgrp = pso->so_pgrp; 1286 so->so_rcvtimeo = pso->so_rcvtimeo; 1287 so->so_sndtimeo = pso->so_sndtimeo; 1288 so->so_xpg_rcvbuf = pso->so_xpg_rcvbuf; 1289 /* 1290 * Make note of the socket level options. TCP and IP level 1291 * options are already inherited. We could do all this after 1292 * accept is successful but doing it here simplifies code and 1293 * no harm done for error case. 1294 */ 1295 so->so_options = pso->so_options & (SO_DEBUG|SO_REUSEADDR| 1296 SO_KEEPALIVE|SO_DONTROUTE|SO_BROADCAST|SO_USELOOPBACK| 1297 SO_OOBINLINE|SO_DGRAM_ERRIND|SO_LINGER); 1298 so->so_proto_props = pso->so_proto_props; 1299 so->so_mode = pso->so_mode; 1300 so->so_pollev = pso->so_pollev & SO_POLLEV_ALWAYS; 1301 1302 mutex_exit(&pso->so_lock); 1303 1304 /* 1305 * If the parent has any filters, try to inherit them. 1306 */ 1307 if (pso->so_filter_active > 0 && 1308 (error = sof_sonode_inherit_filters(so, pso)) != 0) 1309 return (error); 1310 1311 } else { 1312 struct sockparams *sp = so->so_sockparams; 1313 sock_upcalls_t *upcalls_to_use; 1314 1315 /* 1316 * Attach automatic filters, if there are any. 1317 */ 1318 if (!list_is_empty(&sp->sp_auto_filters) && 1319 (error = sof_sonode_autoattach_filters(so, cr)) != 0) 1320 return (error); 1321 1322 /* OK to attach filters */ 1323 so->so_state |= SS_FILOP_OK; 1324 1325 /* 1326 * Based on the version number select the right upcalls to 1327 * pass down. Currently we only have one version so choose 1328 * default 1329 */ 1330 upcalls_to_use = &so_upcalls; 1331 1332 /* active open, so create a lower handle */ 1333 so->so_proto_handle = 1334 sp->sp_smod_info->smod_proto_create_func(so->so_family, 1335 so->so_type, so->so_protocol, &so->so_downcalls, 1336 &so->so_mode, &error, flags, cr); 1337 1338 if (so->so_proto_handle == NULL) { 1339 ASSERT(error != 0); 1340 /* 1341 * To be safe; if a lower handle cannot be created, and 1342 * the proto does not give a reason why, assume there 1343 * was a lack of memory. 1344 */ 1345 return ((error == 0) ? ENOMEM : error); 1346 } 1347 ASSERT(so->so_downcalls != NULL); 1348 ASSERT(so->so_downcalls->sd_send != NULL || 1349 so->so_downcalls->sd_send_uio != NULL); 1350 if (so->so_downcalls->sd_recv_uio != NULL) { 1351 ASSERT(so->so_downcalls->sd_poll != NULL); 1352 so->so_pollev |= SO_POLLEV_ALWAYS; 1353 } 1354 1355 (*so->so_downcalls->sd_activate)(so->so_proto_handle, 1356 (sock_upper_handle_t)so, upcalls_to_use, 0, cr); 1357 1358 /* Wildcard */ 1359 1360 /* 1361 * FIXME No need for this, the protocol can deal with it in 1362 * sd_create(). Should update ICMP. 1363 */ 1364 if (so->so_protocol != so->so_sockparams->sp_protocol) { 1365 int protocol = so->so_protocol; 1366 int error; 1367 /* 1368 * Issue SO_PROTOTYPE setsockopt. 1369 */ 1370 error = socket_setsockopt(so, SOL_SOCKET, SO_PROTOTYPE, 1371 &protocol, (t_uscalar_t)sizeof (protocol), cr); 1372 if (error) { 1373 (void) (*so->so_downcalls->sd_close) 1374 (so->so_proto_handle, 0, cr); 1375 1376 mutex_enter(&so->so_lock); 1377 so_rcv_flush(so); 1378 mutex_exit(&so->so_lock); 1379 /* 1380 * Setsockopt often fails with ENOPROTOOPT but 1381 * socket() should fail with 1382 * EPROTONOSUPPORT/EPROTOTYPE. 1383 */ 1384 return (EPROTONOSUPPORT); 1385 } 1386 } 1387 } 1388 1389 if (uioasync.enabled) 1390 sod_sock_init(so); 1391 1392 /* put an extra reference on the socket for the protocol */ 1393 VN_HOLD(SOTOV(so)); 1394 1395 return (0); 1396 } 1397 1398 /* 1399 * int socket_ioctl_common(struct sonode *so, int cmd, intptr_t arg, int mode, 1400 * struct cred *cr, int32_t *rvalp) 1401 * 1402 * Handle ioctls that manipulate basic socket state; non-blocking, 1403 * async, etc. 1404 * 1405 * Returns: 1406 * < 0 - ioctl was not handle 1407 * >= 0 - ioctl was handled, if > 0, then it is an errno 1408 * 1409 * Notes: 1410 * Assumes the standard receive buffer is used to obtain info for 1411 * NREAD. 1412 */ 1413 /* ARGSUSED */ 1414 int 1415 socket_ioctl_common(struct sonode *so, int cmd, intptr_t arg, int mode, 1416 struct cred *cr, int32_t *rvalp) 1417 { 1418 switch (cmd) { 1419 case SIOCSQPTR: 1420 /* 1421 * SIOCSQPTR is valid only when helper stream is created 1422 * by the protocol. 1423 */ 1424 1425 return (EOPNOTSUPP); 1426 case FIONBIO: { 1427 int32_t value; 1428 1429 if (so_copyin((void *)arg, &value, sizeof (int32_t), 1430 (mode & (int)FKIOCTL))) 1431 return (EFAULT); 1432 1433 mutex_enter(&so->so_lock); 1434 if (value) { 1435 so->so_state |= SS_NDELAY; 1436 } else { 1437 so->so_state &= ~SS_NDELAY; 1438 } 1439 mutex_exit(&so->so_lock); 1440 return (0); 1441 } 1442 case FIOASYNC: { 1443 int32_t value; 1444 1445 if (so_copyin((void *)arg, &value, sizeof (int32_t), 1446 (mode & (int)FKIOCTL))) 1447 return (EFAULT); 1448 1449 mutex_enter(&so->so_lock); 1450 1451 if (value) { 1452 /* Turn on SIGIO */ 1453 so->so_state |= SS_ASYNC; 1454 } else { 1455 /* Turn off SIGIO */ 1456 so->so_state &= ~SS_ASYNC; 1457 } 1458 mutex_exit(&so->so_lock); 1459 1460 return (0); 1461 } 1462 1463 case SIOCSPGRP: 1464 case FIOSETOWN: { 1465 int error; 1466 pid_t pid; 1467 1468 if (so_copyin((void *)arg, &pid, sizeof (pid_t), 1469 (mode & (int)FKIOCTL))) 1470 return (EFAULT); 1471 1472 mutex_enter(&so->so_lock); 1473 error = (pid != so->so_pgrp) ? socket_chgpgrp(so, pid) : 0; 1474 mutex_exit(&so->so_lock); 1475 return (error); 1476 } 1477 case SIOCGPGRP: 1478 case FIOGETOWN: 1479 if (so_copyout(&so->so_pgrp, (void *)arg, 1480 sizeof (pid_t), (mode & (int)FKIOCTL))) 1481 return (EFAULT); 1482 1483 return (0); 1484 case SIOCATMARK: { 1485 int retval; 1486 1487 /* 1488 * Only protocols that support urgent data can handle ATMARK. 1489 */ 1490 if ((so->so_mode & SM_EXDATA) == 0) 1491 return (EINVAL); 1492 1493 /* 1494 * If the protocol is maintaining its own buffer, then the 1495 * request must be passed down. 1496 */ 1497 if (so->so_downcalls->sd_recv_uio != NULL) 1498 return (-1); 1499 1500 retval = (so->so_state & SS_RCVATMARK) != 0; 1501 1502 if (so_copyout(&retval, (void *)arg, sizeof (int), 1503 (mode & (int)FKIOCTL))) { 1504 return (EFAULT); 1505 } 1506 return (0); 1507 } 1508 1509 case FIONREAD: { 1510 int retval; 1511 1512 /* 1513 * If the protocol is maintaining its own buffer, then the 1514 * request must be passed down. 1515 */ 1516 if (so->so_downcalls->sd_recv_uio != NULL) 1517 return (-1); 1518 1519 retval = MIN(so->so_rcv_queued, INT_MAX); 1520 1521 if (so_copyout(&retval, (void *)arg, 1522 sizeof (retval), (mode & (int)FKIOCTL))) { 1523 return (EFAULT); 1524 } 1525 return (0); 1526 } 1527 1528 case _I_GETPEERCRED: { 1529 int error = 0; 1530 1531 if ((mode & FKIOCTL) == 0) 1532 return (EINVAL); 1533 1534 mutex_enter(&so->so_lock); 1535 if ((so->so_mode & SM_CONNREQUIRED) == 0) { 1536 error = ENOTSUP; 1537 } else if ((so->so_state & SS_ISCONNECTED) == 0) { 1538 error = ENOTCONN; 1539 } else if (so->so_peercred != NULL) { 1540 k_peercred_t *kp = (k_peercred_t *)arg; 1541 kp->pc_cr = so->so_peercred; 1542 kp->pc_cpid = so->so_cpid; 1543 crhold(so->so_peercred); 1544 } else { 1545 error = EINVAL; 1546 } 1547 mutex_exit(&so->so_lock); 1548 return (error); 1549 } 1550 default: 1551 return (-1); 1552 } 1553 } 1554 1555 /* 1556 * Handle the I_NREAD STREAM ioctl. 1557 */ 1558 static int 1559 so_strioc_nread(struct sonode *so, intptr_t arg, int mode, int32_t *rvalp) 1560 { 1561 size_t size = 0; 1562 int retval; 1563 int count = 0; 1564 mblk_t *mp; 1565 clock_t wakeup = drv_usectohz(10); 1566 1567 if (so->so_downcalls == NULL || 1568 so->so_downcalls->sd_recv_uio != NULL) 1569 return (EINVAL); 1570 1571 mutex_enter(&so->so_lock); 1572 /* Wait for reader to get out of the way. */ 1573 while (so->so_flag & SOREADLOCKED) { 1574 /* 1575 * If reader is waiting for data, then there should be nothing 1576 * on the rcv queue. 1577 */ 1578 if (so->so_rcv_wakeup) 1579 goto out; 1580 1581 /* Do a timed sleep, in case the reader goes to sleep. */ 1582 (void) cv_reltimedwait(&so->so_read_cv, &so->so_lock, wakeup, 1583 TR_CLOCK_TICK); 1584 } 1585 1586 /* 1587 * Since we are holding so_lock no new reader will come in, and the 1588 * protocol will not be able to enqueue data. So it's safe to walk 1589 * both rcv queues. 1590 */ 1591 mp = so->so_rcv_q_head; 1592 if (mp != NULL) { 1593 size = msgdsize(so->so_rcv_q_head); 1594 for (; mp != NULL; mp = mp->b_next) 1595 count++; 1596 } else { 1597 /* 1598 * In case the processing list was empty, get the size of the 1599 * next msg in line. 1600 */ 1601 size = msgdsize(so->so_rcv_head); 1602 } 1603 1604 for (mp = so->so_rcv_head; mp != NULL; mp = mp->b_next) 1605 count++; 1606 out: 1607 mutex_exit(&so->so_lock); 1608 1609 /* 1610 * Drop down from size_t to the "int" required by the 1611 * interface. Cap at INT_MAX. 1612 */ 1613 retval = MIN(size, INT_MAX); 1614 if (so_copyout(&retval, (void *)arg, sizeof (retval), 1615 (mode & (int)FKIOCTL))) { 1616 return (EFAULT); 1617 } else { 1618 *rvalp = count; 1619 return (0); 1620 } 1621 } 1622 1623 /* 1624 * Process STREAM ioctls. 1625 * 1626 * Returns: 1627 * < 0 - ioctl was not handle 1628 * >= 0 - ioctl was handled, if > 0, then it is an errno 1629 */ 1630 int 1631 socket_strioc_common(struct sonode *so, int cmd, intptr_t arg, int mode, 1632 struct cred *cr, int32_t *rvalp) 1633 { 1634 int retval; 1635 1636 /* Only STREAM iotcls are handled here */ 1637 if ((cmd & 0xffffff00U) != STR) 1638 return (-1); 1639 1640 switch (cmd) { 1641 case I_CANPUT: 1642 /* 1643 * We return an error for I_CANPUT so that isastream(3C) will 1644 * not report the socket as being a STREAM. 1645 */ 1646 return (EOPNOTSUPP); 1647 case I_NREAD: 1648 /* Avoid doing a fallback for I_NREAD. */ 1649 return (so_strioc_nread(so, arg, mode, rvalp)); 1650 case I_LOOK: 1651 /* Avoid doing a fallback for I_LOOK. */ 1652 if (so_copyout("sockmod", (void *)arg, strlen("sockmod") + 1, 1653 (mode & (int)FKIOCTL))) { 1654 return (EFAULT); 1655 } 1656 return (0); 1657 default: 1658 break; 1659 } 1660 1661 /* 1662 * Try to fall back to TPI, and if successful, reissue the ioctl. 1663 */ 1664 if ((retval = so_tpi_fallback(so, cr)) == 0) { 1665 /* Reissue the ioctl */ 1666 ASSERT(so->so_rcv_q_head == NULL); 1667 return (SOP_IOCTL(so, cmd, arg, mode, cr, rvalp)); 1668 } else { 1669 return (retval); 1670 } 1671 } 1672 1673 /* 1674 * This is called for all socket types to verify that the buffer size is large 1675 * enough for the option, and if we can, handle the request as well. Most 1676 * options will be forwarded to the protocol. 1677 */ 1678 int 1679 socket_getopt_common(struct sonode *so, int level, int option_name, 1680 void *optval, socklen_t *optlenp, int flags) 1681 { 1682 if (level != SOL_SOCKET) 1683 return (-1); 1684 1685 switch (option_name) { 1686 case SO_ERROR: 1687 case SO_DOMAIN: 1688 case SO_TYPE: 1689 case SO_ACCEPTCONN: { 1690 int32_t value; 1691 socklen_t optlen = *optlenp; 1692 1693 if (optlen < (t_uscalar_t)sizeof (int32_t)) { 1694 return (EINVAL); 1695 } 1696 1697 switch (option_name) { 1698 case SO_ERROR: 1699 mutex_enter(&so->so_lock); 1700 value = sogeterr(so, B_TRUE); 1701 mutex_exit(&so->so_lock); 1702 break; 1703 case SO_DOMAIN: 1704 value = so->so_family; 1705 break; 1706 case SO_TYPE: 1707 value = so->so_type; 1708 break; 1709 case SO_ACCEPTCONN: 1710 if (so->so_state & SS_ACCEPTCONN) 1711 value = SO_ACCEPTCONN; 1712 else 1713 value = 0; 1714 break; 1715 } 1716 1717 bcopy(&value, optval, sizeof (value)); 1718 *optlenp = sizeof (value); 1719 1720 return (0); 1721 } 1722 case SO_SNDTIMEO: 1723 case SO_RCVTIMEO: { 1724 clock_t value; 1725 socklen_t optlen = *optlenp; 1726 1727 if (get_udatamodel() == DATAMODEL_NONE || 1728 get_udatamodel() == DATAMODEL_NATIVE) { 1729 if (optlen < sizeof (struct timeval)) 1730 return (EINVAL); 1731 } else { 1732 if (optlen < sizeof (struct timeval32)) 1733 return (EINVAL); 1734 } 1735 if (option_name == SO_RCVTIMEO) 1736 value = drv_hztousec(so->so_rcvtimeo); 1737 else 1738 value = drv_hztousec(so->so_sndtimeo); 1739 1740 if (get_udatamodel() == DATAMODEL_NONE || 1741 get_udatamodel() == DATAMODEL_NATIVE) { 1742 ((struct timeval *)(optval))->tv_sec = 1743 value / (1000 * 1000); 1744 ((struct timeval *)(optval))->tv_usec = 1745 value % (1000 * 1000); 1746 *optlenp = sizeof (struct timeval); 1747 } else { 1748 ((struct timeval32 *)(optval))->tv_sec = 1749 value / (1000 * 1000); 1750 ((struct timeval32 *)(optval))->tv_usec = 1751 value % (1000 * 1000); 1752 *optlenp = sizeof (struct timeval32); 1753 } 1754 return (0); 1755 } 1756 case SO_DEBUG: 1757 case SO_REUSEADDR: 1758 case SO_KEEPALIVE: 1759 case SO_DONTROUTE: 1760 case SO_BROADCAST: 1761 case SO_USELOOPBACK: 1762 case SO_OOBINLINE: 1763 case SO_SNDBUF: 1764 #ifdef notyet 1765 case SO_SNDLOWAT: 1766 case SO_RCVLOWAT: 1767 #endif /* notyet */ 1768 case SO_DGRAM_ERRIND: { 1769 socklen_t optlen = *optlenp; 1770 1771 if (optlen < (t_uscalar_t)sizeof (int32_t)) 1772 return (EINVAL); 1773 break; 1774 } 1775 case SO_RCVBUF: { 1776 socklen_t optlen = *optlenp; 1777 1778 if (optlen < (t_uscalar_t)sizeof (int32_t)) 1779 return (EINVAL); 1780 1781 if ((flags & _SOGETSOCKOPT_XPG4_2) && so->so_xpg_rcvbuf != 0) { 1782 /* 1783 * XXX If SO_RCVBUF has been set and this is an 1784 * XPG 4.2 application then do not ask the transport 1785 * since the transport might adjust the value and not 1786 * return exactly what was set by the application. 1787 * For non-XPG 4.2 application we return the value 1788 * that the transport is actually using. 1789 */ 1790 *(int32_t *)optval = so->so_xpg_rcvbuf; 1791 *optlenp = sizeof (so->so_xpg_rcvbuf); 1792 return (0); 1793 } 1794 /* 1795 * If the option has not been set then get a default 1796 * value from the transport. 1797 */ 1798 break; 1799 } 1800 case SO_LINGER: { 1801 socklen_t optlen = *optlenp; 1802 1803 if (optlen < (t_uscalar_t)sizeof (struct linger)) 1804 return (EINVAL); 1805 break; 1806 } 1807 case SO_SND_BUFINFO: { 1808 socklen_t optlen = *optlenp; 1809 1810 if (optlen < (t_uscalar_t)sizeof (struct so_snd_bufinfo)) 1811 return (EINVAL); 1812 ((struct so_snd_bufinfo *)(optval))->sbi_wroff = 1813 (so->so_proto_props).sopp_wroff; 1814 ((struct so_snd_bufinfo *)(optval))->sbi_maxblk = 1815 (so->so_proto_props).sopp_maxblk; 1816 ((struct so_snd_bufinfo *)(optval))->sbi_maxpsz = 1817 (so->so_proto_props).sopp_maxpsz; 1818 ((struct so_snd_bufinfo *)(optval))->sbi_tail = 1819 (so->so_proto_props).sopp_tail; 1820 *optlenp = sizeof (struct so_snd_bufinfo); 1821 return (0); 1822 } 1823 case SO_SND_COPYAVOID: { 1824 sof_instance_t *inst; 1825 1826 /* 1827 * Avoid zero-copy if there is a filter with a data_out 1828 * callback. We could let the operation succeed, but then 1829 * the filter would have to copy the data anyway. 1830 */ 1831 for (inst = so->so_filter_top; inst != NULL; 1832 inst = inst->sofi_next) { 1833 if (SOF_INTERESTED(inst, data_out)) 1834 return (EOPNOTSUPP); 1835 } 1836 break; 1837 } 1838 1839 default: 1840 break; 1841 } 1842 1843 /* Unknown Option */ 1844 return (-1); 1845 } 1846 1847 void 1848 socket_sonode_destroy(struct sonode *so) 1849 { 1850 sonode_fini(so); 1851 kmem_cache_free(socket_cache, so); 1852 } 1853 1854 int 1855 so_zcopy_wait(struct sonode *so) 1856 { 1857 int error = 0; 1858 1859 mutex_enter(&so->so_lock); 1860 while (!(so->so_copyflag & STZCNOTIFY)) { 1861 if (so->so_state & SS_CLOSING) { 1862 mutex_exit(&so->so_lock); 1863 return (EINTR); 1864 } 1865 if (cv_wait_sig(&so->so_copy_cv, &so->so_lock) == 0) { 1866 error = EINTR; 1867 break; 1868 } 1869 } 1870 so->so_copyflag &= ~STZCNOTIFY; 1871 mutex_exit(&so->so_lock); 1872 return (error); 1873 } 1874 1875 void 1876 so_timer_callback(void *arg) 1877 { 1878 struct sonode *so = (struct sonode *)arg; 1879 1880 mutex_enter(&so->so_lock); 1881 1882 so->so_rcv_timer_tid = 0; 1883 if (so->so_rcv_queued > 0) { 1884 so_notify_data(so, so->so_rcv_queued); 1885 } else { 1886 mutex_exit(&so->so_lock); 1887 } 1888 } 1889 1890 #ifdef DEBUG 1891 /* 1892 * Verify that the length stored in so_rcv_queued and the length of data blocks 1893 * queued is same. 1894 */ 1895 static boolean_t 1896 so_check_length(sonode_t *so) 1897 { 1898 mblk_t *mp = so->so_rcv_q_head; 1899 int len = 0; 1900 1901 ASSERT(MUTEX_HELD(&so->so_lock)); 1902 1903 if (mp != NULL) { 1904 len = msgdsize(mp); 1905 while ((mp = mp->b_next) != NULL) 1906 len += msgdsize(mp); 1907 } 1908 mp = so->so_rcv_head; 1909 if (mp != NULL) { 1910 len += msgdsize(mp); 1911 while ((mp = mp->b_next) != NULL) 1912 len += msgdsize(mp); 1913 } 1914 return ((len == so->so_rcv_queued) ? B_TRUE : B_FALSE); 1915 } 1916 #endif 1917 1918 int 1919 so_get_mod_version(struct sockparams *sp) 1920 { 1921 ASSERT(sp != NULL && sp->sp_smod_info != NULL); 1922 return (sp->sp_smod_info->smod_version); 1923 } 1924 1925 /* 1926 * so_start_fallback() 1927 * 1928 * Block new socket operations from coming in, and wait for active operations 1929 * to complete. Threads that are sleeping will be woken up so they can get 1930 * out of the way. 1931 * 1932 * The caller must be a reader on so_fallback_rwlock. 1933 */ 1934 static boolean_t 1935 so_start_fallback(struct sonode *so) 1936 { 1937 ASSERT(RW_READ_HELD(&so->so_fallback_rwlock)); 1938 1939 mutex_enter(&so->so_lock); 1940 if (so->so_state & SS_FALLBACK_PENDING) { 1941 mutex_exit(&so->so_lock); 1942 return (B_FALSE); 1943 } 1944 so->so_state |= SS_FALLBACK_PENDING; 1945 /* 1946 * Poke all threads that might be sleeping. Any operation that comes 1947 * in after the cv_broadcast will observe the fallback pending flag 1948 * which cause the call to return where it would normally sleep. 1949 */ 1950 cv_broadcast(&so->so_state_cv); /* threads in connect() */ 1951 cv_broadcast(&so->so_rcv_cv); /* threads in recvmsg() */ 1952 cv_broadcast(&so->so_snd_cv); /* threads in sendmsg() */ 1953 mutex_enter(&so->so_acceptq_lock); 1954 cv_broadcast(&so->so_acceptq_cv); /* threads in accept() */ 1955 mutex_exit(&so->so_acceptq_lock); 1956 mutex_exit(&so->so_lock); 1957 1958 /* 1959 * The main reason for the rw_tryupgrade call is to provide 1960 * observability during the fallback process. We want to 1961 * be able to see if there are pending operations. 1962 */ 1963 if (rw_tryupgrade(&so->so_fallback_rwlock) == 0) { 1964 /* 1965 * It is safe to drop and reaquire the fallback lock, because 1966 * we are guaranteed that another fallback cannot take place. 1967 */ 1968 rw_exit(&so->so_fallback_rwlock); 1969 DTRACE_PROBE1(pending__ops__wait, (struct sonode *), so); 1970 rw_enter(&so->so_fallback_rwlock, RW_WRITER); 1971 DTRACE_PROBE1(pending__ops__complete, (struct sonode *), so); 1972 } 1973 1974 return (B_TRUE); 1975 } 1976 1977 /* 1978 * so_end_fallback() 1979 * 1980 * Allow socket opertions back in. 1981 * 1982 * The caller must be a writer on so_fallback_rwlock. 1983 */ 1984 static void 1985 so_end_fallback(struct sonode *so) 1986 { 1987 ASSERT(RW_ISWRITER(&so->so_fallback_rwlock)); 1988 1989 mutex_enter(&so->so_lock); 1990 so->so_state &= ~(SS_FALLBACK_PENDING|SS_FALLBACK_DRAIN); 1991 mutex_exit(&so->so_lock); 1992 1993 rw_downgrade(&so->so_fallback_rwlock); 1994 } 1995 1996 /* 1997 * so_quiesced_cb() 1998 * 1999 * Callback passed to the protocol during fallback. It is called once 2000 * the endpoint is quiescent. 2001 * 2002 * No requests from the user, no notifications from the protocol, so it 2003 * is safe to synchronize the state. Data can also be moved without 2004 * risk for reordering. 2005 * 2006 * We do not need to hold so_lock, since there can be only one thread 2007 * operating on the sonode. 2008 */ 2009 static mblk_t * 2010 so_quiesced_cb(sock_upper_handle_t sock_handle, sock_quiesce_arg_t *arg, 2011 struct T_capability_ack *tcap, 2012 struct sockaddr *laddr, socklen_t laddrlen, 2013 struct sockaddr *faddr, socklen_t faddrlen, short opts) 2014 { 2015 struct sonode *so = (struct sonode *)sock_handle; 2016 boolean_t atmark; 2017 mblk_t *retmp = NULL, **tailmpp = &retmp; 2018 2019 if (tcap != NULL) 2020 sotpi_update_state(so, tcap, laddr, laddrlen, faddr, faddrlen, 2021 opts); 2022 2023 /* 2024 * Some protocols do not quiece the data path during fallback. Once 2025 * we set the SS_FALLBACK_DRAIN flag any attempt to queue data will 2026 * fail and the protocol is responsible for saving the data for later 2027 * delivery (i.e., once the fallback has completed). 2028 */ 2029 mutex_enter(&so->so_lock); 2030 so->so_state |= SS_FALLBACK_DRAIN; 2031 SOCKET_TIMER_CANCEL(so); 2032 mutex_exit(&so->so_lock); 2033 2034 if (so->so_rcv_head != NULL) { 2035 if (so->so_rcv_q_last_head == NULL) 2036 so->so_rcv_q_head = so->so_rcv_head; 2037 else 2038 so->so_rcv_q_last_head->b_next = so->so_rcv_head; 2039 so->so_rcv_q_last_head = so->so_rcv_last_head; 2040 } 2041 2042 atmark = (so->so_state & SS_RCVATMARK) != 0; 2043 /* 2044 * Clear any OOB state having to do with pending data. The TPI 2045 * code path will set the appropriate oob state when we move the 2046 * oob data to the STREAM head. We leave SS_HADOOBDATA since the oob 2047 * data has already been consumed. 2048 */ 2049 so->so_state &= ~(SS_RCVATMARK|SS_OOBPEND|SS_HAVEOOBDATA); 2050 2051 ASSERT(so->so_oobmsg != NULL || so->so_oobmark <= so->so_rcv_queued); 2052 2053 /* 2054 * Move data to the STREAM head. 2055 */ 2056 while (so->so_rcv_q_head != NULL) { 2057 mblk_t *mp = so->so_rcv_q_head; 2058 size_t mlen = msgdsize(mp); 2059 2060 so->so_rcv_q_head = mp->b_next; 2061 mp->b_next = NULL; 2062 mp->b_prev = NULL; 2063 2064 /* 2065 * Send T_EXDATA_IND if we are at the oob mark. 2066 */ 2067 if (atmark) { 2068 struct T_exdata_ind *tei; 2069 mblk_t *mp1 = arg->soqa_exdata_mp; 2070 2071 arg->soqa_exdata_mp = NULL; 2072 ASSERT(mp1 != NULL); 2073 mp1->b_datap->db_type = M_PROTO; 2074 tei = (struct T_exdata_ind *)mp1->b_rptr; 2075 tei->PRIM_type = T_EXDATA_IND; 2076 tei->MORE_flag = 0; 2077 mp1->b_wptr = (uchar_t *)&tei[1]; 2078 2079 if (IS_SO_OOB_INLINE(so)) { 2080 mp1->b_cont = mp; 2081 } else { 2082 ASSERT(so->so_oobmsg != NULL); 2083 mp1->b_cont = so->so_oobmsg; 2084 so->so_oobmsg = NULL; 2085 2086 /* process current mp next time around */ 2087 mp->b_next = so->so_rcv_q_head; 2088 so->so_rcv_q_head = mp; 2089 mlen = 0; 2090 } 2091 mp = mp1; 2092 2093 /* we have consumed the oob mark */ 2094 atmark = B_FALSE; 2095 } else if (so->so_oobmark > 0) { 2096 /* 2097 * Check if the OOB mark is within the current 2098 * mblk chain. In that case we have to split it up. 2099 */ 2100 if (so->so_oobmark < mlen) { 2101 mblk_t *urg_mp = mp; 2102 2103 atmark = B_TRUE; 2104 mp = NULL; 2105 mlen = so->so_oobmark; 2106 2107 /* 2108 * It is assumed that the OOB mark does 2109 * not land within a mblk. 2110 */ 2111 do { 2112 so->so_oobmark -= MBLKL(urg_mp); 2113 mp = urg_mp; 2114 urg_mp = urg_mp->b_cont; 2115 } while (so->so_oobmark > 0); 2116 mp->b_cont = NULL; 2117 if (urg_mp != NULL) { 2118 urg_mp->b_next = so->so_rcv_q_head; 2119 so->so_rcv_q_head = urg_mp; 2120 } 2121 } else { 2122 so->so_oobmark -= mlen; 2123 if (so->so_oobmark == 0) 2124 atmark = B_TRUE; 2125 } 2126 } 2127 2128 /* 2129 * Queue data on the STREAM head. 2130 */ 2131 so->so_rcv_queued -= mlen; 2132 *tailmpp = mp; 2133 tailmpp = &mp->b_next; 2134 } 2135 so->so_rcv_head = NULL; 2136 so->so_rcv_last_head = NULL; 2137 so->so_rcv_q_head = NULL; 2138 so->so_rcv_q_last_head = NULL; 2139 2140 /* 2141 * Check if the oob byte is at the end of the data stream, or if the 2142 * oob byte has not yet arrived. In the latter case we have to send a 2143 * SIGURG and a mark indicator to the STREAM head. The mark indicator 2144 * is needed to guarantee correct behavior for SIOCATMARK. See block 2145 * comment in socktpi.h for more details. 2146 */ 2147 if (atmark || so->so_oobmark > 0) { 2148 mblk_t *mp; 2149 2150 if (atmark && so->so_oobmsg != NULL) { 2151 struct T_exdata_ind *tei; 2152 2153 mp = arg->soqa_exdata_mp; 2154 arg->soqa_exdata_mp = NULL; 2155 ASSERT(mp != NULL); 2156 mp->b_datap->db_type = M_PROTO; 2157 tei = (struct T_exdata_ind *)mp->b_rptr; 2158 tei->PRIM_type = T_EXDATA_IND; 2159 tei->MORE_flag = 0; 2160 mp->b_wptr = (uchar_t *)&tei[1]; 2161 2162 mp->b_cont = so->so_oobmsg; 2163 so->so_oobmsg = NULL; 2164 2165 *tailmpp = mp; 2166 tailmpp = &mp->b_next; 2167 } else { 2168 /* Send up the signal */ 2169 mp = arg->soqa_exdata_mp; 2170 arg->soqa_exdata_mp = NULL; 2171 ASSERT(mp != NULL); 2172 DB_TYPE(mp) = M_PCSIG; 2173 *mp->b_wptr++ = (uchar_t)SIGURG; 2174 *tailmpp = mp; 2175 tailmpp = &mp->b_next; 2176 2177 /* Send up the mark indicator */ 2178 mp = arg->soqa_urgmark_mp; 2179 arg->soqa_urgmark_mp = NULL; 2180 mp->b_flag = atmark ? MSGMARKNEXT : MSGNOTMARKNEXT; 2181 *tailmpp = mp; 2182 tailmpp = &mp->b_next; 2183 2184 so->so_oobmark = 0; 2185 } 2186 } 2187 ASSERT(so->so_oobmark == 0); 2188 ASSERT(so->so_rcv_queued == 0); 2189 2190 return (retmp); 2191 } 2192 2193 #ifdef DEBUG 2194 /* 2195 * Do an integrity check of the sonode. This should be done if a 2196 * fallback fails after sonode has initially been converted to use 2197 * TPI and subsequently have to be reverted. 2198 * 2199 * Failure to pass the integrity check will panic the system. 2200 */ 2201 void 2202 so_integrity_check(struct sonode *cur, struct sonode *orig) 2203 { 2204 VERIFY(cur->so_vnode == orig->so_vnode); 2205 VERIFY(cur->so_ops == orig->so_ops); 2206 /* 2207 * For so_state we can only VERIFY the state flags in CHECK_STATE. 2208 * The other state flags might be affected by a notification from the 2209 * protocol. 2210 */ 2211 #define CHECK_STATE (SS_CANTRCVMORE|SS_CANTSENDMORE|SS_NDELAY|SS_NONBLOCK| \ 2212 SS_ASYNC|SS_ACCEPTCONN|SS_SAVEDEOR|SS_RCVATMARK|SS_OOBPEND| \ 2213 SS_HAVEOOBDATA|SS_HADOOBDATA|SS_SENTLASTREADSIG|SS_SENTLASTWRITESIG) 2214 VERIFY((cur->so_state & (orig->so_state & CHECK_STATE)) == 2215 (orig->so_state & CHECK_STATE)); 2216 VERIFY(cur->so_mode == orig->so_mode); 2217 VERIFY(cur->so_flag == orig->so_flag); 2218 VERIFY(cur->so_count == orig->so_count); 2219 /* Cannot VERIFY so_proto_connid; proto can update it */ 2220 VERIFY(cur->so_sockparams == orig->so_sockparams); 2221 /* an error might have been recorded, but it can not be lost */ 2222 VERIFY(cur->so_error != 0 || orig->so_error == 0); 2223 VERIFY(cur->so_family == orig->so_family); 2224 VERIFY(cur->so_type == orig->so_type); 2225 VERIFY(cur->so_protocol == orig->so_protocol); 2226 VERIFY(cur->so_version == orig->so_version); 2227 /* New conns might have arrived, but none should have been lost */ 2228 VERIFY(cur->so_acceptq_len >= orig->so_acceptq_len); 2229 VERIFY(list_head(&cur->so_acceptq_list) == 2230 list_head(&orig->so_acceptq_list)); 2231 VERIFY(cur->so_backlog == orig->so_backlog); 2232 /* New OOB migth have arrived, but mark should not have been lost */ 2233 VERIFY(cur->so_oobmark >= orig->so_oobmark); 2234 /* Cannot VERIFY so_oobmsg; the proto might have sent up a new one */ 2235 VERIFY(cur->so_pgrp == orig->so_pgrp); 2236 VERIFY(cur->so_peercred == orig->so_peercred); 2237 VERIFY(cur->so_cpid == orig->so_cpid); 2238 VERIFY(cur->so_zoneid == orig->so_zoneid); 2239 /* New data migth have arrived, but none should have been lost */ 2240 VERIFY(cur->so_rcv_queued >= orig->so_rcv_queued); 2241 VERIFY(cur->so_rcv_q_head == orig->so_rcv_q_head); 2242 VERIFY(cur->so_rcv_head == orig->so_rcv_head); 2243 VERIFY(cur->so_proto_handle == orig->so_proto_handle); 2244 VERIFY(cur->so_downcalls == orig->so_downcalls); 2245 /* Cannot VERIFY so_proto_props; they can be updated by proto */ 2246 } 2247 #endif 2248 2249 /* 2250 * so_tpi_fallback() 2251 * 2252 * This is the fallback initation routine; things start here. 2253 * 2254 * Basic strategy: 2255 * o Block new socket operations from coming in 2256 * o Allocate/initate info needed by TPI 2257 * o Quiesce the connection, at which point we sync 2258 * state and move data 2259 * o Change operations (sonodeops) associated with the socket 2260 * o Unblock threads waiting for the fallback to finish 2261 */ 2262 int 2263 so_tpi_fallback(struct sonode *so, struct cred *cr) 2264 { 2265 int error; 2266 queue_t *q; 2267 struct sockparams *sp; 2268 struct sockparams *newsp = NULL; 2269 so_proto_fallback_func_t fbfunc; 2270 const char *devpath; 2271 boolean_t direct; 2272 struct sonode *nso; 2273 sock_quiesce_arg_t arg = { NULL, NULL }; 2274 #ifdef DEBUG 2275 struct sonode origso; 2276 #endif 2277 error = 0; 2278 sp = so->so_sockparams; 2279 fbfunc = sp->sp_smod_info->smod_proto_fallback_func; 2280 2281 /* 2282 * Cannot fallback if the socket has active filters or a krecv callback. 2283 */ 2284 if (so->so_filter_active > 0 || so->so_krecv_cb != NULL) 2285 return (EINVAL); 2286 2287 switch (so->so_family) { 2288 case AF_INET: 2289 devpath = sp->sp_smod_info->smod_fallback_devpath_v4; 2290 break; 2291 case AF_INET6: 2292 devpath = sp->sp_smod_info->smod_fallback_devpath_v6; 2293 break; 2294 default: 2295 return (EINVAL); 2296 } 2297 2298 /* 2299 * Fallback can only happen if the socket module has a TPI device 2300 * and fallback function. 2301 */ 2302 if (devpath == NULL || fbfunc == NULL) 2303 return (EINVAL); 2304 2305 /* 2306 * Initiate fallback; upon success we know that no new requests 2307 * will come in from the user. 2308 */ 2309 if (!so_start_fallback(so)) 2310 return (EAGAIN); 2311 #ifdef DEBUG 2312 /* 2313 * Make a copy of the sonode in case we need to make an integrity 2314 * check later on. 2315 */ 2316 bcopy(so, &origso, sizeof (*so)); 2317 #endif 2318 2319 sp->sp_stats.sps_nfallback.value.ui64++; 2320 2321 newsp = sockparams_hold_ephemeral_bydev(so->so_family, so->so_type, 2322 so->so_protocol, devpath, KM_SLEEP, &error); 2323 if (error != 0) 2324 goto out; 2325 2326 if (so->so_direct != NULL) { 2327 sodirect_t *sodp = so->so_direct; 2328 mutex_enter(&so->so_lock); 2329 2330 so->so_direct->sod_enabled = B_FALSE; 2331 so->so_state &= ~SS_SODIRECT; 2332 ASSERT(sodp->sod_uioafh == NULL); 2333 mutex_exit(&so->so_lock); 2334 } 2335 2336 /* Turn sonode into a TPI socket */ 2337 error = sotpi_convert_sonode(so, newsp, &direct, &q, cr); 2338 if (error != 0) 2339 goto out; 2340 /* 2341 * When it comes to urgent data we have two cases to deal with; 2342 * (1) The oob byte has already arrived, or (2) the protocol has 2343 * notified that oob data is pending, but it has not yet arrived. 2344 * 2345 * For (1) all we need to do is send a T_EXDATA_IND to indicate were 2346 * in the byte stream the oob byte is. For (2) we have to send a 2347 * SIGURG (M_PCSIG), followed by a zero-length mblk indicating whether 2348 * the oob byte will be the next byte from the protocol. 2349 * 2350 * So in the worst case we need two mblks, one for the signal, another 2351 * for mark indication. In that case we use the exdata_mp for the sig. 2352 */ 2353 arg.soqa_exdata_mp = allocb_wait(sizeof (struct T_exdata_ind), 2354 BPRI_MED, STR_NOSIG, NULL); 2355 arg.soqa_urgmark_mp = allocb_wait(0, BPRI_MED, STR_NOSIG, NULL); 2356 2357 /* 2358 * Now tell the protocol to start using TPI. so_quiesced_cb be 2359 * called once it's safe to synchronize state. 2360 */ 2361 DTRACE_PROBE1(proto__fallback__begin, struct sonode *, so); 2362 error = (*fbfunc)(so->so_proto_handle, q, direct, so_quiesced_cb, 2363 &arg); 2364 DTRACE_PROBE1(proto__fallback__end, struct sonode *, so); 2365 2366 if (error != 0) { 2367 /* protocol was unable to do a fallback, revert the sonode */ 2368 sotpi_revert_sonode(so, cr); 2369 goto out; 2370 } 2371 2372 /* 2373 * Walk the accept queue and notify the proto that they should 2374 * fall back to TPI. The protocol will send up the T_CONN_IND. 2375 */ 2376 nso = list_head(&so->so_acceptq_list); 2377 while (nso != NULL) { 2378 int rval; 2379 struct sonode *next; 2380 2381 if (arg.soqa_exdata_mp == NULL) { 2382 arg.soqa_exdata_mp = 2383 allocb_wait(sizeof (struct T_exdata_ind), 2384 BPRI_MED, STR_NOSIG, NULL); 2385 } 2386 if (arg.soqa_urgmark_mp == NULL) { 2387 arg.soqa_urgmark_mp = allocb_wait(0, BPRI_MED, 2388 STR_NOSIG, NULL); 2389 } 2390 2391 DTRACE_PROBE1(proto__fallback__begin, struct sonode *, nso); 2392 rval = (*fbfunc)(nso->so_proto_handle, NULL, direct, 2393 so_quiesced_cb, &arg); 2394 DTRACE_PROBE1(proto__fallback__end, struct sonode *, nso); 2395 if (rval != 0) { 2396 /* Abort the connection */ 2397 zcmn_err(getzoneid(), CE_WARN, 2398 "Failed to convert socket in accept queue to TPI. " 2399 "Pid = %d\n", curproc->p_pid); 2400 next = list_next(&so->so_acceptq_list, nso); 2401 list_remove(&so->so_acceptq_list, nso); 2402 so->so_acceptq_len--; 2403 2404 (void) socket_close(nso, 0, CRED()); 2405 socket_destroy(nso); 2406 nso = next; 2407 } else { 2408 nso = list_next(&so->so_acceptq_list, nso); 2409 } 2410 } 2411 2412 /* 2413 * Now flush the acceptq, this will destroy all sockets. They will 2414 * be recreated in sotpi_accept(). 2415 */ 2416 so_acceptq_flush(so, B_FALSE); 2417 2418 mutex_enter(&so->so_lock); 2419 so->so_state |= SS_FALLBACK_COMP; 2420 mutex_exit(&so->so_lock); 2421 2422 /* 2423 * Swap the sonode ops. Socket opertations that come in once this 2424 * is done will proceed without blocking. 2425 */ 2426 so->so_ops = &sotpi_sonodeops; 2427 2428 /* 2429 * Wake up any threads stuck in poll. This is needed since the poll 2430 * head changes when the fallback happens (moves from the sonode to 2431 * the STREAMS head). 2432 */ 2433 pollwakeup(&so->so_poll_list, POLLERR); 2434 2435 /* 2436 * When this non-STREAM socket was created we placed an extra ref on 2437 * the associated vnode to support asynchronous close. Drop that ref 2438 * here. 2439 */ 2440 ASSERT(SOTOV(so)->v_count >= 2); 2441 VN_RELE(SOTOV(so)); 2442 out: 2443 so_end_fallback(so); 2444 2445 if (error != 0) { 2446 #ifdef DEBUG 2447 so_integrity_check(so, &origso); 2448 #endif 2449 zcmn_err(getzoneid(), CE_WARN, 2450 "Failed to convert socket to TPI (err=%d). Pid = %d\n", 2451 error, curproc->p_pid); 2452 if (newsp != NULL) 2453 SOCKPARAMS_DEC_REF(newsp); 2454 } 2455 if (arg.soqa_exdata_mp != NULL) 2456 freemsg(arg.soqa_exdata_mp); 2457 if (arg.soqa_urgmark_mp != NULL) 2458 freemsg(arg.soqa_urgmark_mp); 2459 2460 return (error); 2461 } 2462 2463 int 2464 so_krecv_set(sonode_t *so, so_krecv_f cb, void *arg) 2465 { 2466 int ret; 2467 2468 if (cb == NULL && arg != NULL) 2469 return (EINVAL); 2470 2471 SO_BLOCK_FALLBACK(so, so_krecv_set(so, cb, arg)); 2472 2473 mutex_enter(&so->so_lock); 2474 if (so->so_state & SS_FALLBACK_COMP) { 2475 mutex_exit(&so->so_lock); 2476 SO_UNBLOCK_FALLBACK(so); 2477 return (ENOTSUP); 2478 } 2479 2480 ret = so_lock_read(so, 0); 2481 VERIFY(ret == 0); 2482 /* 2483 * Other consumers may actually care about getting extant data delivered 2484 * to them, when they come along, they should figure out the best API 2485 * for that. 2486 */ 2487 so_rcv_flush(so); 2488 2489 so->so_krecv_cb = cb; 2490 so->so_krecv_arg = arg; 2491 2492 so_unlock_read(so); 2493 mutex_exit(&so->so_lock); 2494 SO_UNBLOCK_FALLBACK(so); 2495 2496 return (0); 2497 } 2498 2499 void 2500 so_krecv_unblock(sonode_t *so) 2501 { 2502 mutex_enter(&so->so_lock); 2503 VERIFY(so->so_krecv_cb != NULL); 2504 2505 so->so_rcv_queued = 0; 2506 (void) so_check_flow_control(so); 2507 /* 2508 * so_check_flow_control() always drops so->so_lock, so we won't 2509 * need to drop it ourselves. 2510 */ 2511 } 2512