1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Shared Memory Communications over RDMA (SMC-R) and RoCE 4 * 5 * Socket Closing - normal and abnormal 6 * 7 * Copyright IBM Corp. 2016 8 * 9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> 10 */ 11 12 #include <linux/workqueue.h> 13 #include <linux/sched/signal.h> 14 15 #include <net/sock.h> 16 #include <net/tcp.h> 17 18 #include "smc.h" 19 #include "smc_tx.h" 20 #include "smc_cdc.h" 21 #include "smc_close.h" 22 23 /* release the clcsock that is assigned to the smc_sock */ 24 void smc_clcsock_release(struct smc_sock *smc) 25 { 26 struct socket *tcp; 27 28 if (smc->listen_smc && current_work() != &smc->smc_listen_work) 29 cancel_work_sync(&smc->smc_listen_work); 30 mutex_lock(&smc->clcsock_release_lock); 31 if (smc->clcsock) { 32 tcp = smc->clcsock; 33 smc->clcsock = NULL; 34 sock_release(tcp); 35 } 36 mutex_unlock(&smc->clcsock_release_lock); 37 } 38 39 static void smc_close_cleanup_listen(struct sock *parent) 40 { 41 struct sock *sk; 42 43 /* Close non-accepted connections */ 44 while ((sk = smc_accept_dequeue(parent, NULL))) 45 smc_close_non_accepted(sk); 46 } 47 48 /* wait for sndbuf data being transmitted */ 49 static void smc_close_stream_wait(struct smc_sock *smc, long timeout) 50 { 51 DEFINE_WAIT_FUNC(wait, woken_wake_function); 52 struct sock *sk = &smc->sk; 53 54 if (!timeout) 55 return; 56 57 if (!smc_tx_prepared_sends(&smc->conn)) 58 return; 59 60 /* Send out corked data remaining in sndbuf */ 61 smc_tx_pending(&smc->conn); 62 63 smc->wait_close_tx_prepared = 1; 64 add_wait_queue(sk_sleep(sk), &wait); 65 while (!signal_pending(current) && timeout) { 66 int rc; 67 68 rc = sk_wait_event(sk, &timeout, 69 !smc_tx_prepared_sends(&smc->conn) || 70 READ_ONCE(sk->sk_err) == ECONNABORTED || 71 READ_ONCE(sk->sk_err) == ECONNRESET || 72 smc->conn.killed, 73 &wait); 74 if (rc) 75 break; 76 } 77 remove_wait_queue(sk_sleep(sk), &wait); 78 smc->wait_close_tx_prepared = 0; 79 } 80 81 void smc_close_wake_tx_prepared(struct smc_sock *smc) 82 { 83 if (smc->wait_close_tx_prepared) 84 /* wake up socket closing */ 85 smc->sk.sk_state_change(&smc->sk); 86 } 87 88 static int smc_close_wr(struct smc_connection *conn) 89 { 90 conn->local_tx_ctrl.conn_state_flags.peer_done_writing = 1; 91 92 return smc_cdc_get_slot_and_msg_send(conn); 93 } 94 95 static int smc_close_final(struct smc_connection *conn) 96 { 97 if (atomic_read(&conn->bytes_to_rcv)) 98 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; 99 else 100 conn->local_tx_ctrl.conn_state_flags.peer_conn_closed = 1; 101 if (conn->killed) 102 return -EPIPE; 103 104 return smc_cdc_get_slot_and_msg_send(conn); 105 } 106 107 int smc_close_abort(struct smc_connection *conn) 108 { 109 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; 110 111 return smc_cdc_get_slot_and_msg_send(conn); 112 } 113 114 static void smc_close_cancel_work(struct smc_sock *smc) 115 { 116 struct sock *sk = &smc->sk; 117 118 release_sock(sk); 119 if (cancel_work_sync(&smc->conn.close_work)) 120 sock_put(sk); 121 if (cancel_delayed_work_sync(&smc->conn.tx_work)) 122 sock_put(sk); 123 lock_sock(sk); 124 } 125 126 /* terminate smc socket abnormally - active abort 127 * link group is terminated, i.e. RDMA communication no longer possible 128 */ 129 void smc_close_active_abort(struct smc_sock *smc) 130 { 131 struct sock *sk = &smc->sk; 132 bool release_clcsock = false; 133 134 if (sk->sk_state != SMC_INIT && smc->clcsock && smc->clcsock->sk) { 135 sk->sk_err = ECONNABORTED; 136 if (smc->clcsock && smc->clcsock->sk) 137 tcp_abort(smc->clcsock->sk, ECONNABORTED); 138 } 139 switch (sk->sk_state) { 140 case SMC_ACTIVE: 141 case SMC_APPCLOSEWAIT1: 142 case SMC_APPCLOSEWAIT2: 143 sk->sk_state = SMC_PEERABORTWAIT; 144 smc_close_cancel_work(smc); 145 if (sk->sk_state != SMC_PEERABORTWAIT) 146 break; 147 sk->sk_state = SMC_CLOSED; 148 sock_put(sk); /* (postponed) passive closing */ 149 break; 150 case SMC_PEERCLOSEWAIT1: 151 case SMC_PEERCLOSEWAIT2: 152 case SMC_PEERFINCLOSEWAIT: 153 sk->sk_state = SMC_PEERABORTWAIT; 154 smc_close_cancel_work(smc); 155 if (sk->sk_state != SMC_PEERABORTWAIT) 156 break; 157 sk->sk_state = SMC_CLOSED; 158 smc_conn_free(&smc->conn); 159 release_clcsock = true; 160 sock_put(sk); /* passive closing */ 161 break; 162 case SMC_PROCESSABORT: 163 case SMC_APPFINCLOSEWAIT: 164 sk->sk_state = SMC_PEERABORTWAIT; 165 smc_close_cancel_work(smc); 166 if (sk->sk_state != SMC_PEERABORTWAIT) 167 break; 168 sk->sk_state = SMC_CLOSED; 169 smc_conn_free(&smc->conn); 170 release_clcsock = true; 171 break; 172 case SMC_INIT: 173 case SMC_PEERABORTWAIT: 174 case SMC_CLOSED: 175 break; 176 } 177 178 smc_sock_set_flag(sk, SOCK_DEAD); 179 sk->sk_state_change(sk); 180 181 if (release_clcsock) { 182 release_sock(sk); 183 smc_clcsock_release(smc); 184 lock_sock(sk); 185 } 186 } 187 188 static inline bool smc_close_sent_any_close(struct smc_connection *conn) 189 { 190 return conn->local_tx_ctrl.conn_state_flags.peer_conn_abort || 191 conn->local_tx_ctrl.conn_state_flags.peer_conn_closed; 192 } 193 194 int smc_close_active(struct smc_sock *smc) 195 { 196 struct smc_cdc_conn_state_flags *txflags = 197 &smc->conn.local_tx_ctrl.conn_state_flags; 198 struct smc_connection *conn = &smc->conn; 199 struct sock *sk = &smc->sk; 200 int old_state; 201 long timeout; 202 int rc = 0; 203 int rc1 = 0; 204 205 timeout = current->flags & PF_EXITING ? 206 0 : sock_flag(sk, SOCK_LINGER) ? 207 sk->sk_lingertime : SMC_MAX_STREAM_WAIT_TIMEOUT; 208 209 old_state = sk->sk_state; 210 again: 211 switch (sk->sk_state) { 212 case SMC_INIT: 213 sk->sk_state = SMC_CLOSED; 214 break; 215 case SMC_LISTEN: 216 sk->sk_state = SMC_CLOSED; 217 sk->sk_state_change(sk); /* wake up accept */ 218 if (smc->clcsock && smc->clcsock->sk) { 219 write_lock_bh(&smc->clcsock->sk->sk_callback_lock); 220 smc_clcsock_restore_cb(&smc->clcsock->sk->sk_data_ready, 221 &smc->clcsk_data_ready); 222 rcu_assign_sk_user_data(smc->clcsock->sk, NULL); 223 write_unlock_bh(&smc->clcsock->sk->sk_callback_lock); 224 rc = kernel_sock_shutdown(smc->clcsock, SHUT_RDWR); 225 } 226 smc_close_cleanup_listen(sk); 227 release_sock(sk); 228 flush_work(&smc->tcp_listen_work); 229 lock_sock(sk); 230 break; 231 case SMC_ACTIVE: 232 smc_close_stream_wait(smc, timeout); 233 release_sock(sk); 234 if (cancel_delayed_work_sync(&conn->tx_work)) 235 sock_put(sk); 236 lock_sock(sk); 237 if (sk->sk_state == SMC_ACTIVE) { 238 /* send close request */ 239 rc = smc_close_final(conn); 240 sk->sk_state = SMC_PEERCLOSEWAIT1; 241 242 /* actively shutdown clcsock before peer close it, 243 * prevent peer from entering TIME_WAIT state. 244 */ 245 if (smc->clcsock && smc->clcsock->sk) { 246 rc1 = kernel_sock_shutdown(smc->clcsock, 247 SHUT_RDWR); 248 rc = rc ? rc : rc1; 249 } 250 } else { 251 /* peer event has changed the state */ 252 goto again; 253 } 254 break; 255 case SMC_APPFINCLOSEWAIT: 256 /* socket already shutdown wr or both (active close) */ 257 if (txflags->peer_done_writing && 258 !smc_close_sent_any_close(conn)) { 259 /* just shutdown wr done, send close request */ 260 rc = smc_close_final(conn); 261 } 262 sk->sk_state = SMC_CLOSED; 263 break; 264 case SMC_APPCLOSEWAIT1: 265 case SMC_APPCLOSEWAIT2: 266 if (!smc_cdc_rxed_any_close(conn)) 267 smc_close_stream_wait(smc, timeout); 268 release_sock(sk); 269 if (cancel_delayed_work_sync(&conn->tx_work)) 270 sock_put(sk); 271 lock_sock(sk); 272 if (sk->sk_state != SMC_APPCLOSEWAIT1 && 273 sk->sk_state != SMC_APPCLOSEWAIT2) 274 goto again; 275 /* confirm close from peer */ 276 rc = smc_close_final(conn); 277 if (smc_cdc_rxed_any_close(conn)) { 278 /* peer has closed the socket already */ 279 sk->sk_state = SMC_CLOSED; 280 sock_put(sk); /* postponed passive closing */ 281 } else { 282 /* peer has just issued a shutdown write */ 283 sk->sk_state = SMC_PEERFINCLOSEWAIT; 284 } 285 break; 286 case SMC_PEERCLOSEWAIT1: 287 case SMC_PEERCLOSEWAIT2: 288 if (txflags->peer_done_writing && 289 !smc_close_sent_any_close(conn)) { 290 /* just shutdown wr done, send close request */ 291 rc = smc_close_final(conn); 292 } 293 /* peer sending PeerConnectionClosed will cause transition */ 294 break; 295 case SMC_PEERFINCLOSEWAIT: 296 /* peer sending PeerConnectionClosed will cause transition */ 297 break; 298 case SMC_PROCESSABORT: 299 rc = smc_close_abort(conn); 300 sk->sk_state = SMC_CLOSED; 301 break; 302 case SMC_PEERABORTWAIT: 303 sk->sk_state = SMC_CLOSED; 304 break; 305 case SMC_CLOSED: 306 /* nothing to do, add tracing in future patch */ 307 break; 308 } 309 310 if (old_state != sk->sk_state) 311 sk->sk_state_change(sk); 312 return rc; 313 } 314 315 static void smc_close_passive_abort_received(struct smc_sock *smc) 316 { 317 struct smc_cdc_conn_state_flags *txflags = 318 &smc->conn.local_tx_ctrl.conn_state_flags; 319 struct sock *sk = &smc->sk; 320 321 switch (sk->sk_state) { 322 case SMC_INIT: 323 case SMC_ACTIVE: 324 case SMC_APPCLOSEWAIT1: 325 sk->sk_state = SMC_PROCESSABORT; 326 sock_put(sk); /* passive closing */ 327 break; 328 case SMC_APPFINCLOSEWAIT: 329 sk->sk_state = SMC_PROCESSABORT; 330 break; 331 case SMC_PEERCLOSEWAIT1: 332 case SMC_PEERCLOSEWAIT2: 333 if (txflags->peer_done_writing && 334 !smc_close_sent_any_close(&smc->conn)) 335 /* just shutdown, but not yet closed locally */ 336 sk->sk_state = SMC_PROCESSABORT; 337 else 338 sk->sk_state = SMC_CLOSED; 339 sock_put(sk); /* passive closing */ 340 break; 341 case SMC_APPCLOSEWAIT2: 342 case SMC_PEERFINCLOSEWAIT: 343 sk->sk_state = SMC_CLOSED; 344 sock_put(sk); /* passive closing */ 345 break; 346 case SMC_PEERABORTWAIT: 347 sk->sk_state = SMC_CLOSED; 348 break; 349 case SMC_PROCESSABORT: 350 /* nothing to do, add tracing in future patch */ 351 break; 352 } 353 } 354 355 /* Either some kind of closing has been received: peer_conn_closed, 356 * peer_conn_abort, or peer_done_writing 357 * or the link group of the connection terminates abnormally. 358 */ 359 static void smc_close_passive_work(struct work_struct *work) 360 { 361 struct smc_connection *conn = container_of(work, 362 struct smc_connection, 363 close_work); 364 struct smc_sock *smc = container_of(conn, struct smc_sock, conn); 365 struct smc_cdc_conn_state_flags *rxflags; 366 bool release_clcsock = false; 367 struct sock *sk = &smc->sk; 368 int old_state; 369 370 lock_sock(sk); 371 old_state = sk->sk_state; 372 373 rxflags = &conn->local_rx_ctrl.conn_state_flags; 374 if (rxflags->peer_conn_abort) { 375 /* peer has not received all data */ 376 smc_close_passive_abort_received(smc); 377 release_sock(sk); 378 if (cancel_delayed_work_sync(&conn->tx_work)) 379 sock_put(sk); 380 lock_sock(sk); 381 goto wakeup; 382 } 383 384 switch (sk->sk_state) { 385 case SMC_INIT: 386 sk->sk_state = SMC_APPCLOSEWAIT1; 387 break; 388 case SMC_ACTIVE: 389 sk->sk_state = SMC_APPCLOSEWAIT1; 390 /* postpone sock_put() for passive closing to cover 391 * received SEND_SHUTDOWN as well 392 */ 393 break; 394 case SMC_PEERCLOSEWAIT1: 395 if (rxflags->peer_done_writing) 396 sk->sk_state = SMC_PEERCLOSEWAIT2; 397 fallthrough; 398 /* to check for closing */ 399 case SMC_PEERCLOSEWAIT2: 400 if (!smc_cdc_rxed_any_close(conn)) 401 break; 402 if (sock_flag(sk, SOCK_DEAD) && 403 smc_close_sent_any_close(conn)) { 404 /* smc_release has already been called locally */ 405 sk->sk_state = SMC_CLOSED; 406 } else { 407 /* just shutdown, but not yet closed locally */ 408 sk->sk_state = SMC_APPFINCLOSEWAIT; 409 } 410 sock_put(sk); /* passive closing */ 411 break; 412 case SMC_PEERFINCLOSEWAIT: 413 if (smc_cdc_rxed_any_close(conn)) { 414 sk->sk_state = SMC_CLOSED; 415 sock_put(sk); /* passive closing */ 416 } 417 break; 418 case SMC_APPCLOSEWAIT1: 419 case SMC_APPCLOSEWAIT2: 420 /* postpone sock_put() for passive closing to cover 421 * received SEND_SHUTDOWN as well 422 */ 423 break; 424 case SMC_APPFINCLOSEWAIT: 425 case SMC_PEERABORTWAIT: 426 case SMC_PROCESSABORT: 427 case SMC_CLOSED: 428 /* nothing to do, add tracing in future patch */ 429 break; 430 } 431 432 wakeup: 433 sk->sk_data_ready(sk); /* wakeup blocked rcvbuf consumers */ 434 sk->sk_write_space(sk); /* wakeup blocked sndbuf producers */ 435 436 if (old_state != sk->sk_state) { 437 sk->sk_state_change(sk); 438 if ((sk->sk_state == SMC_CLOSED) && 439 (sock_flag(sk, SOCK_DEAD) || !sk->sk_socket)) { 440 smc_conn_free(conn); 441 if (smc->clcsock) 442 release_clcsock = true; 443 } 444 } 445 release_sock(sk); 446 if (release_clcsock) 447 smc_clcsock_release(smc); 448 sock_put(sk); /* sock_hold done by schedulers of close_work */ 449 } 450 451 int smc_close_shutdown_write(struct smc_sock *smc) 452 { 453 struct smc_connection *conn = &smc->conn; 454 struct sock *sk = &smc->sk; 455 int old_state; 456 long timeout; 457 int rc = 0; 458 459 timeout = current->flags & PF_EXITING ? 460 0 : sock_flag(sk, SOCK_LINGER) ? 461 sk->sk_lingertime : SMC_MAX_STREAM_WAIT_TIMEOUT; 462 463 old_state = sk->sk_state; 464 again: 465 switch (sk->sk_state) { 466 case SMC_ACTIVE: 467 smc_close_stream_wait(smc, timeout); 468 release_sock(sk); 469 if (cancel_delayed_work_sync(&conn->tx_work)) 470 sock_put(sk); 471 lock_sock(sk); 472 if (sk->sk_state != SMC_ACTIVE) 473 goto again; 474 /* send close wr request */ 475 rc = smc_close_wr(conn); 476 sk->sk_state = SMC_PEERCLOSEWAIT1; 477 break; 478 case SMC_APPCLOSEWAIT1: 479 /* passive close */ 480 if (!smc_cdc_rxed_any_close(conn)) 481 smc_close_stream_wait(smc, timeout); 482 release_sock(sk); 483 if (cancel_delayed_work_sync(&conn->tx_work)) 484 sock_put(sk); 485 lock_sock(sk); 486 if (sk->sk_state != SMC_APPCLOSEWAIT1) 487 goto again; 488 /* confirm close from peer */ 489 rc = smc_close_wr(conn); 490 sk->sk_state = SMC_APPCLOSEWAIT2; 491 break; 492 case SMC_APPCLOSEWAIT2: 493 case SMC_PEERFINCLOSEWAIT: 494 case SMC_PEERCLOSEWAIT1: 495 case SMC_PEERCLOSEWAIT2: 496 case SMC_APPFINCLOSEWAIT: 497 case SMC_PROCESSABORT: 498 case SMC_PEERABORTWAIT: 499 /* nothing to do, add tracing in future patch */ 500 break; 501 } 502 503 if (old_state != sk->sk_state) 504 sk->sk_state_change(sk); 505 return rc; 506 } 507 508 /* Initialize close properties on connection establishment. */ 509 void smc_close_init(struct smc_sock *smc) 510 { 511 INIT_WORK(&smc->conn.close_work, smc_close_passive_work); 512 } 513