1 /* 2 * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 #include <linux/kernel.h> 34 #include <linux/list.h> 35 #include <linux/slab.h> 36 #include <linux/export.h> 37 #include <net/ipv6.h> 38 #include <net/inet6_hashtables.h> 39 #include <net/addrconf.h> 40 41 #include "rds.h" 42 #include "loop.h" 43 44 #define RDS_CONNECTION_HASH_BITS 12 45 #define RDS_CONNECTION_HASH_ENTRIES (1 << RDS_CONNECTION_HASH_BITS) 46 #define RDS_CONNECTION_HASH_MASK (RDS_CONNECTION_HASH_ENTRIES - 1) 47 48 /* converting this to RCU is a chore for another day.. */ 49 static DEFINE_SPINLOCK(rds_conn_lock); 50 static unsigned long rds_conn_count; 51 static struct hlist_head rds_conn_hash[RDS_CONNECTION_HASH_ENTRIES]; 52 static struct kmem_cache *rds_conn_slab; 53 54 static struct hlist_head *rds_conn_bucket(const struct in6_addr *laddr, 55 const struct in6_addr *faddr) 56 { 57 static u32 rds6_hash_secret __read_mostly; 58 static u32 rds_hash_secret __read_mostly; 59 60 __be32 lhash, fhash; 61 u32 hash; 62 63 net_get_random_once(&rds_hash_secret, sizeof(rds_hash_secret)); 64 net_get_random_once(&rds6_hash_secret, sizeof(rds6_hash_secret)); 65 66 lhash = laddr->s6_addr32[3]; 67 #if IS_ENABLED(CONFIG_IPV6) 68 fhash = (__force __be32)__ipv6_addr_jhash(faddr, rds6_hash_secret); 69 #else 70 fhash = faddr->s6_addr32[3]; 71 #endif 72 hash = __inet_ehashfn(lhash, 0, fhash, 0, rds_hash_secret); 73 74 return &rds_conn_hash[hash & RDS_CONNECTION_HASH_MASK]; 75 } 76 77 #define rds_conn_info_set(var, test, suffix) do { \ 78 if (test) \ 79 var |= RDS_INFO_CONNECTION_FLAG_##suffix; \ 80 } while (0) 81 82 /* rcu read lock must be held or the connection spinlock */ 83 static struct rds_connection *rds_conn_lookup(struct net *net, 84 struct hlist_head *head, 85 const struct in6_addr *laddr, 86 const struct in6_addr *faddr, 87 struct rds_transport *trans, 88 u8 tos, int dev_if) 89 { 90 struct rds_connection *conn, *ret = NULL; 91 92 hlist_for_each_entry_rcu(conn, head, c_hash_node) { 93 if (ipv6_addr_equal(&conn->c_faddr, faddr) && 94 ipv6_addr_equal(&conn->c_laddr, laddr) && 95 conn->c_trans == trans && 96 conn->c_tos == tos && 97 net == rds_conn_net(conn) && 98 conn->c_dev_if == dev_if) { 99 ret = conn; 100 break; 101 } 102 } 103 rdsdebug("returning conn %p for %pI6c -> %pI6c\n", ret, 104 laddr, faddr); 105 return ret; 106 } 107 108 /* 109 * This is called by transports as they're bringing down a connection. 110 * It clears partial message state so that the transport can start sending 111 * and receiving over this connection again in the future. It is up to 112 * the transport to have serialized this call with its send and recv. 113 */ 114 static void rds_conn_path_reset(struct rds_conn_path *cp) 115 { 116 struct rds_connection *conn = cp->cp_conn; 117 118 rdsdebug("connection %pI6c to %pI6c reset\n", 119 &conn->c_laddr, &conn->c_faddr); 120 121 rds_stats_inc(s_conn_reset); 122 rds_send_path_reset(cp); 123 cp->cp_flags = 0; 124 125 /* Do not clear next_rx_seq here, else we cannot distinguish 126 * retransmitted packets from new packets, and will hand all 127 * of them to the application. That is not consistent with the 128 * reliability guarantees of RDS. */ 129 } 130 131 static void __rds_conn_path_init(struct rds_connection *conn, 132 struct rds_conn_path *cp, bool is_outgoing) 133 { 134 spin_lock_init(&cp->cp_lock); 135 cp->cp_next_tx_seq = 1; 136 init_waitqueue_head(&cp->cp_waitq); 137 INIT_LIST_HEAD(&cp->cp_send_queue); 138 INIT_LIST_HEAD(&cp->cp_retrans); 139 140 cp->cp_conn = conn; 141 atomic_set(&cp->cp_state, RDS_CONN_DOWN); 142 cp->cp_send_gen = 0; 143 cp->cp_reconnect_jiffies = 0; 144 cp->cp_conn->c_proposed_version = RDS_PROTOCOL_VERSION; 145 INIT_DELAYED_WORK(&cp->cp_send_w, rds_send_worker); 146 INIT_DELAYED_WORK(&cp->cp_recv_w, rds_recv_worker); 147 INIT_DELAYED_WORK(&cp->cp_conn_w, rds_connect_worker); 148 INIT_WORK(&cp->cp_down_w, rds_shutdown_worker); 149 mutex_init(&cp->cp_cm_lock); 150 cp->cp_flags = 0; 151 } 152 153 /* 154 * There is only every one 'conn' for a given pair of addresses in the 155 * system at a time. They contain messages to be retransmitted and so 156 * span the lifetime of the actual underlying transport connections. 157 * 158 * For now they are not garbage collected once they're created. They 159 * are torn down as the module is removed, if ever. 160 */ 161 static struct rds_connection *__rds_conn_create(struct net *net, 162 const struct in6_addr *laddr, 163 const struct in6_addr *faddr, 164 struct rds_transport *trans, 165 gfp_t gfp, u8 tos, 166 int is_outgoing, 167 int dev_if) 168 { 169 struct rds_connection *conn, *parent = NULL; 170 struct hlist_head *head = rds_conn_bucket(laddr, faddr); 171 struct rds_transport *loop_trans; 172 struct rds_conn_path *free_cp = NULL; 173 unsigned long flags; 174 int ret, i; 175 int npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1); 176 177 rcu_read_lock(); 178 conn = rds_conn_lookup(net, head, laddr, faddr, trans, tos, dev_if); 179 if (conn && 180 conn->c_loopback && 181 conn->c_trans != &rds_loop_transport && 182 ipv6_addr_equal(laddr, faddr) && 183 !is_outgoing) { 184 /* This is a looped back IB connection, and we're 185 * called by the code handling the incoming connect. 186 * We need a second connection object into which we 187 * can stick the other QP. */ 188 parent = conn; 189 conn = parent->c_passive; 190 } 191 rcu_read_unlock(); 192 if (conn) 193 goto out; 194 195 conn = kmem_cache_zalloc(rds_conn_slab, gfp); 196 if (!conn) { 197 conn = ERR_PTR(-ENOMEM); 198 goto out; 199 } 200 conn->c_path = kzalloc_objs(struct rds_conn_path, npaths, gfp); 201 if (!conn->c_path) { 202 kmem_cache_free(rds_conn_slab, conn); 203 conn = ERR_PTR(-ENOMEM); 204 goto out; 205 } 206 207 INIT_HLIST_NODE(&conn->c_hash_node); 208 conn->c_laddr = *laddr; 209 conn->c_isv6 = !ipv6_addr_v4mapped(laddr); 210 conn->c_faddr = *faddr; 211 conn->c_dev_if = dev_if; 212 conn->c_tos = tos; 213 214 #if IS_ENABLED(CONFIG_IPV6) 215 /* If the local address is link local, set c_bound_if to be the 216 * index used for this connection. Otherwise, set it to 0 as 217 * the socket is not bound to an interface. c_bound_if is used 218 * to look up a socket when a packet is received 219 */ 220 if (ipv6_addr_type(laddr) & IPV6_ADDR_LINKLOCAL) 221 conn->c_bound_if = dev_if; 222 else 223 #endif 224 conn->c_bound_if = 0; 225 226 rds_conn_net_set(conn, net); 227 228 ret = rds_cong_get_maps(conn); 229 if (ret) { 230 kfree(conn->c_path); 231 kmem_cache_free(rds_conn_slab, conn); 232 conn = ERR_PTR(ret); 233 goto out; 234 } 235 236 /* 237 * This is where a connection becomes loopback. If *any* RDS sockets 238 * can bind to the destination address then we'd rather the messages 239 * flow through loopback rather than either transport. 240 */ 241 loop_trans = rds_trans_get_preferred(net, faddr, conn->c_dev_if); 242 if (loop_trans) { 243 rds_trans_put(loop_trans); 244 conn->c_loopback = 1; 245 if (trans->t_prefer_loopback) { 246 if (likely(is_outgoing)) { 247 /* "outgoing" connection to local address. 248 * Protocol says it wants the connection 249 * handled by the loopback transport. 250 * This is what TCP does. 251 */ 252 trans = &rds_loop_transport; 253 } else { 254 /* No transport currently in use 255 * should end up here, but if it 256 * does, reset/destroy the connection. 257 */ 258 kfree(conn->c_path); 259 kmem_cache_free(rds_conn_slab, conn); 260 conn = ERR_PTR(-EOPNOTSUPP); 261 goto out; 262 } 263 } 264 } 265 266 conn->c_trans = trans; 267 268 init_waitqueue_head(&conn->c_hs_waitq); 269 for (i = 0; i < npaths; i++) { 270 __rds_conn_path_init(conn, &conn->c_path[i], 271 is_outgoing); 272 conn->c_path[i].cp_index = i; 273 conn->c_path[i].cp_wq = 274 alloc_ordered_workqueue("krds_cp_wq#%lu/%d", 0, 275 rds_conn_count, i); 276 if (!conn->c_path[i].cp_wq) 277 conn->c_path[i].cp_wq = rds_wq; 278 } 279 rcu_read_lock(); 280 if (rds_destroy_pending(conn)) 281 ret = -ENETDOWN; 282 else 283 ret = trans->conn_alloc(conn, GFP_ATOMIC); 284 if (ret) { 285 rcu_read_unlock(); 286 free_cp = conn->c_path; 287 kmem_cache_free(rds_conn_slab, conn); 288 conn = ERR_PTR(ret); 289 goto out; 290 } 291 292 rdsdebug("allocated conn %p for %pI6c -> %pI6c over %s %s\n", 293 conn, laddr, faddr, 294 strnlen(trans->t_name, sizeof(trans->t_name)) ? 295 trans->t_name : "[unknown]", is_outgoing ? "(outgoing)" : ""); 296 297 /* 298 * Since we ran without holding the conn lock, someone could 299 * have created the same conn (either normal or passive) in the 300 * interim. We check while holding the lock. If we won, we complete 301 * init and return our conn. If we lost, we rollback and return the 302 * other one. 303 */ 304 spin_lock_irqsave(&rds_conn_lock, flags); 305 if (parent) { 306 /* Creating passive conn */ 307 if (parent->c_passive) { 308 trans->conn_free(conn->c_path[0].cp_transport_data); 309 free_cp = conn->c_path; 310 kmem_cache_free(rds_conn_slab, conn); 311 conn = parent->c_passive; 312 } else { 313 parent->c_passive = conn; 314 rds_cong_add_conn(conn); 315 rds_conn_count++; 316 } 317 } else { 318 /* Creating normal conn */ 319 struct rds_connection *found; 320 321 found = rds_conn_lookup(net, head, laddr, faddr, trans, 322 tos, dev_if); 323 if (found) { 324 struct rds_conn_path *cp; 325 int i; 326 327 for (i = 0; i < npaths; i++) { 328 cp = &conn->c_path[i]; 329 /* The ->conn_alloc invocation may have 330 * allocated resource for all paths, so all 331 * of them may have to be freed here. 332 */ 333 if (cp->cp_transport_data) 334 trans->conn_free(cp->cp_transport_data); 335 } 336 free_cp = conn->c_path; 337 kmem_cache_free(rds_conn_slab, conn); 338 conn = found; 339 } else { 340 conn->c_my_gen_num = rds_gen_num; 341 conn->c_peer_gen_num = 0; 342 hlist_add_head_rcu(&conn->c_hash_node, head); 343 rds_cong_add_conn(conn); 344 rds_conn_count++; 345 } 346 } 347 spin_unlock_irqrestore(&rds_conn_lock, flags); 348 rcu_read_unlock(); 349 350 out: 351 if (free_cp) { 352 for (i = 0; i < npaths; i++) 353 if (free_cp[i].cp_wq != rds_wq) 354 destroy_workqueue(free_cp[i].cp_wq); 355 kfree(free_cp); 356 } 357 358 return conn; 359 } 360 361 struct rds_connection *rds_conn_create(struct net *net, 362 const struct in6_addr *laddr, 363 const struct in6_addr *faddr, 364 struct rds_transport *trans, u8 tos, 365 gfp_t gfp, int dev_if) 366 { 367 return __rds_conn_create(net, laddr, faddr, trans, gfp, tos, 0, dev_if); 368 } 369 EXPORT_SYMBOL_GPL(rds_conn_create); 370 371 struct rds_connection *rds_conn_create_outgoing(struct net *net, 372 const struct in6_addr *laddr, 373 const struct in6_addr *faddr, 374 struct rds_transport *trans, 375 u8 tos, gfp_t gfp, int dev_if) 376 { 377 return __rds_conn_create(net, laddr, faddr, trans, gfp, tos, 1, dev_if); 378 } 379 EXPORT_SYMBOL_GPL(rds_conn_create_outgoing); 380 381 void rds_conn_shutdown(struct rds_conn_path *cp) 382 { 383 struct rds_connection *conn = cp->cp_conn; 384 385 /* shut it down unless it's down already */ 386 if (!rds_conn_path_transition(cp, RDS_CONN_DOWN, RDS_CONN_DOWN)) { 387 /* 388 * Quiesce the connection mgmt handlers before we start tearing 389 * things down. We don't hold the mutex for the entire 390 * duration of the shutdown operation, else we may be 391 * deadlocking with the CM handler. Instead, the CM event 392 * handler is supposed to check for state DISCONNECTING 393 */ 394 mutex_lock(&cp->cp_cm_lock); 395 if (!rds_conn_path_transition(cp, RDS_CONN_UP, 396 RDS_CONN_DISCONNECTING) && 397 !rds_conn_path_transition(cp, RDS_CONN_ERROR, 398 RDS_CONN_DISCONNECTING) && 399 !rds_conn_path_transition(cp, RDS_CONN_RESETTING, 400 RDS_CONN_DISCONNECTING)) { 401 rds_conn_path_error(cp, 402 "shutdown called in state %d\n", 403 atomic_read(&cp->cp_state)); 404 mutex_unlock(&cp->cp_cm_lock); 405 return; 406 } 407 mutex_unlock(&cp->cp_cm_lock); 408 409 wait_event(cp->cp_waitq, 410 !test_bit(RDS_IN_XMIT, &cp->cp_flags)); 411 wait_event(cp->cp_waitq, 412 !test_bit(RDS_RECV_REFILL, &cp->cp_flags)); 413 414 conn->c_trans->conn_path_shutdown(cp); 415 rds_conn_path_reset(cp); 416 417 if (!rds_conn_path_transition(cp, RDS_CONN_DISCONNECTING, 418 RDS_CONN_DOWN) && 419 !rds_conn_path_transition(cp, RDS_CONN_ERROR, 420 RDS_CONN_DOWN)) { 421 /* This can happen - eg when we're in the middle of tearing 422 * down the connection, and someone unloads the rds module. 423 * Quite reproducible with loopback connections. 424 * Mostly harmless. 425 * 426 * Note that this also happens with rds-tcp because 427 * we could have triggered rds_conn_path_drop in irq 428 * mode from rds_tcp_state change on the receipt of 429 * a FIN, thus we need to recheck for RDS_CONN_ERROR 430 * here. 431 */ 432 rds_conn_path_error(cp, "%s: failed to transition " 433 "to state DOWN, current state " 434 "is %d\n", __func__, 435 atomic_read(&cp->cp_state)); 436 return; 437 } 438 } 439 440 /* Then reconnect if it's still live. 441 * The passive side of an IB loopback connection is never added 442 * to the conn hash, so we never trigger a reconnect on this 443 * conn - the reconnect is always triggered by the active peer. */ 444 cancel_delayed_work_sync(&cp->cp_conn_w); 445 446 clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); 447 rcu_read_lock(); 448 if (!hlist_unhashed(&conn->c_hash_node)) { 449 rcu_read_unlock(); 450 if (conn->c_trans->t_mp_capable && 451 cp->cp_index == 0) 452 rds_send_ping(conn, 0); 453 rds_queue_reconnect(cp); 454 } else { 455 rcu_read_unlock(); 456 } 457 458 if (conn->c_trans->conn_slots_available) 459 conn->c_trans->conn_slots_available(conn, false); 460 } 461 462 /* destroy a single rds_conn_path. rds_conn_destroy() iterates over 463 * all paths using rds_conn_path_destroy() 464 */ 465 static void rds_conn_path_destroy(struct rds_conn_path *cp) 466 { 467 struct rds_message *rm, *rtmp; 468 469 if (!cp->cp_transport_data) 470 return; 471 472 /* make sure lingering queued work won't try to ref the conn */ 473 cancel_delayed_work_sync(&cp->cp_send_w); 474 cancel_delayed_work_sync(&cp->cp_recv_w); 475 476 rds_conn_path_drop(cp, true); 477 flush_work(&cp->cp_down_w); 478 479 /* tear down queued messages */ 480 list_for_each_entry_safe(rm, rtmp, 481 &cp->cp_send_queue, 482 m_conn_item) { 483 list_del_init(&rm->m_conn_item); 484 BUG_ON(!list_empty(&rm->m_sock_item)); 485 rds_message_put(rm); 486 } 487 if (cp->cp_xmit_rm) 488 rds_message_put(cp->cp_xmit_rm); 489 490 WARN_ON(delayed_work_pending(&cp->cp_send_w)); 491 WARN_ON(delayed_work_pending(&cp->cp_recv_w)); 492 WARN_ON(delayed_work_pending(&cp->cp_conn_w)); 493 WARN_ON(work_pending(&cp->cp_down_w)); 494 495 if (cp->cp_wq != rds_wq) { 496 destroy_workqueue(cp->cp_wq); 497 cp->cp_wq = NULL; 498 } 499 500 cp->cp_conn->c_trans->conn_free(cp->cp_transport_data); 501 } 502 503 /* 504 * Stop and free a connection. 505 * 506 * This can only be used in very limited circumstances. It assumes that once 507 * the conn has been shutdown that no one else is referencing the connection. 508 * We can only ensure this in the rmmod path in the current code. 509 */ 510 void rds_conn_destroy(struct rds_connection *conn) 511 { 512 unsigned long flags; 513 int i; 514 struct rds_conn_path *cp; 515 int npaths = (conn->c_trans->t_mp_capable ? RDS_MPATH_WORKERS : 1); 516 517 rdsdebug("freeing conn %p for %pI4 -> " 518 "%pI4\n", conn, &conn->c_laddr, 519 &conn->c_faddr); 520 521 /* Ensure conn will not be scheduled for reconnect */ 522 spin_lock_irq(&rds_conn_lock); 523 hlist_del_init_rcu(&conn->c_hash_node); 524 spin_unlock_irq(&rds_conn_lock); 525 synchronize_rcu(); 526 527 /* shut the connection down */ 528 for (i = 0; i < npaths; i++) { 529 cp = &conn->c_path[i]; 530 rds_conn_path_destroy(cp); 531 BUG_ON(!list_empty(&cp->cp_retrans)); 532 } 533 534 /* 535 * The congestion maps aren't freed up here. They're 536 * freed by rds_cong_exit() after all the connections 537 * have been freed. 538 */ 539 rds_cong_remove_conn(conn); 540 541 kfree(conn->c_path); 542 kmem_cache_free(rds_conn_slab, conn); 543 544 spin_lock_irqsave(&rds_conn_lock, flags); 545 rds_conn_count--; 546 spin_unlock_irqrestore(&rds_conn_lock, flags); 547 } 548 EXPORT_SYMBOL_GPL(rds_conn_destroy); 549 550 static void __rds_inc_msg_cp(struct rds_incoming *inc, 551 struct rds_info_iterator *iter, 552 void *saddr, void *daddr, int flip, bool isv6) 553 { 554 #if IS_ENABLED(CONFIG_IPV6) 555 if (isv6) 556 rds6_inc_info_copy(inc, iter, saddr, daddr, flip); 557 else 558 #endif 559 rds_inc_info_copy(inc, iter, *(__be32 *)saddr, 560 *(__be32 *)daddr, flip); 561 } 562 563 static void rds_conn_message_info_cmn(struct socket *sock, unsigned int len, 564 struct rds_info_iterator *iter, 565 struct rds_info_lengths *lens, 566 int want_send, bool isv6) 567 { 568 struct hlist_head *head; 569 struct list_head *list; 570 struct rds_connection *conn; 571 struct rds_message *rm; 572 unsigned int total = 0; 573 unsigned long flags; 574 size_t i; 575 int j; 576 577 if (isv6) 578 len /= sizeof(struct rds6_info_message); 579 else 580 len /= sizeof(struct rds_info_message); 581 582 rcu_read_lock(); 583 584 for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash); 585 i++, head++) { 586 hlist_for_each_entry_rcu(conn, head, c_hash_node) { 587 struct rds_conn_path *cp; 588 int npaths; 589 590 if (!isv6 && conn->c_isv6) 591 continue; 592 593 npaths = (conn->c_trans->t_mp_capable ? 594 RDS_MPATH_WORKERS : 1); 595 596 for (j = 0; j < npaths; j++) { 597 cp = &conn->c_path[j]; 598 if (want_send) 599 list = &cp->cp_send_queue; 600 else 601 list = &cp->cp_retrans; 602 603 spin_lock_irqsave(&cp->cp_lock, flags); 604 605 /* XXX too lazy to maintain counts.. */ 606 list_for_each_entry(rm, list, m_conn_item) { 607 total++; 608 if (total <= len) 609 __rds_inc_msg_cp(&rm->m_inc, 610 iter, 611 &conn->c_laddr, 612 &conn->c_faddr, 613 0, isv6); 614 } 615 616 spin_unlock_irqrestore(&cp->cp_lock, flags); 617 } 618 } 619 } 620 rcu_read_unlock(); 621 622 lens->nr = total; 623 if (isv6) 624 lens->each = sizeof(struct rds6_info_message); 625 else 626 lens->each = sizeof(struct rds_info_message); 627 } 628 629 static void rds_conn_message_info(struct socket *sock, unsigned int len, 630 struct rds_info_iterator *iter, 631 struct rds_info_lengths *lens, 632 int want_send) 633 { 634 rds_conn_message_info_cmn(sock, len, iter, lens, want_send, false); 635 } 636 637 #if IS_ENABLED(CONFIG_IPV6) 638 static void rds6_conn_message_info(struct socket *sock, unsigned int len, 639 struct rds_info_iterator *iter, 640 struct rds_info_lengths *lens, 641 int want_send) 642 { 643 rds_conn_message_info_cmn(sock, len, iter, lens, want_send, true); 644 } 645 #endif 646 647 static void rds_conn_message_info_send(struct socket *sock, unsigned int len, 648 struct rds_info_iterator *iter, 649 struct rds_info_lengths *lens) 650 { 651 rds_conn_message_info(sock, len, iter, lens, 1); 652 } 653 654 #if IS_ENABLED(CONFIG_IPV6) 655 static void rds6_conn_message_info_send(struct socket *sock, unsigned int len, 656 struct rds_info_iterator *iter, 657 struct rds_info_lengths *lens) 658 { 659 rds6_conn_message_info(sock, len, iter, lens, 1); 660 } 661 #endif 662 663 static void rds_conn_message_info_retrans(struct socket *sock, 664 unsigned int len, 665 struct rds_info_iterator *iter, 666 struct rds_info_lengths *lens) 667 { 668 rds_conn_message_info(sock, len, iter, lens, 0); 669 } 670 671 #if IS_ENABLED(CONFIG_IPV6) 672 static void rds6_conn_message_info_retrans(struct socket *sock, 673 unsigned int len, 674 struct rds_info_iterator *iter, 675 struct rds_info_lengths *lens) 676 { 677 rds6_conn_message_info(sock, len, iter, lens, 0); 678 } 679 #endif 680 681 void rds_for_each_conn_info(struct socket *sock, unsigned int len, 682 struct rds_info_iterator *iter, 683 struct rds_info_lengths *lens, 684 int (*visitor)(struct rds_connection *, void *), 685 u64 *buffer, 686 size_t item_len) 687 { 688 struct hlist_head *head; 689 struct rds_connection *conn; 690 size_t i; 691 692 rcu_read_lock(); 693 694 lens->nr = 0; 695 lens->each = item_len; 696 697 for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash); 698 i++, head++) { 699 hlist_for_each_entry_rcu(conn, head, c_hash_node) { 700 701 /* XXX no c_lock usage.. */ 702 if (!visitor(conn, buffer)) 703 continue; 704 705 /* We copy as much as we can fit in the buffer, 706 * but we count all items so that the caller 707 * can resize the buffer. */ 708 if (len >= item_len) { 709 rds_info_copy(iter, buffer, item_len); 710 len -= item_len; 711 } 712 lens->nr++; 713 } 714 } 715 rcu_read_unlock(); 716 } 717 EXPORT_SYMBOL_GPL(rds_for_each_conn_info); 718 719 static void rds_walk_conn_path_info(struct socket *sock, unsigned int len, 720 struct rds_info_iterator *iter, 721 struct rds_info_lengths *lens, 722 int (*visitor)(struct rds_conn_path *, void *), 723 u64 *buffer, 724 size_t item_len) 725 { 726 struct hlist_head *head; 727 struct rds_connection *conn; 728 size_t i; 729 730 rcu_read_lock(); 731 732 lens->nr = 0; 733 lens->each = item_len; 734 735 for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash); 736 i++, head++) { 737 hlist_for_each_entry_rcu(conn, head, c_hash_node) { 738 struct rds_conn_path *cp; 739 740 /* XXX We only copy the information from the first 741 * path for now. The problem is that if there are 742 * more than one underlying paths, we cannot report 743 * information of all of them using the existing 744 * API. For example, there is only one next_tx_seq, 745 * which path's next_tx_seq should we report? It is 746 * a bug in the design of MPRDS. 747 */ 748 cp = conn->c_path; 749 750 /* XXX no cp_lock usage.. */ 751 if (!visitor(cp, buffer)) 752 continue; 753 754 /* We copy as much as we can fit in the buffer, 755 * but we count all items so that the caller 756 * can resize the buffer. 757 */ 758 if (len >= item_len) { 759 rds_info_copy(iter, buffer, item_len); 760 len -= item_len; 761 } 762 lens->nr++; 763 } 764 } 765 rcu_read_unlock(); 766 } 767 768 static int rds_conn_info_visitor(struct rds_conn_path *cp, void *buffer) 769 { 770 struct rds_info_connection *cinfo = buffer; 771 struct rds_connection *conn = cp->cp_conn; 772 773 if (conn->c_isv6) 774 return 0; 775 776 cinfo->next_tx_seq = cp->cp_next_tx_seq; 777 cinfo->next_rx_seq = cp->cp_next_rx_seq; 778 cinfo->laddr = conn->c_laddr.s6_addr32[3]; 779 cinfo->faddr = conn->c_faddr.s6_addr32[3]; 780 cinfo->tos = conn->c_tos; 781 strscpy_pad(cinfo->transport, conn->c_trans->t_name); 782 cinfo->flags = 0; 783 784 rds_conn_info_set(cinfo->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags), 785 SENDING); 786 /* XXX Future: return the state rather than these funky bits */ 787 rds_conn_info_set(cinfo->flags, 788 atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING, 789 CONNECTING); 790 rds_conn_info_set(cinfo->flags, 791 atomic_read(&cp->cp_state) == RDS_CONN_UP, 792 CONNECTED); 793 return 1; 794 } 795 796 #if IS_ENABLED(CONFIG_IPV6) 797 static int rds6_conn_info_visitor(struct rds_conn_path *cp, void *buffer) 798 { 799 struct rds6_info_connection *cinfo6 = buffer; 800 struct rds_connection *conn = cp->cp_conn; 801 802 cinfo6->next_tx_seq = cp->cp_next_tx_seq; 803 cinfo6->next_rx_seq = cp->cp_next_rx_seq; 804 cinfo6->laddr = conn->c_laddr; 805 cinfo6->faddr = conn->c_faddr; 806 strscpy_pad(cinfo6->transport, conn->c_trans->t_name); 807 cinfo6->flags = 0; 808 809 rds_conn_info_set(cinfo6->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags), 810 SENDING); 811 /* XXX Future: return the state rather than these funky bits */ 812 rds_conn_info_set(cinfo6->flags, 813 atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING, 814 CONNECTING); 815 rds_conn_info_set(cinfo6->flags, 816 atomic_read(&cp->cp_state) == RDS_CONN_UP, 817 CONNECTED); 818 /* Just return 1 as there is no error case. This is a helper function 819 * for rds_walk_conn_path_info() and it wants a return value. 820 */ 821 return 1; 822 } 823 #endif 824 825 static void rds_conn_info(struct socket *sock, unsigned int len, 826 struct rds_info_iterator *iter, 827 struct rds_info_lengths *lens) 828 { 829 u64 buffer[(sizeof(struct rds_info_connection) + 7) / 8]; 830 831 rds_walk_conn_path_info(sock, len, iter, lens, 832 rds_conn_info_visitor, 833 buffer, 834 sizeof(struct rds_info_connection)); 835 } 836 837 #if IS_ENABLED(CONFIG_IPV6) 838 static void rds6_conn_info(struct socket *sock, unsigned int len, 839 struct rds_info_iterator *iter, 840 struct rds_info_lengths *lens) 841 { 842 u64 buffer[(sizeof(struct rds6_info_connection) + 7) / 8]; 843 844 rds_walk_conn_path_info(sock, len, iter, lens, 845 rds6_conn_info_visitor, 846 buffer, 847 sizeof(struct rds6_info_connection)); 848 } 849 #endif 850 851 int rds_conn_init(void) 852 { 853 int ret; 854 855 ret = rds_loop_net_init(); /* register pernet callback */ 856 if (ret) 857 return ret; 858 859 rds_conn_slab = KMEM_CACHE(rds_connection, 0); 860 if (!rds_conn_slab) { 861 rds_loop_net_exit(); 862 return -ENOMEM; 863 } 864 865 rds_info_register_func(RDS_INFO_CONNECTIONS, rds_conn_info); 866 rds_info_register_func(RDS_INFO_SEND_MESSAGES, 867 rds_conn_message_info_send); 868 rds_info_register_func(RDS_INFO_RETRANS_MESSAGES, 869 rds_conn_message_info_retrans); 870 #if IS_ENABLED(CONFIG_IPV6) 871 rds_info_register_func(RDS6_INFO_CONNECTIONS, rds6_conn_info); 872 rds_info_register_func(RDS6_INFO_SEND_MESSAGES, 873 rds6_conn_message_info_send); 874 rds_info_register_func(RDS6_INFO_RETRANS_MESSAGES, 875 rds6_conn_message_info_retrans); 876 #endif 877 return 0; 878 } 879 880 void rds_conn_exit(void) 881 { 882 rds_loop_net_exit(); /* unregister pernet callback */ 883 rds_loop_exit(); 884 885 WARN_ON(!hlist_empty(rds_conn_hash)); 886 887 kmem_cache_destroy(rds_conn_slab); 888 889 rds_info_deregister_func(RDS_INFO_CONNECTIONS, rds_conn_info); 890 rds_info_deregister_func(RDS_INFO_SEND_MESSAGES, 891 rds_conn_message_info_send); 892 rds_info_deregister_func(RDS_INFO_RETRANS_MESSAGES, 893 rds_conn_message_info_retrans); 894 #if IS_ENABLED(CONFIG_IPV6) 895 rds_info_deregister_func(RDS6_INFO_CONNECTIONS, rds6_conn_info); 896 rds_info_deregister_func(RDS6_INFO_SEND_MESSAGES, 897 rds6_conn_message_info_send); 898 rds_info_deregister_func(RDS6_INFO_RETRANS_MESSAGES, 899 rds6_conn_message_info_retrans); 900 #endif 901 } 902 903 /* 904 * Force a disconnect 905 */ 906 void rds_conn_path_drop(struct rds_conn_path *cp, bool destroy) 907 { 908 atomic_set(&cp->cp_state, RDS_CONN_ERROR); 909 910 rcu_read_lock(); 911 if (!destroy && rds_destroy_pending(cp->cp_conn)) { 912 rcu_read_unlock(); 913 return; 914 } 915 queue_work(cp->cp_wq, &cp->cp_down_w); 916 rcu_read_unlock(); 917 } 918 EXPORT_SYMBOL_GPL(rds_conn_path_drop); 919 920 void rds_conn_drop(struct rds_connection *conn) 921 { 922 WARN_ON(conn->c_trans->t_mp_capable); 923 rds_conn_path_drop(&conn->c_path[0], false); 924 } 925 EXPORT_SYMBOL_GPL(rds_conn_drop); 926 927 /* 928 * If the connection is down, trigger a connect. We may have scheduled a 929 * delayed reconnect however - in this case we should not interfere. 930 */ 931 void rds_conn_path_connect_if_down(struct rds_conn_path *cp) 932 { 933 rcu_read_lock(); 934 if (rds_destroy_pending(cp->cp_conn)) { 935 rcu_read_unlock(); 936 return; 937 } 938 if (rds_conn_path_state(cp) == RDS_CONN_DOWN && 939 !test_and_set_bit(RDS_RECONNECT_PENDING, &cp->cp_flags)) 940 queue_delayed_work(cp->cp_wq, &cp->cp_conn_w, 0); 941 rcu_read_unlock(); 942 } 943 EXPORT_SYMBOL_GPL(rds_conn_path_connect_if_down); 944 945 /* Check connectivity of all paths 946 */ 947 void rds_check_all_paths(struct rds_connection *conn) 948 { 949 int i = 0; 950 951 do { 952 rds_conn_path_connect_if_down(&conn->c_path[i]); 953 } while (++i < conn->c_npaths); 954 } 955 956 void rds_conn_connect_if_down(struct rds_connection *conn) 957 { 958 WARN_ON(conn->c_trans->t_mp_capable); 959 rds_conn_path_connect_if_down(&conn->c_path[0]); 960 } 961 EXPORT_SYMBOL_GPL(rds_conn_connect_if_down); 962 963 void 964 __rds_conn_path_error(struct rds_conn_path *cp, const char *fmt, ...) 965 { 966 va_list ap; 967 968 va_start(ap, fmt); 969 vprintk(fmt, ap); 970 va_end(ap); 971 972 rds_conn_path_drop(cp, false); 973 } 974