1 /* 2 * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 #include <linux/kernel.h> 34 #include <linux/list.h> 35 #include <linux/slab.h> 36 #include <linux/export.h> 37 #include <net/ipv6.h> 38 #include <net/inet6_hashtables.h> 39 #include <net/addrconf.h> 40 41 #include "rds.h" 42 #include "loop.h" 43 44 #define RDS_CONNECTION_HASH_BITS 12 45 #define RDS_CONNECTION_HASH_ENTRIES (1 << RDS_CONNECTION_HASH_BITS) 46 #define RDS_CONNECTION_HASH_MASK (RDS_CONNECTION_HASH_ENTRIES - 1) 47 48 /* converting this to RCU is a chore for another day.. */ 49 static DEFINE_SPINLOCK(rds_conn_lock); 50 static unsigned long rds_conn_count; 51 static struct hlist_head rds_conn_hash[RDS_CONNECTION_HASH_ENTRIES]; 52 static struct kmem_cache *rds_conn_slab; 53 54 static struct hlist_head *rds_conn_bucket(const struct in6_addr *laddr, 55 const struct in6_addr *faddr) 56 { 57 static u32 rds6_hash_secret __read_mostly; 58 static u32 rds_hash_secret __read_mostly; 59 60 __be32 lhash, fhash; 61 u32 hash; 62 63 net_get_random_once(&rds_hash_secret, sizeof(rds_hash_secret)); 64 net_get_random_once(&rds6_hash_secret, sizeof(rds6_hash_secret)); 65 66 lhash = laddr->s6_addr32[3]; 67 #if IS_ENABLED(CONFIG_IPV6) 68 fhash = (__force __be32)__ipv6_addr_jhash(faddr, rds6_hash_secret); 69 #else 70 fhash = faddr->s6_addr32[3]; 71 #endif 72 hash = __inet_ehashfn(lhash, 0, fhash, 0, rds_hash_secret); 73 74 return &rds_conn_hash[hash & RDS_CONNECTION_HASH_MASK]; 75 } 76 77 #define rds_conn_info_set(var, test, suffix) do { \ 78 if (test) \ 79 var |= RDS_INFO_CONNECTION_FLAG_##suffix; \ 80 } while (0) 81 82 /* rcu read lock must be held or the connection spinlock */ 83 static struct rds_connection *rds_conn_lookup(struct net *net, 84 struct hlist_head *head, 85 const struct in6_addr *laddr, 86 const struct in6_addr *faddr, 87 struct rds_transport *trans, 88 u8 tos, int dev_if) 89 { 90 struct rds_connection *conn, *ret = NULL; 91 92 hlist_for_each_entry_rcu(conn, head, c_hash_node) { 93 if (ipv6_addr_equal(&conn->c_faddr, faddr) && 94 ipv6_addr_equal(&conn->c_laddr, laddr) && 95 conn->c_trans == trans && 96 conn->c_tos == tos && 97 net == rds_conn_net(conn) && 98 conn->c_dev_if == dev_if) { 99 ret = conn; 100 break; 101 } 102 } 103 rdsdebug("returning conn %p for %pI6c -> %pI6c\n", ret, 104 laddr, faddr); 105 return ret; 106 } 107 108 /* 109 * This is called by transports as they're bringing down a connection. 110 * It clears partial message state so that the transport can start sending 111 * and receiving over this connection again in the future. It is up to 112 * the transport to have serialized this call with its send and recv. 113 */ 114 static void rds_conn_path_reset(struct rds_conn_path *cp) 115 { 116 struct rds_connection *conn = cp->cp_conn; 117 118 rdsdebug("connection %pI6c to %pI6c reset\n", 119 &conn->c_laddr, &conn->c_faddr); 120 121 rds_stats_inc(s_conn_reset); 122 rds_send_path_reset(cp); 123 cp->cp_flags = 0; 124 125 /* Do not clear next_rx_seq here, else we cannot distinguish 126 * retransmitted packets from new packets, and will hand all 127 * of them to the application. That is not consistent with the 128 * reliability guarantees of RDS. */ 129 } 130 131 static void __rds_conn_path_init(struct rds_connection *conn, 132 struct rds_conn_path *cp, bool is_outgoing) 133 { 134 spin_lock_init(&cp->cp_lock); 135 cp->cp_next_tx_seq = 1; 136 init_waitqueue_head(&cp->cp_waitq); 137 INIT_LIST_HEAD(&cp->cp_send_queue); 138 INIT_LIST_HEAD(&cp->cp_retrans); 139 140 cp->cp_conn = conn; 141 atomic_set(&cp->cp_state, RDS_CONN_DOWN); 142 cp->cp_send_gen = 0; 143 cp->cp_reconnect_jiffies = 0; 144 cp->cp_conn->c_proposed_version = RDS_PROTOCOL_VERSION; 145 INIT_DELAYED_WORK(&cp->cp_send_w, rds_send_worker); 146 INIT_DELAYED_WORK(&cp->cp_recv_w, rds_recv_worker); 147 INIT_DELAYED_WORK(&cp->cp_conn_w, rds_connect_worker); 148 INIT_WORK(&cp->cp_down_w, rds_shutdown_worker); 149 mutex_init(&cp->cp_cm_lock); 150 cp->cp_flags = 0; 151 } 152 153 /* 154 * There is only every one 'conn' for a given pair of addresses in the 155 * system at a time. They contain messages to be retransmitted and so 156 * span the lifetime of the actual underlying transport connections. 157 * 158 * For now they are not garbage collected once they're created. They 159 * are torn down as the module is removed, if ever. 160 */ 161 static struct rds_connection *__rds_conn_create(struct net *net, 162 const struct in6_addr *laddr, 163 const struct in6_addr *faddr, 164 struct rds_transport *trans, 165 gfp_t gfp, u8 tos, 166 int is_outgoing, 167 int dev_if) 168 { 169 struct rds_connection *conn, *parent = NULL; 170 struct hlist_head *head = rds_conn_bucket(laddr, faddr); 171 struct rds_transport *loop_trans; 172 struct rds_conn_path *free_cp = NULL; 173 unsigned long flags; 174 int ret, i; 175 int npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1); 176 177 rcu_read_lock(); 178 conn = rds_conn_lookup(net, head, laddr, faddr, trans, tos, dev_if); 179 if (conn && 180 conn->c_loopback && 181 conn->c_trans != &rds_loop_transport && 182 ipv6_addr_equal(laddr, faddr) && 183 !is_outgoing) { 184 /* This is a looped back IB connection, and we're 185 * called by the code handling the incoming connect. 186 * We need a second connection object into which we 187 * can stick the other QP. */ 188 parent = conn; 189 conn = parent->c_passive; 190 } 191 rcu_read_unlock(); 192 if (conn) 193 goto out; 194 195 conn = kmem_cache_zalloc(rds_conn_slab, gfp); 196 if (!conn) { 197 conn = ERR_PTR(-ENOMEM); 198 goto out; 199 } 200 conn->c_path = kzalloc_objs(struct rds_conn_path, npaths, gfp); 201 if (!conn->c_path) { 202 kmem_cache_free(rds_conn_slab, conn); 203 conn = ERR_PTR(-ENOMEM); 204 goto out; 205 } 206 207 INIT_HLIST_NODE(&conn->c_hash_node); 208 conn->c_laddr = *laddr; 209 conn->c_isv6 = !ipv6_addr_v4mapped(laddr); 210 conn->c_faddr = *faddr; 211 conn->c_dev_if = dev_if; 212 conn->c_tos = tos; 213 214 #if IS_ENABLED(CONFIG_IPV6) 215 /* If the local address is link local, set c_bound_if to be the 216 * index used for this connection. Otherwise, set it to 0 as 217 * the socket is not bound to an interface. c_bound_if is used 218 * to look up a socket when a packet is received 219 */ 220 if (ipv6_addr_type(laddr) & IPV6_ADDR_LINKLOCAL) 221 conn->c_bound_if = dev_if; 222 else 223 #endif 224 conn->c_bound_if = 0; 225 226 rds_conn_net_set(conn, net); 227 228 ret = rds_cong_get_maps(conn); 229 if (ret) { 230 kfree(conn->c_path); 231 kmem_cache_free(rds_conn_slab, conn); 232 conn = ERR_PTR(ret); 233 goto out; 234 } 235 236 /* 237 * This is where a connection becomes loopback. If *any* RDS sockets 238 * can bind to the destination address then we'd rather the messages 239 * flow through loopback rather than either transport. 240 */ 241 loop_trans = rds_trans_get_preferred(net, faddr, conn->c_dev_if); 242 if (loop_trans) { 243 rds_trans_put(loop_trans); 244 conn->c_loopback = 1; 245 if (trans->t_prefer_loopback) { 246 if (likely(is_outgoing)) { 247 /* "outgoing" connection to local address. 248 * Protocol says it wants the connection 249 * handled by the loopback transport. 250 * This is what TCP does. 251 */ 252 trans = &rds_loop_transport; 253 } else { 254 /* No transport currently in use 255 * should end up here, but if it 256 * does, reset/destroy the connection. 257 */ 258 kfree(conn->c_path); 259 kmem_cache_free(rds_conn_slab, conn); 260 conn = ERR_PTR(-EOPNOTSUPP); 261 goto out; 262 } 263 } 264 } 265 266 conn->c_trans = trans; 267 268 init_waitqueue_head(&conn->c_hs_waitq); 269 for (i = 0; i < npaths; i++) { 270 __rds_conn_path_init(conn, &conn->c_path[i], 271 is_outgoing); 272 conn->c_path[i].cp_index = i; 273 conn->c_path[i].cp_wq = 274 alloc_ordered_workqueue("krds_cp_wq#%lu/%d", 0, 275 rds_conn_count, i); 276 if (!conn->c_path[i].cp_wq) 277 conn->c_path[i].cp_wq = rds_wq; 278 } 279 rcu_read_lock(); 280 if (rds_destroy_pending(conn)) 281 ret = -ENETDOWN; 282 else 283 ret = trans->conn_alloc(conn, GFP_ATOMIC); 284 if (ret) { 285 rcu_read_unlock(); 286 free_cp = conn->c_path; 287 kmem_cache_free(rds_conn_slab, conn); 288 conn = ERR_PTR(ret); 289 goto out; 290 } 291 292 rdsdebug("allocated conn %p for %pI6c -> %pI6c over %s %s\n", 293 conn, laddr, faddr, 294 strnlen(trans->t_name, sizeof(trans->t_name)) ? 295 trans->t_name : "[unknown]", is_outgoing ? "(outgoing)" : ""); 296 297 /* 298 * Since we ran without holding the conn lock, someone could 299 * have created the same conn (either normal or passive) in the 300 * interim. We check while holding the lock. If we won, we complete 301 * init and return our conn. If we lost, we rollback and return the 302 * other one. 303 */ 304 spin_lock_irqsave(&rds_conn_lock, flags); 305 if (parent) { 306 /* Creating passive conn */ 307 if (parent->c_passive) { 308 trans->conn_free(conn->c_path[0].cp_transport_data); 309 free_cp = conn->c_path; 310 kmem_cache_free(rds_conn_slab, conn); 311 conn = parent->c_passive; 312 } else { 313 parent->c_passive = conn; 314 rds_cong_add_conn(conn); 315 rds_conn_count++; 316 } 317 } else { 318 /* Creating normal conn */ 319 struct rds_connection *found; 320 321 found = rds_conn_lookup(net, head, laddr, faddr, trans, 322 tos, dev_if); 323 if (found) { 324 struct rds_conn_path *cp; 325 int i; 326 327 for (i = 0; i < npaths; i++) { 328 cp = &conn->c_path[i]; 329 /* The ->conn_alloc invocation may have 330 * allocated resource for all paths, so all 331 * of them may have to be freed here. 332 */ 333 if (cp->cp_transport_data) 334 trans->conn_free(cp->cp_transport_data); 335 } 336 free_cp = conn->c_path; 337 kmem_cache_free(rds_conn_slab, conn); 338 conn = found; 339 } else { 340 conn->c_my_gen_num = rds_gen_num; 341 conn->c_peer_gen_num = 0; 342 hlist_add_head_rcu(&conn->c_hash_node, head); 343 rds_cong_add_conn(conn); 344 rds_conn_count++; 345 } 346 } 347 spin_unlock_irqrestore(&rds_conn_lock, flags); 348 rcu_read_unlock(); 349 350 out: 351 if (free_cp) { 352 for (i = 0; i < npaths; i++) 353 if (free_cp[i].cp_wq != rds_wq) 354 destroy_workqueue(free_cp[i].cp_wq); 355 kfree(free_cp); 356 } 357 358 return conn; 359 } 360 361 struct rds_connection *rds_conn_create(struct net *net, 362 const struct in6_addr *laddr, 363 const struct in6_addr *faddr, 364 struct rds_transport *trans, u8 tos, 365 gfp_t gfp, int dev_if) 366 { 367 return __rds_conn_create(net, laddr, faddr, trans, gfp, tos, 0, dev_if); 368 } 369 EXPORT_SYMBOL_GPL(rds_conn_create); 370 371 struct rds_connection *rds_conn_create_outgoing(struct net *net, 372 const struct in6_addr *laddr, 373 const struct in6_addr *faddr, 374 struct rds_transport *trans, 375 u8 tos, gfp_t gfp, int dev_if) 376 { 377 return __rds_conn_create(net, laddr, faddr, trans, gfp, tos, 1, dev_if); 378 } 379 EXPORT_SYMBOL_GPL(rds_conn_create_outgoing); 380 381 void rds_conn_shutdown(struct rds_conn_path *cp) 382 { 383 struct rds_connection *conn = cp->cp_conn; 384 385 /* shut it down unless it's down already */ 386 if (!rds_conn_path_transition(cp, RDS_CONN_DOWN, RDS_CONN_DOWN)) { 387 /* 388 * Quiesce the connection mgmt handlers before we start tearing 389 * things down. We don't hold the mutex for the entire 390 * duration of the shutdown operation, else we may be 391 * deadlocking with the CM handler. Instead, the CM event 392 * handler is supposed to check for state DISCONNECTING 393 */ 394 mutex_lock(&cp->cp_cm_lock); 395 if (!rds_conn_path_transition(cp, RDS_CONN_UP, 396 RDS_CONN_DISCONNECTING) && 397 !rds_conn_path_transition(cp, RDS_CONN_ERROR, 398 RDS_CONN_DISCONNECTING) && 399 !rds_conn_path_transition(cp, RDS_CONN_RESETTING, 400 RDS_CONN_DISCONNECTING)) { 401 rds_conn_path_error(cp, 402 "shutdown called in state %d\n", 403 atomic_read(&cp->cp_state)); 404 mutex_unlock(&cp->cp_cm_lock); 405 return; 406 } 407 mutex_unlock(&cp->cp_cm_lock); 408 409 wait_event(cp->cp_waitq, 410 !test_bit(RDS_IN_XMIT, &cp->cp_flags)); 411 wait_event(cp->cp_waitq, 412 !test_bit(RDS_RECV_REFILL, &cp->cp_flags)); 413 414 conn->c_trans->conn_path_shutdown(cp); 415 rds_conn_path_reset(cp); 416 417 if (!rds_conn_path_transition(cp, RDS_CONN_DISCONNECTING, 418 RDS_CONN_DOWN) && 419 !rds_conn_path_transition(cp, RDS_CONN_ERROR, 420 RDS_CONN_DOWN)) { 421 /* This can happen - eg when we're in the middle of tearing 422 * down the connection, and someone unloads the rds module. 423 * Quite reproducible with loopback connections. 424 * Mostly harmless. 425 * 426 * Note that this also happens with rds-tcp because 427 * we could have triggered rds_conn_path_drop in irq 428 * mode from rds_tcp_state change on the receipt of 429 * a FIN, thus we need to recheck for RDS_CONN_ERROR 430 * here. 431 */ 432 rds_conn_path_error(cp, "%s: failed to transition " 433 "to state DOWN, current state " 434 "is %d\n", __func__, 435 atomic_read(&cp->cp_state)); 436 return; 437 } 438 } 439 440 /* Then reconnect if it's still live. 441 * The passive side of an IB loopback connection is never added 442 * to the conn hash, so we never trigger a reconnect on this 443 * conn - the reconnect is always triggered by the active peer. */ 444 cancel_delayed_work_sync(&cp->cp_conn_w); 445 446 clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); 447 rcu_read_lock(); 448 if (!hlist_unhashed(&conn->c_hash_node)) { 449 rcu_read_unlock(); 450 if (conn->c_trans->t_mp_capable && 451 cp->cp_index == 0) 452 rds_send_ping(conn, 0); 453 rds_queue_reconnect(cp); 454 } else { 455 rcu_read_unlock(); 456 } 457 458 /* we do not hold the socket lock here but it is safe because 459 * fan-out is disabled when calling conn_slots_available() 460 */ 461 if (conn->c_trans->conn_slots_available) 462 conn->c_trans->conn_slots_available(conn, false); 463 } 464 465 /* destroy a single rds_conn_path. rds_conn_destroy() iterates over 466 * all paths using rds_conn_path_destroy() 467 */ 468 static void rds_conn_path_destroy(struct rds_conn_path *cp) 469 { 470 struct rds_message *rm, *rtmp; 471 472 if (!cp->cp_transport_data) 473 return; 474 475 /* make sure lingering queued work won't try to ref the conn */ 476 cancel_delayed_work_sync(&cp->cp_send_w); 477 cancel_delayed_work_sync(&cp->cp_recv_w); 478 479 rds_conn_path_drop(cp, true); 480 flush_work(&cp->cp_down_w); 481 482 /* tear down queued messages */ 483 list_for_each_entry_safe(rm, rtmp, 484 &cp->cp_send_queue, 485 m_conn_item) { 486 list_del_init(&rm->m_conn_item); 487 BUG_ON(!list_empty(&rm->m_sock_item)); 488 rds_message_put(rm); 489 } 490 if (cp->cp_xmit_rm) 491 rds_message_put(cp->cp_xmit_rm); 492 493 WARN_ON(delayed_work_pending(&cp->cp_send_w)); 494 WARN_ON(delayed_work_pending(&cp->cp_recv_w)); 495 WARN_ON(delayed_work_pending(&cp->cp_conn_w)); 496 WARN_ON(work_pending(&cp->cp_down_w)); 497 498 if (cp->cp_wq != rds_wq) { 499 destroy_workqueue(cp->cp_wq); 500 cp->cp_wq = NULL; 501 } 502 503 cp->cp_conn->c_trans->conn_free(cp->cp_transport_data); 504 } 505 506 /* 507 * Stop and free a connection. 508 * 509 * This can only be used in very limited circumstances. It assumes that once 510 * the conn has been shutdown that no one else is referencing the connection. 511 * We can only ensure this in the rmmod path in the current code. 512 */ 513 void rds_conn_destroy(struct rds_connection *conn) 514 { 515 unsigned long flags; 516 int i; 517 struct rds_conn_path *cp; 518 int npaths = (conn->c_trans->t_mp_capable ? RDS_MPATH_WORKERS : 1); 519 520 rdsdebug("freeing conn %p for %pI4 -> " 521 "%pI4\n", conn, &conn->c_laddr, 522 &conn->c_faddr); 523 524 /* Ensure conn will not be scheduled for reconnect */ 525 spin_lock_irq(&rds_conn_lock); 526 hlist_del_init_rcu(&conn->c_hash_node); 527 spin_unlock_irq(&rds_conn_lock); 528 synchronize_rcu(); 529 530 /* shut the connection down */ 531 for (i = 0; i < npaths; i++) { 532 cp = &conn->c_path[i]; 533 rds_conn_path_destroy(cp); 534 BUG_ON(!list_empty(&cp->cp_retrans)); 535 } 536 537 /* 538 * The congestion maps aren't freed up here. They're 539 * freed by rds_cong_exit() after all the connections 540 * have been freed. 541 */ 542 rds_cong_remove_conn(conn); 543 544 kfree(conn->c_path); 545 kmem_cache_free(rds_conn_slab, conn); 546 547 spin_lock_irqsave(&rds_conn_lock, flags); 548 rds_conn_count--; 549 spin_unlock_irqrestore(&rds_conn_lock, flags); 550 } 551 EXPORT_SYMBOL_GPL(rds_conn_destroy); 552 553 static void __rds_inc_msg_cp(struct rds_incoming *inc, 554 struct rds_info_iterator *iter, 555 void *saddr, void *daddr, int flip, bool isv6) 556 { 557 #if IS_ENABLED(CONFIG_IPV6) 558 if (isv6) 559 rds6_inc_info_copy(inc, iter, saddr, daddr, flip); 560 else 561 #endif 562 rds_inc_info_copy(inc, iter, *(__be32 *)saddr, 563 *(__be32 *)daddr, flip); 564 } 565 566 static void rds_conn_message_info_cmn(struct socket *sock, unsigned int len, 567 struct rds_info_iterator *iter, 568 struct rds_info_lengths *lens, 569 int want_send, bool isv6) 570 { 571 struct hlist_head *head; 572 struct list_head *list; 573 struct rds_connection *conn; 574 struct rds_message *rm; 575 unsigned int total = 0; 576 unsigned long flags; 577 size_t i; 578 int j; 579 580 if (isv6) 581 len /= sizeof(struct rds6_info_message); 582 else 583 len /= sizeof(struct rds_info_message); 584 585 rcu_read_lock(); 586 587 for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash); 588 i++, head++) { 589 hlist_for_each_entry_rcu(conn, head, c_hash_node) { 590 struct rds_conn_path *cp; 591 int npaths; 592 593 if (!isv6 && conn->c_isv6) 594 continue; 595 596 npaths = (conn->c_trans->t_mp_capable ? 597 RDS_MPATH_WORKERS : 1); 598 599 for (j = 0; j < npaths; j++) { 600 cp = &conn->c_path[j]; 601 if (want_send) 602 list = &cp->cp_send_queue; 603 else 604 list = &cp->cp_retrans; 605 606 spin_lock_irqsave(&cp->cp_lock, flags); 607 608 /* XXX too lazy to maintain counts.. */ 609 list_for_each_entry(rm, list, m_conn_item) { 610 total++; 611 if (total <= len) 612 __rds_inc_msg_cp(&rm->m_inc, 613 iter, 614 &conn->c_laddr, 615 &conn->c_faddr, 616 0, isv6); 617 } 618 619 spin_unlock_irqrestore(&cp->cp_lock, flags); 620 } 621 } 622 } 623 rcu_read_unlock(); 624 625 lens->nr = total; 626 if (isv6) 627 lens->each = sizeof(struct rds6_info_message); 628 else 629 lens->each = sizeof(struct rds_info_message); 630 } 631 632 static void rds_conn_message_info(struct socket *sock, unsigned int len, 633 struct rds_info_iterator *iter, 634 struct rds_info_lengths *lens, 635 int want_send) 636 { 637 rds_conn_message_info_cmn(sock, len, iter, lens, want_send, false); 638 } 639 640 #if IS_ENABLED(CONFIG_IPV6) 641 static void rds6_conn_message_info(struct socket *sock, unsigned int len, 642 struct rds_info_iterator *iter, 643 struct rds_info_lengths *lens, 644 int want_send) 645 { 646 rds_conn_message_info_cmn(sock, len, iter, lens, want_send, true); 647 } 648 #endif 649 650 static void rds_conn_message_info_send(struct socket *sock, unsigned int len, 651 struct rds_info_iterator *iter, 652 struct rds_info_lengths *lens) 653 { 654 rds_conn_message_info(sock, len, iter, lens, 1); 655 } 656 657 #if IS_ENABLED(CONFIG_IPV6) 658 static void rds6_conn_message_info_send(struct socket *sock, unsigned int len, 659 struct rds_info_iterator *iter, 660 struct rds_info_lengths *lens) 661 { 662 rds6_conn_message_info(sock, len, iter, lens, 1); 663 } 664 #endif 665 666 static void rds_conn_message_info_retrans(struct socket *sock, 667 unsigned int len, 668 struct rds_info_iterator *iter, 669 struct rds_info_lengths *lens) 670 { 671 rds_conn_message_info(sock, len, iter, lens, 0); 672 } 673 674 #if IS_ENABLED(CONFIG_IPV6) 675 static void rds6_conn_message_info_retrans(struct socket *sock, 676 unsigned int len, 677 struct rds_info_iterator *iter, 678 struct rds_info_lengths *lens) 679 { 680 rds6_conn_message_info(sock, len, iter, lens, 0); 681 } 682 #endif 683 684 void rds_for_each_conn_info(struct socket *sock, unsigned int len, 685 struct rds_info_iterator *iter, 686 struct rds_info_lengths *lens, 687 int (*visitor)(struct rds_connection *, void *), 688 u64 *buffer, 689 size_t item_len) 690 { 691 struct hlist_head *head; 692 struct rds_connection *conn; 693 size_t i; 694 695 rcu_read_lock(); 696 697 lens->nr = 0; 698 lens->each = item_len; 699 700 for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash); 701 i++, head++) { 702 hlist_for_each_entry_rcu(conn, head, c_hash_node) { 703 704 /* XXX no c_lock usage.. */ 705 if (!visitor(conn, buffer)) 706 continue; 707 708 /* We copy as much as we can fit in the buffer, 709 * but we count all items so that the caller 710 * can resize the buffer. */ 711 if (len >= item_len) { 712 rds_info_copy(iter, buffer, item_len); 713 len -= item_len; 714 } 715 lens->nr++; 716 } 717 } 718 rcu_read_unlock(); 719 } 720 EXPORT_SYMBOL_GPL(rds_for_each_conn_info); 721 722 static void rds_walk_conn_path_info(struct socket *sock, unsigned int len, 723 struct rds_info_iterator *iter, 724 struct rds_info_lengths *lens, 725 int (*visitor)(struct rds_conn_path *, void *), 726 u64 *buffer, 727 size_t item_len) 728 { 729 struct hlist_head *head; 730 struct rds_connection *conn; 731 size_t i; 732 733 rcu_read_lock(); 734 735 lens->nr = 0; 736 lens->each = item_len; 737 738 for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash); 739 i++, head++) { 740 hlist_for_each_entry_rcu(conn, head, c_hash_node) { 741 struct rds_conn_path *cp; 742 743 /* XXX We only copy the information from the first 744 * path for now. The problem is that if there are 745 * more than one underlying paths, we cannot report 746 * information of all of them using the existing 747 * API. For example, there is only one next_tx_seq, 748 * which path's next_tx_seq should we report? It is 749 * a bug in the design of MPRDS. 750 */ 751 cp = conn->c_path; 752 753 /* XXX no cp_lock usage.. */ 754 if (!visitor(cp, buffer)) 755 continue; 756 757 /* We copy as much as we can fit in the buffer, 758 * but we count all items so that the caller 759 * can resize the buffer. 760 */ 761 if (len >= item_len) { 762 rds_info_copy(iter, buffer, item_len); 763 len -= item_len; 764 } 765 lens->nr++; 766 } 767 } 768 rcu_read_unlock(); 769 } 770 771 static int rds_conn_info_visitor(struct rds_conn_path *cp, void *buffer) 772 { 773 struct rds_info_connection *cinfo = buffer; 774 struct rds_connection *conn = cp->cp_conn; 775 776 if (conn->c_isv6) 777 return 0; 778 779 cinfo->next_tx_seq = cp->cp_next_tx_seq; 780 cinfo->next_rx_seq = cp->cp_next_rx_seq; 781 cinfo->laddr = conn->c_laddr.s6_addr32[3]; 782 cinfo->faddr = conn->c_faddr.s6_addr32[3]; 783 cinfo->tos = conn->c_tos; 784 strscpy_pad(cinfo->transport, conn->c_trans->t_name); 785 cinfo->flags = 0; 786 787 rds_conn_info_set(cinfo->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags), 788 SENDING); 789 /* XXX Future: return the state rather than these funky bits */ 790 rds_conn_info_set(cinfo->flags, 791 atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING, 792 CONNECTING); 793 rds_conn_info_set(cinfo->flags, 794 atomic_read(&cp->cp_state) == RDS_CONN_UP, 795 CONNECTED); 796 return 1; 797 } 798 799 #if IS_ENABLED(CONFIG_IPV6) 800 static int rds6_conn_info_visitor(struct rds_conn_path *cp, void *buffer) 801 { 802 struct rds6_info_connection *cinfo6 = buffer; 803 struct rds_connection *conn = cp->cp_conn; 804 805 cinfo6->next_tx_seq = cp->cp_next_tx_seq; 806 cinfo6->next_rx_seq = cp->cp_next_rx_seq; 807 cinfo6->laddr = conn->c_laddr; 808 cinfo6->faddr = conn->c_faddr; 809 strscpy_pad(cinfo6->transport, conn->c_trans->t_name); 810 cinfo6->flags = 0; 811 812 rds_conn_info_set(cinfo6->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags), 813 SENDING); 814 /* XXX Future: return the state rather than these funky bits */ 815 rds_conn_info_set(cinfo6->flags, 816 atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING, 817 CONNECTING); 818 rds_conn_info_set(cinfo6->flags, 819 atomic_read(&cp->cp_state) == RDS_CONN_UP, 820 CONNECTED); 821 /* Just return 1 as there is no error case. This is a helper function 822 * for rds_walk_conn_path_info() and it wants a return value. 823 */ 824 return 1; 825 } 826 #endif 827 828 static void rds_conn_info(struct socket *sock, unsigned int len, 829 struct rds_info_iterator *iter, 830 struct rds_info_lengths *lens) 831 { 832 u64 buffer[(sizeof(struct rds_info_connection) + 7) / 8]; 833 834 rds_walk_conn_path_info(sock, len, iter, lens, 835 rds_conn_info_visitor, 836 buffer, 837 sizeof(struct rds_info_connection)); 838 } 839 840 #if IS_ENABLED(CONFIG_IPV6) 841 static void rds6_conn_info(struct socket *sock, unsigned int len, 842 struct rds_info_iterator *iter, 843 struct rds_info_lengths *lens) 844 { 845 u64 buffer[(sizeof(struct rds6_info_connection) + 7) / 8]; 846 847 rds_walk_conn_path_info(sock, len, iter, lens, 848 rds6_conn_info_visitor, 849 buffer, 850 sizeof(struct rds6_info_connection)); 851 } 852 #endif 853 854 int rds_conn_init(void) 855 { 856 int ret; 857 858 ret = rds_loop_net_init(); /* register pernet callback */ 859 if (ret) 860 return ret; 861 862 rds_conn_slab = KMEM_CACHE(rds_connection, 0); 863 if (!rds_conn_slab) { 864 rds_loop_net_exit(); 865 return -ENOMEM; 866 } 867 868 rds_info_register_func(RDS_INFO_CONNECTIONS, rds_conn_info); 869 rds_info_register_func(RDS_INFO_SEND_MESSAGES, 870 rds_conn_message_info_send); 871 rds_info_register_func(RDS_INFO_RETRANS_MESSAGES, 872 rds_conn_message_info_retrans); 873 #if IS_ENABLED(CONFIG_IPV6) 874 rds_info_register_func(RDS6_INFO_CONNECTIONS, rds6_conn_info); 875 rds_info_register_func(RDS6_INFO_SEND_MESSAGES, 876 rds6_conn_message_info_send); 877 rds_info_register_func(RDS6_INFO_RETRANS_MESSAGES, 878 rds6_conn_message_info_retrans); 879 #endif 880 return 0; 881 } 882 883 void rds_conn_exit(void) 884 { 885 rds_loop_net_exit(); /* unregister pernet callback */ 886 rds_loop_exit(); 887 888 WARN_ON(!hlist_empty(rds_conn_hash)); 889 890 kmem_cache_destroy(rds_conn_slab); 891 892 rds_info_deregister_func(RDS_INFO_CONNECTIONS, rds_conn_info); 893 rds_info_deregister_func(RDS_INFO_SEND_MESSAGES, 894 rds_conn_message_info_send); 895 rds_info_deregister_func(RDS_INFO_RETRANS_MESSAGES, 896 rds_conn_message_info_retrans); 897 #if IS_ENABLED(CONFIG_IPV6) 898 rds_info_deregister_func(RDS6_INFO_CONNECTIONS, rds6_conn_info); 899 rds_info_deregister_func(RDS6_INFO_SEND_MESSAGES, 900 rds6_conn_message_info_send); 901 rds_info_deregister_func(RDS6_INFO_RETRANS_MESSAGES, 902 rds6_conn_message_info_retrans); 903 #endif 904 } 905 906 /* 907 * Force a disconnect 908 */ 909 void rds_conn_path_drop(struct rds_conn_path *cp, bool destroy) 910 { 911 atomic_set(&cp->cp_state, RDS_CONN_ERROR); 912 913 rcu_read_lock(); 914 if (!destroy && rds_destroy_pending(cp->cp_conn)) { 915 rcu_read_unlock(); 916 return; 917 } 918 queue_work(cp->cp_wq, &cp->cp_down_w); 919 rcu_read_unlock(); 920 } 921 EXPORT_SYMBOL_GPL(rds_conn_path_drop); 922 923 void rds_conn_drop(struct rds_connection *conn) 924 { 925 WARN_ON(conn->c_trans->t_mp_capable); 926 rds_conn_path_drop(&conn->c_path[0], false); 927 } 928 EXPORT_SYMBOL_GPL(rds_conn_drop); 929 930 /* 931 * If the connection is down, trigger a connect. We may have scheduled a 932 * delayed reconnect however - in this case we should not interfere. 933 */ 934 void rds_conn_path_connect_if_down(struct rds_conn_path *cp) 935 { 936 rcu_read_lock(); 937 if (rds_destroy_pending(cp->cp_conn)) { 938 rcu_read_unlock(); 939 return; 940 } 941 if (rds_conn_path_state(cp) == RDS_CONN_DOWN && 942 !test_and_set_bit(RDS_RECONNECT_PENDING, &cp->cp_flags)) 943 queue_delayed_work(cp->cp_wq, &cp->cp_conn_w, 0); 944 rcu_read_unlock(); 945 } 946 EXPORT_SYMBOL_GPL(rds_conn_path_connect_if_down); 947 948 /* Check connectivity of all paths 949 */ 950 void rds_check_all_paths(struct rds_connection *conn) 951 { 952 int i = 0; 953 954 do { 955 rds_conn_path_connect_if_down(&conn->c_path[i]); 956 } while (++i < conn->c_npaths); 957 } 958 959 void rds_conn_connect_if_down(struct rds_connection *conn) 960 { 961 WARN_ON(conn->c_trans->t_mp_capable); 962 rds_conn_path_connect_if_down(&conn->c_path[0]); 963 } 964 EXPORT_SYMBOL_GPL(rds_conn_connect_if_down); 965 966 void 967 __rds_conn_path_error(struct rds_conn_path *cp, const char *fmt, ...) 968 { 969 va_list ap; 970 971 va_start(ap, fmt); 972 vprintk(fmt, ap); 973 va_end(ap); 974 975 rds_conn_path_drop(cp, false); 976 } 977