124f52149SLeon Romanovsky // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2e51060f0SSean Hefty /* 3e51060f0SSean Hefty * Copyright (c) 2005 Voltaire Inc. All rights reserved. 4e51060f0SSean Hefty * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved. 524f52149SLeon Romanovsky * Copyright (c) 1999-2019, Mellanox Technologies, Inc. All rights reserved. 6e51060f0SSean Hefty * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 7e51060f0SSean Hefty */ 8e51060f0SSean Hefty 9e51060f0SSean Hefty #include <linux/completion.h> 10e51060f0SSean Hefty #include <linux/in.h> 11e51060f0SSean Hefty #include <linux/in6.h> 12e51060f0SSean Hefty #include <linux/mutex.h> 13e51060f0SSean Hefty #include <linux/random.h> 14bee3c3c9SMoni Shoua #include <linux/igmp.h> 1563826753SMatthew Wilcox #include <linux/xarray.h> 1607ebafbaSTom Tucker #include <linux/inetdevice.h> 175a0e3ad6STejun Heo #include <linux/slab.h> 18e4dd23d7SPaul Gortmaker #include <linux/module.h> 19366cddb4SAmir Vadai #include <net/route.h> 20e51060f0SSean Hefty 214be74b42SHaggai Eran #include <net/net_namespace.h> 224be74b42SHaggai Eran #include <net/netns/generic.h> 23e51060f0SSean Hefty #include <net/tcp.h> 241f5175adSAleksey Senin #include <net/ipv6.h> 25f887f2acSHaggai Eran #include <net/ip_fib.h> 26f887f2acSHaggai Eran #include <net/ip6_route.h> 27e51060f0SSean Hefty 28e51060f0SSean Hefty #include <rdma/rdma_cm.h> 29e51060f0SSean Hefty #include <rdma/rdma_cm_ib.h> 30753f618aSNir Muchtar #include <rdma/rdma_netlink.h> 312e2d190cSSean Hefty #include <rdma/ib.h> 32e51060f0SSean Hefty #include <rdma/ib_cache.h> 33e51060f0SSean Hefty #include <rdma/ib_cm.h> 34e51060f0SSean Hefty #include <rdma/ib_sa.h> 3507ebafbaSTom Tucker #include <rdma/iw_cm.h> 36e51060f0SSean Hefty 37218a773fSMatan Barak #include "core_priv.h" 38a3b641afSSteve Wise #include "cma_priv.h" 39ed999f82SChuck Lever #include "cma_trace.h" 40218a773fSMatan Barak 41e51060f0SSean Hefty MODULE_AUTHOR("Sean Hefty"); 42e51060f0SSean Hefty MODULE_DESCRIPTION("Generic RDMA CM Agent"); 43e51060f0SSean Hefty MODULE_LICENSE("Dual BSD/GPL"); 44e51060f0SSean Hefty 45e51060f0SSean Hefty #define CMA_CM_RESPONSE_TIMEOUT 20 46ab15c95aSAlex Vesker #define CMA_QUERY_CLASSPORT_INFO_TIMEOUT 3000 47d5bb7599SMichael S. Tsirkin #define CMA_MAX_CM_RETRIES 15 48dcb3f974SSean Hefty #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24) 493c86aa70SEli Cohen #define CMA_IBOE_PACKET_LIFETIME 18 505ab2d89bSLeon Romanovsky #define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP 51e51060f0SSean Hefty 522b1b5b60SSagi Grimberg static const char * const cma_events[] = { 532b1b5b60SSagi Grimberg [RDMA_CM_EVENT_ADDR_RESOLVED] = "address resolved", 542b1b5b60SSagi Grimberg [RDMA_CM_EVENT_ADDR_ERROR] = "address error", 552b1b5b60SSagi Grimberg [RDMA_CM_EVENT_ROUTE_RESOLVED] = "route resolved ", 562b1b5b60SSagi Grimberg [RDMA_CM_EVENT_ROUTE_ERROR] = "route error", 572b1b5b60SSagi Grimberg [RDMA_CM_EVENT_CONNECT_REQUEST] = "connect request", 582b1b5b60SSagi Grimberg [RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response", 592b1b5b60SSagi Grimberg [RDMA_CM_EVENT_CONNECT_ERROR] = "connect error", 602b1b5b60SSagi Grimberg [RDMA_CM_EVENT_UNREACHABLE] = "unreachable", 612b1b5b60SSagi Grimberg [RDMA_CM_EVENT_REJECTED] = "rejected", 622b1b5b60SSagi Grimberg [RDMA_CM_EVENT_ESTABLISHED] = "established", 632b1b5b60SSagi Grimberg [RDMA_CM_EVENT_DISCONNECTED] = "disconnected", 642b1b5b60SSagi Grimberg [RDMA_CM_EVENT_DEVICE_REMOVAL] = "device removal", 652b1b5b60SSagi Grimberg [RDMA_CM_EVENT_MULTICAST_JOIN] = "multicast join", 662b1b5b60SSagi Grimberg [RDMA_CM_EVENT_MULTICAST_ERROR] = "multicast error", 672b1b5b60SSagi Grimberg [RDMA_CM_EVENT_ADDR_CHANGE] = "address change", 682b1b5b60SSagi Grimberg [RDMA_CM_EVENT_TIMEWAIT_EXIT] = "timewait exit", 692b1b5b60SSagi Grimberg }; 702b1b5b60SSagi Grimberg 71db7489e0SBart Van Assche const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event) 722b1b5b60SSagi Grimberg { 732b1b5b60SSagi Grimberg size_t index = event; 742b1b5b60SSagi Grimberg 752b1b5b60SSagi Grimberg return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ? 762b1b5b60SSagi Grimberg cma_events[index] : "unrecognized event"; 772b1b5b60SSagi Grimberg } 782b1b5b60SSagi Grimberg EXPORT_SYMBOL(rdma_event_msg); 792b1b5b60SSagi Grimberg 8077a5db13SSteve Wise const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id, 8177a5db13SSteve Wise int reason) 8277a5db13SSteve Wise { 8377a5db13SSteve Wise if (rdma_ib_or_roce(id->device, id->port_num)) 8477a5db13SSteve Wise return ibcm_reject_msg(reason); 8577a5db13SSteve Wise 8677a5db13SSteve Wise if (rdma_protocol_iwarp(id->device, id->port_num)) 8777a5db13SSteve Wise return iwcm_reject_msg(reason); 8877a5db13SSteve Wise 8977a5db13SSteve Wise WARN_ON_ONCE(1); 9077a5db13SSteve Wise return "unrecognized transport"; 9177a5db13SSteve Wise } 9277a5db13SSteve Wise EXPORT_SYMBOL(rdma_reject_msg); 9377a5db13SSteve Wise 94dd302ee4SLeon Romanovsky /** 95dd302ee4SLeon Romanovsky * rdma_is_consumer_reject - return true if the consumer rejected the connect 96dd302ee4SLeon Romanovsky * request. 97dd302ee4SLeon Romanovsky * @id: Communication identifier that received the REJECT event. 98dd302ee4SLeon Romanovsky * @reason: Value returned in the REJECT event status field. 99dd302ee4SLeon Romanovsky */ 100dd302ee4SLeon Romanovsky static bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason) 1015042a73dSSteve Wise { 1025042a73dSSteve Wise if (rdma_ib_or_roce(id->device, id->port_num)) 1035042a73dSSteve Wise return reason == IB_CM_REJ_CONSUMER_DEFINED; 1045042a73dSSteve Wise 1055042a73dSSteve Wise if (rdma_protocol_iwarp(id->device, id->port_num)) 1065042a73dSSteve Wise return reason == -ECONNREFUSED; 1075042a73dSSteve Wise 1085042a73dSSteve Wise WARN_ON_ONCE(1); 1095042a73dSSteve Wise return false; 1105042a73dSSteve Wise } 1115042a73dSSteve Wise 1125f244104SSteve Wise const void *rdma_consumer_reject_data(struct rdma_cm_id *id, 1135f244104SSteve Wise struct rdma_cm_event *ev, u8 *data_len) 1145f244104SSteve Wise { 1155f244104SSteve Wise const void *p; 1165f244104SSteve Wise 1175f244104SSteve Wise if (rdma_is_consumer_reject(id, ev->status)) { 1185f244104SSteve Wise *data_len = ev->param.conn.private_data_len; 1195f244104SSteve Wise p = ev->param.conn.private_data; 1205f244104SSteve Wise } else { 1215f244104SSteve Wise *data_len = 0; 1225f244104SSteve Wise p = NULL; 1235f244104SSteve Wise } 1245f244104SSteve Wise return p; 1255f244104SSteve Wise } 1265f244104SSteve Wise EXPORT_SYMBOL(rdma_consumer_reject_data); 1275f244104SSteve Wise 128fbdb0a91SSteve Wise /** 129fbdb0a91SSteve Wise * rdma_iw_cm_id() - return the iw_cm_id pointer for this cm_id. 130fbdb0a91SSteve Wise * @id: Communication Identifier 131fbdb0a91SSteve Wise */ 132fbdb0a91SSteve Wise struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *id) 133fbdb0a91SSteve Wise { 134fbdb0a91SSteve Wise struct rdma_id_private *id_priv; 135fbdb0a91SSteve Wise 136fbdb0a91SSteve Wise id_priv = container_of(id, struct rdma_id_private, id); 137fbdb0a91SSteve Wise if (id->device->node_type == RDMA_NODE_RNIC) 138fbdb0a91SSteve Wise return id_priv->cm_id.iw; 139fbdb0a91SSteve Wise return NULL; 140fbdb0a91SSteve Wise } 141fbdb0a91SSteve Wise EXPORT_SYMBOL(rdma_iw_cm_id); 142fbdb0a91SSteve Wise 143fbdb0a91SSteve Wise /** 144fbdb0a91SSteve Wise * rdma_res_to_id() - return the rdma_cm_id pointer for this restrack. 145fbdb0a91SSteve Wise * @res: rdma resource tracking entry pointer 146fbdb0a91SSteve Wise */ 147fbdb0a91SSteve Wise struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res) 148fbdb0a91SSteve Wise { 149fbdb0a91SSteve Wise struct rdma_id_private *id_priv = 150fbdb0a91SSteve Wise container_of(res, struct rdma_id_private, res); 151fbdb0a91SSteve Wise 152fbdb0a91SSteve Wise return &id_priv->id; 153fbdb0a91SSteve Wise } 154fbdb0a91SSteve Wise EXPORT_SYMBOL(rdma_res_to_id); 155fbdb0a91SSteve Wise 15611a0ae4cSJason Gunthorpe static int cma_add_one(struct ib_device *device); 1577c1eb45aSHaggai Eran static void cma_remove_one(struct ib_device *device, void *client_data); 158e51060f0SSean Hefty 159e51060f0SSean Hefty static struct ib_client cma_client = { 160e51060f0SSean Hefty .name = "cma", 161e51060f0SSean Hefty .add = cma_add_one, 162e51060f0SSean Hefty .remove = cma_remove_one 163e51060f0SSean Hefty }; 164e51060f0SSean Hefty 165c1a0b23bSMichael S. Tsirkin static struct ib_sa_client sa_client; 166e51060f0SSean Hefty static LIST_HEAD(dev_list); 167e51060f0SSean Hefty static LIST_HEAD(listen_any_list); 168e51060f0SSean Hefty static DEFINE_MUTEX(lock); 169e51060f0SSean Hefty static struct workqueue_struct *cma_wq; 170c7d03a00SAlexey Dobriyan static unsigned int cma_pernet_id; 171e51060f0SSean Hefty 1724be74b42SHaggai Eran struct cma_pernet { 17363826753SMatthew Wilcox struct xarray tcp_ps; 17463826753SMatthew Wilcox struct xarray udp_ps; 17563826753SMatthew Wilcox struct xarray ipoib_ps; 17663826753SMatthew Wilcox struct xarray ib_ps; 1774be74b42SHaggai Eran }; 1784be74b42SHaggai Eran 1794be74b42SHaggai Eran static struct cma_pernet *cma_pernet(struct net *net) 180aac978e1SHaggai Eran { 1814be74b42SHaggai Eran return net_generic(net, cma_pernet_id); 1824be74b42SHaggai Eran } 1834be74b42SHaggai Eran 18463826753SMatthew Wilcox static 18563826753SMatthew Wilcox struct xarray *cma_pernet_xa(struct net *net, enum rdma_ucm_port_space ps) 1864be74b42SHaggai Eran { 1874be74b42SHaggai Eran struct cma_pernet *pernet = cma_pernet(net); 1884be74b42SHaggai Eran 189aac978e1SHaggai Eran switch (ps) { 190aac978e1SHaggai Eran case RDMA_PS_TCP: 1914be74b42SHaggai Eran return &pernet->tcp_ps; 192aac978e1SHaggai Eran case RDMA_PS_UDP: 1934be74b42SHaggai Eran return &pernet->udp_ps; 194aac978e1SHaggai Eran case RDMA_PS_IPOIB: 1954be74b42SHaggai Eran return &pernet->ipoib_ps; 196aac978e1SHaggai Eran case RDMA_PS_IB: 1974be74b42SHaggai Eran return &pernet->ib_ps; 198aac978e1SHaggai Eran default: 199aac978e1SHaggai Eran return NULL; 200aac978e1SHaggai Eran } 201aac978e1SHaggai Eran } 202aac978e1SHaggai Eran 203e51060f0SSean Hefty struct cma_device { 204e51060f0SSean Hefty struct list_head list; 205e51060f0SSean Hefty struct ib_device *device; 206e51060f0SSean Hefty struct completion comp; 207be439912SParav Pandit refcount_t refcount; 208e51060f0SSean Hefty struct list_head id_list; 209045959dbSMatan Barak enum ib_gid_type *default_gid_type; 21089052d78SMajd Dibbiny u8 *default_roce_tos; 211e51060f0SSean Hefty }; 212e51060f0SSean Hefty 213e51060f0SSean Hefty struct rdma_bind_list { 2142253fc0cSSteve Wise enum rdma_ucm_port_space ps; 215e51060f0SSean Hefty struct hlist_head owners; 216e51060f0SSean Hefty unsigned short port; 217e51060f0SSean Hefty }; 218e51060f0SSean Hefty 219ab15c95aSAlex Vesker struct class_port_info_context { 220ab15c95aSAlex Vesker struct ib_class_port_info *class_port_info; 221ab15c95aSAlex Vesker struct ib_device *device; 222ab15c95aSAlex Vesker struct completion done; 223ab15c95aSAlex Vesker struct ib_sa_query *sa_query; 224ab15c95aSAlex Vesker u8 port_num; 225ab15c95aSAlex Vesker }; 226ab15c95aSAlex Vesker 2272253fc0cSSteve Wise static int cma_ps_alloc(struct net *net, enum rdma_ucm_port_space ps, 228aac978e1SHaggai Eran struct rdma_bind_list *bind_list, int snum) 229aac978e1SHaggai Eran { 23063826753SMatthew Wilcox struct xarray *xa = cma_pernet_xa(net, ps); 231aac978e1SHaggai Eran 23263826753SMatthew Wilcox return xa_insert(xa, snum, bind_list, GFP_KERNEL); 233aac978e1SHaggai Eran } 234aac978e1SHaggai Eran 2354be74b42SHaggai Eran static struct rdma_bind_list *cma_ps_find(struct net *net, 2362253fc0cSSteve Wise enum rdma_ucm_port_space ps, int snum) 237aac978e1SHaggai Eran { 23863826753SMatthew Wilcox struct xarray *xa = cma_pernet_xa(net, ps); 239aac978e1SHaggai Eran 24063826753SMatthew Wilcox return xa_load(xa, snum); 241aac978e1SHaggai Eran } 242aac978e1SHaggai Eran 2432253fc0cSSteve Wise static void cma_ps_remove(struct net *net, enum rdma_ucm_port_space ps, 2442253fc0cSSteve Wise int snum) 245aac978e1SHaggai Eran { 24663826753SMatthew Wilcox struct xarray *xa = cma_pernet_xa(net, ps); 247aac978e1SHaggai Eran 24863826753SMatthew Wilcox xa_erase(xa, snum); 249aac978e1SHaggai Eran } 250aac978e1SHaggai Eran 25168602120SSean Hefty enum { 25268602120SSean Hefty CMA_OPTION_AFONLY, 25368602120SSean Hefty }; 25468602120SSean Hefty 2555ff8c8faSParav Pandit void cma_dev_get(struct cma_device *cma_dev) 256218a773fSMatan Barak { 257be439912SParav Pandit refcount_inc(&cma_dev->refcount); 258218a773fSMatan Barak } 259218a773fSMatan Barak 2605ff8c8faSParav Pandit void cma_dev_put(struct cma_device *cma_dev) 2615ff8c8faSParav Pandit { 262be439912SParav Pandit if (refcount_dec_and_test(&cma_dev->refcount)) 2635ff8c8faSParav Pandit complete(&cma_dev->comp); 2645ff8c8faSParav Pandit } 2655ff8c8faSParav Pandit 266045959dbSMatan Barak struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter filter, 267045959dbSMatan Barak void *cookie) 268045959dbSMatan Barak { 269045959dbSMatan Barak struct cma_device *cma_dev; 270045959dbSMatan Barak struct cma_device *found_cma_dev = NULL; 271045959dbSMatan Barak 272045959dbSMatan Barak mutex_lock(&lock); 273045959dbSMatan Barak 274045959dbSMatan Barak list_for_each_entry(cma_dev, &dev_list, list) 275045959dbSMatan Barak if (filter(cma_dev->device, cookie)) { 276045959dbSMatan Barak found_cma_dev = cma_dev; 277045959dbSMatan Barak break; 278045959dbSMatan Barak } 279045959dbSMatan Barak 280045959dbSMatan Barak if (found_cma_dev) 2815ff8c8faSParav Pandit cma_dev_get(found_cma_dev); 282045959dbSMatan Barak mutex_unlock(&lock); 283045959dbSMatan Barak return found_cma_dev; 284045959dbSMatan Barak } 285045959dbSMatan Barak 286045959dbSMatan Barak int cma_get_default_gid_type(struct cma_device *cma_dev, 287045959dbSMatan Barak unsigned int port) 288045959dbSMatan Barak { 28924dc831bSYuval Shaia if (!rdma_is_port_valid(cma_dev->device, port)) 290045959dbSMatan Barak return -EINVAL; 291045959dbSMatan Barak 292045959dbSMatan Barak return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)]; 293045959dbSMatan Barak } 294045959dbSMatan Barak 295045959dbSMatan Barak int cma_set_default_gid_type(struct cma_device *cma_dev, 296045959dbSMatan Barak unsigned int port, 297045959dbSMatan Barak enum ib_gid_type default_gid_type) 298045959dbSMatan Barak { 299045959dbSMatan Barak unsigned long supported_gids; 300045959dbSMatan Barak 30124dc831bSYuval Shaia if (!rdma_is_port_valid(cma_dev->device, port)) 302045959dbSMatan Barak return -EINVAL; 303045959dbSMatan Barak 304045959dbSMatan Barak supported_gids = roce_gid_type_mask_support(cma_dev->device, port); 305045959dbSMatan Barak 306045959dbSMatan Barak if (!(supported_gids & 1 << default_gid_type)) 307045959dbSMatan Barak return -EINVAL; 308045959dbSMatan Barak 309045959dbSMatan Barak cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] = 310045959dbSMatan Barak default_gid_type; 311045959dbSMatan Barak 312045959dbSMatan Barak return 0; 313045959dbSMatan Barak } 314045959dbSMatan Barak 31589052d78SMajd Dibbiny int cma_get_default_roce_tos(struct cma_device *cma_dev, unsigned int port) 31689052d78SMajd Dibbiny { 31789052d78SMajd Dibbiny if (!rdma_is_port_valid(cma_dev->device, port)) 31889052d78SMajd Dibbiny return -EINVAL; 31989052d78SMajd Dibbiny 32089052d78SMajd Dibbiny return cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)]; 32189052d78SMajd Dibbiny } 32289052d78SMajd Dibbiny 32389052d78SMajd Dibbiny int cma_set_default_roce_tos(struct cma_device *cma_dev, unsigned int port, 32489052d78SMajd Dibbiny u8 default_roce_tos) 32589052d78SMajd Dibbiny { 32689052d78SMajd Dibbiny if (!rdma_is_port_valid(cma_dev->device, port)) 32789052d78SMajd Dibbiny return -EINVAL; 32889052d78SMajd Dibbiny 32989052d78SMajd Dibbiny cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)] = 33089052d78SMajd Dibbiny default_roce_tos; 33189052d78SMajd Dibbiny 33289052d78SMajd Dibbiny return 0; 33389052d78SMajd Dibbiny } 334045959dbSMatan Barak struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev) 335045959dbSMatan Barak { 336045959dbSMatan Barak return cma_dev->device; 337045959dbSMatan Barak } 338045959dbSMatan Barak 339e51060f0SSean Hefty /* 340e51060f0SSean Hefty * Device removal can occur at anytime, so we need extra handling to 341e51060f0SSean Hefty * serialize notifying the user of device removal with other callbacks. 342e51060f0SSean Hefty * We do this by disabling removal notification while a callback is in process, 343e51060f0SSean Hefty * and reporting it after the callback completes. 344e51060f0SSean Hefty */ 345e51060f0SSean Hefty 346c8f6a362SSean Hefty struct cma_multicast { 347c8f6a362SSean Hefty struct rdma_id_private *id_priv; 348c8f6a362SSean Hefty union { 349c8f6a362SSean Hefty struct ib_sa_multicast *ib; 350c8f6a362SSean Hefty } multicast; 351c8f6a362SSean Hefty struct list_head list; 352c8f6a362SSean Hefty void *context; 3533f446754SRoland Dreier struct sockaddr_storage addr; 3543c86aa70SEli Cohen struct kref mcref; 355ab15c95aSAlex Vesker u8 join_state; 356c8f6a362SSean Hefty }; 357c8f6a362SSean Hefty 358e51060f0SSean Hefty struct cma_work { 359e51060f0SSean Hefty struct work_struct work; 360e51060f0SSean Hefty struct rdma_id_private *id; 361550e5ca7SNir Muchtar enum rdma_cm_state old_state; 362550e5ca7SNir Muchtar enum rdma_cm_state new_state; 363e51060f0SSean Hefty struct rdma_cm_event event; 364e51060f0SSean Hefty }; 365e51060f0SSean Hefty 366dd5bdff8SOr Gerlitz struct cma_ndev_work { 367dd5bdff8SOr Gerlitz struct work_struct work; 368dd5bdff8SOr Gerlitz struct rdma_id_private *id; 369dd5bdff8SOr Gerlitz struct rdma_cm_event event; 370dd5bdff8SOr Gerlitz }; 371dd5bdff8SOr Gerlitz 3723c86aa70SEli Cohen struct iboe_mcast_work { 3733c86aa70SEli Cohen struct work_struct work; 3743c86aa70SEli Cohen struct rdma_id_private *id; 3753c86aa70SEli Cohen struct cma_multicast *mc; 3763c86aa70SEli Cohen }; 3773c86aa70SEli Cohen 378e51060f0SSean Hefty union cma_ip_addr { 379e51060f0SSean Hefty struct in6_addr ip6; 380e51060f0SSean Hefty struct { 3811b90c137SAl Viro __be32 pad[3]; 3821b90c137SAl Viro __be32 addr; 383e51060f0SSean Hefty } ip4; 384e51060f0SSean Hefty }; 385e51060f0SSean Hefty 386e51060f0SSean Hefty struct cma_hdr { 387e51060f0SSean Hefty u8 cma_version; 388e51060f0SSean Hefty u8 ip_version; /* IP version: 7:4 */ 3891b90c137SAl Viro __be16 port; 390e51060f0SSean Hefty union cma_ip_addr src_addr; 391e51060f0SSean Hefty union cma_ip_addr dst_addr; 392e51060f0SSean Hefty }; 393e51060f0SSean Hefty 394e51060f0SSean Hefty #define CMA_VERSION 0x00 395e51060f0SSean Hefty 3964c21b5bcSHaggai Eran struct cma_req_info { 3972918c1a9SParav Pandit struct sockaddr_storage listen_addr_storage; 3982918c1a9SParav Pandit struct sockaddr_storage src_addr_storage; 3994c21b5bcSHaggai Eran struct ib_device *device; 4004c21b5bcSHaggai Eran union ib_gid local_gid; 4014c21b5bcSHaggai Eran __be64 service_id; 40205e0b86cSParav Pandit int port; 40305e0b86cSParav Pandit bool has_gid; 4044c21b5bcSHaggai Eran u16 pkey; 4054c21b5bcSHaggai Eran }; 4064c21b5bcSHaggai Eran 407550e5ca7SNir Muchtar static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp) 408e51060f0SSean Hefty { 409e51060f0SSean Hefty unsigned long flags; 410e51060f0SSean Hefty int ret; 411e51060f0SSean Hefty 412e51060f0SSean Hefty spin_lock_irqsave(&id_priv->lock, flags); 413e51060f0SSean Hefty ret = (id_priv->state == comp); 414e51060f0SSean Hefty spin_unlock_irqrestore(&id_priv->lock, flags); 415e51060f0SSean Hefty return ret; 416e51060f0SSean Hefty } 417e51060f0SSean Hefty 418e51060f0SSean Hefty static int cma_comp_exch(struct rdma_id_private *id_priv, 419550e5ca7SNir Muchtar enum rdma_cm_state comp, enum rdma_cm_state exch) 420e51060f0SSean Hefty { 421e51060f0SSean Hefty unsigned long flags; 422e51060f0SSean Hefty int ret; 423e51060f0SSean Hefty 424e51060f0SSean Hefty spin_lock_irqsave(&id_priv->lock, flags); 425e51060f0SSean Hefty if ((ret = (id_priv->state == comp))) 426e51060f0SSean Hefty id_priv->state = exch; 427e51060f0SSean Hefty spin_unlock_irqrestore(&id_priv->lock, flags); 428e51060f0SSean Hefty return ret; 429e51060f0SSean Hefty } 430e51060f0SSean Hefty 431550e5ca7SNir Muchtar static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv, 432550e5ca7SNir Muchtar enum rdma_cm_state exch) 433e51060f0SSean Hefty { 434e51060f0SSean Hefty unsigned long flags; 435550e5ca7SNir Muchtar enum rdma_cm_state old; 436e51060f0SSean Hefty 437e51060f0SSean Hefty spin_lock_irqsave(&id_priv->lock, flags); 438e51060f0SSean Hefty old = id_priv->state; 439e51060f0SSean Hefty id_priv->state = exch; 440e51060f0SSean Hefty spin_unlock_irqrestore(&id_priv->lock, flags); 441e51060f0SSean Hefty return old; 442e51060f0SSean Hefty } 443e51060f0SSean Hefty 4444c21b5bcSHaggai Eran static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr) 445e51060f0SSean Hefty { 446e51060f0SSean Hefty return hdr->ip_version >> 4; 447e51060f0SSean Hefty } 448e51060f0SSean Hefty 449e51060f0SSean Hefty static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver) 450e51060f0SSean Hefty { 451e51060f0SSean Hefty hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF); 452e51060f0SSean Hefty } 453e51060f0SSean Hefty 454bee3c3c9SMoni Shoua static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join) 455bee3c3c9SMoni Shoua { 456bee3c3c9SMoni Shoua struct in_device *in_dev = NULL; 457bee3c3c9SMoni Shoua 458bee3c3c9SMoni Shoua if (ndev) { 459bee3c3c9SMoni Shoua rtnl_lock(); 460bee3c3c9SMoni Shoua in_dev = __in_dev_get_rtnl(ndev); 461bee3c3c9SMoni Shoua if (in_dev) { 462bee3c3c9SMoni Shoua if (join) 463bee3c3c9SMoni Shoua ip_mc_inc_group(in_dev, 464bee3c3c9SMoni Shoua *(__be32 *)(mgid->raw + 12)); 465bee3c3c9SMoni Shoua else 466bee3c3c9SMoni Shoua ip_mc_dec_group(in_dev, 467bee3c3c9SMoni Shoua *(__be32 *)(mgid->raw + 12)); 468bee3c3c9SMoni Shoua } 469bee3c3c9SMoni Shoua rtnl_unlock(); 470bee3c3c9SMoni Shoua } 471bee3c3c9SMoni Shoua return (in_dev) ? 0 : -ENODEV; 472bee3c3c9SMoni Shoua } 473bee3c3c9SMoni Shoua 474045959dbSMatan Barak static void _cma_attach_to_dev(struct rdma_id_private *id_priv, 475e51060f0SSean Hefty struct cma_device *cma_dev) 476e51060f0SSean Hefty { 4775ff8c8faSParav Pandit cma_dev_get(cma_dev); 478e51060f0SSean Hefty id_priv->cma_dev = cma_dev; 479e51060f0SSean Hefty id_priv->id.device = cma_dev->device; 4803c86aa70SEli Cohen id_priv->id.route.addr.dev_addr.transport = 4813c86aa70SEli Cohen rdma_node_get_transport(cma_dev->device->node_type); 482e51060f0SSean Hefty list_add_tail(&id_priv->list, &cma_dev->id_list); 483917cb8a7SSteve Wise if (id_priv->res.kern_name) 484af8d7037SShamir Rabinovitch rdma_restrack_kadd(&id_priv->res); 485917cb8a7SSteve Wise else 486917cb8a7SSteve Wise rdma_restrack_uadd(&id_priv->res); 487e51060f0SSean Hefty } 488e51060f0SSean Hefty 489045959dbSMatan Barak static void cma_attach_to_dev(struct rdma_id_private *id_priv, 490045959dbSMatan Barak struct cma_device *cma_dev) 491045959dbSMatan Barak { 492045959dbSMatan Barak _cma_attach_to_dev(id_priv, cma_dev); 493045959dbSMatan Barak id_priv->gid_type = 494045959dbSMatan Barak cma_dev->default_gid_type[id_priv->id.port_num - 495045959dbSMatan Barak rdma_start_port(cma_dev->device)]; 496045959dbSMatan Barak } 497045959dbSMatan Barak 4983c86aa70SEli Cohen static inline void release_mc(struct kref *kref) 4993c86aa70SEli Cohen { 5003c86aa70SEli Cohen struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref); 5013c86aa70SEli Cohen 5023c86aa70SEli Cohen kfree(mc->multicast.ib); 5033c86aa70SEli Cohen kfree(mc); 5043c86aa70SEli Cohen } 5053c86aa70SEli Cohen 506a396d43aSSean Hefty static void cma_release_dev(struct rdma_id_private *id_priv) 507e51060f0SSean Hefty { 508a396d43aSSean Hefty mutex_lock(&lock); 509e51060f0SSean Hefty list_del(&id_priv->list); 5105ff8c8faSParav Pandit cma_dev_put(id_priv->cma_dev); 511e51060f0SSean Hefty id_priv->cma_dev = NULL; 512a396d43aSSean Hefty mutex_unlock(&lock); 513e51060f0SSean Hefty } 514e51060f0SSean Hefty 515f4753834SSean Hefty static inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv) 516f4753834SSean Hefty { 517f4753834SSean Hefty return (struct sockaddr *) &id_priv->id.route.addr.src_addr; 518f4753834SSean Hefty } 519f4753834SSean Hefty 520f4753834SSean Hefty static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv) 521f4753834SSean Hefty { 522f4753834SSean Hefty return (struct sockaddr *) &id_priv->id.route.addr.dst_addr; 523f4753834SSean Hefty } 524f4753834SSean Hefty 525f4753834SSean Hefty static inline unsigned short cma_family(struct rdma_id_private *id_priv) 526f4753834SSean Hefty { 527f4753834SSean Hefty return id_priv->id.route.addr.src_addr.ss_family; 528f4753834SSean Hefty } 529f4753834SSean Hefty 5305c438135SSean Hefty static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey) 531c8f6a362SSean Hefty { 532c8f6a362SSean Hefty struct ib_sa_mcmember_rec rec; 533c8f6a362SSean Hefty int ret = 0; 534c8f6a362SSean Hefty 5355c438135SSean Hefty if (id_priv->qkey) { 5365c438135SSean Hefty if (qkey && id_priv->qkey != qkey) 5375c438135SSean Hefty return -EINVAL; 538d2ca39f2SYossi Etigin return 0; 5395c438135SSean Hefty } 5405c438135SSean Hefty 5415c438135SSean Hefty if (qkey) { 5425c438135SSean Hefty id_priv->qkey = qkey; 5435c438135SSean Hefty return 0; 5445c438135SSean Hefty } 545d2ca39f2SYossi Etigin 546d2ca39f2SYossi Etigin switch (id_priv->id.ps) { 547c8f6a362SSean Hefty case RDMA_PS_UDP: 5485c438135SSean Hefty case RDMA_PS_IB: 549d2ca39f2SYossi Etigin id_priv->qkey = RDMA_UDP_QKEY; 550c8f6a362SSean Hefty break; 551c8f6a362SSean Hefty case RDMA_PS_IPOIB: 552d2ca39f2SYossi Etigin ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid); 553d2ca39f2SYossi Etigin ret = ib_sa_get_mcmember_rec(id_priv->id.device, 554d2ca39f2SYossi Etigin id_priv->id.port_num, &rec.mgid, 555d2ca39f2SYossi Etigin &rec); 556d2ca39f2SYossi Etigin if (!ret) 557d2ca39f2SYossi Etigin id_priv->qkey = be32_to_cpu(rec.qkey); 558c8f6a362SSean Hefty break; 559c8f6a362SSean Hefty default: 560c8f6a362SSean Hefty break; 561c8f6a362SSean Hefty } 562c8f6a362SSean Hefty return ret; 563c8f6a362SSean Hefty } 564c8f6a362SSean Hefty 565680f920aSSean Hefty static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr) 566680f920aSSean Hefty { 567680f920aSSean Hefty dev_addr->dev_type = ARPHRD_INFINIBAND; 568680f920aSSean Hefty rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr); 569680f920aSSean Hefty ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey)); 570680f920aSSean Hefty } 571680f920aSSean Hefty 572680f920aSSean Hefty static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr) 573680f920aSSean Hefty { 574680f920aSSean Hefty int ret; 575680f920aSSean Hefty 576680f920aSSean Hefty if (addr->sa_family != AF_IB) { 577575c7e58SParav Pandit ret = rdma_translate_ip(addr, dev_addr); 578680f920aSSean Hefty } else { 579680f920aSSean Hefty cma_translate_ib((struct sockaddr_ib *) addr, dev_addr); 580680f920aSSean Hefty ret = 0; 581680f920aSSean Hefty } 582680f920aSSean Hefty 583680f920aSSean Hefty return ret; 584680f920aSSean Hefty } 585680f920aSSean Hefty 5864ed13a5fSParav Pandit static const struct ib_gid_attr * 5874ed13a5fSParav Pandit cma_validate_port(struct ib_device *device, u8 port, 588045959dbSMatan Barak enum ib_gid_type gid_type, 5892493a57bSParav Pandit union ib_gid *gid, 5902493a57bSParav Pandit struct rdma_id_private *id_priv) 5917c11147dSMichael Wang { 5922493a57bSParav Pandit struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 5932493a57bSParav Pandit int bound_if_index = dev_addr->bound_dev_if; 5944ed13a5fSParav Pandit const struct ib_gid_attr *sgid_attr; 5952493a57bSParav Pandit int dev_type = dev_addr->dev_type; 596abae1b71SMatan Barak struct net_device *ndev = NULL; 5977c11147dSMichael Wang 59841c61401SParav Pandit if (!rdma_dev_access_netns(device, id_priv->id.route.addr.dev_addr.net)) 59941c61401SParav Pandit return ERR_PTR(-ENODEV); 60041c61401SParav Pandit 6017c11147dSMichael Wang if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port)) 6024ed13a5fSParav Pandit return ERR_PTR(-ENODEV); 6037c11147dSMichael Wang 6047c11147dSMichael Wang if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port)) 6054ed13a5fSParav Pandit return ERR_PTR(-ENODEV); 6067c11147dSMichael Wang 60700db63c1SParav Pandit if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) { 60866c74d74SParav Pandit ndev = dev_get_by_index(dev_addr->net, bound_if_index); 60900db63c1SParav Pandit if (!ndev) 6104ed13a5fSParav Pandit return ERR_PTR(-ENODEV); 61100db63c1SParav Pandit } else { 612045959dbSMatan Barak gid_type = IB_GID_TYPE_IB; 61300db63c1SParav Pandit } 614abae1b71SMatan Barak 6154ed13a5fSParav Pandit sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev); 616abae1b71SMatan Barak if (ndev) 617abae1b71SMatan Barak dev_put(ndev); 6184ed13a5fSParav Pandit return sgid_attr; 6194ed13a5fSParav Pandit } 6207c11147dSMichael Wang 6214ed13a5fSParav Pandit static void cma_bind_sgid_attr(struct rdma_id_private *id_priv, 6224ed13a5fSParav Pandit const struct ib_gid_attr *sgid_attr) 6234ed13a5fSParav Pandit { 6244ed13a5fSParav Pandit WARN_ON(id_priv->id.route.addr.dev_addr.sgid_attr); 6254ed13a5fSParav Pandit id_priv->id.route.addr.dev_addr.sgid_attr = sgid_attr; 6267c11147dSMichael Wang } 6277c11147dSMichael Wang 628ff11c6cdSParav Pandit /** 629ff11c6cdSParav Pandit * cma_acquire_dev_by_src_ip - Acquire cma device, port, gid attribute 630ff11c6cdSParav Pandit * based on source ip address. 631ff11c6cdSParav Pandit * @id_priv: cm_id which should be bound to cma device 632ff11c6cdSParav Pandit * 633ff11c6cdSParav Pandit * cma_acquire_dev_by_src_ip() binds cm id to cma device, port and GID attribute 634ff11c6cdSParav Pandit * based on source IP address. It returns 0 on success or error code otherwise. 635ff11c6cdSParav Pandit * It is applicable to active and passive side cm_id. 636ff11c6cdSParav Pandit */ 637ff11c6cdSParav Pandit static int cma_acquire_dev_by_src_ip(struct rdma_id_private *id_priv) 638ff11c6cdSParav Pandit { 639ff11c6cdSParav Pandit struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 640ff11c6cdSParav Pandit const struct ib_gid_attr *sgid_attr; 641ff11c6cdSParav Pandit union ib_gid gid, iboe_gid, *gidp; 642ff11c6cdSParav Pandit struct cma_device *cma_dev; 643ff11c6cdSParav Pandit enum ib_gid_type gid_type; 644ff11c6cdSParav Pandit int ret = -ENODEV; 645ea1075edSJason Gunthorpe unsigned int port; 646ff11c6cdSParav Pandit 647ff11c6cdSParav Pandit if (dev_addr->dev_type != ARPHRD_INFINIBAND && 648ff11c6cdSParav Pandit id_priv->id.ps == RDMA_PS_IPOIB) 649ff11c6cdSParav Pandit return -EINVAL; 650ff11c6cdSParav Pandit 651ff11c6cdSParav Pandit rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 652ff11c6cdSParav Pandit &iboe_gid); 653ff11c6cdSParav Pandit 654ff11c6cdSParav Pandit memcpy(&gid, dev_addr->src_dev_addr + 655ff11c6cdSParav Pandit rdma_addr_gid_offset(dev_addr), sizeof(gid)); 656ff11c6cdSParav Pandit 657ff11c6cdSParav Pandit mutex_lock(&lock); 658ff11c6cdSParav Pandit list_for_each_entry(cma_dev, &dev_list, list) { 659ea1075edSJason Gunthorpe rdma_for_each_port (cma_dev->device, port) { 660ff11c6cdSParav Pandit gidp = rdma_protocol_roce(cma_dev->device, port) ? 661ff11c6cdSParav Pandit &iboe_gid : &gid; 662ff11c6cdSParav Pandit gid_type = cma_dev->default_gid_type[port - 1]; 663ff11c6cdSParav Pandit sgid_attr = cma_validate_port(cma_dev->device, port, 664ff11c6cdSParav Pandit gid_type, gidp, id_priv); 665ff11c6cdSParav Pandit if (!IS_ERR(sgid_attr)) { 666ff11c6cdSParav Pandit id_priv->id.port_num = port; 667ff11c6cdSParav Pandit cma_bind_sgid_attr(id_priv, sgid_attr); 668ff11c6cdSParav Pandit cma_attach_to_dev(id_priv, cma_dev); 669ff11c6cdSParav Pandit ret = 0; 670ff11c6cdSParav Pandit goto out; 671ff11c6cdSParav Pandit } 672ff11c6cdSParav Pandit } 673ff11c6cdSParav Pandit } 674ff11c6cdSParav Pandit out: 675ff11c6cdSParav Pandit mutex_unlock(&lock); 676ff11c6cdSParav Pandit return ret; 677ff11c6cdSParav Pandit } 678ff11c6cdSParav Pandit 67941ab1cb7SParav Pandit /** 68041ab1cb7SParav Pandit * cma_ib_acquire_dev - Acquire cma device, port and SGID attribute 68141ab1cb7SParav Pandit * @id_priv: cm id to bind to cma device 68241ab1cb7SParav Pandit * @listen_id_priv: listener cm id to match against 68341ab1cb7SParav Pandit * @req: Pointer to req structure containaining incoming 68441ab1cb7SParav Pandit * request information 68541ab1cb7SParav Pandit * cma_ib_acquire_dev() acquires cma device, port and SGID attribute when 68641ab1cb7SParav Pandit * rdma device matches for listen_id and incoming request. It also verifies 68741ab1cb7SParav Pandit * that a GID table entry is present for the source address. 68841ab1cb7SParav Pandit * Returns 0 on success, or returns error code otherwise. 68941ab1cb7SParav Pandit */ 69041ab1cb7SParav Pandit static int cma_ib_acquire_dev(struct rdma_id_private *id_priv, 69141ab1cb7SParav Pandit const struct rdma_id_private *listen_id_priv, 69241ab1cb7SParav Pandit struct cma_req_info *req) 69341ab1cb7SParav Pandit { 69441ab1cb7SParav Pandit struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 69541ab1cb7SParav Pandit const struct ib_gid_attr *sgid_attr; 69641ab1cb7SParav Pandit enum ib_gid_type gid_type; 69741ab1cb7SParav Pandit union ib_gid gid; 69841ab1cb7SParav Pandit 69941ab1cb7SParav Pandit if (dev_addr->dev_type != ARPHRD_INFINIBAND && 70041ab1cb7SParav Pandit id_priv->id.ps == RDMA_PS_IPOIB) 70141ab1cb7SParav Pandit return -EINVAL; 70241ab1cb7SParav Pandit 70341ab1cb7SParav Pandit if (rdma_protocol_roce(req->device, req->port)) 70441ab1cb7SParav Pandit rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 70541ab1cb7SParav Pandit &gid); 70641ab1cb7SParav Pandit else 70741ab1cb7SParav Pandit memcpy(&gid, dev_addr->src_dev_addr + 70841ab1cb7SParav Pandit rdma_addr_gid_offset(dev_addr), sizeof(gid)); 70941ab1cb7SParav Pandit 71041ab1cb7SParav Pandit gid_type = listen_id_priv->cma_dev->default_gid_type[req->port - 1]; 71141ab1cb7SParav Pandit sgid_attr = cma_validate_port(req->device, req->port, 71241ab1cb7SParav Pandit gid_type, &gid, id_priv); 71341ab1cb7SParav Pandit if (IS_ERR(sgid_attr)) 71441ab1cb7SParav Pandit return PTR_ERR(sgid_attr); 71541ab1cb7SParav Pandit 71641ab1cb7SParav Pandit id_priv->id.port_num = req->port; 71741ab1cb7SParav Pandit cma_bind_sgid_attr(id_priv, sgid_attr); 71841ab1cb7SParav Pandit /* Need to acquire lock to protect against reader 71941ab1cb7SParav Pandit * of cma_dev->id_list such as cma_netdev_callback() and 72041ab1cb7SParav Pandit * cma_process_remove(). 72141ab1cb7SParav Pandit */ 72241ab1cb7SParav Pandit mutex_lock(&lock); 72341ab1cb7SParav Pandit cma_attach_to_dev(id_priv, listen_id_priv->cma_dev); 72441ab1cb7SParav Pandit mutex_unlock(&lock); 72541ab1cb7SParav Pandit return 0; 72641ab1cb7SParav Pandit } 72741ab1cb7SParav Pandit 72841ab1cb7SParav Pandit static int cma_iw_acquire_dev(struct rdma_id_private *id_priv, 729e7ff98aeSParav Pandit const struct rdma_id_private *listen_id_priv) 730e51060f0SSean Hefty { 731c8f6a362SSean Hefty struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 7324ed13a5fSParav Pandit const struct ib_gid_attr *sgid_attr; 733e51060f0SSean Hefty struct cma_device *cma_dev; 7344ed13a5fSParav Pandit enum ib_gid_type gid_type; 735e51060f0SSean Hefty int ret = -ENODEV; 736cc055dd3SParav Pandit unsigned int port; 73741ab1cb7SParav Pandit union ib_gid gid; 738e51060f0SSean Hefty 7397c11147dSMichael Wang if (dev_addr->dev_type != ARPHRD_INFINIBAND && 7402efdd6a0SMoni Shoua id_priv->id.ps == RDMA_PS_IPOIB) 7412efdd6a0SMoni Shoua return -EINVAL; 7422efdd6a0SMoni Shoua 7433c86aa70SEli Cohen memcpy(&gid, dev_addr->src_dev_addr + 74441ab1cb7SParav Pandit rdma_addr_gid_offset(dev_addr), sizeof(gid)); 74541ab1cb7SParav Pandit 74641ab1cb7SParav Pandit mutex_lock(&lock); 7477c11147dSMichael Wang 748be9130ccSDoug Ledford cma_dev = listen_id_priv->cma_dev; 749be9130ccSDoug Ledford port = listen_id_priv->id.port_num; 75079d684f0SParav Pandit gid_type = listen_id_priv->gid_type; 7514ed13a5fSParav Pandit sgid_attr = cma_validate_port(cma_dev->device, port, 75241ab1cb7SParav Pandit gid_type, &gid, id_priv); 7534ed13a5fSParav Pandit if (!IS_ERR(sgid_attr)) { 7547c11147dSMichael Wang id_priv->id.port_num = port; 7554ed13a5fSParav Pandit cma_bind_sgid_attr(id_priv, sgid_attr); 7564ed13a5fSParav Pandit ret = 0; 757be9130ccSDoug Ledford goto out; 758be9130ccSDoug Ledford } 7597c11147dSMichael Wang 760e51060f0SSean Hefty list_for_each_entry(cma_dev, &dev_list, list) { 761cc055dd3SParav Pandit rdma_for_each_port (cma_dev->device, port) { 762ff11c6cdSParav Pandit if (listen_id_priv->cma_dev == cma_dev && 763be9130ccSDoug Ledford listen_id_priv->id.port_num == port) 764be9130ccSDoug Ledford continue; 7653c86aa70SEli Cohen 76679d684f0SParav Pandit gid_type = cma_dev->default_gid_type[port - 1]; 7674ed13a5fSParav Pandit sgid_attr = cma_validate_port(cma_dev->device, port, 76841ab1cb7SParav Pandit gid_type, &gid, id_priv); 7694ed13a5fSParav Pandit if (!IS_ERR(sgid_attr)) { 7707c11147dSMichael Wang id_priv->id.port_num = port; 7714ed13a5fSParav Pandit cma_bind_sgid_attr(id_priv, sgid_attr); 7724ed13a5fSParav Pandit ret = 0; 7733c86aa70SEli Cohen goto out; 77463f05be2Sshefty } 775e51060f0SSean Hefty } 776e51060f0SSean Hefty } 7773c86aa70SEli Cohen 7783c86aa70SEli Cohen out: 7793c86aa70SEli Cohen if (!ret) 7803c86aa70SEli Cohen cma_attach_to_dev(id_priv, cma_dev); 7813c86aa70SEli Cohen 782a396d43aSSean Hefty mutex_unlock(&lock); 783e51060f0SSean Hefty return ret; 784e51060f0SSean Hefty } 785e51060f0SSean Hefty 786f17df3b0SSean Hefty /* 787f17df3b0SSean Hefty * Select the source IB device and address to reach the destination IB address. 788f17df3b0SSean Hefty */ 789f17df3b0SSean Hefty static int cma_resolve_ib_dev(struct rdma_id_private *id_priv) 790f17df3b0SSean Hefty { 791f17df3b0SSean Hefty struct cma_device *cma_dev, *cur_dev; 792f17df3b0SSean Hefty struct sockaddr_ib *addr; 793f17df3b0SSean Hefty union ib_gid gid, sgid, *dgid; 794cc055dd3SParav Pandit unsigned int p; 795f17df3b0SSean Hefty u16 pkey, index; 79693b1f29dSJack Wang enum ib_port_state port_state; 797f17df3b0SSean Hefty int i; 798f17df3b0SSean Hefty 799f17df3b0SSean Hefty cma_dev = NULL; 800f17df3b0SSean Hefty addr = (struct sockaddr_ib *) cma_dst_addr(id_priv); 801f17df3b0SSean Hefty dgid = (union ib_gid *) &addr->sib_addr; 802f17df3b0SSean Hefty pkey = ntohs(addr->sib_pkey); 803f17df3b0SSean Hefty 804954a8e3aSParav Pandit mutex_lock(&lock); 805f17df3b0SSean Hefty list_for_each_entry(cur_dev, &dev_list, list) { 806cc055dd3SParav Pandit rdma_for_each_port (cur_dev->device, p) { 80730a74ef4SMichael Wang if (!rdma_cap_af_ib(cur_dev->device, p)) 808f17df3b0SSean Hefty continue; 809f17df3b0SSean Hefty 810f17df3b0SSean Hefty if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index)) 811f17df3b0SSean Hefty continue; 812f17df3b0SSean Hefty 81393b1f29dSJack Wang if (ib_get_cached_port_state(cur_dev->device, p, &port_state)) 81493b1f29dSJack Wang continue; 8151dfce294SParav Pandit for (i = 0; !rdma_query_gid(cur_dev->device, 8161dfce294SParav Pandit p, i, &gid); 81755ee3ab2SMatan Barak i++) { 818f17df3b0SSean Hefty if (!memcmp(&gid, dgid, sizeof(gid))) { 819f17df3b0SSean Hefty cma_dev = cur_dev; 820f17df3b0SSean Hefty sgid = gid; 8218fb488d7SPaul Bolle id_priv->id.port_num = p; 822f17df3b0SSean Hefty goto found; 823f17df3b0SSean Hefty } 824f17df3b0SSean Hefty 825f17df3b0SSean Hefty if (!cma_dev && (gid.global.subnet_prefix == 82693b1f29dSJack Wang dgid->global.subnet_prefix) && 82793b1f29dSJack Wang port_state == IB_PORT_ACTIVE) { 828f17df3b0SSean Hefty cma_dev = cur_dev; 829f17df3b0SSean Hefty sgid = gid; 8308fb488d7SPaul Bolle id_priv->id.port_num = p; 831954a8e3aSParav Pandit goto found; 832f17df3b0SSean Hefty } 833f17df3b0SSean Hefty } 834f17df3b0SSean Hefty } 835f17df3b0SSean Hefty } 836954a8e3aSParav Pandit mutex_unlock(&lock); 837f17df3b0SSean Hefty return -ENODEV; 838f17df3b0SSean Hefty 839f17df3b0SSean Hefty found: 840f17df3b0SSean Hefty cma_attach_to_dev(id_priv, cma_dev); 841954a8e3aSParav Pandit mutex_unlock(&lock); 842f17df3b0SSean Hefty addr = (struct sockaddr_ib *)cma_src_addr(id_priv); 843954a8e3aSParav Pandit memcpy(&addr->sib_addr, &sgid, sizeof(sgid)); 844f17df3b0SSean Hefty cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr); 845f17df3b0SSean Hefty return 0; 846f17df3b0SSean Hefty } 847f17df3b0SSean Hefty 848e368d23fSParav Pandit static void cma_id_get(struct rdma_id_private *id_priv) 849e368d23fSParav Pandit { 85043fb5892SParav Pandit refcount_inc(&id_priv->refcount); 851e368d23fSParav Pandit } 852e368d23fSParav Pandit 853e368d23fSParav Pandit static void cma_id_put(struct rdma_id_private *id_priv) 854e51060f0SSean Hefty { 85543fb5892SParav Pandit if (refcount_dec_and_test(&id_priv->refcount)) 856e51060f0SSean Hefty complete(&id_priv->comp); 857e51060f0SSean Hefty } 858e51060f0SSean Hefty 85900313983SSteve Wise struct rdma_cm_id *__rdma_create_id(struct net *net, 860fa20105eSGuy Shapiro rdma_cm_event_handler event_handler, 8612253fc0cSSteve Wise void *context, enum rdma_ucm_port_space ps, 86200313983SSteve Wise enum ib_qp_type qp_type, const char *caller) 863e51060f0SSean Hefty { 864e51060f0SSean Hefty struct rdma_id_private *id_priv; 865e51060f0SSean Hefty 866e51060f0SSean Hefty id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL); 867e51060f0SSean Hefty if (!id_priv) 868e51060f0SSean Hefty return ERR_PTR(-ENOMEM); 869e51060f0SSean Hefty 8702165fc26SLeon Romanovsky rdma_restrack_set_task(&id_priv->res, caller); 871e11fef9fSParav Pandit id_priv->res.type = RDMA_RESTRACK_CM_ID; 872550e5ca7SNir Muchtar id_priv->state = RDMA_CM_IDLE; 873e51060f0SSean Hefty id_priv->id.context = context; 874e51060f0SSean Hefty id_priv->id.event_handler = event_handler; 875e51060f0SSean Hefty id_priv->id.ps = ps; 876b26f9b99SSean Hefty id_priv->id.qp_type = qp_type; 87789052d78SMajd Dibbiny id_priv->tos_set = false; 8782c1619edSDanit Goldberg id_priv->timeout_set = false; 87979d684f0SParav Pandit id_priv->gid_type = IB_GID_TYPE_IB; 880e51060f0SSean Hefty spin_lock_init(&id_priv->lock); 881c5483388SSean Hefty mutex_init(&id_priv->qp_mutex); 882e51060f0SSean Hefty init_completion(&id_priv->comp); 88343fb5892SParav Pandit refcount_set(&id_priv->refcount, 1); 884de910bd9SOr Gerlitz mutex_init(&id_priv->handler_mutex); 885e51060f0SSean Hefty INIT_LIST_HEAD(&id_priv->listen_list); 886c8f6a362SSean Hefty INIT_LIST_HEAD(&id_priv->mc_list); 887e51060f0SSean Hefty get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num); 888fa20105eSGuy Shapiro id_priv->id.route.addr.dev_addr.net = get_net(net); 88923a9cd2aSMoni Shoua id_priv->seq_num &= 0x00ffffff; 890e51060f0SSean Hefty 891ed999f82SChuck Lever trace_cm_id_create(id_priv); 892e51060f0SSean Hefty return &id_priv->id; 893e51060f0SSean Hefty } 89400313983SSteve Wise EXPORT_SYMBOL(__rdma_create_id); 895e51060f0SSean Hefty 896c8f6a362SSean Hefty static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 897e51060f0SSean Hefty { 898e51060f0SSean Hefty struct ib_qp_attr qp_attr; 899c8f6a362SSean Hefty int qp_attr_mask, ret; 900e51060f0SSean Hefty 901c8f6a362SSean Hefty qp_attr.qp_state = IB_QPS_INIT; 902c8f6a362SSean Hefty ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 903e51060f0SSean Hefty if (ret) 904e51060f0SSean Hefty return ret; 905e51060f0SSean Hefty 906c8f6a362SSean Hefty ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask); 907c8f6a362SSean Hefty if (ret) 908c8f6a362SSean Hefty return ret; 909c8f6a362SSean Hefty 910c8f6a362SSean Hefty qp_attr.qp_state = IB_QPS_RTR; 911c8f6a362SSean Hefty ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE); 912c8f6a362SSean Hefty if (ret) 913c8f6a362SSean Hefty return ret; 914c8f6a362SSean Hefty 915c8f6a362SSean Hefty qp_attr.qp_state = IB_QPS_RTS; 916c8f6a362SSean Hefty qp_attr.sq_psn = 0; 917c8f6a362SSean Hefty ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN); 918c8f6a362SSean Hefty 919c8f6a362SSean Hefty return ret; 920e51060f0SSean Hefty } 921e51060f0SSean Hefty 922c8f6a362SSean Hefty static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp) 92307ebafbaSTom Tucker { 92407ebafbaSTom Tucker struct ib_qp_attr qp_attr; 925c8f6a362SSean Hefty int qp_attr_mask, ret; 92607ebafbaSTom Tucker 92707ebafbaSTom Tucker qp_attr.qp_state = IB_QPS_INIT; 928c8f6a362SSean Hefty ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 929c8f6a362SSean Hefty if (ret) 930c8f6a362SSean Hefty return ret; 93107ebafbaSTom Tucker 932c8f6a362SSean Hefty return ib_modify_qp(qp, &qp_attr, qp_attr_mask); 93307ebafbaSTom Tucker } 93407ebafbaSTom Tucker 935e51060f0SSean Hefty int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd, 936e51060f0SSean Hefty struct ib_qp_init_attr *qp_init_attr) 937e51060f0SSean Hefty { 938e51060f0SSean Hefty struct rdma_id_private *id_priv; 939e51060f0SSean Hefty struct ib_qp *qp; 940e51060f0SSean Hefty int ret; 941e51060f0SSean Hefty 942e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 943ed999f82SChuck Lever if (id->device != pd->device) { 944ed999f82SChuck Lever ret = -EINVAL; 945ed999f82SChuck Lever goto out_err; 946ed999f82SChuck Lever } 947e51060f0SSean Hefty 9480691a286SChristoph Hellwig qp_init_attr->port_num = id->port_num; 949e51060f0SSean Hefty qp = ib_create_qp(pd, qp_init_attr); 950ed999f82SChuck Lever if (IS_ERR(qp)) { 951ed999f82SChuck Lever ret = PTR_ERR(qp); 952ed999f82SChuck Lever goto out_err; 953ed999f82SChuck Lever } 954e51060f0SSean Hefty 955b26f9b99SSean Hefty if (id->qp_type == IB_QPT_UD) 956c8f6a362SSean Hefty ret = cma_init_ud_qp(id_priv, qp); 957c8f6a362SSean Hefty else 958c8f6a362SSean Hefty ret = cma_init_conn_qp(id_priv, qp); 959e51060f0SSean Hefty if (ret) 960ed999f82SChuck Lever goto out_destroy; 961e51060f0SSean Hefty 962e51060f0SSean Hefty id->qp = qp; 963e51060f0SSean Hefty id_priv->qp_num = qp->qp_num; 964e51060f0SSean Hefty id_priv->srq = (qp->srq != NULL); 965ed999f82SChuck Lever trace_cm_qp_create(id_priv, pd, qp_init_attr, 0); 966e51060f0SSean Hefty return 0; 967ed999f82SChuck Lever out_destroy: 968e51060f0SSean Hefty ib_destroy_qp(qp); 969ed999f82SChuck Lever out_err: 970ed999f82SChuck Lever trace_cm_qp_create(id_priv, pd, qp_init_attr, ret); 971e51060f0SSean Hefty return ret; 972e51060f0SSean Hefty } 973e51060f0SSean Hefty EXPORT_SYMBOL(rdma_create_qp); 974e51060f0SSean Hefty 975e51060f0SSean Hefty void rdma_destroy_qp(struct rdma_cm_id *id) 976e51060f0SSean Hefty { 977c5483388SSean Hefty struct rdma_id_private *id_priv; 978c5483388SSean Hefty 979c5483388SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 980ed999f82SChuck Lever trace_cm_qp_destroy(id_priv); 981c5483388SSean Hefty mutex_lock(&id_priv->qp_mutex); 982c5483388SSean Hefty ib_destroy_qp(id_priv->id.qp); 983c5483388SSean Hefty id_priv->id.qp = NULL; 984c5483388SSean Hefty mutex_unlock(&id_priv->qp_mutex); 985e51060f0SSean Hefty } 986e51060f0SSean Hefty EXPORT_SYMBOL(rdma_destroy_qp); 987e51060f0SSean Hefty 9885851bb89SSean Hefty static int cma_modify_qp_rtr(struct rdma_id_private *id_priv, 9895851bb89SSean Hefty struct rdma_conn_param *conn_param) 990e51060f0SSean Hefty { 991e51060f0SSean Hefty struct ib_qp_attr qp_attr; 992e51060f0SSean Hefty int qp_attr_mask, ret; 993e51060f0SSean Hefty 994c5483388SSean Hefty mutex_lock(&id_priv->qp_mutex); 995c5483388SSean Hefty if (!id_priv->id.qp) { 996c5483388SSean Hefty ret = 0; 997c5483388SSean Hefty goto out; 998c5483388SSean Hefty } 999e51060f0SSean Hefty 1000e51060f0SSean Hefty /* Need to update QP attributes from default values. */ 1001e51060f0SSean Hefty qp_attr.qp_state = IB_QPS_INIT; 1002c5483388SSean Hefty ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 1003e51060f0SSean Hefty if (ret) 1004c5483388SSean Hefty goto out; 1005e51060f0SSean Hefty 1006c5483388SSean Hefty ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 1007e51060f0SSean Hefty if (ret) 1008c5483388SSean Hefty goto out; 1009e51060f0SSean Hefty 1010e51060f0SSean Hefty qp_attr.qp_state = IB_QPS_RTR; 1011c5483388SSean Hefty ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 1012e51060f0SSean Hefty if (ret) 1013c5483388SSean Hefty goto out; 1014e51060f0SSean Hefty 1015fef60902SMichael Wang BUG_ON(id_priv->cma_dev->device != id_priv->id.device); 1016fef60902SMichael Wang 10175851bb89SSean Hefty if (conn_param) 10185851bb89SSean Hefty qp_attr.max_dest_rd_atomic = conn_param->responder_resources; 1019c5483388SSean Hefty ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 1020c5483388SSean Hefty out: 1021c5483388SSean Hefty mutex_unlock(&id_priv->qp_mutex); 1022c5483388SSean Hefty return ret; 1023e51060f0SSean Hefty } 1024e51060f0SSean Hefty 10255851bb89SSean Hefty static int cma_modify_qp_rts(struct rdma_id_private *id_priv, 10265851bb89SSean Hefty struct rdma_conn_param *conn_param) 1027e51060f0SSean Hefty { 1028e51060f0SSean Hefty struct ib_qp_attr qp_attr; 1029e51060f0SSean Hefty int qp_attr_mask, ret; 1030e51060f0SSean Hefty 1031c5483388SSean Hefty mutex_lock(&id_priv->qp_mutex); 1032c5483388SSean Hefty if (!id_priv->id.qp) { 1033c5483388SSean Hefty ret = 0; 1034c5483388SSean Hefty goto out; 1035e51060f0SSean Hefty } 1036e51060f0SSean Hefty 1037c5483388SSean Hefty qp_attr.qp_state = IB_QPS_RTS; 1038c5483388SSean Hefty ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask); 1039c5483388SSean Hefty if (ret) 1040c5483388SSean Hefty goto out; 1041c5483388SSean Hefty 10425851bb89SSean Hefty if (conn_param) 10435851bb89SSean Hefty qp_attr.max_rd_atomic = conn_param->initiator_depth; 1044c5483388SSean Hefty ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask); 1045c5483388SSean Hefty out: 1046c5483388SSean Hefty mutex_unlock(&id_priv->qp_mutex); 1047c5483388SSean Hefty return ret; 1048c5483388SSean Hefty } 1049c5483388SSean Hefty 1050c5483388SSean Hefty static int cma_modify_qp_err(struct rdma_id_private *id_priv) 1051e51060f0SSean Hefty { 1052e51060f0SSean Hefty struct ib_qp_attr qp_attr; 1053c5483388SSean Hefty int ret; 1054e51060f0SSean Hefty 1055c5483388SSean Hefty mutex_lock(&id_priv->qp_mutex); 1056c5483388SSean Hefty if (!id_priv->id.qp) { 1057c5483388SSean Hefty ret = 0; 1058c5483388SSean Hefty goto out; 1059c5483388SSean Hefty } 1060e51060f0SSean Hefty 1061e51060f0SSean Hefty qp_attr.qp_state = IB_QPS_ERR; 1062c5483388SSean Hefty ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE); 1063c5483388SSean Hefty out: 1064c5483388SSean Hefty mutex_unlock(&id_priv->qp_mutex); 1065c5483388SSean Hefty return ret; 1066e51060f0SSean Hefty } 1067e51060f0SSean Hefty 1068c8f6a362SSean Hefty static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv, 1069c8f6a362SSean Hefty struct ib_qp_attr *qp_attr, int *qp_attr_mask) 1070c8f6a362SSean Hefty { 1071c8f6a362SSean Hefty struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 1072c8f6a362SSean Hefty int ret; 10733c86aa70SEli Cohen u16 pkey; 10743c86aa70SEli Cohen 1075227128fcSMichael Wang if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num)) 10763c86aa70SEli Cohen pkey = 0xffff; 1077fef60902SMichael Wang else 1078fef60902SMichael Wang pkey = ib_addr_get_pkey(dev_addr); 1079c8f6a362SSean Hefty 1080c8f6a362SSean Hefty ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num, 10813c86aa70SEli Cohen pkey, &qp_attr->pkey_index); 1082c8f6a362SSean Hefty if (ret) 1083c8f6a362SSean Hefty return ret; 1084c8f6a362SSean Hefty 1085c8f6a362SSean Hefty qp_attr->port_num = id_priv->id.port_num; 1086c8f6a362SSean Hefty *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT; 1087c8f6a362SSean Hefty 1088b26f9b99SSean Hefty if (id_priv->id.qp_type == IB_QPT_UD) { 10895c438135SSean Hefty ret = cma_set_qkey(id_priv, 0); 1090d2ca39f2SYossi Etigin if (ret) 1091d2ca39f2SYossi Etigin return ret; 1092d2ca39f2SYossi Etigin 1093c8f6a362SSean Hefty qp_attr->qkey = id_priv->qkey; 1094c8f6a362SSean Hefty *qp_attr_mask |= IB_QP_QKEY; 1095c8f6a362SSean Hefty } else { 1096c8f6a362SSean Hefty qp_attr->qp_access_flags = 0; 1097c8f6a362SSean Hefty *qp_attr_mask |= IB_QP_ACCESS_FLAGS; 1098c8f6a362SSean Hefty } 1099c8f6a362SSean Hefty return 0; 1100c8f6a362SSean Hefty } 1101c8f6a362SSean Hefty 1102e51060f0SSean Hefty int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr, 1103e51060f0SSean Hefty int *qp_attr_mask) 1104e51060f0SSean Hefty { 1105e51060f0SSean Hefty struct rdma_id_private *id_priv; 1106c8f6a362SSean Hefty int ret = 0; 1107e51060f0SSean Hefty 1108e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 110972219ceaSMichael Wang if (rdma_cap_ib_cm(id->device, id->port_num)) { 1110b26f9b99SSean Hefty if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD)) 1111c8f6a362SSean Hefty ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask); 1112c8f6a362SSean Hefty else 1113e51060f0SSean Hefty ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr, 1114e51060f0SSean Hefty qp_attr_mask); 1115dd5f03beSMatan Barak 1116e51060f0SSean Hefty if (qp_attr->qp_state == IB_QPS_RTR) 1117e51060f0SSean Hefty qp_attr->rq_psn = id_priv->seq_num; 111804215330SMichael Wang } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 1119c8f6a362SSean Hefty if (!id_priv->cm_id.iw) { 11208f076531SDotan Barak qp_attr->qp_access_flags = 0; 1121c8f6a362SSean Hefty *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS; 1122c8f6a362SSean Hefty } else 112307ebafbaSTom Tucker ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr, 112407ebafbaSTom Tucker qp_attr_mask); 1125a62ab66bSIsmail, Mustafa qp_attr->port_num = id_priv->id.port_num; 1126a62ab66bSIsmail, Mustafa *qp_attr_mask |= IB_QP_PORT; 112721655afcSMichael Wang } else 1128e51060f0SSean Hefty ret = -ENOSYS; 1129e51060f0SSean Hefty 11302c1619edSDanit Goldberg if ((*qp_attr_mask & IB_QP_TIMEOUT) && id_priv->timeout_set) 11312c1619edSDanit Goldberg qp_attr->timeout = id_priv->timeout; 11322c1619edSDanit Goldberg 1133e51060f0SSean Hefty return ret; 1134e51060f0SSean Hefty } 1135e51060f0SSean Hefty EXPORT_SYMBOL(rdma_init_qp_attr); 1136e51060f0SSean Hefty 1137ca3a8aceSParav Pandit static inline bool cma_zero_addr(const struct sockaddr *addr) 1138e51060f0SSean Hefty { 11392e2d190cSSean Hefty switch (addr->sa_family) { 11402e2d190cSSean Hefty case AF_INET: 11412e2d190cSSean Hefty return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr); 11422e2d190cSSean Hefty case AF_INET6: 11432e2d190cSSean Hefty return ipv6_addr_any(&((struct sockaddr_in6 *)addr)->sin6_addr); 11442e2d190cSSean Hefty case AF_IB: 11452e2d190cSSean Hefty return ib_addr_any(&((struct sockaddr_ib *)addr)->sib_addr); 11462e2d190cSSean Hefty default: 1147ca3a8aceSParav Pandit return false; 1148e51060f0SSean Hefty } 1149e51060f0SSean Hefty } 1150e51060f0SSean Hefty 1151ca3a8aceSParav Pandit static inline bool cma_loopback_addr(const struct sockaddr *addr) 1152e51060f0SSean Hefty { 11532e2d190cSSean Hefty switch (addr->sa_family) { 11542e2d190cSSean Hefty case AF_INET: 1155ca3a8aceSParav Pandit return ipv4_is_loopback( 1156ca3a8aceSParav Pandit ((struct sockaddr_in *)addr)->sin_addr.s_addr); 11572e2d190cSSean Hefty case AF_INET6: 1158ca3a8aceSParav Pandit return ipv6_addr_loopback( 1159ca3a8aceSParav Pandit &((struct sockaddr_in6 *)addr)->sin6_addr); 11602e2d190cSSean Hefty case AF_IB: 1161ca3a8aceSParav Pandit return ib_addr_loopback( 1162ca3a8aceSParav Pandit &((struct sockaddr_ib *)addr)->sib_addr); 11632e2d190cSSean Hefty default: 1164ca3a8aceSParav Pandit return false; 11652e2d190cSSean Hefty } 1166e51060f0SSean Hefty } 1167e51060f0SSean Hefty 1168ca3a8aceSParav Pandit static inline bool cma_any_addr(const struct sockaddr *addr) 1169e51060f0SSean Hefty { 1170e51060f0SSean Hefty return cma_zero_addr(addr) || cma_loopback_addr(addr); 1171e51060f0SSean Hefty } 1172e51060f0SSean Hefty 11735d7ed2f2SParav Pandit static int cma_addr_cmp(const struct sockaddr *src, const struct sockaddr *dst) 117443b752daSHefty, Sean { 117543b752daSHefty, Sean if (src->sa_family != dst->sa_family) 117643b752daSHefty, Sean return -1; 117743b752daSHefty, Sean 117843b752daSHefty, Sean switch (src->sa_family) { 117943b752daSHefty, Sean case AF_INET: 118043b752daSHefty, Sean return ((struct sockaddr_in *)src)->sin_addr.s_addr != 118143b752daSHefty, Sean ((struct sockaddr_in *)dst)->sin_addr.s_addr; 11825d7ed2f2SParav Pandit case AF_INET6: { 11835d7ed2f2SParav Pandit struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *)src; 11845d7ed2f2SParav Pandit struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *)dst; 11855d7ed2f2SParav Pandit bool link_local; 11865d7ed2f2SParav Pandit 11875d7ed2f2SParav Pandit if (ipv6_addr_cmp(&src_addr6->sin6_addr, 11885d7ed2f2SParav Pandit &dst_addr6->sin6_addr)) 11895d7ed2f2SParav Pandit return 1; 11905d7ed2f2SParav Pandit link_local = ipv6_addr_type(&dst_addr6->sin6_addr) & 11915d7ed2f2SParav Pandit IPV6_ADDR_LINKLOCAL; 11925d7ed2f2SParav Pandit /* Link local must match their scope_ids */ 11935d7ed2f2SParav Pandit return link_local ? (src_addr6->sin6_scope_id != 11945d7ed2f2SParav Pandit dst_addr6->sin6_scope_id) : 11955d7ed2f2SParav Pandit 0; 11965d7ed2f2SParav Pandit } 11975d7ed2f2SParav Pandit 11982e2d190cSSean Hefty default: 11992e2d190cSSean Hefty return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr, 12002e2d190cSSean Hefty &((struct sockaddr_ib *) dst)->sib_addr); 120143b752daSHefty, Sean } 120243b752daSHefty, Sean } 120343b752daSHefty, Sean 12042df7dba8SParav Pandit static __be16 cma_port(const struct sockaddr *addr) 1205628e5f6dSSean Hefty { 120658afdcb7SSean Hefty struct sockaddr_ib *sib; 120758afdcb7SSean Hefty 120858afdcb7SSean Hefty switch (addr->sa_family) { 120958afdcb7SSean Hefty case AF_INET: 1210628e5f6dSSean Hefty return ((struct sockaddr_in *) addr)->sin_port; 121158afdcb7SSean Hefty case AF_INET6: 1212628e5f6dSSean Hefty return ((struct sockaddr_in6 *) addr)->sin6_port; 121358afdcb7SSean Hefty case AF_IB: 121458afdcb7SSean Hefty sib = (struct sockaddr_ib *) addr; 121558afdcb7SSean Hefty return htons((u16) (be64_to_cpu(sib->sib_sid) & 121658afdcb7SSean Hefty be64_to_cpu(sib->sib_sid_mask))); 121758afdcb7SSean Hefty default: 121858afdcb7SSean Hefty return 0; 121958afdcb7SSean Hefty } 1220628e5f6dSSean Hefty } 1221628e5f6dSSean Hefty 12222df7dba8SParav Pandit static inline int cma_any_port(const struct sockaddr *addr) 1223e51060f0SSean Hefty { 1224628e5f6dSSean Hefty return !cma_port(addr); 1225e51060f0SSean Hefty } 1226e51060f0SSean Hefty 12270c505f70SHaggai Eran static void cma_save_ib_info(struct sockaddr *src_addr, 12280c505f70SHaggai Eran struct sockaddr *dst_addr, 1229e7ff98aeSParav Pandit const struct rdma_cm_id *listen_id, 1230e7ff98aeSParav Pandit const struct sa_path_rec *path) 1231e51060f0SSean Hefty { 1232fbaa1a6dSSean Hefty struct sockaddr_ib *listen_ib, *ib; 1233e51060f0SSean Hefty 1234fbaa1a6dSSean Hefty listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr; 12350c505f70SHaggai Eran if (src_addr) { 12360c505f70SHaggai Eran ib = (struct sockaddr_ib *)src_addr; 12370c505f70SHaggai Eran ib->sib_family = AF_IB; 1238c07678bbSMatthew Finlay if (path) { 1239fbaa1a6dSSean Hefty ib->sib_pkey = path->pkey; 1240fbaa1a6dSSean Hefty ib->sib_flowinfo = path->flow_label; 1241fbaa1a6dSSean Hefty memcpy(&ib->sib_addr, &path->sgid, 16); 1242d3957b86SMajd Dibbiny ib->sib_sid = path->service_id; 12430c505f70SHaggai Eran ib->sib_scope_id = 0; 1244c07678bbSMatthew Finlay } else { 1245c07678bbSMatthew Finlay ib->sib_pkey = listen_ib->sib_pkey; 1246c07678bbSMatthew Finlay ib->sib_flowinfo = listen_ib->sib_flowinfo; 1247c07678bbSMatthew Finlay ib->sib_addr = listen_ib->sib_addr; 1248fbaa1a6dSSean Hefty ib->sib_sid = listen_ib->sib_sid; 1249fbaa1a6dSSean Hefty ib->sib_scope_id = listen_ib->sib_scope_id; 12500c505f70SHaggai Eran } 12510c505f70SHaggai Eran ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL); 12520c505f70SHaggai Eran } 12530c505f70SHaggai Eran if (dst_addr) { 12540c505f70SHaggai Eran ib = (struct sockaddr_ib *)dst_addr; 12550c505f70SHaggai Eran ib->sib_family = AF_IB; 1256c07678bbSMatthew Finlay if (path) { 1257fbaa1a6dSSean Hefty ib->sib_pkey = path->pkey; 1258fbaa1a6dSSean Hefty ib->sib_flowinfo = path->flow_label; 1259fbaa1a6dSSean Hefty memcpy(&ib->sib_addr, &path->dgid, 16); 1260fbaa1a6dSSean Hefty } 1261c07678bbSMatthew Finlay } 126228521440SJason Gunthorpe } 126328521440SJason Gunthorpe 1264c50e90d0SArnd Bergmann static void cma_save_ip4_info(struct sockaddr_in *src_addr, 1265c50e90d0SArnd Bergmann struct sockaddr_in *dst_addr, 12660c505f70SHaggai Eran struct cma_hdr *hdr, 12670c505f70SHaggai Eran __be16 local_port) 1268fbaa1a6dSSean Hefty { 12690c505f70SHaggai Eran if (src_addr) { 1270c50e90d0SArnd Bergmann *src_addr = (struct sockaddr_in) { 1271c50e90d0SArnd Bergmann .sin_family = AF_INET, 1272c50e90d0SArnd Bergmann .sin_addr.s_addr = hdr->dst_addr.ip4.addr, 1273c50e90d0SArnd Bergmann .sin_port = local_port, 1274c50e90d0SArnd Bergmann }; 12750c505f70SHaggai Eran } 1276fbaa1a6dSSean Hefty 12770c505f70SHaggai Eran if (dst_addr) { 1278c50e90d0SArnd Bergmann *dst_addr = (struct sockaddr_in) { 1279c50e90d0SArnd Bergmann .sin_family = AF_INET, 1280c50e90d0SArnd Bergmann .sin_addr.s_addr = hdr->src_addr.ip4.addr, 1281c50e90d0SArnd Bergmann .sin_port = hdr->port, 1282c50e90d0SArnd Bergmann }; 1283fbaa1a6dSSean Hefty } 12840c505f70SHaggai Eran } 1285fbaa1a6dSSean Hefty 1286c50e90d0SArnd Bergmann static void cma_save_ip6_info(struct sockaddr_in6 *src_addr, 1287c50e90d0SArnd Bergmann struct sockaddr_in6 *dst_addr, 12880c505f70SHaggai Eran struct cma_hdr *hdr, 12890c505f70SHaggai Eran __be16 local_port) 1290fbaa1a6dSSean Hefty { 12910c505f70SHaggai Eran if (src_addr) { 1292c50e90d0SArnd Bergmann *src_addr = (struct sockaddr_in6) { 1293c50e90d0SArnd Bergmann .sin6_family = AF_INET6, 1294c50e90d0SArnd Bergmann .sin6_addr = hdr->dst_addr.ip6, 1295c50e90d0SArnd Bergmann .sin6_port = local_port, 1296c50e90d0SArnd Bergmann }; 12970c505f70SHaggai Eran } 1298fbaa1a6dSSean Hefty 12990c505f70SHaggai Eran if (dst_addr) { 1300c50e90d0SArnd Bergmann *dst_addr = (struct sockaddr_in6) { 1301c50e90d0SArnd Bergmann .sin6_family = AF_INET6, 1302c50e90d0SArnd Bergmann .sin6_addr = hdr->src_addr.ip6, 1303c50e90d0SArnd Bergmann .sin6_port = hdr->port, 1304c50e90d0SArnd Bergmann }; 1305fbaa1a6dSSean Hefty } 13060c505f70SHaggai Eran } 1307fbaa1a6dSSean Hefty 13080c505f70SHaggai Eran static u16 cma_port_from_service_id(__be64 service_id) 13090c505f70SHaggai Eran { 13100c505f70SHaggai Eran return (u16)be64_to_cpu(service_id); 13110c505f70SHaggai Eran } 13120c505f70SHaggai Eran 13130c505f70SHaggai Eran static int cma_save_ip_info(struct sockaddr *src_addr, 13140c505f70SHaggai Eran struct sockaddr *dst_addr, 1315e7ff98aeSParav Pandit const struct ib_cm_event *ib_event, 13160c505f70SHaggai Eran __be64 service_id) 1317fbaa1a6dSSean Hefty { 1318fbaa1a6dSSean Hefty struct cma_hdr *hdr; 13190c505f70SHaggai Eran __be16 port; 1320e51060f0SSean Hefty 1321fbaa1a6dSSean Hefty hdr = ib_event->private_data; 1322fbaa1a6dSSean Hefty if (hdr->cma_version != CMA_VERSION) 1323fbaa1a6dSSean Hefty return -EINVAL; 1324e51060f0SSean Hefty 13250c505f70SHaggai Eran port = htons(cma_port_from_service_id(service_id)); 13260c505f70SHaggai Eran 1327fbaa1a6dSSean Hefty switch (cma_get_ip_ver(hdr)) { 1328e51060f0SSean Hefty case 4: 1329c50e90d0SArnd Bergmann cma_save_ip4_info((struct sockaddr_in *)src_addr, 1330c50e90d0SArnd Bergmann (struct sockaddr_in *)dst_addr, hdr, port); 1331e51060f0SSean Hefty break; 1332e51060f0SSean Hefty case 6: 1333c50e90d0SArnd Bergmann cma_save_ip6_info((struct sockaddr_in6 *)src_addr, 1334c50e90d0SArnd Bergmann (struct sockaddr_in6 *)dst_addr, hdr, port); 1335e51060f0SSean Hefty break; 1336e51060f0SSean Hefty default: 13374c21b5bcSHaggai Eran return -EAFNOSUPPORT; 1338e51060f0SSean Hefty } 13390c505f70SHaggai Eran 1340fbaa1a6dSSean Hefty return 0; 1341e51060f0SSean Hefty } 1342e51060f0SSean Hefty 13430c505f70SHaggai Eran static int cma_save_net_info(struct sockaddr *src_addr, 13440c505f70SHaggai Eran struct sockaddr *dst_addr, 1345e7ff98aeSParav Pandit const struct rdma_cm_id *listen_id, 1346e7ff98aeSParav Pandit const struct ib_cm_event *ib_event, 13470c505f70SHaggai Eran sa_family_t sa_family, __be64 service_id) 13480c505f70SHaggai Eran { 13490c505f70SHaggai Eran if (sa_family == AF_IB) { 13500c505f70SHaggai Eran if (ib_event->event == IB_CM_REQ_RECEIVED) 13510c505f70SHaggai Eran cma_save_ib_info(src_addr, dst_addr, listen_id, 13520c505f70SHaggai Eran ib_event->param.req_rcvd.primary_path); 13530c505f70SHaggai Eran else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) 13540c505f70SHaggai Eran cma_save_ib_info(src_addr, dst_addr, listen_id, NULL); 13550c505f70SHaggai Eran return 0; 13560c505f70SHaggai Eran } 13570c505f70SHaggai Eran 13580c505f70SHaggai Eran return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id); 13590c505f70SHaggai Eran } 13600c505f70SHaggai Eran 13614c21b5bcSHaggai Eran static int cma_save_req_info(const struct ib_cm_event *ib_event, 13624c21b5bcSHaggai Eran struct cma_req_info *req) 13634c21b5bcSHaggai Eran { 13644c21b5bcSHaggai Eran const struct ib_cm_req_event_param *req_param = 13654c21b5bcSHaggai Eran &ib_event->param.req_rcvd; 13664c21b5bcSHaggai Eran const struct ib_cm_sidr_req_event_param *sidr_param = 13674c21b5bcSHaggai Eran &ib_event->param.sidr_req_rcvd; 13684c21b5bcSHaggai Eran 13694c21b5bcSHaggai Eran switch (ib_event->event) { 13704c21b5bcSHaggai Eran case IB_CM_REQ_RECEIVED: 13714c21b5bcSHaggai Eran req->device = req_param->listen_id->device; 13724c21b5bcSHaggai Eran req->port = req_param->port; 13734c21b5bcSHaggai Eran memcpy(&req->local_gid, &req_param->primary_path->sgid, 13744c21b5bcSHaggai Eran sizeof(req->local_gid)); 13754c21b5bcSHaggai Eran req->has_gid = true; 1376d3957b86SMajd Dibbiny req->service_id = req_param->primary_path->service_id; 1377ab3964adSHaggai Eran req->pkey = be16_to_cpu(req_param->primary_path->pkey); 137884424a7fSHaggai Eran if (req->pkey != req_param->bth_pkey) 137984424a7fSHaggai Eran pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n" 138084424a7fSHaggai Eran "RDMA CMA: in the future this may cause the request to be dropped\n", 138184424a7fSHaggai Eran req_param->bth_pkey, req->pkey); 13824c21b5bcSHaggai Eran break; 13834c21b5bcSHaggai Eran case IB_CM_SIDR_REQ_RECEIVED: 13844c21b5bcSHaggai Eran req->device = sidr_param->listen_id->device; 13854c21b5bcSHaggai Eran req->port = sidr_param->port; 13864c21b5bcSHaggai Eran req->has_gid = false; 13874c21b5bcSHaggai Eran req->service_id = sidr_param->service_id; 1388ab3964adSHaggai Eran req->pkey = sidr_param->pkey; 138984424a7fSHaggai Eran if (req->pkey != sidr_param->bth_pkey) 139084424a7fSHaggai Eran pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n" 139184424a7fSHaggai Eran "RDMA CMA: in the future this may cause the request to be dropped\n", 139284424a7fSHaggai Eran sidr_param->bth_pkey, req->pkey); 13934c21b5bcSHaggai Eran break; 13944c21b5bcSHaggai Eran default: 13954c21b5bcSHaggai Eran return -EINVAL; 13964c21b5bcSHaggai Eran } 13974c21b5bcSHaggai Eran 13984c21b5bcSHaggai Eran return 0; 13994c21b5bcSHaggai Eran } 14004c21b5bcSHaggai Eran 1401f887f2acSHaggai Eran static bool validate_ipv4_net_dev(struct net_device *net_dev, 1402f887f2acSHaggai Eran const struct sockaddr_in *dst_addr, 1403f887f2acSHaggai Eran const struct sockaddr_in *src_addr) 1404f887f2acSHaggai Eran { 1405f887f2acSHaggai Eran __be32 daddr = dst_addr->sin_addr.s_addr, 1406f887f2acSHaggai Eran saddr = src_addr->sin_addr.s_addr; 1407f887f2acSHaggai Eran struct fib_result res; 1408f887f2acSHaggai Eran struct flowi4 fl4; 1409f887f2acSHaggai Eran int err; 1410f887f2acSHaggai Eran bool ret; 1411f887f2acSHaggai Eran 1412f887f2acSHaggai Eran if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) || 1413f887f2acSHaggai Eran ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) || 1414f887f2acSHaggai Eran ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) || 1415f887f2acSHaggai Eran ipv4_is_loopback(saddr)) 1416f887f2acSHaggai Eran return false; 1417f887f2acSHaggai Eran 1418f887f2acSHaggai Eran memset(&fl4, 0, sizeof(fl4)); 1419f887f2acSHaggai Eran fl4.flowi4_iif = net_dev->ifindex; 1420f887f2acSHaggai Eran fl4.daddr = daddr; 1421f887f2acSHaggai Eran fl4.saddr = saddr; 1422f887f2acSHaggai Eran 1423f887f2acSHaggai Eran rcu_read_lock(); 1424f887f2acSHaggai Eran err = fib_lookup(dev_net(net_dev), &fl4, &res, 0); 1425d3632493SBart Van Assche ret = err == 0 && FIB_RES_DEV(res) == net_dev; 1426f887f2acSHaggai Eran rcu_read_unlock(); 1427f887f2acSHaggai Eran 1428f887f2acSHaggai Eran return ret; 1429f887f2acSHaggai Eran } 1430f887f2acSHaggai Eran 1431f887f2acSHaggai Eran static bool validate_ipv6_net_dev(struct net_device *net_dev, 1432f887f2acSHaggai Eran const struct sockaddr_in6 *dst_addr, 1433f887f2acSHaggai Eran const struct sockaddr_in6 *src_addr) 1434f887f2acSHaggai Eran { 1435f887f2acSHaggai Eran #if IS_ENABLED(CONFIG_IPV6) 1436f887f2acSHaggai Eran const int strict = ipv6_addr_type(&dst_addr->sin6_addr) & 1437f887f2acSHaggai Eran IPV6_ADDR_LINKLOCAL; 1438f887f2acSHaggai Eran struct rt6_info *rt = rt6_lookup(dev_net(net_dev), &dst_addr->sin6_addr, 1439f887f2acSHaggai Eran &src_addr->sin6_addr, net_dev->ifindex, 1440b75cc8f9SDavid Ahern NULL, strict); 1441f887f2acSHaggai Eran bool ret; 1442f887f2acSHaggai Eran 1443f887f2acSHaggai Eran if (!rt) 1444f887f2acSHaggai Eran return false; 1445f887f2acSHaggai Eran 1446f887f2acSHaggai Eran ret = rt->rt6i_idev->dev == net_dev; 1447f887f2acSHaggai Eran ip6_rt_put(rt); 1448f887f2acSHaggai Eran 1449f887f2acSHaggai Eran return ret; 1450f887f2acSHaggai Eran #else 1451f887f2acSHaggai Eran return false; 1452f887f2acSHaggai Eran #endif 1453f887f2acSHaggai Eran } 1454f887f2acSHaggai Eran 1455f887f2acSHaggai Eran static bool validate_net_dev(struct net_device *net_dev, 1456f887f2acSHaggai Eran const struct sockaddr *daddr, 1457f887f2acSHaggai Eran const struct sockaddr *saddr) 1458f887f2acSHaggai Eran { 1459f887f2acSHaggai Eran const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr; 1460f887f2acSHaggai Eran const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr; 1461f887f2acSHaggai Eran const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr; 1462f887f2acSHaggai Eran const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr; 1463f887f2acSHaggai Eran 1464f887f2acSHaggai Eran switch (daddr->sa_family) { 1465f887f2acSHaggai Eran case AF_INET: 1466f887f2acSHaggai Eran return saddr->sa_family == AF_INET && 1467f887f2acSHaggai Eran validate_ipv4_net_dev(net_dev, daddr4, saddr4); 1468f887f2acSHaggai Eran 1469f887f2acSHaggai Eran case AF_INET6: 1470f887f2acSHaggai Eran return saddr->sa_family == AF_INET6 && 1471f887f2acSHaggai Eran validate_ipv6_net_dev(net_dev, daddr6, saddr6); 1472f887f2acSHaggai Eran 1473f887f2acSHaggai Eran default: 1474f887f2acSHaggai Eran return false; 1475f887f2acSHaggai Eran } 1476f887f2acSHaggai Eran } 1477f887f2acSHaggai Eran 1478cee10433SParav Pandit static struct net_device * 1479cee10433SParav Pandit roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event) 1480cee10433SParav Pandit { 1481cee10433SParav Pandit const struct ib_gid_attr *sgid_attr = NULL; 1482adb4a57aSParav Pandit struct net_device *ndev; 1483cee10433SParav Pandit 1484cee10433SParav Pandit if (ib_event->event == IB_CM_REQ_RECEIVED) 1485cee10433SParav Pandit sgid_attr = ib_event->param.req_rcvd.ppath_sgid_attr; 1486cee10433SParav Pandit else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) 1487cee10433SParav Pandit sgid_attr = ib_event->param.sidr_req_rcvd.sgid_attr; 1488cee10433SParav Pandit 1489cee10433SParav Pandit if (!sgid_attr) 1490cee10433SParav Pandit return NULL; 1491adb4a57aSParav Pandit 1492adb4a57aSParav Pandit rcu_read_lock(); 1493adb4a57aSParav Pandit ndev = rdma_read_gid_attr_ndev_rcu(sgid_attr); 1494adb4a57aSParav Pandit if (IS_ERR(ndev)) 1495adb4a57aSParav Pandit ndev = NULL; 1496adb4a57aSParav Pandit else 1497adb4a57aSParav Pandit dev_hold(ndev); 1498adb4a57aSParav Pandit rcu_read_unlock(); 1499adb4a57aSParav Pandit return ndev; 1500cee10433SParav Pandit } 1501cee10433SParav Pandit 1502e7ff98aeSParav Pandit static struct net_device *cma_get_net_dev(const struct ib_cm_event *ib_event, 15032918c1a9SParav Pandit struct cma_req_info *req) 15044c21b5bcSHaggai Eran { 15052918c1a9SParav Pandit struct sockaddr *listen_addr = 15062918c1a9SParav Pandit (struct sockaddr *)&req->listen_addr_storage; 15072918c1a9SParav Pandit struct sockaddr *src_addr = (struct sockaddr *)&req->src_addr_storage; 15084c21b5bcSHaggai Eran struct net_device *net_dev; 15094c21b5bcSHaggai Eran const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL; 15104c21b5bcSHaggai Eran int err; 15114c21b5bcSHaggai Eran 1512f887f2acSHaggai Eran err = cma_save_ip_info(listen_addr, src_addr, ib_event, 1513f887f2acSHaggai Eran req->service_id); 15144c21b5bcSHaggai Eran if (err) 15154c21b5bcSHaggai Eran return ERR_PTR(err); 15164c21b5bcSHaggai Eran 1517cee10433SParav Pandit if (rdma_protocol_roce(req->device, req->port)) 1518cee10433SParav Pandit net_dev = roce_get_net_dev_by_cm_event(ib_event); 1519cee10433SParav Pandit else 1520cee10433SParav Pandit net_dev = ib_get_net_dev_by_params(req->device, req->port, 1521cee10433SParav Pandit req->pkey, 15224c21b5bcSHaggai Eran gid, listen_addr); 15234c21b5bcSHaggai Eran if (!net_dev) 15244c21b5bcSHaggai Eran return ERR_PTR(-ENODEV); 15254c21b5bcSHaggai Eran 15264c21b5bcSHaggai Eran return net_dev; 15274c21b5bcSHaggai Eran } 15284c21b5bcSHaggai Eran 15292253fc0cSSteve Wise static enum rdma_ucm_port_space rdma_ps_from_service_id(__be64 service_id) 15304c21b5bcSHaggai Eran { 15314c21b5bcSHaggai Eran return (be64_to_cpu(service_id) >> 16) & 0xffff; 15324c21b5bcSHaggai Eran } 15334c21b5bcSHaggai Eran 15344c21b5bcSHaggai Eran static bool cma_match_private_data(struct rdma_id_private *id_priv, 15354c21b5bcSHaggai Eran const struct cma_hdr *hdr) 15364c21b5bcSHaggai Eran { 15374c21b5bcSHaggai Eran struct sockaddr *addr = cma_src_addr(id_priv); 15384c21b5bcSHaggai Eran __be32 ip4_addr; 15394c21b5bcSHaggai Eran struct in6_addr ip6_addr; 15404c21b5bcSHaggai Eran 15414c21b5bcSHaggai Eran if (cma_any_addr(addr) && !id_priv->afonly) 15424c21b5bcSHaggai Eran return true; 15434c21b5bcSHaggai Eran 15444c21b5bcSHaggai Eran switch (addr->sa_family) { 15454c21b5bcSHaggai Eran case AF_INET: 15464c21b5bcSHaggai Eran ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr; 15474c21b5bcSHaggai Eran if (cma_get_ip_ver(hdr) != 4) 15484c21b5bcSHaggai Eran return false; 15494c21b5bcSHaggai Eran if (!cma_any_addr(addr) && 15504c21b5bcSHaggai Eran hdr->dst_addr.ip4.addr != ip4_addr) 15514c21b5bcSHaggai Eran return false; 15524c21b5bcSHaggai Eran break; 15534c21b5bcSHaggai Eran case AF_INET6: 15544c21b5bcSHaggai Eran ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr; 15554c21b5bcSHaggai Eran if (cma_get_ip_ver(hdr) != 6) 15564c21b5bcSHaggai Eran return false; 15574c21b5bcSHaggai Eran if (!cma_any_addr(addr) && 15584c21b5bcSHaggai Eran memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr))) 15594c21b5bcSHaggai Eran return false; 15604c21b5bcSHaggai Eran break; 15614c21b5bcSHaggai Eran case AF_IB: 15624c21b5bcSHaggai Eran return true; 15634c21b5bcSHaggai Eran default: 15644c21b5bcSHaggai Eran return false; 15654c21b5bcSHaggai Eran } 15664c21b5bcSHaggai Eran 15674c21b5bcSHaggai Eran return true; 15684c21b5bcSHaggai Eran } 15694c21b5bcSHaggai Eran 1570b8cab5daSHaggai Eran static bool cma_protocol_roce(const struct rdma_cm_id *id) 1571b8cab5daSHaggai Eran { 1572b8cab5daSHaggai Eran struct ib_device *device = id->device; 1573b8cab5daSHaggai Eran const int port_num = id->port_num ?: rdma_start_port(device); 1574b8cab5daSHaggai Eran 15755ac08a34SParav Pandit return rdma_protocol_roce(device, port_num); 1576b8cab5daSHaggai Eran } 1577b8cab5daSHaggai Eran 157878fb282bSParav Pandit static bool cma_is_req_ipv6_ll(const struct cma_req_info *req) 157978fb282bSParav Pandit { 158078fb282bSParav Pandit const struct sockaddr *daddr = 158178fb282bSParav Pandit (const struct sockaddr *)&req->listen_addr_storage; 158278fb282bSParav Pandit const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr; 158378fb282bSParav Pandit 158478fb282bSParav Pandit /* Returns true if the req is for IPv6 link local */ 158578fb282bSParav Pandit return (daddr->sa_family == AF_INET6 && 158678fb282bSParav Pandit (ipv6_addr_type(&daddr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)); 158778fb282bSParav Pandit } 158878fb282bSParav Pandit 1589fac51590SMatan Barak static bool cma_match_net_dev(const struct rdma_cm_id *id, 1590fac51590SMatan Barak const struct net_device *net_dev, 159178fb282bSParav Pandit const struct cma_req_info *req) 15924c21b5bcSHaggai Eran { 1593fac51590SMatan Barak const struct rdma_addr *addr = &id->route.addr; 15944c21b5bcSHaggai Eran 15954c21b5bcSHaggai Eran if (!net_dev) 1596d274e45cSParav Pandit /* This request is an AF_IB request */ 159778fb282bSParav Pandit return (!id->port_num || id->port_num == req->port) && 1598d274e45cSParav Pandit (addr->src_addr.ss_family == AF_IB); 15994c21b5bcSHaggai Eran 1600643d213aSParav Pandit /* 160178fb282bSParav Pandit * If the request is not for IPv6 link local, allow matching 160278fb282bSParav Pandit * request to any netdevice of the one or multiport rdma device. 160378fb282bSParav Pandit */ 160478fb282bSParav Pandit if (!cma_is_req_ipv6_ll(req)) 160578fb282bSParav Pandit return true; 160678fb282bSParav Pandit /* 1607643d213aSParav Pandit * Net namespaces must match, and if the listner is listening 1608643d213aSParav Pandit * on a specific netdevice than netdevice must match as well. 1609643d213aSParav Pandit */ 1610643d213aSParav Pandit if (net_eq(dev_net(net_dev), addr->dev_addr.net) && 1611643d213aSParav Pandit (!!addr->dev_addr.bound_dev_if == 1612643d213aSParav Pandit (addr->dev_addr.bound_dev_if == net_dev->ifindex))) 1613643d213aSParav Pandit return true; 1614643d213aSParav Pandit else 1615643d213aSParav Pandit return false; 16164c21b5bcSHaggai Eran } 16174c21b5bcSHaggai Eran 16184c21b5bcSHaggai Eran static struct rdma_id_private *cma_find_listener( 16194c21b5bcSHaggai Eran const struct rdma_bind_list *bind_list, 16204c21b5bcSHaggai Eran const struct ib_cm_id *cm_id, 16214c21b5bcSHaggai Eran const struct ib_cm_event *ib_event, 16224c21b5bcSHaggai Eran const struct cma_req_info *req, 16234c21b5bcSHaggai Eran const struct net_device *net_dev) 16244c21b5bcSHaggai Eran { 16254c21b5bcSHaggai Eran struct rdma_id_private *id_priv, *id_priv_dev; 16264c21b5bcSHaggai Eran 16274c21b5bcSHaggai Eran if (!bind_list) 16284c21b5bcSHaggai Eran return ERR_PTR(-EINVAL); 16294c21b5bcSHaggai Eran 16304c21b5bcSHaggai Eran hlist_for_each_entry(id_priv, &bind_list->owners, node) { 16314c21b5bcSHaggai Eran if (cma_match_private_data(id_priv, ib_event->private_data)) { 16324c21b5bcSHaggai Eran if (id_priv->id.device == cm_id->device && 163378fb282bSParav Pandit cma_match_net_dev(&id_priv->id, net_dev, req)) 16344c21b5bcSHaggai Eran return id_priv; 16354c21b5bcSHaggai Eran list_for_each_entry(id_priv_dev, 16364c21b5bcSHaggai Eran &id_priv->listen_list, 16374c21b5bcSHaggai Eran listen_list) { 16384c21b5bcSHaggai Eran if (id_priv_dev->id.device == cm_id->device && 163978fb282bSParav Pandit cma_match_net_dev(&id_priv_dev->id, 164078fb282bSParav Pandit net_dev, req)) 16414c21b5bcSHaggai Eran return id_priv_dev; 16424c21b5bcSHaggai Eran } 16434c21b5bcSHaggai Eran } 16444c21b5bcSHaggai Eran } 16454c21b5bcSHaggai Eran 16464c21b5bcSHaggai Eran return ERR_PTR(-EINVAL); 16474c21b5bcSHaggai Eran } 16484c21b5bcSHaggai Eran 1649e7ff98aeSParav Pandit static struct rdma_id_private * 165085463316SParav Pandit cma_ib_id_from_event(struct ib_cm_id *cm_id, 1651e7ff98aeSParav Pandit const struct ib_cm_event *ib_event, 165241ab1cb7SParav Pandit struct cma_req_info *req, 16530b3ca768SHaggai Eran struct net_device **net_dev) 16544c21b5bcSHaggai Eran { 16554c21b5bcSHaggai Eran struct rdma_bind_list *bind_list; 16564c21b5bcSHaggai Eran struct rdma_id_private *id_priv; 16574c21b5bcSHaggai Eran int err; 16584c21b5bcSHaggai Eran 165941ab1cb7SParav Pandit err = cma_save_req_info(ib_event, req); 16604c21b5bcSHaggai Eran if (err) 16614c21b5bcSHaggai Eran return ERR_PTR(err); 16624c21b5bcSHaggai Eran 166341ab1cb7SParav Pandit *net_dev = cma_get_net_dev(ib_event, req); 16640b3ca768SHaggai Eran if (IS_ERR(*net_dev)) { 16650b3ca768SHaggai Eran if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) { 16664c21b5bcSHaggai Eran /* Assuming the protocol is AF_IB */ 16670b3ca768SHaggai Eran *net_dev = NULL; 16684c21b5bcSHaggai Eran } else { 16690b3ca768SHaggai Eran return ERR_CAST(*net_dev); 16704c21b5bcSHaggai Eran } 16714c21b5bcSHaggai Eran } 16724c21b5bcSHaggai Eran 16732918c1a9SParav Pandit /* 16742918c1a9SParav Pandit * Net namespace might be getting deleted while route lookup, 16752918c1a9SParav Pandit * cm_id lookup is in progress. Therefore, perform netdevice 16762918c1a9SParav Pandit * validation, cm_id lookup under rcu lock. 16772918c1a9SParav Pandit * RCU lock along with netdevice state check, synchronizes with 16782918c1a9SParav Pandit * netdevice migrating to different net namespace and also avoids 16792918c1a9SParav Pandit * case where net namespace doesn't get deleted while lookup is in 16802918c1a9SParav Pandit * progress. 16812918c1a9SParav Pandit * If the device state is not IFF_UP, its properties such as ifindex 16822918c1a9SParav Pandit * and nd_net cannot be trusted to remain valid without rcu lock. 16832918c1a9SParav Pandit * net/core/dev.c change_net_namespace() ensures to synchronize with 16842918c1a9SParav Pandit * ongoing operations on net device after device is closed using 16852918c1a9SParav Pandit * synchronize_net(). 16862918c1a9SParav Pandit */ 16872918c1a9SParav Pandit rcu_read_lock(); 16882918c1a9SParav Pandit if (*net_dev) { 16892918c1a9SParav Pandit /* 16902918c1a9SParav Pandit * If netdevice is down, it is likely that it is administratively 16912918c1a9SParav Pandit * down or it might be migrating to different namespace. 16922918c1a9SParav Pandit * In that case avoid further processing, as the net namespace 16932918c1a9SParav Pandit * or ifindex may change. 16942918c1a9SParav Pandit */ 16952918c1a9SParav Pandit if (((*net_dev)->flags & IFF_UP) == 0) { 16962918c1a9SParav Pandit id_priv = ERR_PTR(-EHOSTUNREACH); 16972918c1a9SParav Pandit goto err; 16982918c1a9SParav Pandit } 16992918c1a9SParav Pandit 17002918c1a9SParav Pandit if (!validate_net_dev(*net_dev, 170141ab1cb7SParav Pandit (struct sockaddr *)&req->listen_addr_storage, 170241ab1cb7SParav Pandit (struct sockaddr *)&req->src_addr_storage)) { 17032918c1a9SParav Pandit id_priv = ERR_PTR(-EHOSTUNREACH); 17042918c1a9SParav Pandit goto err; 17052918c1a9SParav Pandit } 17062918c1a9SParav Pandit } 17072918c1a9SParav Pandit 1708fa20105eSGuy Shapiro bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net, 170941ab1cb7SParav Pandit rdma_ps_from_service_id(req->service_id), 171041ab1cb7SParav Pandit cma_port_from_service_id(req->service_id)); 171141ab1cb7SParav Pandit id_priv = cma_find_listener(bind_list, cm_id, ib_event, req, *net_dev); 17122918c1a9SParav Pandit err: 17132918c1a9SParav Pandit rcu_read_unlock(); 1714b3b51f9fSHaggai Eran if (IS_ERR(id_priv) && *net_dev) { 1715be688195SHaggai Eran dev_put(*net_dev); 1716be688195SHaggai Eran *net_dev = NULL; 1717be688195SHaggai Eran } 17184c21b5bcSHaggai Eran return id_priv; 17194c21b5bcSHaggai Eran } 17204c21b5bcSHaggai Eran 1721c0b64f58SBart Van Assche static inline u8 cma_user_data_offset(struct rdma_id_private *id_priv) 1722e51060f0SSean Hefty { 1723e8160e15SSean Hefty return cma_family(id_priv) == AF_IB ? 0 : sizeof(struct cma_hdr); 1724e51060f0SSean Hefty } 1725e51060f0SSean Hefty 1726e51060f0SSean Hefty static void cma_cancel_route(struct rdma_id_private *id_priv) 1727e51060f0SSean Hefty { 1728fe53ba2fSMichael Wang if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) { 1729e51060f0SSean Hefty if (id_priv->query) 1730e51060f0SSean Hefty ib_sa_cancel_query(id_priv->query_id, id_priv->query); 1731e51060f0SSean Hefty } 1732e51060f0SSean Hefty } 1733e51060f0SSean Hefty 1734e51060f0SSean Hefty static void cma_cancel_listens(struct rdma_id_private *id_priv) 1735e51060f0SSean Hefty { 1736e51060f0SSean Hefty struct rdma_id_private *dev_id_priv; 1737e51060f0SSean Hefty 1738d02d1f53SSean Hefty /* 1739d02d1f53SSean Hefty * Remove from listen_any_list to prevent added devices from spawning 1740d02d1f53SSean Hefty * additional listen requests. 1741d02d1f53SSean Hefty */ 1742e51060f0SSean Hefty mutex_lock(&lock); 1743e51060f0SSean Hefty list_del(&id_priv->list); 1744e51060f0SSean Hefty 1745e51060f0SSean Hefty while (!list_empty(&id_priv->listen_list)) { 1746e51060f0SSean Hefty dev_id_priv = list_entry(id_priv->listen_list.next, 1747e51060f0SSean Hefty struct rdma_id_private, listen_list); 1748d02d1f53SSean Hefty /* sync with device removal to avoid duplicate destruction */ 1749d02d1f53SSean Hefty list_del_init(&dev_id_priv->list); 1750d02d1f53SSean Hefty list_del(&dev_id_priv->listen_list); 1751d02d1f53SSean Hefty mutex_unlock(&lock); 1752d02d1f53SSean Hefty 1753d02d1f53SSean Hefty rdma_destroy_id(&dev_id_priv->id); 1754d02d1f53SSean Hefty mutex_lock(&lock); 1755e51060f0SSean Hefty } 1756e51060f0SSean Hefty mutex_unlock(&lock); 1757e51060f0SSean Hefty } 1758e51060f0SSean Hefty 1759e51060f0SSean Hefty static void cma_cancel_operation(struct rdma_id_private *id_priv, 1760550e5ca7SNir Muchtar enum rdma_cm_state state) 1761e51060f0SSean Hefty { 1762e51060f0SSean Hefty switch (state) { 1763550e5ca7SNir Muchtar case RDMA_CM_ADDR_QUERY: 1764e51060f0SSean Hefty rdma_addr_cancel(&id_priv->id.route.addr.dev_addr); 1765e51060f0SSean Hefty break; 1766550e5ca7SNir Muchtar case RDMA_CM_ROUTE_QUERY: 1767e51060f0SSean Hefty cma_cancel_route(id_priv); 1768e51060f0SSean Hefty break; 1769550e5ca7SNir Muchtar case RDMA_CM_LISTEN: 1770f4753834SSean Hefty if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev) 1771e51060f0SSean Hefty cma_cancel_listens(id_priv); 1772e51060f0SSean Hefty break; 1773e51060f0SSean Hefty default: 1774e51060f0SSean Hefty break; 1775e51060f0SSean Hefty } 1776e51060f0SSean Hefty } 1777e51060f0SSean Hefty 1778e51060f0SSean Hefty static void cma_release_port(struct rdma_id_private *id_priv) 1779e51060f0SSean Hefty { 1780e51060f0SSean Hefty struct rdma_bind_list *bind_list = id_priv->bind_list; 1781fa20105eSGuy Shapiro struct net *net = id_priv->id.route.addr.dev_addr.net; 1782e51060f0SSean Hefty 1783e51060f0SSean Hefty if (!bind_list) 1784e51060f0SSean Hefty return; 1785e51060f0SSean Hefty 1786e51060f0SSean Hefty mutex_lock(&lock); 1787e51060f0SSean Hefty hlist_del(&id_priv->node); 1788e51060f0SSean Hefty if (hlist_empty(&bind_list->owners)) { 1789fa20105eSGuy Shapiro cma_ps_remove(net, bind_list->ps, bind_list->port); 1790e51060f0SSean Hefty kfree(bind_list); 1791e51060f0SSean Hefty } 1792e51060f0SSean Hefty mutex_unlock(&lock); 1793e51060f0SSean Hefty } 1794e51060f0SSean Hefty 179588145678SParav Pandit static void cma_leave_roce_mc_group(struct rdma_id_private *id_priv, 179688145678SParav Pandit struct cma_multicast *mc) 179788145678SParav Pandit { 1798c0126915SJason Gunthorpe struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 179988145678SParav Pandit struct net_device *ndev = NULL; 180088145678SParav Pandit 180188145678SParav Pandit if (dev_addr->bound_dev_if) 1802c0126915SJason Gunthorpe ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); 180388145678SParav Pandit if (ndev) { 1804c0126915SJason Gunthorpe cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid, false); 180588145678SParav Pandit dev_put(ndev); 180688145678SParav Pandit } 180788145678SParav Pandit kref_put(&mc->mcref, release_mc); 180888145678SParav Pandit } 180988145678SParav Pandit 1810c8f6a362SSean Hefty static void cma_leave_mc_groups(struct rdma_id_private *id_priv) 1811c8f6a362SSean Hefty { 1812c8f6a362SSean Hefty struct cma_multicast *mc; 1813c8f6a362SSean Hefty 1814c8f6a362SSean Hefty while (!list_empty(&id_priv->mc_list)) { 1815c8f6a362SSean Hefty mc = container_of(id_priv->mc_list.next, 1816c8f6a362SSean Hefty struct cma_multicast, list); 1817c8f6a362SSean Hefty list_del(&mc->list); 1818a31ad3b0SMichael Wang if (rdma_cap_ib_mcast(id_priv->cma_dev->device, 18195c9a5282SMichael Wang id_priv->id.port_num)) { 1820c8f6a362SSean Hefty ib_sa_free_multicast(mc->multicast.ib); 1821c8f6a362SSean Hefty kfree(mc); 1822bee3c3c9SMoni Shoua } else { 182388145678SParav Pandit cma_leave_roce_mc_group(id_priv, mc); 1824c8f6a362SSean Hefty } 1825c8f6a362SSean Hefty } 1826bee3c3c9SMoni Shoua } 1827c8f6a362SSean Hefty 1828e51060f0SSean Hefty void rdma_destroy_id(struct rdma_cm_id *id) 1829e51060f0SSean Hefty { 1830e51060f0SSean Hefty struct rdma_id_private *id_priv; 1831550e5ca7SNir Muchtar enum rdma_cm_state state; 1832e51060f0SSean Hefty 1833e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 1834ed999f82SChuck Lever trace_cm_id_destroy(id_priv); 1835550e5ca7SNir Muchtar state = cma_exch(id_priv, RDMA_CM_DESTROYING); 1836e51060f0SSean Hefty cma_cancel_operation(id_priv, state); 1837e51060f0SSean Hefty 1838a396d43aSSean Hefty /* 1839a396d43aSSean Hefty * Wait for any active callback to finish. New callbacks will find 1840a396d43aSSean Hefty * the id_priv state set to destroying and abort. 1841a396d43aSSean Hefty */ 1842a396d43aSSean Hefty mutex_lock(&id_priv->handler_mutex); 1843a396d43aSSean Hefty mutex_unlock(&id_priv->handler_mutex); 1844a396d43aSSean Hefty 184500313983SSteve Wise rdma_restrack_del(&id_priv->res); 1846ed7a01fdSLeon Romanovsky if (id_priv->cma_dev) { 184772219ceaSMichael Wang if (rdma_cap_ib_cm(id_priv->id.device, 1)) { 18480c9361fcSJack Morgenstein if (id_priv->cm_id.ib) 1849e51060f0SSean Hefty ib_destroy_cm_id(id_priv->cm_id.ib); 185004215330SMichael Wang } else if (rdma_cap_iw_cm(id_priv->id.device, 1)) { 18510c9361fcSJack Morgenstein if (id_priv->cm_id.iw) 185207ebafbaSTom Tucker iw_destroy_cm_id(id_priv->cm_id.iw); 1853e51060f0SSean Hefty } 1854c8f6a362SSean Hefty cma_leave_mc_groups(id_priv); 1855a396d43aSSean Hefty cma_release_dev(id_priv); 1856e51060f0SSean Hefty } 1857e51060f0SSean Hefty 1858e51060f0SSean Hefty cma_release_port(id_priv); 1859e368d23fSParav Pandit cma_id_put(id_priv); 1860e51060f0SSean Hefty wait_for_completion(&id_priv->comp); 1861e51060f0SSean Hefty 1862d02d1f53SSean Hefty if (id_priv->internal_id) 1863e368d23fSParav Pandit cma_id_put(id_priv->id.context); 1864d02d1f53SSean Hefty 1865e51060f0SSean Hefty kfree(id_priv->id.route.path_rec); 18664ed13a5fSParav Pandit 18674ed13a5fSParav Pandit if (id_priv->id.route.addr.dev_addr.sgid_attr) 18684ed13a5fSParav Pandit rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr); 18694ed13a5fSParav Pandit 1870fa20105eSGuy Shapiro put_net(id_priv->id.route.addr.dev_addr.net); 1871e51060f0SSean Hefty kfree(id_priv); 1872e51060f0SSean Hefty } 1873e51060f0SSean Hefty EXPORT_SYMBOL(rdma_destroy_id); 1874e51060f0SSean Hefty 1875e51060f0SSean Hefty static int cma_rep_recv(struct rdma_id_private *id_priv) 1876e51060f0SSean Hefty { 1877e51060f0SSean Hefty int ret; 1878e51060f0SSean Hefty 18795851bb89SSean Hefty ret = cma_modify_qp_rtr(id_priv, NULL); 1880e51060f0SSean Hefty if (ret) 1881e51060f0SSean Hefty goto reject; 1882e51060f0SSean Hefty 18835851bb89SSean Hefty ret = cma_modify_qp_rts(id_priv, NULL); 1884e51060f0SSean Hefty if (ret) 1885e51060f0SSean Hefty goto reject; 1886e51060f0SSean Hefty 1887ed999f82SChuck Lever trace_cm_send_rtu(id_priv); 1888e51060f0SSean Hefty ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0); 1889e51060f0SSean Hefty if (ret) 1890e51060f0SSean Hefty goto reject; 1891e51060f0SSean Hefty 1892e51060f0SSean Hefty return 0; 1893e51060f0SSean Hefty reject: 1894498683c6SMoni Shoua pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret); 1895c5483388SSean Hefty cma_modify_qp_err(id_priv); 1896ed999f82SChuck Lever trace_cm_send_rej(id_priv); 1897e51060f0SSean Hefty ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED, 1898e51060f0SSean Hefty NULL, 0, NULL, 0); 1899e51060f0SSean Hefty return ret; 1900e51060f0SSean Hefty } 1901e51060f0SSean Hefty 1902a1b1b61fSSean Hefty static void cma_set_rep_event_data(struct rdma_cm_event *event, 1903e7ff98aeSParav Pandit const struct ib_cm_rep_event_param *rep_data, 1904a1b1b61fSSean Hefty void *private_data) 1905a1b1b61fSSean Hefty { 1906a1b1b61fSSean Hefty event->param.conn.private_data = private_data; 1907a1b1b61fSSean Hefty event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE; 1908a1b1b61fSSean Hefty event->param.conn.responder_resources = rep_data->responder_resources; 1909a1b1b61fSSean Hefty event->param.conn.initiator_depth = rep_data->initiator_depth; 1910a1b1b61fSSean Hefty event->param.conn.flow_control = rep_data->flow_control; 1911a1b1b61fSSean Hefty event->param.conn.rnr_retry_count = rep_data->rnr_retry_count; 1912a1b1b61fSSean Hefty event->param.conn.srq = rep_data->srq; 1913a1b1b61fSSean Hefty event->param.conn.qp_num = rep_data->remote_qpn; 1914a1b1b61fSSean Hefty } 1915a1b1b61fSSean Hefty 1916ed999f82SChuck Lever static int cma_cm_event_handler(struct rdma_id_private *id_priv, 1917ed999f82SChuck Lever struct rdma_cm_event *event) 1918ed999f82SChuck Lever { 1919ed999f82SChuck Lever int ret; 1920ed999f82SChuck Lever 1921ed999f82SChuck Lever trace_cm_event_handler(id_priv, event); 1922ed999f82SChuck Lever ret = id_priv->id.event_handler(&id_priv->id, event); 1923ed999f82SChuck Lever trace_cm_event_done(id_priv, event, ret); 1924ed999f82SChuck Lever return ret; 1925ed999f82SChuck Lever } 1926ed999f82SChuck Lever 1927e7ff98aeSParav Pandit static int cma_ib_handler(struct ib_cm_id *cm_id, 1928e7ff98aeSParav Pandit const struct ib_cm_event *ib_event) 1929e51060f0SSean Hefty { 1930e51060f0SSean Hefty struct rdma_id_private *id_priv = cm_id->context; 19317582df82SParav Pandit struct rdma_cm_event event = {}; 1932a1b1b61fSSean Hefty int ret = 0; 1933e51060f0SSean Hefty 193437e07cdaSBart Van Assche mutex_lock(&id_priv->handler_mutex); 193538ca83a5SAmir Vadai if ((ib_event->event != IB_CM_TIMEWAIT_EXIT && 193637e07cdaSBart Van Assche id_priv->state != RDMA_CM_CONNECT) || 193738ca83a5SAmir Vadai (ib_event->event == IB_CM_TIMEWAIT_EXIT && 193837e07cdaSBart Van Assche id_priv->state != RDMA_CM_DISCONNECT)) 193937e07cdaSBart Van Assche goto out; 1940e51060f0SSean Hefty 1941e51060f0SSean Hefty switch (ib_event->event) { 1942e51060f0SSean Hefty case IB_CM_REQ_ERROR: 1943e51060f0SSean Hefty case IB_CM_REP_ERROR: 1944a1b1b61fSSean Hefty event.event = RDMA_CM_EVENT_UNREACHABLE; 1945a1b1b61fSSean Hefty event.status = -ETIMEDOUT; 1946e51060f0SSean Hefty break; 1947e51060f0SSean Hefty case IB_CM_REP_RECEIVED: 194861c0ddbeSMoni Shoua if (cma_comp(id_priv, RDMA_CM_CONNECT) && 1949ed999f82SChuck Lever (id_priv->id.qp_type != IB_QPT_UD)) { 1950ed999f82SChuck Lever trace_cm_send_mra(id_priv); 195161c0ddbeSMoni Shoua ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); 1952ed999f82SChuck Lever } 195301602f11SSean Hefty if (id_priv->id.qp) { 1954a1b1b61fSSean Hefty event.status = cma_rep_recv(id_priv); 1955a1b1b61fSSean Hefty event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR : 1956e51060f0SSean Hefty RDMA_CM_EVENT_ESTABLISHED; 195701602f11SSean Hefty } else { 1958a1b1b61fSSean Hefty event.event = RDMA_CM_EVENT_CONNECT_RESPONSE; 195901602f11SSean Hefty } 1960a1b1b61fSSean Hefty cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd, 1961a1b1b61fSSean Hefty ib_event->private_data); 1962e51060f0SSean Hefty break; 1963e51060f0SSean Hefty case IB_CM_RTU_RECEIVED: 19640fe313b0SSean Hefty case IB_CM_USER_ESTABLISHED: 19650fe313b0SSean Hefty event.event = RDMA_CM_EVENT_ESTABLISHED; 1966e51060f0SSean Hefty break; 1967e51060f0SSean Hefty case IB_CM_DREQ_ERROR: 1968a1b1b61fSSean Hefty event.status = -ETIMEDOUT; /* fall through */ 1969e51060f0SSean Hefty case IB_CM_DREQ_RECEIVED: 1970e51060f0SSean Hefty case IB_CM_DREP_RECEIVED: 1971550e5ca7SNir Muchtar if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT, 1972550e5ca7SNir Muchtar RDMA_CM_DISCONNECT)) 1973e51060f0SSean Hefty goto out; 1974a1b1b61fSSean Hefty event.event = RDMA_CM_EVENT_DISCONNECTED; 1975e51060f0SSean Hefty break; 1976e51060f0SSean Hefty case IB_CM_TIMEWAIT_EXIT: 197738ca83a5SAmir Vadai event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT; 197838ca83a5SAmir Vadai break; 1979e51060f0SSean Hefty case IB_CM_MRA_RECEIVED: 1980e51060f0SSean Hefty /* ignore event */ 1981e51060f0SSean Hefty goto out; 1982e51060f0SSean Hefty case IB_CM_REJ_RECEIVED: 1983498683c6SMoni Shoua pr_debug_ratelimited("RDMA CM: REJECTED: %s\n", rdma_reject_msg(&id_priv->id, 1984498683c6SMoni Shoua ib_event->param.rej_rcvd.reason)); 1985c5483388SSean Hefty cma_modify_qp_err(id_priv); 1986a1b1b61fSSean Hefty event.status = ib_event->param.rej_rcvd.reason; 1987a1b1b61fSSean Hefty event.event = RDMA_CM_EVENT_REJECTED; 1988a1b1b61fSSean Hefty event.param.conn.private_data = ib_event->private_data; 1989a1b1b61fSSean Hefty event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE; 1990e51060f0SSean Hefty break; 1991e51060f0SSean Hefty default: 1992aba25a3eSParav Pandit pr_err("RDMA CMA: unexpected IB CM event: %d\n", 1993e51060f0SSean Hefty ib_event->event); 1994e51060f0SSean Hefty goto out; 1995e51060f0SSean Hefty } 1996e51060f0SSean Hefty 1997ed999f82SChuck Lever ret = cma_cm_event_handler(id_priv, &event); 1998e51060f0SSean Hefty if (ret) { 1999e51060f0SSean Hefty /* Destroy the CM ID by returning a non-zero value. */ 2000e51060f0SSean Hefty id_priv->cm_id.ib = NULL; 2001550e5ca7SNir Muchtar cma_exch(id_priv, RDMA_CM_DESTROYING); 2002de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 2003e51060f0SSean Hefty rdma_destroy_id(&id_priv->id); 2004e51060f0SSean Hefty return ret; 2005e51060f0SSean Hefty } 2006e51060f0SSean Hefty out: 2007de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 2008e51060f0SSean Hefty return ret; 2009e51060f0SSean Hefty } 2010e51060f0SSean Hefty 2011e7ff98aeSParav Pandit static struct rdma_id_private * 201285463316SParav Pandit cma_ib_new_conn_id(const struct rdma_cm_id *listen_id, 2013e7ff98aeSParav Pandit const struct ib_cm_event *ib_event, 20140b3ca768SHaggai Eran struct net_device *net_dev) 2015e51060f0SSean Hefty { 201600313983SSteve Wise struct rdma_id_private *listen_id_priv; 2017e51060f0SSean Hefty struct rdma_id_private *id_priv; 2018e51060f0SSean Hefty struct rdma_cm_id *id; 2019e51060f0SSean Hefty struct rdma_route *rt; 20200c505f70SHaggai Eran const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family; 20219fdca4daSDasaratharaman Chandramouli struct sa_path_rec *path = ib_event->param.req_rcvd.primary_path; 2022d3957b86SMajd Dibbiny const __be64 service_id = 2023d3957b86SMajd Dibbiny ib_event->param.req_rcvd.primary_path->service_id; 202464c5e613SOr Gerlitz int ret; 2025e51060f0SSean Hefty 202600313983SSteve Wise listen_id_priv = container_of(listen_id, struct rdma_id_private, id); 202700313983SSteve Wise id = __rdma_create_id(listen_id->route.addr.dev_addr.net, 2028fa20105eSGuy Shapiro listen_id->event_handler, listen_id->context, 202900313983SSteve Wise listen_id->ps, ib_event->param.req_rcvd.qp_type, 203000313983SSteve Wise listen_id_priv->res.kern_name); 20313f168d2bSKrishna Kumar if (IS_ERR(id)) 20320c9361fcSJack Morgenstein return NULL; 20333f168d2bSKrishna Kumar 2034f4753834SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 20350c505f70SHaggai Eran if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr, 20360c505f70SHaggai Eran (struct sockaddr *)&id->route.addr.dst_addr, 20370c505f70SHaggai Eran listen_id, ib_event, ss_family, service_id)) 2038fbaa1a6dSSean Hefty goto err; 20393f168d2bSKrishna Kumar 20403f168d2bSKrishna Kumar rt = &id->route; 20413f168d2bSKrishna Kumar rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1; 20426da2ec56SKees Cook rt->path_rec = kmalloc_array(rt->num_paths, sizeof(*rt->path_rec), 20433f168d2bSKrishna Kumar GFP_KERNEL); 20443f168d2bSKrishna Kumar if (!rt->path_rec) 20450c9361fcSJack Morgenstein goto err; 20463f168d2bSKrishna Kumar 20479fdca4daSDasaratharaman Chandramouli rt->path_rec[0] = *path; 2048e51060f0SSean Hefty if (rt->num_paths == 2) 2049e51060f0SSean Hefty rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path; 2050e51060f0SSean Hefty 20510b3ca768SHaggai Eran if (net_dev) { 205277addc52SParav Pandit rdma_copy_src_l2_addr(&rt->addr.dev_addr, net_dev); 20530b3ca768SHaggai Eran } else { 2054b8cab5daSHaggai Eran if (!cma_protocol_roce(listen_id) && 2055b8cab5daSHaggai Eran cma_any_addr(cma_src_addr(id_priv))) { 2056b8cab5daSHaggai Eran rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND; 2057b8cab5daSHaggai Eran rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid); 2058b8cab5daSHaggai Eran ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey)); 2059b8cab5daSHaggai Eran } else if (!cma_any_addr(cma_src_addr(id_priv))) { 2060b8cab5daSHaggai Eran ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr); 2061b8cab5daSHaggai Eran if (ret) 2062b8cab5daSHaggai Eran goto err; 2063b8cab5daSHaggai Eran } 20646f8372b6SSean Hefty } 20656f8372b6SSean Hefty rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid); 2066e51060f0SSean Hefty 2067550e5ca7SNir Muchtar id_priv->state = RDMA_CM_CONNECT; 2068e51060f0SSean Hefty return id_priv; 20693f168d2bSKrishna Kumar 20703f168d2bSKrishna Kumar err: 20710c9361fcSJack Morgenstein rdma_destroy_id(id); 2072e51060f0SSean Hefty return NULL; 2073e51060f0SSean Hefty } 2074e51060f0SSean Hefty 2075e7ff98aeSParav Pandit static struct rdma_id_private * 207685463316SParav Pandit cma_ib_new_udp_id(const struct rdma_cm_id *listen_id, 2077e7ff98aeSParav Pandit const struct ib_cm_event *ib_event, 20780b3ca768SHaggai Eran struct net_device *net_dev) 2079628e5f6dSSean Hefty { 2080e7ff98aeSParav Pandit const struct rdma_id_private *listen_id_priv; 2081628e5f6dSSean Hefty struct rdma_id_private *id_priv; 2082628e5f6dSSean Hefty struct rdma_cm_id *id; 20830c505f70SHaggai Eran const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family; 2084fa20105eSGuy Shapiro struct net *net = listen_id->route.addr.dev_addr.net; 2085628e5f6dSSean Hefty int ret; 2086628e5f6dSSean Hefty 208700313983SSteve Wise listen_id_priv = container_of(listen_id, struct rdma_id_private, id); 208800313983SSteve Wise id = __rdma_create_id(net, listen_id->event_handler, listen_id->context, 208900313983SSteve Wise listen_id->ps, IB_QPT_UD, 209000313983SSteve Wise listen_id_priv->res.kern_name); 2091628e5f6dSSean Hefty if (IS_ERR(id)) 2092628e5f6dSSean Hefty return NULL; 2093628e5f6dSSean Hefty 2094f4753834SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 20950c505f70SHaggai Eran if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr, 20960c505f70SHaggai Eran (struct sockaddr *)&id->route.addr.dst_addr, 20970c505f70SHaggai Eran listen_id, ib_event, ss_family, 20980c505f70SHaggai Eran ib_event->param.sidr_req_rcvd.service_id)) 2099628e5f6dSSean Hefty goto err; 2100628e5f6dSSean Hefty 21010b3ca768SHaggai Eran if (net_dev) { 210277addc52SParav Pandit rdma_copy_src_l2_addr(&id->route.addr.dev_addr, net_dev); 21030b3ca768SHaggai Eran } else { 2104b8cab5daSHaggai Eran if (!cma_any_addr(cma_src_addr(id_priv))) { 2105b8cab5daSHaggai Eran ret = cma_translate_addr(cma_src_addr(id_priv), 21060b3ca768SHaggai Eran &id->route.addr.dev_addr); 2107b8cab5daSHaggai Eran if (ret) 2108b8cab5daSHaggai Eran goto err; 2109b8cab5daSHaggai Eran } 21106f8372b6SSean Hefty } 2111628e5f6dSSean Hefty 2112550e5ca7SNir Muchtar id_priv->state = RDMA_CM_CONNECT; 2113628e5f6dSSean Hefty return id_priv; 2114628e5f6dSSean Hefty err: 2115628e5f6dSSean Hefty rdma_destroy_id(id); 2116628e5f6dSSean Hefty return NULL; 2117628e5f6dSSean Hefty } 2118628e5f6dSSean Hefty 2119a1b1b61fSSean Hefty static void cma_set_req_event_data(struct rdma_cm_event *event, 2120e7ff98aeSParav Pandit const struct ib_cm_req_event_param *req_data, 2121a1b1b61fSSean Hefty void *private_data, int offset) 2122a1b1b61fSSean Hefty { 2123a1b1b61fSSean Hefty event->param.conn.private_data = private_data + offset; 2124a1b1b61fSSean Hefty event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset; 2125a1b1b61fSSean Hefty event->param.conn.responder_resources = req_data->responder_resources; 2126a1b1b61fSSean Hefty event->param.conn.initiator_depth = req_data->initiator_depth; 2127a1b1b61fSSean Hefty event->param.conn.flow_control = req_data->flow_control; 2128a1b1b61fSSean Hefty event->param.conn.retry_count = req_data->retry_count; 2129a1b1b61fSSean Hefty event->param.conn.rnr_retry_count = req_data->rnr_retry_count; 2130a1b1b61fSSean Hefty event->param.conn.srq = req_data->srq; 2131a1b1b61fSSean Hefty event->param.conn.qp_num = req_data->remote_qpn; 2132a1b1b61fSSean Hefty } 2133a1b1b61fSSean Hefty 213485463316SParav Pandit static int cma_ib_check_req_qp_type(const struct rdma_cm_id *id, 2135e7ff98aeSParav Pandit const struct ib_cm_event *ib_event) 21369595480cSHefty, Sean { 21374dd81e89SSean Hefty return (((ib_event->event == IB_CM_REQ_RECEIVED) && 21389595480cSHefty, Sean (ib_event->param.req_rcvd.qp_type == id->qp_type)) || 21399595480cSHefty, Sean ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) && 21409595480cSHefty, Sean (id->qp_type == IB_QPT_UD)) || 21419595480cSHefty, Sean (!id->qp_type)); 21429595480cSHefty, Sean } 21439595480cSHefty, Sean 214485463316SParav Pandit static int cma_ib_req_handler(struct ib_cm_id *cm_id, 2145e7ff98aeSParav Pandit const struct ib_cm_event *ib_event) 2146e51060f0SSean Hefty { 214737e07cdaSBart Van Assche struct rdma_id_private *listen_id, *conn_id = NULL; 21487582df82SParav Pandit struct rdma_cm_event event = {}; 214941ab1cb7SParav Pandit struct cma_req_info req = {}; 21500b3ca768SHaggai Eran struct net_device *net_dev; 2151c0b64f58SBart Van Assche u8 offset; 2152c0b64f58SBart Van Assche int ret; 2153e51060f0SSean Hefty 215441ab1cb7SParav Pandit listen_id = cma_ib_id_from_event(cm_id, ib_event, &req, &net_dev); 21554c21b5bcSHaggai Eran if (IS_ERR(listen_id)) 21564c21b5bcSHaggai Eran return PTR_ERR(listen_id); 21574c21b5bcSHaggai Eran 2158ed999f82SChuck Lever trace_cm_req_handler(listen_id, ib_event->event); 215985463316SParav Pandit if (!cma_ib_check_req_qp_type(&listen_id->id, ib_event)) { 21600b3ca768SHaggai Eran ret = -EINVAL; 21610b3ca768SHaggai Eran goto net_dev_put; 21620b3ca768SHaggai Eran } 21639595480cSHefty, Sean 216437e07cdaSBart Van Assche mutex_lock(&listen_id->handler_mutex); 216537e07cdaSBart Van Assche if (listen_id->state != RDMA_CM_LISTEN) { 21660b3ca768SHaggai Eran ret = -ECONNABORTED; 216737e07cdaSBart Van Assche goto err1; 21680b3ca768SHaggai Eran } 2169e51060f0SSean Hefty 2170e8160e15SSean Hefty offset = cma_user_data_offset(listen_id); 2171628e5f6dSSean Hefty event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 21729595480cSHefty, Sean if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) { 217385463316SParav Pandit conn_id = cma_ib_new_udp_id(&listen_id->id, ib_event, net_dev); 2174628e5f6dSSean Hefty event.param.ud.private_data = ib_event->private_data + offset; 2175628e5f6dSSean Hefty event.param.ud.private_data_len = 2176628e5f6dSSean Hefty IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset; 2177628e5f6dSSean Hefty } else { 217885463316SParav Pandit conn_id = cma_ib_new_conn_id(&listen_id->id, ib_event, net_dev); 2179628e5f6dSSean Hefty cma_set_req_event_data(&event, &ib_event->param.req_rcvd, 2180628e5f6dSSean Hefty ib_event->private_data, offset); 2181628e5f6dSSean Hefty } 2182e51060f0SSean Hefty if (!conn_id) { 2183e51060f0SSean Hefty ret = -ENOMEM; 2184b6cec8aaSSean Hefty goto err1; 2185e51060f0SSean Hefty } 2186e51060f0SSean Hefty 2187de910bd9SOr Gerlitz mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 218841ab1cb7SParav Pandit ret = cma_ib_acquire_dev(conn_id, listen_id, &req); 2189a1a733f6SKrishna Kumar if (ret) 2190b6cec8aaSSean Hefty goto err2; 2191e51060f0SSean Hefty 2192e51060f0SSean Hefty conn_id->cm_id.ib = cm_id; 2193e51060f0SSean Hefty cm_id->context = conn_id; 2194e51060f0SSean Hefty cm_id->cm_handler = cma_ib_handler; 2195e51060f0SSean Hefty 219625ae21a1SSean Hefty /* 219725ae21a1SSean Hefty * Protect against the user destroying conn_id from another thread 219825ae21a1SSean Hefty * until we're done accessing it. 219925ae21a1SSean Hefty */ 2200e368d23fSParav Pandit cma_id_get(conn_id); 2201ed999f82SChuck Lever ret = cma_cm_event_handler(conn_id, &event); 2202b6cec8aaSSean Hefty if (ret) 2203b6cec8aaSSean Hefty goto err3; 2204ead595aeSSean Hefty /* 2205ead595aeSSean Hefty * Acquire mutex to prevent user executing rdma_destroy_id() 2206ead595aeSSean Hefty * while we're accessing the cm_id. 2207ead595aeSSean Hefty */ 2208ead595aeSSean Hefty mutex_lock(&lock); 2209dd5f03beSMatan Barak if (cma_comp(conn_id, RDMA_CM_CONNECT) && 2210ed999f82SChuck Lever (conn_id->id.qp_type != IB_QPT_UD)) { 2211ed999f82SChuck Lever trace_cm_send_mra(cm_id->context); 2212ead595aeSSean Hefty ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0); 2213ed999f82SChuck Lever } 2214ead595aeSSean Hefty mutex_unlock(&lock); 2215de910bd9SOr Gerlitz mutex_unlock(&conn_id->handler_mutex); 2216b6cec8aaSSean Hefty mutex_unlock(&listen_id->handler_mutex); 2217e368d23fSParav Pandit cma_id_put(conn_id); 22180b3ca768SHaggai Eran if (net_dev) 22190b3ca768SHaggai Eran dev_put(net_dev); 2220b6cec8aaSSean Hefty return 0; 2221a1a733f6SKrishna Kumar 2222b6cec8aaSSean Hefty err3: 2223e368d23fSParav Pandit cma_id_put(conn_id); 2224e51060f0SSean Hefty /* Destroy the CM ID by returning a non-zero value. */ 2225e51060f0SSean Hefty conn_id->cm_id.ib = NULL; 2226b6cec8aaSSean Hefty err2: 2227550e5ca7SNir Muchtar cma_exch(conn_id, RDMA_CM_DESTROYING); 2228de910bd9SOr Gerlitz mutex_unlock(&conn_id->handler_mutex); 2229b6cec8aaSSean Hefty err1: 2230de910bd9SOr Gerlitz mutex_unlock(&listen_id->handler_mutex); 2231b6cec8aaSSean Hefty if (conn_id) 2232b6cec8aaSSean Hefty rdma_destroy_id(&conn_id->id); 22330b3ca768SHaggai Eran 22340b3ca768SHaggai Eran net_dev_put: 22350b3ca768SHaggai Eran if (net_dev) 22360b3ca768SHaggai Eran dev_put(net_dev); 22370b3ca768SHaggai Eran 2238e51060f0SSean Hefty return ret; 2239e51060f0SSean Hefty } 2240e51060f0SSean Hefty 2241cf53936fSSean Hefty __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr) 2242e51060f0SSean Hefty { 2243496ce3ceSSean Hefty if (addr->sa_family == AF_IB) 2244496ce3ceSSean Hefty return ((struct sockaddr_ib *) addr)->sib_sid; 2245496ce3ceSSean Hefty 2246cf53936fSSean Hefty return cpu_to_be64(((u64)id->ps << 16) + be16_to_cpu(cma_port(addr))); 2247e51060f0SSean Hefty } 2248cf53936fSSean Hefty EXPORT_SYMBOL(rdma_get_service_id); 2249e51060f0SSean Hefty 2250411460acSParav Pandit void rdma_read_gids(struct rdma_cm_id *cm_id, union ib_gid *sgid, 2251411460acSParav Pandit union ib_gid *dgid) 2252411460acSParav Pandit { 2253411460acSParav Pandit struct rdma_addr *addr = &cm_id->route.addr; 2254411460acSParav Pandit 2255411460acSParav Pandit if (!cm_id->device) { 2256411460acSParav Pandit if (sgid) 2257411460acSParav Pandit memset(sgid, 0, sizeof(*sgid)); 2258411460acSParav Pandit if (dgid) 2259411460acSParav Pandit memset(dgid, 0, sizeof(*dgid)); 2260411460acSParav Pandit return; 2261411460acSParav Pandit } 2262411460acSParav Pandit 2263411460acSParav Pandit if (rdma_protocol_roce(cm_id->device, cm_id->port_num)) { 2264411460acSParav Pandit if (sgid) 2265411460acSParav Pandit rdma_ip2gid((struct sockaddr *)&addr->src_addr, sgid); 2266411460acSParav Pandit if (dgid) 2267411460acSParav Pandit rdma_ip2gid((struct sockaddr *)&addr->dst_addr, dgid); 2268411460acSParav Pandit } else { 2269411460acSParav Pandit if (sgid) 2270411460acSParav Pandit rdma_addr_get_sgid(&addr->dev_addr, sgid); 2271411460acSParav Pandit if (dgid) 2272411460acSParav Pandit rdma_addr_get_dgid(&addr->dev_addr, dgid); 2273411460acSParav Pandit } 2274411460acSParav Pandit } 2275411460acSParav Pandit EXPORT_SYMBOL(rdma_read_gids); 2276411460acSParav Pandit 227707ebafbaSTom Tucker static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event) 227807ebafbaSTom Tucker { 227907ebafbaSTom Tucker struct rdma_id_private *id_priv = iw_id->context; 22807582df82SParav Pandit struct rdma_cm_event event = {}; 228107ebafbaSTom Tucker int ret = 0; 228224d44a39SSteve Wise struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr; 228324d44a39SSteve Wise struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; 228407ebafbaSTom Tucker 228537e07cdaSBart Van Assche mutex_lock(&id_priv->handler_mutex); 228637e07cdaSBart Van Assche if (id_priv->state != RDMA_CM_CONNECT) 228737e07cdaSBart Van Assche goto out; 228807ebafbaSTom Tucker 228907ebafbaSTom Tucker switch (iw_event->event) { 229007ebafbaSTom Tucker case IW_CM_EVENT_CLOSE: 2291a1b1b61fSSean Hefty event.event = RDMA_CM_EVENT_DISCONNECTED; 229207ebafbaSTom Tucker break; 229307ebafbaSTom Tucker case IW_CM_EVENT_CONNECT_REPLY: 229424d44a39SSteve Wise memcpy(cma_src_addr(id_priv), laddr, 229524d44a39SSteve Wise rdma_addr_size(laddr)); 229624d44a39SSteve Wise memcpy(cma_dst_addr(id_priv), raddr, 229724d44a39SSteve Wise rdma_addr_size(raddr)); 2298881a045fSSteve Wise switch (iw_event->status) { 2299881a045fSSteve Wise case 0: 2300a1b1b61fSSean Hefty event.event = RDMA_CM_EVENT_ESTABLISHED; 23013ebeebc3SKumar Sanghvi event.param.conn.initiator_depth = iw_event->ird; 23023ebeebc3SKumar Sanghvi event.param.conn.responder_resources = iw_event->ord; 230307ebafbaSTom Tucker break; 2304881a045fSSteve Wise case -ECONNRESET: 2305881a045fSSteve Wise case -ECONNREFUSED: 2306881a045fSSteve Wise event.event = RDMA_CM_EVENT_REJECTED; 2307881a045fSSteve Wise break; 2308881a045fSSteve Wise case -ETIMEDOUT: 2309881a045fSSteve Wise event.event = RDMA_CM_EVENT_UNREACHABLE; 2310881a045fSSteve Wise break; 2311881a045fSSteve Wise default: 2312881a045fSSteve Wise event.event = RDMA_CM_EVENT_CONNECT_ERROR; 2313881a045fSSteve Wise break; 2314881a045fSSteve Wise } 2315881a045fSSteve Wise break; 231607ebafbaSTom Tucker case IW_CM_EVENT_ESTABLISHED: 2317a1b1b61fSSean Hefty event.event = RDMA_CM_EVENT_ESTABLISHED; 23183ebeebc3SKumar Sanghvi event.param.conn.initiator_depth = iw_event->ird; 23193ebeebc3SKumar Sanghvi event.param.conn.responder_resources = iw_event->ord; 232007ebafbaSTom Tucker break; 232107ebafbaSTom Tucker default: 2322671a6cc2SLeon Romanovsky goto out; 232307ebafbaSTom Tucker } 232407ebafbaSTom Tucker 2325a1b1b61fSSean Hefty event.status = iw_event->status; 2326a1b1b61fSSean Hefty event.param.conn.private_data = iw_event->private_data; 2327a1b1b61fSSean Hefty event.param.conn.private_data_len = iw_event->private_data_len; 2328ed999f82SChuck Lever ret = cma_cm_event_handler(id_priv, &event); 232907ebafbaSTom Tucker if (ret) { 233007ebafbaSTom Tucker /* Destroy the CM ID by returning a non-zero value. */ 233107ebafbaSTom Tucker id_priv->cm_id.iw = NULL; 2332550e5ca7SNir Muchtar cma_exch(id_priv, RDMA_CM_DESTROYING); 2333de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 233407ebafbaSTom Tucker rdma_destroy_id(&id_priv->id); 233507ebafbaSTom Tucker return ret; 233607ebafbaSTom Tucker } 233707ebafbaSTom Tucker 233837e07cdaSBart Van Assche out: 2339de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 234007ebafbaSTom Tucker return ret; 234107ebafbaSTom Tucker } 234207ebafbaSTom Tucker 234307ebafbaSTom Tucker static int iw_conn_req_handler(struct iw_cm_id *cm_id, 234407ebafbaSTom Tucker struct iw_cm_event *iw_event) 234507ebafbaSTom Tucker { 234607ebafbaSTom Tucker struct rdma_cm_id *new_cm_id; 234707ebafbaSTom Tucker struct rdma_id_private *listen_id, *conn_id; 23487582df82SParav Pandit struct rdma_cm_event event = {}; 234937e07cdaSBart Van Assche int ret = -ECONNABORTED; 235024d44a39SSteve Wise struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr; 235124d44a39SSteve Wise struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr; 235207ebafbaSTom Tucker 23537582df82SParav Pandit event.event = RDMA_CM_EVENT_CONNECT_REQUEST; 23547582df82SParav Pandit event.param.conn.private_data = iw_event->private_data; 23557582df82SParav Pandit event.param.conn.private_data_len = iw_event->private_data_len; 23567582df82SParav Pandit event.param.conn.initiator_depth = iw_event->ird; 23577582df82SParav Pandit event.param.conn.responder_resources = iw_event->ord; 23587582df82SParav Pandit 235907ebafbaSTom Tucker listen_id = cm_id->context; 236037e07cdaSBart Van Assche 236137e07cdaSBart Van Assche mutex_lock(&listen_id->handler_mutex); 236237e07cdaSBart Van Assche if (listen_id->state != RDMA_CM_LISTEN) 236337e07cdaSBart Van Assche goto out; 236407ebafbaSTom Tucker 236507ebafbaSTom Tucker /* Create a new RDMA id for the new IW CM ID */ 236600313983SSteve Wise new_cm_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net, 2367fa20105eSGuy Shapiro listen_id->id.event_handler, 236807ebafbaSTom Tucker listen_id->id.context, 236900313983SSteve Wise RDMA_PS_TCP, IB_QPT_RC, 237000313983SSteve Wise listen_id->res.kern_name); 237110f32065SJulia Lawall if (IS_ERR(new_cm_id)) { 237207ebafbaSTom Tucker ret = -ENOMEM; 237307ebafbaSTom Tucker goto out; 237407ebafbaSTom Tucker } 237507ebafbaSTom Tucker conn_id = container_of(new_cm_id, struct rdma_id_private, id); 2376de910bd9SOr Gerlitz mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING); 2377550e5ca7SNir Muchtar conn_id->state = RDMA_CM_CONNECT; 237807ebafbaSTom Tucker 2379575c7e58SParav Pandit ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr); 238007ebafbaSTom Tucker if (ret) { 2381de910bd9SOr Gerlitz mutex_unlock(&conn_id->handler_mutex); 238207ebafbaSTom Tucker rdma_destroy_id(new_cm_id); 238307ebafbaSTom Tucker goto out; 238407ebafbaSTom Tucker } 238507ebafbaSTom Tucker 238641ab1cb7SParav Pandit ret = cma_iw_acquire_dev(conn_id, listen_id); 238707ebafbaSTom Tucker if (ret) { 2388de910bd9SOr Gerlitz mutex_unlock(&conn_id->handler_mutex); 238907ebafbaSTom Tucker rdma_destroy_id(new_cm_id); 239007ebafbaSTom Tucker goto out; 239107ebafbaSTom Tucker } 239207ebafbaSTom Tucker 239307ebafbaSTom Tucker conn_id->cm_id.iw = cm_id; 239407ebafbaSTom Tucker cm_id->context = conn_id; 239507ebafbaSTom Tucker cm_id->cm_handler = cma_iw_handler; 239607ebafbaSTom Tucker 239724d44a39SSteve Wise memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr)); 239824d44a39SSteve Wise memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr)); 239907ebafbaSTom Tucker 240025ae21a1SSean Hefty /* 240125ae21a1SSean Hefty * Protect against the user destroying conn_id from another thread 240225ae21a1SSean Hefty * until we're done accessing it. 240325ae21a1SSean Hefty */ 2404e368d23fSParav Pandit cma_id_get(conn_id); 2405ed999f82SChuck Lever ret = cma_cm_event_handler(conn_id, &event); 240607ebafbaSTom Tucker if (ret) { 240707ebafbaSTom Tucker /* User wants to destroy the CM ID */ 240807ebafbaSTom Tucker conn_id->cm_id.iw = NULL; 2409550e5ca7SNir Muchtar cma_exch(conn_id, RDMA_CM_DESTROYING); 2410de910bd9SOr Gerlitz mutex_unlock(&conn_id->handler_mutex); 2411b66f31efSBart Van Assche mutex_unlock(&listen_id->handler_mutex); 2412e368d23fSParav Pandit cma_id_put(conn_id); 241307ebafbaSTom Tucker rdma_destroy_id(&conn_id->id); 2414b66f31efSBart Van Assche return ret; 241507ebafbaSTom Tucker } 241607ebafbaSTom Tucker 2417de910bd9SOr Gerlitz mutex_unlock(&conn_id->handler_mutex); 2418e368d23fSParav Pandit cma_id_put(conn_id); 2419de910bd9SOr Gerlitz 242007ebafbaSTom Tucker out: 2421de910bd9SOr Gerlitz mutex_unlock(&listen_id->handler_mutex); 242207ebafbaSTom Tucker return ret; 242307ebafbaSTom Tucker } 242407ebafbaSTom Tucker 2425e51060f0SSean Hefty static int cma_ib_listen(struct rdma_id_private *id_priv) 2426e51060f0SSean Hefty { 2427e51060f0SSean Hefty struct sockaddr *addr; 24280c9361fcSJack Morgenstein struct ib_cm_id *id; 2429e51060f0SSean Hefty __be64 svc_id; 2430e51060f0SSean Hefty 2431f4753834SSean Hefty addr = cma_src_addr(id_priv); 2432cf53936fSSean Hefty svc_id = rdma_get_service_id(&id_priv->id, addr); 243385463316SParav Pandit id = ib_cm_insert_listen(id_priv->id.device, 243485463316SParav Pandit cma_ib_req_handler, svc_id); 243551efe394SHaggai Eran if (IS_ERR(id)) 243651efe394SHaggai Eran return PTR_ERR(id); 243751efe394SHaggai Eran id_priv->cm_id.ib = id; 2438e51060f0SSean Hefty 243951efe394SHaggai Eran return 0; 2440e51060f0SSean Hefty } 2441e51060f0SSean Hefty 244207ebafbaSTom Tucker static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog) 244307ebafbaSTom Tucker { 244407ebafbaSTom Tucker int ret; 24450c9361fcSJack Morgenstein struct iw_cm_id *id; 244607ebafbaSTom Tucker 24470c9361fcSJack Morgenstein id = iw_create_cm_id(id_priv->id.device, 244807ebafbaSTom Tucker iw_conn_req_handler, 244907ebafbaSTom Tucker id_priv); 24500c9361fcSJack Morgenstein if (IS_ERR(id)) 24510c9361fcSJack Morgenstein return PTR_ERR(id); 24520c9361fcSJack Morgenstein 245368cdba06SSteve Wise id->tos = id_priv->tos; 2454926ba19bSSteve Wise id->tos_set = id_priv->tos_set; 24550c9361fcSJack Morgenstein id_priv->cm_id.iw = id; 245607ebafbaSTom Tucker 245724d44a39SSteve Wise memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv), 245824d44a39SSteve Wise rdma_addr_size(cma_src_addr(id_priv))); 245907ebafbaSTom Tucker 246007ebafbaSTom Tucker ret = iw_cm_listen(id_priv->cm_id.iw, backlog); 246107ebafbaSTom Tucker 246207ebafbaSTom Tucker if (ret) { 246307ebafbaSTom Tucker iw_destroy_cm_id(id_priv->cm_id.iw); 246407ebafbaSTom Tucker id_priv->cm_id.iw = NULL; 246507ebafbaSTom Tucker } 246607ebafbaSTom Tucker 246707ebafbaSTom Tucker return ret; 246807ebafbaSTom Tucker } 246907ebafbaSTom Tucker 2470e51060f0SSean Hefty static int cma_listen_handler(struct rdma_cm_id *id, 2471e51060f0SSean Hefty struct rdma_cm_event *event) 2472e51060f0SSean Hefty { 2473e51060f0SSean Hefty struct rdma_id_private *id_priv = id->context; 2474e51060f0SSean Hefty 2475e51060f0SSean Hefty id->context = id_priv->id.context; 2476e51060f0SSean Hefty id->event_handler = id_priv->id.event_handler; 2477ed999f82SChuck Lever trace_cm_event_handler(id_priv, event); 2478e51060f0SSean Hefty return id_priv->id.event_handler(id, event); 2479e51060f0SSean Hefty } 2480e51060f0SSean Hefty 2481e51060f0SSean Hefty static void cma_listen_on_dev(struct rdma_id_private *id_priv, 2482e51060f0SSean Hefty struct cma_device *cma_dev) 2483e51060f0SSean Hefty { 2484e51060f0SSean Hefty struct rdma_id_private *dev_id_priv; 2485e51060f0SSean Hefty struct rdma_cm_id *id; 2486fa20105eSGuy Shapiro struct net *net = id_priv->id.route.addr.dev_addr.net; 2487e51060f0SSean Hefty int ret; 2488e51060f0SSean Hefty 248972219ceaSMichael Wang if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1)) 249094d0c939SSean Hefty return; 249194d0c939SSean Hefty 249200313983SSteve Wise id = __rdma_create_id(net, cma_listen_handler, id_priv, id_priv->id.ps, 249300313983SSteve Wise id_priv->id.qp_type, id_priv->res.kern_name); 2494e51060f0SSean Hefty if (IS_ERR(id)) 2495e51060f0SSean Hefty return; 2496e51060f0SSean Hefty 2497e51060f0SSean Hefty dev_id_priv = container_of(id, struct rdma_id_private, id); 2498e51060f0SSean Hefty 2499550e5ca7SNir Muchtar dev_id_priv->state = RDMA_CM_ADDR_BOUND; 2500f4753834SSean Hefty memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv), 2501f4753834SSean Hefty rdma_addr_size(cma_src_addr(id_priv))); 2502e51060f0SSean Hefty 2503045959dbSMatan Barak _cma_attach_to_dev(dev_id_priv, cma_dev); 2504e51060f0SSean Hefty list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list); 2505e368d23fSParav Pandit cma_id_get(id_priv); 2506d02d1f53SSean Hefty dev_id_priv->internal_id = 1; 25075b0ec991SSean Hefty dev_id_priv->afonly = id_priv->afonly; 25089491128fSSteve Wise dev_id_priv->tos_set = id_priv->tos_set; 25099491128fSSteve Wise dev_id_priv->tos = id_priv->tos; 2510e51060f0SSean Hefty 2511e51060f0SSean Hefty ret = rdma_listen(id, id_priv->backlog); 2512e51060f0SSean Hefty if (ret) 251343c7c851SJason Gunthorpe dev_warn(&cma_dev->device->dev, 251443c7c851SJason Gunthorpe "RDMA CMA: cma_listen_on_dev, error %d\n", ret); 2515e51060f0SSean Hefty } 2516e51060f0SSean Hefty 2517e51060f0SSean Hefty static void cma_listen_on_all(struct rdma_id_private *id_priv) 2518e51060f0SSean Hefty { 2519e51060f0SSean Hefty struct cma_device *cma_dev; 2520e51060f0SSean Hefty 2521e51060f0SSean Hefty mutex_lock(&lock); 2522e51060f0SSean Hefty list_add_tail(&id_priv->list, &listen_any_list); 2523e51060f0SSean Hefty list_for_each_entry(cma_dev, &dev_list, list) 2524e51060f0SSean Hefty cma_listen_on_dev(id_priv, cma_dev); 2525e51060f0SSean Hefty mutex_unlock(&lock); 2526e51060f0SSean Hefty } 2527e51060f0SSean Hefty 2528a81c994dSSean Hefty void rdma_set_service_type(struct rdma_cm_id *id, int tos) 2529a81c994dSSean Hefty { 2530a81c994dSSean Hefty struct rdma_id_private *id_priv; 2531a81c994dSSean Hefty 2532a81c994dSSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 2533a81c994dSSean Hefty id_priv->tos = (u8) tos; 253489052d78SMajd Dibbiny id_priv->tos_set = true; 2535a81c994dSSean Hefty } 2536a81c994dSSean Hefty EXPORT_SYMBOL(rdma_set_service_type); 2537a81c994dSSean Hefty 25382c1619edSDanit Goldberg /** 25392c1619edSDanit Goldberg * rdma_set_ack_timeout() - Set the ack timeout of QP associated 25402c1619edSDanit Goldberg * with a connection identifier. 25412c1619edSDanit Goldberg * @id: Communication identifier to associated with service type. 25422c1619edSDanit Goldberg * @timeout: Ack timeout to set a QP, expressed as 4.096 * 2^(timeout) usec. 25432c1619edSDanit Goldberg * 25442c1619edSDanit Goldberg * This function should be called before rdma_connect() on active side, 25452c1619edSDanit Goldberg * and on passive side before rdma_accept(). It is applicable to primary 25462c1619edSDanit Goldberg * path only. The timeout will affect the local side of the QP, it is not 2547e1ee1e62SDag Moxnes * negotiated with remote side and zero disables the timer. In case it is 2548e1ee1e62SDag Moxnes * set before rdma_resolve_route, the value will also be used to determine 2549e1ee1e62SDag Moxnes * PacketLifeTime for RoCE. 25502c1619edSDanit Goldberg * 25512c1619edSDanit Goldberg * Return: 0 for success 25522c1619edSDanit Goldberg */ 25532c1619edSDanit Goldberg int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout) 25542c1619edSDanit Goldberg { 25552c1619edSDanit Goldberg struct rdma_id_private *id_priv; 25562c1619edSDanit Goldberg 25572c1619edSDanit Goldberg if (id->qp_type != IB_QPT_RC) 25582c1619edSDanit Goldberg return -EINVAL; 25592c1619edSDanit Goldberg 25602c1619edSDanit Goldberg id_priv = container_of(id, struct rdma_id_private, id); 25612c1619edSDanit Goldberg id_priv->timeout = timeout; 25622c1619edSDanit Goldberg id_priv->timeout_set = true; 25632c1619edSDanit Goldberg 25642c1619edSDanit Goldberg return 0; 25652c1619edSDanit Goldberg } 25662c1619edSDanit Goldberg EXPORT_SYMBOL(rdma_set_ack_timeout); 25672c1619edSDanit Goldberg 2568c2f8fc4eSDasaratharaman Chandramouli static void cma_query_handler(int status, struct sa_path_rec *path_rec, 2569e51060f0SSean Hefty void *context) 2570e51060f0SSean Hefty { 2571e51060f0SSean Hefty struct cma_work *work = context; 2572e51060f0SSean Hefty struct rdma_route *route; 2573e51060f0SSean Hefty 2574e51060f0SSean Hefty route = &work->id->id.route; 2575e51060f0SSean Hefty 2576e51060f0SSean Hefty if (!status) { 2577e51060f0SSean Hefty route->num_paths = 1; 2578e51060f0SSean Hefty *route->path_rec = *path_rec; 2579e51060f0SSean Hefty } else { 2580550e5ca7SNir Muchtar work->old_state = RDMA_CM_ROUTE_QUERY; 2581550e5ca7SNir Muchtar work->new_state = RDMA_CM_ADDR_RESOLVED; 2582e51060f0SSean Hefty work->event.event = RDMA_CM_EVENT_ROUTE_ERROR; 25838f0472d3SSean Hefty work->event.status = status; 2584498683c6SMoni Shoua pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n", 2585498683c6SMoni Shoua status); 2586e51060f0SSean Hefty } 2587e51060f0SSean Hefty 2588e51060f0SSean Hefty queue_work(cma_wq, &work->work); 2589e51060f0SSean Hefty } 2590e51060f0SSean Hefty 2591dbace111SLeon Romanovsky static int cma_query_ib_route(struct rdma_id_private *id_priv, 2592dbace111SLeon Romanovsky unsigned long timeout_ms, struct cma_work *work) 2593e51060f0SSean Hefty { 2594f4753834SSean Hefty struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 2595c2f8fc4eSDasaratharaman Chandramouli struct sa_path_rec path_rec; 2596a81c994dSSean Hefty ib_sa_comp_mask comp_mask; 2597a81c994dSSean Hefty struct sockaddr_in6 *sin6; 2598f68194caSSean Hefty struct sockaddr_ib *sib; 2599e51060f0SSean Hefty 2600e51060f0SSean Hefty memset(&path_rec, 0, sizeof path_rec); 26014c33bd19SDasaratharaman Chandramouli 26024c33bd19SDasaratharaman Chandramouli if (rdma_cap_opa_ah(id_priv->id.device, id_priv->id.port_num)) 26034c33bd19SDasaratharaman Chandramouli path_rec.rec_type = SA_PATH_REC_TYPE_OPA; 26044c33bd19SDasaratharaman Chandramouli else 26059fdca4daSDasaratharaman Chandramouli path_rec.rec_type = SA_PATH_REC_TYPE_IB; 2606f4753834SSean Hefty rdma_addr_get_sgid(dev_addr, &path_rec.sgid); 2607f4753834SSean Hefty rdma_addr_get_dgid(dev_addr, &path_rec.dgid); 2608f4753834SSean Hefty path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 2609e51060f0SSean Hefty path_rec.numb_path = 1; 2610962063e6SSean Hefty path_rec.reversible = 1; 2611d3957b86SMajd Dibbiny path_rec.service_id = rdma_get_service_id(&id_priv->id, 2612d3957b86SMajd Dibbiny cma_dst_addr(id_priv)); 2613a81c994dSSean Hefty 2614a81c994dSSean Hefty comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID | 2615a81c994dSSean Hefty IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH | 2616a81c994dSSean Hefty IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID; 2617a81c994dSSean Hefty 2618f68194caSSean Hefty switch (cma_family(id_priv)) { 2619f68194caSSean Hefty case AF_INET: 2620a81c994dSSean Hefty path_rec.qos_class = cpu_to_be16((u16) id_priv->tos); 2621a81c994dSSean Hefty comp_mask |= IB_SA_PATH_REC_QOS_CLASS; 2622f68194caSSean Hefty break; 2623f68194caSSean Hefty case AF_INET6: 2624f4753834SSean Hefty sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv); 2625a81c994dSSean Hefty path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20); 2626a81c994dSSean Hefty comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; 2627f68194caSSean Hefty break; 2628f68194caSSean Hefty case AF_IB: 2629f68194caSSean Hefty sib = (struct sockaddr_ib *) cma_src_addr(id_priv); 2630f68194caSSean Hefty path_rec.traffic_class = (u8) (be32_to_cpu(sib->sib_flowinfo) >> 20); 2631f68194caSSean Hefty comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS; 2632f68194caSSean Hefty break; 2633a81c994dSSean Hefty } 2634e51060f0SSean Hefty 2635c1a0b23bSMichael S. Tsirkin id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device, 2636e51060f0SSean Hefty id_priv->id.port_num, &path_rec, 2637a81c994dSSean Hefty comp_mask, timeout_ms, 2638a81c994dSSean Hefty GFP_KERNEL, cma_query_handler, 2639a81c994dSSean Hefty work, &id_priv->query); 2640e51060f0SSean Hefty 2641e51060f0SSean Hefty return (id_priv->query_id < 0) ? id_priv->query_id : 0; 2642e51060f0SSean Hefty } 2643e51060f0SSean Hefty 2644c4028958SDavid Howells static void cma_work_handler(struct work_struct *_work) 2645e51060f0SSean Hefty { 2646c4028958SDavid Howells struct cma_work *work = container_of(_work, struct cma_work, work); 2647e51060f0SSean Hefty struct rdma_id_private *id_priv = work->id; 2648e51060f0SSean Hefty int destroy = 0; 2649e51060f0SSean Hefty 2650de910bd9SOr Gerlitz mutex_lock(&id_priv->handler_mutex); 2651e51060f0SSean Hefty if (!cma_comp_exch(id_priv, work->old_state, work->new_state)) 2652e51060f0SSean Hefty goto out; 2653e51060f0SSean Hefty 2654ed999f82SChuck Lever if (cma_cm_event_handler(id_priv, &work->event)) { 2655550e5ca7SNir Muchtar cma_exch(id_priv, RDMA_CM_DESTROYING); 2656e51060f0SSean Hefty destroy = 1; 2657e51060f0SSean Hefty } 2658e51060f0SSean Hefty out: 2659de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 2660e368d23fSParav Pandit cma_id_put(id_priv); 2661e51060f0SSean Hefty if (destroy) 2662e51060f0SSean Hefty rdma_destroy_id(&id_priv->id); 2663e51060f0SSean Hefty kfree(work); 2664e51060f0SSean Hefty } 2665e51060f0SSean Hefty 2666dd5bdff8SOr Gerlitz static void cma_ndev_work_handler(struct work_struct *_work) 2667dd5bdff8SOr Gerlitz { 2668dd5bdff8SOr Gerlitz struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work); 2669dd5bdff8SOr Gerlitz struct rdma_id_private *id_priv = work->id; 2670dd5bdff8SOr Gerlitz int destroy = 0; 2671dd5bdff8SOr Gerlitz 2672dd5bdff8SOr Gerlitz mutex_lock(&id_priv->handler_mutex); 2673550e5ca7SNir Muchtar if (id_priv->state == RDMA_CM_DESTROYING || 2674550e5ca7SNir Muchtar id_priv->state == RDMA_CM_DEVICE_REMOVAL) 2675dd5bdff8SOr Gerlitz goto out; 2676dd5bdff8SOr Gerlitz 2677ed999f82SChuck Lever if (cma_cm_event_handler(id_priv, &work->event)) { 2678550e5ca7SNir Muchtar cma_exch(id_priv, RDMA_CM_DESTROYING); 2679dd5bdff8SOr Gerlitz destroy = 1; 2680dd5bdff8SOr Gerlitz } 2681dd5bdff8SOr Gerlitz 2682dd5bdff8SOr Gerlitz out: 2683dd5bdff8SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 2684e368d23fSParav Pandit cma_id_put(id_priv); 2685dd5bdff8SOr Gerlitz if (destroy) 2686dd5bdff8SOr Gerlitz rdma_destroy_id(&id_priv->id); 2687dd5bdff8SOr Gerlitz kfree(work); 2688dd5bdff8SOr Gerlitz } 2689dd5bdff8SOr Gerlitz 2690981b5a23SParav Pandit static void cma_init_resolve_route_work(struct cma_work *work, 2691981b5a23SParav Pandit struct rdma_id_private *id_priv) 2692981b5a23SParav Pandit { 2693981b5a23SParav Pandit work->id = id_priv; 2694981b5a23SParav Pandit INIT_WORK(&work->work, cma_work_handler); 2695981b5a23SParav Pandit work->old_state = RDMA_CM_ROUTE_QUERY; 2696981b5a23SParav Pandit work->new_state = RDMA_CM_ROUTE_RESOLVED; 2697981b5a23SParav Pandit work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 2698981b5a23SParav Pandit } 2699981b5a23SParav Pandit 2700081ea519SParav Pandit static void enqueue_resolve_addr_work(struct cma_work *work, 2701981b5a23SParav Pandit struct rdma_id_private *id_priv) 2702981b5a23SParav Pandit { 2703e368d23fSParav Pandit /* Balances with cma_id_put() in cma_work_handler */ 2704e368d23fSParav Pandit cma_id_get(id_priv); 2705081ea519SParav Pandit 2706981b5a23SParav Pandit work->id = id_priv; 2707981b5a23SParav Pandit INIT_WORK(&work->work, cma_work_handler); 2708981b5a23SParav Pandit work->old_state = RDMA_CM_ADDR_QUERY; 2709981b5a23SParav Pandit work->new_state = RDMA_CM_ADDR_RESOLVED; 2710981b5a23SParav Pandit work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 2711081ea519SParav Pandit 2712081ea519SParav Pandit queue_work(cma_wq, &work->work); 2713981b5a23SParav Pandit } 2714981b5a23SParav Pandit 2715dbace111SLeon Romanovsky static int cma_resolve_ib_route(struct rdma_id_private *id_priv, 2716dbace111SLeon Romanovsky unsigned long timeout_ms) 2717e51060f0SSean Hefty { 2718e51060f0SSean Hefty struct rdma_route *route = &id_priv->id.route; 2719e51060f0SSean Hefty struct cma_work *work; 2720e51060f0SSean Hefty int ret; 2721e51060f0SSean Hefty 2722e51060f0SSean Hefty work = kzalloc(sizeof *work, GFP_KERNEL); 2723e51060f0SSean Hefty if (!work) 2724e51060f0SSean Hefty return -ENOMEM; 2725e51060f0SSean Hefty 2726981b5a23SParav Pandit cma_init_resolve_route_work(work, id_priv); 2727e51060f0SSean Hefty 2728e51060f0SSean Hefty route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL); 2729e51060f0SSean Hefty if (!route->path_rec) { 2730e51060f0SSean Hefty ret = -ENOMEM; 2731e51060f0SSean Hefty goto err1; 2732e51060f0SSean Hefty } 2733e51060f0SSean Hefty 2734e51060f0SSean Hefty ret = cma_query_ib_route(id_priv, timeout_ms, work); 2735e51060f0SSean Hefty if (ret) 2736e51060f0SSean Hefty goto err2; 2737e51060f0SSean Hefty 2738e51060f0SSean Hefty return 0; 2739e51060f0SSean Hefty err2: 2740e51060f0SSean Hefty kfree(route->path_rec); 2741e51060f0SSean Hefty route->path_rec = NULL; 2742e51060f0SSean Hefty err1: 2743e51060f0SSean Hefty kfree(work); 2744e51060f0SSean Hefty return ret; 2745e51060f0SSean Hefty } 2746e51060f0SSean Hefty 27479327c7afSParav Pandit static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type, 27489327c7afSParav Pandit unsigned long supported_gids, 27499327c7afSParav Pandit enum ib_gid_type default_gid) 27509327c7afSParav Pandit { 27519327c7afSParav Pandit if ((network_type == RDMA_NETWORK_IPV4 || 27529327c7afSParav Pandit network_type == RDMA_NETWORK_IPV6) && 27539327c7afSParav Pandit test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids)) 27549327c7afSParav Pandit return IB_GID_TYPE_ROCE_UDP_ENCAP; 27559327c7afSParav Pandit 27569327c7afSParav Pandit return default_gid; 27579327c7afSParav Pandit } 27589327c7afSParav Pandit 27599327c7afSParav Pandit /* 27609327c7afSParav Pandit * cma_iboe_set_path_rec_l2_fields() is helper function which sets 27619327c7afSParav Pandit * path record type based on GID type. 27629327c7afSParav Pandit * It also sets up other L2 fields which includes destination mac address 27639327c7afSParav Pandit * netdev ifindex, of the path record. 27649327c7afSParav Pandit * It returns the netdev of the bound interface for this path record entry. 27659327c7afSParav Pandit */ 27669327c7afSParav Pandit static struct net_device * 27679327c7afSParav Pandit cma_iboe_set_path_rec_l2_fields(struct rdma_id_private *id_priv) 27689327c7afSParav Pandit { 27699327c7afSParav Pandit struct rdma_route *route = &id_priv->id.route; 27709327c7afSParav Pandit enum ib_gid_type gid_type = IB_GID_TYPE_ROCE; 27719327c7afSParav Pandit struct rdma_addr *addr = &route->addr; 27729327c7afSParav Pandit unsigned long supported_gids; 27739327c7afSParav Pandit struct net_device *ndev; 27749327c7afSParav Pandit 27759327c7afSParav Pandit if (!addr->dev_addr.bound_dev_if) 27769327c7afSParav Pandit return NULL; 27779327c7afSParav Pandit 27789327c7afSParav Pandit ndev = dev_get_by_index(addr->dev_addr.net, 27799327c7afSParav Pandit addr->dev_addr.bound_dev_if); 27809327c7afSParav Pandit if (!ndev) 27819327c7afSParav Pandit return NULL; 27829327c7afSParav Pandit 27839327c7afSParav Pandit supported_gids = roce_gid_type_mask_support(id_priv->id.device, 27849327c7afSParav Pandit id_priv->id.port_num); 27859327c7afSParav Pandit gid_type = cma_route_gid_type(addr->dev_addr.network, 27869327c7afSParav Pandit supported_gids, 27879327c7afSParav Pandit id_priv->gid_type); 27889327c7afSParav Pandit /* Use the hint from IP Stack to select GID Type */ 27899327c7afSParav Pandit if (gid_type < ib_network_to_gid_type(addr->dev_addr.network)) 27909327c7afSParav Pandit gid_type = ib_network_to_gid_type(addr->dev_addr.network); 27919327c7afSParav Pandit route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type); 27929327c7afSParav Pandit 2793114cc9c4SParav Pandit route->path_rec->roce.route_resolved = true; 27949327c7afSParav Pandit sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr); 27959327c7afSParav Pandit return ndev; 27969327c7afSParav Pandit } 27979327c7afSParav Pandit 2798fe75889fSParav Pandit int rdma_set_ib_path(struct rdma_cm_id *id, 2799fe75889fSParav Pandit struct sa_path_rec *path_rec) 2800e51060f0SSean Hefty { 2801e51060f0SSean Hefty struct rdma_id_private *id_priv; 28028d20a1f0SParav Pandit struct net_device *ndev; 2803e51060f0SSean Hefty int ret; 2804e51060f0SSean Hefty 2805e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 2806550e5ca7SNir Muchtar if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, 2807550e5ca7SNir Muchtar RDMA_CM_ROUTE_RESOLVED)) 2808e51060f0SSean Hefty return -EINVAL; 2809e51060f0SSean Hefty 2810fe75889fSParav Pandit id->route.path_rec = kmemdup(path_rec, sizeof(*path_rec), 28119893e742SJulia Lawall GFP_KERNEL); 2812e51060f0SSean Hefty if (!id->route.path_rec) { 2813e51060f0SSean Hefty ret = -ENOMEM; 2814e51060f0SSean Hefty goto err; 2815e51060f0SSean Hefty } 2816e51060f0SSean Hefty 28178d20a1f0SParav Pandit if (rdma_protocol_roce(id->device, id->port_num)) { 28188d20a1f0SParav Pandit ndev = cma_iboe_set_path_rec_l2_fields(id_priv); 28198d20a1f0SParav Pandit if (!ndev) { 28208d20a1f0SParav Pandit ret = -ENODEV; 28218d20a1f0SParav Pandit goto err_free; 28228d20a1f0SParav Pandit } 28238d20a1f0SParav Pandit dev_put(ndev); 28248d20a1f0SParav Pandit } 28258d20a1f0SParav Pandit 2826fe75889fSParav Pandit id->route.num_paths = 1; 2827e51060f0SSean Hefty return 0; 28288d20a1f0SParav Pandit 28298d20a1f0SParav Pandit err_free: 28308d20a1f0SParav Pandit kfree(id->route.path_rec); 28318d20a1f0SParav Pandit id->route.path_rec = NULL; 2832e51060f0SSean Hefty err: 2833550e5ca7SNir Muchtar cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED); 2834e51060f0SSean Hefty return ret; 2835e51060f0SSean Hefty } 2836fe75889fSParav Pandit EXPORT_SYMBOL(rdma_set_ib_path); 2837e51060f0SSean Hefty 2838d6f91252SLeon Romanovsky static int cma_resolve_iw_route(struct rdma_id_private *id_priv) 283907ebafbaSTom Tucker { 284007ebafbaSTom Tucker struct cma_work *work; 284107ebafbaSTom Tucker 284207ebafbaSTom Tucker work = kzalloc(sizeof *work, GFP_KERNEL); 284307ebafbaSTom Tucker if (!work) 284407ebafbaSTom Tucker return -ENOMEM; 284507ebafbaSTom Tucker 2846981b5a23SParav Pandit cma_init_resolve_route_work(work, id_priv); 284707ebafbaSTom Tucker queue_work(cma_wq, &work->work); 284807ebafbaSTom Tucker return 0; 284907ebafbaSTom Tucker } 285007ebafbaSTom Tucker 2851d3bd9396SParav Pandit static int get_vlan_ndev_tc(struct net_device *vlan_ndev, int prio) 2852eb072c4bSEyal Perry { 2853eb072c4bSEyal Perry struct net_device *dev; 2854eb072c4bSEyal Perry 2855d3bd9396SParav Pandit dev = vlan_dev_real_dev(vlan_ndev); 2856eb072c4bSEyal Perry if (dev->num_tc) 2857eb072c4bSEyal Perry return netdev_get_prio_tc_map(dev, prio); 2858eb072c4bSEyal Perry 2859d3bd9396SParav Pandit return (vlan_dev_get_egress_qos_mask(vlan_ndev, prio) & 2860eb072c4bSEyal Perry VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 2861d3bd9396SParav Pandit } 2862d3bd9396SParav Pandit 2863d3bd9396SParav Pandit struct iboe_prio_tc_map { 2864d3bd9396SParav Pandit int input_prio; 2865d3bd9396SParav Pandit int output_tc; 2866d3bd9396SParav Pandit bool found; 2867d3bd9396SParav Pandit }; 2868d3bd9396SParav Pandit 2869d3bd9396SParav Pandit static int get_lower_vlan_dev_tc(struct net_device *dev, void *data) 2870d3bd9396SParav Pandit { 2871d3bd9396SParav Pandit struct iboe_prio_tc_map *map = data; 2872d3bd9396SParav Pandit 2873d3bd9396SParav Pandit if (is_vlan_dev(dev)) 2874d3bd9396SParav Pandit map->output_tc = get_vlan_ndev_tc(dev, map->input_prio); 2875d3bd9396SParav Pandit else if (dev->num_tc) 2876d3bd9396SParav Pandit map->output_tc = netdev_get_prio_tc_map(dev, map->input_prio); 2877d3bd9396SParav Pandit else 2878d3bd9396SParav Pandit map->output_tc = 0; 2879d3bd9396SParav Pandit /* We are interested only in first level VLAN device, so always 2880d3bd9396SParav Pandit * return 1 to stop iterating over next level devices. 2881d3bd9396SParav Pandit */ 2882d3bd9396SParav Pandit map->found = true; 2883d3bd9396SParav Pandit return 1; 2884d3bd9396SParav Pandit } 2885d3bd9396SParav Pandit 2886d3bd9396SParav Pandit static int iboe_tos_to_sl(struct net_device *ndev, int tos) 2887d3bd9396SParav Pandit { 2888d3bd9396SParav Pandit struct iboe_prio_tc_map prio_tc_map = {}; 2889d3bd9396SParav Pandit int prio = rt_tos2priority(tos); 2890d3bd9396SParav Pandit 2891d3bd9396SParav Pandit /* If VLAN device, get it directly from the VLAN netdev */ 2892d3bd9396SParav Pandit if (is_vlan_dev(ndev)) 2893d3bd9396SParav Pandit return get_vlan_ndev_tc(ndev, prio); 2894d3bd9396SParav Pandit 2895d3bd9396SParav Pandit prio_tc_map.input_prio = prio; 2896d3bd9396SParav Pandit rcu_read_lock(); 2897d3bd9396SParav Pandit netdev_walk_all_lower_dev_rcu(ndev, 2898d3bd9396SParav Pandit get_lower_vlan_dev_tc, 2899d3bd9396SParav Pandit &prio_tc_map); 2900d3bd9396SParav Pandit rcu_read_unlock(); 2901d3bd9396SParav Pandit /* If map is found from lower device, use it; Otherwise 2902d3bd9396SParav Pandit * continue with the current netdevice to get priority to tc map. 2903d3bd9396SParav Pandit */ 2904d3bd9396SParav Pandit if (prio_tc_map.found) 2905d3bd9396SParav Pandit return prio_tc_map.output_tc; 2906d3bd9396SParav Pandit else if (ndev->num_tc) 2907d3bd9396SParav Pandit return netdev_get_prio_tc_map(ndev, prio); 2908d3bd9396SParav Pandit else 2909eb072c4bSEyal Perry return 0; 2910eb072c4bSEyal Perry } 2911eb072c4bSEyal Perry 2912*f6653405SMark Zhang static __be32 cma_get_roce_udp_flow_label(struct rdma_id_private *id_priv) 2913*f6653405SMark Zhang { 2914*f6653405SMark Zhang struct sockaddr_in6 *addr6; 2915*f6653405SMark Zhang u16 dport, sport; 2916*f6653405SMark Zhang u32 hash, fl; 2917*f6653405SMark Zhang 2918*f6653405SMark Zhang addr6 = (struct sockaddr_in6 *)cma_src_addr(id_priv); 2919*f6653405SMark Zhang fl = be32_to_cpu(addr6->sin6_flowinfo) & IB_GRH_FLOWLABEL_MASK; 2920*f6653405SMark Zhang if ((cma_family(id_priv) != AF_INET6) || !fl) { 2921*f6653405SMark Zhang dport = be16_to_cpu(cma_port(cma_dst_addr(id_priv))); 2922*f6653405SMark Zhang sport = be16_to_cpu(cma_port(cma_src_addr(id_priv))); 2923*f6653405SMark Zhang hash = (u32)sport * 31 + dport; 2924*f6653405SMark Zhang fl = hash & IB_GRH_FLOWLABEL_MASK; 2925*f6653405SMark Zhang } 2926*f6653405SMark Zhang 2927*f6653405SMark Zhang return cpu_to_be32(fl); 2928*f6653405SMark Zhang } 2929*f6653405SMark Zhang 29303c86aa70SEli Cohen static int cma_resolve_iboe_route(struct rdma_id_private *id_priv) 29313c86aa70SEli Cohen { 29323c86aa70SEli Cohen struct rdma_route *route = &id_priv->id.route; 29333c86aa70SEli Cohen struct rdma_addr *addr = &route->addr; 29343c86aa70SEli Cohen struct cma_work *work; 29353c86aa70SEli Cohen int ret; 29364367ec7fSParav Pandit struct net_device *ndev; 29374367ec7fSParav Pandit 293889052d78SMajd Dibbiny u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num - 293989052d78SMajd Dibbiny rdma_start_port(id_priv->cma_dev->device)]; 294089052d78SMajd Dibbiny u8 tos = id_priv->tos_set ? id_priv->tos : default_roce_tos; 2941dd5f03beSMatan Barak 29423c86aa70SEli Cohen 29433c86aa70SEli Cohen work = kzalloc(sizeof *work, GFP_KERNEL); 29443c86aa70SEli Cohen if (!work) 29453c86aa70SEli Cohen return -ENOMEM; 29463c86aa70SEli Cohen 29473c86aa70SEli Cohen route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL); 29483c86aa70SEli Cohen if (!route->path_rec) { 29493c86aa70SEli Cohen ret = -ENOMEM; 29503c86aa70SEli Cohen goto err1; 29513c86aa70SEli Cohen } 29523c86aa70SEli Cohen 29533c86aa70SEli Cohen route->num_paths = 1; 29543c86aa70SEli Cohen 29559327c7afSParav Pandit ndev = cma_iboe_set_path_rec_l2_fields(id_priv); 295623d70503SWei Yongjun if (!ndev) { 295723d70503SWei Yongjun ret = -ENODEV; 295823d70503SWei Yongjun goto err2; 295923d70503SWei Yongjun } 296020029832SMatan Barak 29617b85627bSMoni Shoua rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 29627b85627bSMoni Shoua &route->path_rec->sgid); 29637b85627bSMoni Shoua rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr, 29647b85627bSMoni Shoua &route->path_rec->dgid); 2965af7bd463SEli Cohen 2966c3efe750SMatan Barak if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB) 2967c865f246SSomnath Kotur /* TODO: get the hoplimit from the inet/inet6 device */ 2968c3efe750SMatan Barak route->path_rec->hop_limit = addr->dev_addr.hoplimit; 2969c3efe750SMatan Barak else 2970af7bd463SEli Cohen route->path_rec->hop_limit = 1; 2971af7bd463SEli Cohen route->path_rec->reversible = 1; 2972af7bd463SEli Cohen route->path_rec->pkey = cpu_to_be16(0xffff); 2973af7bd463SEli Cohen route->path_rec->mtu_selector = IB_SA_EQ; 297489052d78SMajd Dibbiny route->path_rec->sl = iboe_tos_to_sl(ndev, tos); 297589052d78SMajd Dibbiny route->path_rec->traffic_class = tos; 29763c86aa70SEli Cohen route->path_rec->mtu = iboe_get_mtu(ndev->mtu); 29773c86aa70SEli Cohen route->path_rec->rate_selector = IB_SA_EQ; 29783c86aa70SEli Cohen route->path_rec->rate = iboe_get_rate(ndev); 29793c86aa70SEli Cohen dev_put(ndev); 29803c86aa70SEli Cohen route->path_rec->packet_life_time_selector = IB_SA_EQ; 2981e1ee1e62SDag Moxnes /* In case ACK timeout is set, use this value to calculate 2982e1ee1e62SDag Moxnes * PacketLifeTime. As per IBTA 12.7.34, 2983e1ee1e62SDag Moxnes * local ACK timeout = (2 * PacketLifeTime + Local CA’s ACK delay). 2984e1ee1e62SDag Moxnes * Assuming a negligible local ACK delay, we can use 2985e1ee1e62SDag Moxnes * PacketLifeTime = local ACK timeout/2 2986e1ee1e62SDag Moxnes * as a reasonable approximation for RoCE networks. 2987e1ee1e62SDag Moxnes */ 2988e1ee1e62SDag Moxnes route->path_rec->packet_life_time = id_priv->timeout_set ? 2989e1ee1e62SDag Moxnes id_priv->timeout - 1 : CMA_IBOE_PACKET_LIFETIME; 2990e1ee1e62SDag Moxnes 29913c86aa70SEli Cohen if (!route->path_rec->mtu) { 29923c86aa70SEli Cohen ret = -EINVAL; 29933c86aa70SEli Cohen goto err2; 29943c86aa70SEli Cohen } 29953c86aa70SEli Cohen 2996*f6653405SMark Zhang if (rdma_protocol_roce_udp_encap(id_priv->id.device, 2997*f6653405SMark Zhang id_priv->id.port_num)) 2998*f6653405SMark Zhang route->path_rec->flow_label = 2999*f6653405SMark Zhang cma_get_roce_udp_flow_label(id_priv); 3000*f6653405SMark Zhang 3001981b5a23SParav Pandit cma_init_resolve_route_work(work, id_priv); 30023c86aa70SEli Cohen queue_work(cma_wq, &work->work); 30033c86aa70SEli Cohen 30043c86aa70SEli Cohen return 0; 30053c86aa70SEli Cohen 30063c86aa70SEli Cohen err2: 30073c86aa70SEli Cohen kfree(route->path_rec); 30083c86aa70SEli Cohen route->path_rec = NULL; 3009987914abSAvihai Horon route->num_paths = 0; 30103c86aa70SEli Cohen err1: 30113c86aa70SEli Cohen kfree(work); 30123c86aa70SEli Cohen return ret; 30133c86aa70SEli Cohen } 30143c86aa70SEli Cohen 3015dbace111SLeon Romanovsky int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms) 3016e51060f0SSean Hefty { 3017e51060f0SSean Hefty struct rdma_id_private *id_priv; 3018e51060f0SSean Hefty int ret; 3019e51060f0SSean Hefty 3020e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 3021550e5ca7SNir Muchtar if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY)) 3022e51060f0SSean Hefty return -EINVAL; 3023e51060f0SSean Hefty 3024e368d23fSParav Pandit cma_id_get(id_priv); 3025fe53ba2fSMichael Wang if (rdma_cap_ib_sa(id->device, id->port_num)) 3026e51060f0SSean Hefty ret = cma_resolve_ib_route(id_priv, timeout_ms); 30275d9fb044SIra Weiny else if (rdma_protocol_roce(id->device, id->port_num)) 30283c86aa70SEli Cohen ret = cma_resolve_iboe_route(id_priv); 3029c72f2189SMichael Wang else if (rdma_protocol_iwarp(id->device, id->port_num)) 3030d6f91252SLeon Romanovsky ret = cma_resolve_iw_route(id_priv); 3031c72f2189SMichael Wang else 3032e51060f0SSean Hefty ret = -ENOSYS; 3033c72f2189SMichael Wang 3034e51060f0SSean Hefty if (ret) 3035e51060f0SSean Hefty goto err; 3036e51060f0SSean Hefty 3037e51060f0SSean Hefty return 0; 3038e51060f0SSean Hefty err: 3039550e5ca7SNir Muchtar cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED); 3040e368d23fSParav Pandit cma_id_put(id_priv); 3041e51060f0SSean Hefty return ret; 3042e51060f0SSean Hefty } 3043e51060f0SSean Hefty EXPORT_SYMBOL(rdma_resolve_route); 3044e51060f0SSean Hefty 30456a3e362dSSean Hefty static void cma_set_loopback(struct sockaddr *addr) 30466a3e362dSSean Hefty { 30476a3e362dSSean Hefty switch (addr->sa_family) { 30486a3e362dSSean Hefty case AF_INET: 30496a3e362dSSean Hefty ((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK); 30506a3e362dSSean Hefty break; 30516a3e362dSSean Hefty case AF_INET6: 30526a3e362dSSean Hefty ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr, 30536a3e362dSSean Hefty 0, 0, 0, htonl(1)); 30546a3e362dSSean Hefty break; 30556a3e362dSSean Hefty default: 30566a3e362dSSean Hefty ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr, 30576a3e362dSSean Hefty 0, 0, 0, htonl(1)); 30586a3e362dSSean Hefty break; 30596a3e362dSSean Hefty } 30606a3e362dSSean Hefty } 30616a3e362dSSean Hefty 3062e51060f0SSean Hefty static int cma_bind_loopback(struct rdma_id_private *id_priv) 3063e51060f0SSean Hefty { 3064b0569e40SSean Hefty struct cma_device *cma_dev, *cur_dev; 3065f0ee3404SMichael S. Tsirkin union ib_gid gid; 3066102c5ce0SJack Wang enum ib_port_state port_state; 3067cc055dd3SParav Pandit unsigned int p; 3068e51060f0SSean Hefty u16 pkey; 3069e51060f0SSean Hefty int ret; 3070e51060f0SSean Hefty 3071b0569e40SSean Hefty cma_dev = NULL; 3072e51060f0SSean Hefty mutex_lock(&lock); 3073b0569e40SSean Hefty list_for_each_entry(cur_dev, &dev_list, list) { 3074b0569e40SSean Hefty if (cma_family(id_priv) == AF_IB && 307572219ceaSMichael Wang !rdma_cap_ib_cm(cur_dev->device, 1)) 3076b0569e40SSean Hefty continue; 3077b0569e40SSean Hefty 3078b0569e40SSean Hefty if (!cma_dev) 3079b0569e40SSean Hefty cma_dev = cur_dev; 3080b0569e40SSean Hefty 3081cc055dd3SParav Pandit rdma_for_each_port (cur_dev->device, p) { 3082102c5ce0SJack Wang if (!ib_get_cached_port_state(cur_dev->device, p, &port_state) && 3083102c5ce0SJack Wang port_state == IB_PORT_ACTIVE) { 3084b0569e40SSean Hefty cma_dev = cur_dev; 3085b0569e40SSean Hefty goto port_found; 3086b0569e40SSean Hefty } 3087b0569e40SSean Hefty } 3088b0569e40SSean Hefty } 3089b0569e40SSean Hefty 3090b0569e40SSean Hefty if (!cma_dev) { 3091e82153b5SKrishna Kumar ret = -ENODEV; 3092e82153b5SKrishna Kumar goto out; 3093e82153b5SKrishna Kumar } 3094e51060f0SSean Hefty 3095e51060f0SSean Hefty p = 1; 3096e51060f0SSean Hefty 3097e51060f0SSean Hefty port_found: 30981dfce294SParav Pandit ret = rdma_query_gid(cma_dev->device, p, 0, &gid); 3099e51060f0SSean Hefty if (ret) 3100e51060f0SSean Hefty goto out; 3101e51060f0SSean Hefty 3102e51060f0SSean Hefty ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey); 3103e51060f0SSean Hefty if (ret) 3104e51060f0SSean Hefty goto out; 3105e51060f0SSean Hefty 31066f8372b6SSean Hefty id_priv->id.route.addr.dev_addr.dev_type = 310721655afcSMichael Wang (rdma_protocol_ib(cma_dev->device, p)) ? 31086f8372b6SSean Hefty ARPHRD_INFINIBAND : ARPHRD_ETHER; 31096f8372b6SSean Hefty 31106f8372b6SSean Hefty rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid); 3111e51060f0SSean Hefty ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey); 3112e51060f0SSean Hefty id_priv->id.port_num = p; 3113e51060f0SSean Hefty cma_attach_to_dev(id_priv, cma_dev); 3114f4753834SSean Hefty cma_set_loopback(cma_src_addr(id_priv)); 3115e51060f0SSean Hefty out: 3116e51060f0SSean Hefty mutex_unlock(&lock); 3117e51060f0SSean Hefty return ret; 3118e51060f0SSean Hefty } 3119e51060f0SSean Hefty 3120e51060f0SSean Hefty static void addr_handler(int status, struct sockaddr *src_addr, 3121e51060f0SSean Hefty struct rdma_dev_addr *dev_addr, void *context) 3122e51060f0SSean Hefty { 3123e51060f0SSean Hefty struct rdma_id_private *id_priv = context; 31247582df82SParav Pandit struct rdma_cm_event event = {}; 31255fc01fb8SMyungho Jung struct sockaddr *addr; 31265fc01fb8SMyungho Jung struct sockaddr_storage old_addr; 3127e51060f0SSean Hefty 3128de910bd9SOr Gerlitz mutex_lock(&id_priv->handler_mutex); 3129550e5ca7SNir Muchtar if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, 3130550e5ca7SNir Muchtar RDMA_CM_ADDR_RESOLVED)) 313161a73c70SSean Hefty goto out; 313261a73c70SSean Hefty 31335fc01fb8SMyungho Jung /* 31345fc01fb8SMyungho Jung * Store the previous src address, so that if we fail to acquire 31355fc01fb8SMyungho Jung * matching rdma device, old address can be restored back, which helps 31365fc01fb8SMyungho Jung * to cancel the cma listen operation correctly. 31375fc01fb8SMyungho Jung */ 31385fc01fb8SMyungho Jung addr = cma_src_addr(id_priv); 31395fc01fb8SMyungho Jung memcpy(&old_addr, addr, rdma_addr_size(addr)); 31405fc01fb8SMyungho Jung memcpy(addr, src_addr, rdma_addr_size(src_addr)); 3141498683c6SMoni Shoua if (!status && !id_priv->cma_dev) { 3142ff11c6cdSParav Pandit status = cma_acquire_dev_by_src_ip(id_priv); 3143498683c6SMoni Shoua if (status) 3144498683c6SMoni Shoua pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n", 3145498683c6SMoni Shoua status); 3146a6e4d254SHåkon Bugge } else if (status) { 3147498683c6SMoni Shoua pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status); 3148498683c6SMoni Shoua } 3149e51060f0SSean Hefty 3150e51060f0SSean Hefty if (status) { 31515fc01fb8SMyungho Jung memcpy(addr, &old_addr, 31525fc01fb8SMyungho Jung rdma_addr_size((struct sockaddr *)&old_addr)); 3153550e5ca7SNir Muchtar if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, 3154550e5ca7SNir Muchtar RDMA_CM_ADDR_BOUND)) 3155e51060f0SSean Hefty goto out; 3156a1b1b61fSSean Hefty event.event = RDMA_CM_EVENT_ADDR_ERROR; 3157a1b1b61fSSean Hefty event.status = status; 31587b85627bSMoni Shoua } else 3159a1b1b61fSSean Hefty event.event = RDMA_CM_EVENT_ADDR_RESOLVED; 3160e51060f0SSean Hefty 3161ed999f82SChuck Lever if (cma_cm_event_handler(id_priv, &event)) { 3162550e5ca7SNir Muchtar cma_exch(id_priv, RDMA_CM_DESTROYING); 3163de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 3164e51060f0SSean Hefty rdma_destroy_id(&id_priv->id); 3165e51060f0SSean Hefty return; 3166e51060f0SSean Hefty } 3167e51060f0SSean Hefty out: 3168de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 3169e51060f0SSean Hefty } 3170e51060f0SSean Hefty 3171e51060f0SSean Hefty static int cma_resolve_loopback(struct rdma_id_private *id_priv) 3172e51060f0SSean Hefty { 3173e51060f0SSean Hefty struct cma_work *work; 3174f0ee3404SMichael S. Tsirkin union ib_gid gid; 3175e51060f0SSean Hefty int ret; 3176e51060f0SSean Hefty 3177e51060f0SSean Hefty work = kzalloc(sizeof *work, GFP_KERNEL); 3178e51060f0SSean Hefty if (!work) 3179e51060f0SSean Hefty return -ENOMEM; 3180e51060f0SSean Hefty 3181e51060f0SSean Hefty if (!id_priv->cma_dev) { 3182e51060f0SSean Hefty ret = cma_bind_loopback(id_priv); 3183e51060f0SSean Hefty if (ret) 3184e51060f0SSean Hefty goto err; 3185e51060f0SSean Hefty } 3186e51060f0SSean Hefty 31876f8372b6SSean Hefty rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid); 31886f8372b6SSean Hefty rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid); 3189e51060f0SSean Hefty 3190081ea519SParav Pandit enqueue_resolve_addr_work(work, id_priv); 3191e51060f0SSean Hefty return 0; 3192e51060f0SSean Hefty err: 3193e51060f0SSean Hefty kfree(work); 3194e51060f0SSean Hefty return ret; 3195e51060f0SSean Hefty } 3196e51060f0SSean Hefty 3197f17df3b0SSean Hefty static int cma_resolve_ib_addr(struct rdma_id_private *id_priv) 3198f17df3b0SSean Hefty { 3199f17df3b0SSean Hefty struct cma_work *work; 3200f17df3b0SSean Hefty int ret; 3201f17df3b0SSean Hefty 3202f17df3b0SSean Hefty work = kzalloc(sizeof *work, GFP_KERNEL); 3203f17df3b0SSean Hefty if (!work) 3204f17df3b0SSean Hefty return -ENOMEM; 3205f17df3b0SSean Hefty 3206f17df3b0SSean Hefty if (!id_priv->cma_dev) { 3207f17df3b0SSean Hefty ret = cma_resolve_ib_dev(id_priv); 3208f17df3b0SSean Hefty if (ret) 3209f17df3b0SSean Hefty goto err; 3210f17df3b0SSean Hefty } 3211f17df3b0SSean Hefty 3212f17df3b0SSean Hefty rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *) 3213f17df3b0SSean Hefty &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr)); 3214f17df3b0SSean Hefty 3215081ea519SParav Pandit enqueue_resolve_addr_work(work, id_priv); 3216f17df3b0SSean Hefty return 0; 3217f17df3b0SSean Hefty err: 3218f17df3b0SSean Hefty kfree(work); 3219f17df3b0SSean Hefty return ret; 3220f17df3b0SSean Hefty } 3221f17df3b0SSean Hefty 3222e51060f0SSean Hefty static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 32232df7dba8SParav Pandit const struct sockaddr *dst_addr) 3224e51060f0SSean Hefty { 3225d14714dfSSean Hefty if (!src_addr || !src_addr->sa_family) { 3226d14714dfSSean Hefty src_addr = (struct sockaddr *) &id->route.addr.src_addr; 3227f17df3b0SSean Hefty src_addr->sa_family = dst_addr->sa_family; 3228b4cfe397SJack Morgenstein if (IS_ENABLED(CONFIG_IPV6) && 3229b4cfe397SJack Morgenstein dst_addr->sa_family == AF_INET6) { 32306c26a771SSpencer Baugh struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *) src_addr; 32316c26a771SSpencer Baugh struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *) dst_addr; 32326c26a771SSpencer Baugh src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id; 32336c26a771SSpencer Baugh if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL) 32346c26a771SSpencer Baugh id->route.addr.dev_addr.bound_dev_if = dst_addr6->sin6_scope_id; 3235f17df3b0SSean Hefty } else if (dst_addr->sa_family == AF_IB) { 3236f17df3b0SSean Hefty ((struct sockaddr_ib *) src_addr)->sib_pkey = 3237f17df3b0SSean Hefty ((struct sockaddr_ib *) dst_addr)->sib_pkey; 3238d14714dfSSean Hefty } 3239d14714dfSSean Hefty } 3240e51060f0SSean Hefty return rdma_bind_addr(id, src_addr); 3241e51060f0SSean Hefty } 3242e51060f0SSean Hefty 3243e51060f0SSean Hefty int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr, 3244dbace111SLeon Romanovsky const struct sockaddr *dst_addr, unsigned long timeout_ms) 3245e51060f0SSean Hefty { 3246e51060f0SSean Hefty struct rdma_id_private *id_priv; 3247e51060f0SSean Hefty int ret; 3248e51060f0SSean Hefty 3249e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 3250e4103312SParav Pandit memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr)); 3251550e5ca7SNir Muchtar if (id_priv->state == RDMA_CM_IDLE) { 3252e51060f0SSean Hefty ret = cma_bind_addr(id, src_addr, dst_addr); 3253e4103312SParav Pandit if (ret) { 3254e4103312SParav Pandit memset(cma_dst_addr(id_priv), 0, 3255e4103312SParav Pandit rdma_addr_size(dst_addr)); 3256e51060f0SSean Hefty return ret; 3257e51060f0SSean Hefty } 3258e4103312SParav Pandit } 3259e51060f0SSean Hefty 3260e4103312SParav Pandit if (cma_family(id_priv) != dst_addr->sa_family) { 3261e4103312SParav Pandit memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr)); 32624ae7152eSSean Hefty return -EINVAL; 3263e4103312SParav Pandit } 32644ae7152eSSean Hefty 3265e4103312SParav Pandit if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) { 3266e4103312SParav Pandit memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr)); 3267e51060f0SSean Hefty return -EINVAL; 3268e4103312SParav Pandit } 3269e51060f0SSean Hefty 3270f17df3b0SSean Hefty if (cma_any_addr(dst_addr)) { 3271e51060f0SSean Hefty ret = cma_resolve_loopback(id_priv); 3272f17df3b0SSean Hefty } else { 3273f17df3b0SSean Hefty if (dst_addr->sa_family == AF_IB) { 3274f17df3b0SSean Hefty ret = cma_resolve_ib_addr(id_priv); 3275f17df3b0SSean Hefty } else { 32760e9d2c19SParav Pandit ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr, 32770e9d2c19SParav Pandit &id->route.addr.dev_addr, 32780e9d2c19SParav Pandit timeout_ms, addr_handler, 32790e9d2c19SParav Pandit false, id_priv); 3280f17df3b0SSean Hefty } 3281f17df3b0SSean Hefty } 3282e51060f0SSean Hefty if (ret) 3283e51060f0SSean Hefty goto err; 3284e51060f0SSean Hefty 3285e51060f0SSean Hefty return 0; 3286e51060f0SSean Hefty err: 3287550e5ca7SNir Muchtar cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND); 3288e51060f0SSean Hefty return ret; 3289e51060f0SSean Hefty } 3290e51060f0SSean Hefty EXPORT_SYMBOL(rdma_resolve_addr); 3291e51060f0SSean Hefty 3292a9bb7912SHefty, Sean int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse) 3293a9bb7912SHefty, Sean { 3294a9bb7912SHefty, Sean struct rdma_id_private *id_priv; 3295a9bb7912SHefty, Sean unsigned long flags; 3296a9bb7912SHefty, Sean int ret; 3297a9bb7912SHefty, Sean 3298a9bb7912SHefty, Sean id_priv = container_of(id, struct rdma_id_private, id); 3299a9bb7912SHefty, Sean spin_lock_irqsave(&id_priv->lock, flags); 3300c8dea2f9SSean Hefty if (reuse || id_priv->state == RDMA_CM_IDLE) { 3301a9bb7912SHefty, Sean id_priv->reuseaddr = reuse; 3302a9bb7912SHefty, Sean ret = 0; 3303a9bb7912SHefty, Sean } else { 3304a9bb7912SHefty, Sean ret = -EINVAL; 3305a9bb7912SHefty, Sean } 3306a9bb7912SHefty, Sean spin_unlock_irqrestore(&id_priv->lock, flags); 3307a9bb7912SHefty, Sean return ret; 3308a9bb7912SHefty, Sean } 3309a9bb7912SHefty, Sean EXPORT_SYMBOL(rdma_set_reuseaddr); 3310a9bb7912SHefty, Sean 331168602120SSean Hefty int rdma_set_afonly(struct rdma_cm_id *id, int afonly) 331268602120SSean Hefty { 331368602120SSean Hefty struct rdma_id_private *id_priv; 331468602120SSean Hefty unsigned long flags; 331568602120SSean Hefty int ret; 331668602120SSean Hefty 331768602120SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 331868602120SSean Hefty spin_lock_irqsave(&id_priv->lock, flags); 331968602120SSean Hefty if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) { 332068602120SSean Hefty id_priv->options |= (1 << CMA_OPTION_AFONLY); 332168602120SSean Hefty id_priv->afonly = afonly; 332268602120SSean Hefty ret = 0; 332368602120SSean Hefty } else { 332468602120SSean Hefty ret = -EINVAL; 332568602120SSean Hefty } 332668602120SSean Hefty spin_unlock_irqrestore(&id_priv->lock, flags); 332768602120SSean Hefty return ret; 332868602120SSean Hefty } 332968602120SSean Hefty EXPORT_SYMBOL(rdma_set_afonly); 333068602120SSean Hefty 3331e51060f0SSean Hefty static void cma_bind_port(struct rdma_bind_list *bind_list, 3332e51060f0SSean Hefty struct rdma_id_private *id_priv) 3333e51060f0SSean Hefty { 333458afdcb7SSean Hefty struct sockaddr *addr; 333558afdcb7SSean Hefty struct sockaddr_ib *sib; 333658afdcb7SSean Hefty u64 sid, mask; 333758afdcb7SSean Hefty __be16 port; 3338e51060f0SSean Hefty 3339f4753834SSean Hefty addr = cma_src_addr(id_priv); 334058afdcb7SSean Hefty port = htons(bind_list->port); 334158afdcb7SSean Hefty 334258afdcb7SSean Hefty switch (addr->sa_family) { 334358afdcb7SSean Hefty case AF_INET: 334458afdcb7SSean Hefty ((struct sockaddr_in *) addr)->sin_port = port; 334558afdcb7SSean Hefty break; 334658afdcb7SSean Hefty case AF_INET6: 334758afdcb7SSean Hefty ((struct sockaddr_in6 *) addr)->sin6_port = port; 334858afdcb7SSean Hefty break; 334958afdcb7SSean Hefty case AF_IB: 335058afdcb7SSean Hefty sib = (struct sockaddr_ib *) addr; 335158afdcb7SSean Hefty sid = be64_to_cpu(sib->sib_sid); 335258afdcb7SSean Hefty mask = be64_to_cpu(sib->sib_sid_mask); 335358afdcb7SSean Hefty sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port)); 335458afdcb7SSean Hefty sib->sib_sid_mask = cpu_to_be64(~0ULL); 335558afdcb7SSean Hefty break; 335658afdcb7SSean Hefty } 3357e51060f0SSean Hefty id_priv->bind_list = bind_list; 3358e51060f0SSean Hefty hlist_add_head(&id_priv->node, &bind_list->owners); 3359e51060f0SSean Hefty } 3360e51060f0SSean Hefty 33612253fc0cSSteve Wise static int cma_alloc_port(enum rdma_ucm_port_space ps, 3362aac978e1SHaggai Eran struct rdma_id_private *id_priv, unsigned short snum) 3363e51060f0SSean Hefty { 3364e51060f0SSean Hefty struct rdma_bind_list *bind_list; 33653b069c5dSTejun Heo int ret; 3366e51060f0SSean Hefty 3367cb164b8cSSean Hefty bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL); 3368e51060f0SSean Hefty if (!bind_list) 3369e51060f0SSean Hefty return -ENOMEM; 3370e51060f0SSean Hefty 3371fa20105eSGuy Shapiro ret = cma_ps_alloc(id_priv->id.route.addr.dev_addr.net, ps, bind_list, 3372fa20105eSGuy Shapiro snum); 33733b069c5dSTejun Heo if (ret < 0) 33743b069c5dSTejun Heo goto err; 3375e51060f0SSean Hefty 3376e51060f0SSean Hefty bind_list->ps = ps; 3377061ccb52SLeon Romanovsky bind_list->port = snum; 3378e51060f0SSean Hefty cma_bind_port(bind_list, id_priv); 3379e51060f0SSean Hefty return 0; 33803b069c5dSTejun Heo err: 3381aedec080SSean Hefty kfree(bind_list); 33823b069c5dSTejun Heo return ret == -ENOSPC ? -EADDRNOTAVAIL : ret; 3383aedec080SSean Hefty } 3384aedec080SSean Hefty 338519b752a1SMoni Shoua static int cma_port_is_unique(struct rdma_bind_list *bind_list, 338619b752a1SMoni Shoua struct rdma_id_private *id_priv) 338719b752a1SMoni Shoua { 338819b752a1SMoni Shoua struct rdma_id_private *cur_id; 338919b752a1SMoni Shoua struct sockaddr *daddr = cma_dst_addr(id_priv); 339019b752a1SMoni Shoua struct sockaddr *saddr = cma_src_addr(id_priv); 339119b752a1SMoni Shoua __be16 dport = cma_port(daddr); 339219b752a1SMoni Shoua 339319b752a1SMoni Shoua hlist_for_each_entry(cur_id, &bind_list->owners, node) { 339419b752a1SMoni Shoua struct sockaddr *cur_daddr = cma_dst_addr(cur_id); 339519b752a1SMoni Shoua struct sockaddr *cur_saddr = cma_src_addr(cur_id); 339619b752a1SMoni Shoua __be16 cur_dport = cma_port(cur_daddr); 339719b752a1SMoni Shoua 339819b752a1SMoni Shoua if (id_priv == cur_id) 339919b752a1SMoni Shoua continue; 340019b752a1SMoni Shoua 340119b752a1SMoni Shoua /* different dest port -> unique */ 34029dea9a2fSTatyana Nikolova if (!cma_any_port(daddr) && 34039dea9a2fSTatyana Nikolova !cma_any_port(cur_daddr) && 340419b752a1SMoni Shoua (dport != cur_dport)) 340519b752a1SMoni Shoua continue; 340619b752a1SMoni Shoua 340719b752a1SMoni Shoua /* different src address -> unique */ 340819b752a1SMoni Shoua if (!cma_any_addr(saddr) && 340919b752a1SMoni Shoua !cma_any_addr(cur_saddr) && 341019b752a1SMoni Shoua cma_addr_cmp(saddr, cur_saddr)) 341119b752a1SMoni Shoua continue; 341219b752a1SMoni Shoua 341319b752a1SMoni Shoua /* different dst address -> unique */ 34149dea9a2fSTatyana Nikolova if (!cma_any_addr(daddr) && 34159dea9a2fSTatyana Nikolova !cma_any_addr(cur_daddr) && 341619b752a1SMoni Shoua cma_addr_cmp(daddr, cur_daddr)) 341719b752a1SMoni Shoua continue; 341819b752a1SMoni Shoua 341919b752a1SMoni Shoua return -EADDRNOTAVAIL; 342019b752a1SMoni Shoua } 342119b752a1SMoni Shoua return 0; 342219b752a1SMoni Shoua } 342319b752a1SMoni Shoua 34242253fc0cSSteve Wise static int cma_alloc_any_port(enum rdma_ucm_port_space ps, 3425aac978e1SHaggai Eran struct rdma_id_private *id_priv) 3426aedec080SSean Hefty { 34275d7220e8STetsuo Handa static unsigned int last_used_port; 34285d7220e8STetsuo Handa int low, high, remaining; 34295d7220e8STetsuo Handa unsigned int rover; 3430fa20105eSGuy Shapiro struct net *net = id_priv->id.route.addr.dev_addr.net; 3431aedec080SSean Hefty 3432fa20105eSGuy Shapiro inet_get_local_port_range(net, &low, &high); 34335d7220e8STetsuo Handa remaining = (high - low) + 1; 343463862b5bSAruna-Hewapathirane rover = prandom_u32() % remaining + low; 34355d7220e8STetsuo Handa retry: 343619b752a1SMoni Shoua if (last_used_port != rover) { 343719b752a1SMoni Shoua struct rdma_bind_list *bind_list; 343819b752a1SMoni Shoua int ret; 343919b752a1SMoni Shoua 344019b752a1SMoni Shoua bind_list = cma_ps_find(net, ps, (unsigned short)rover); 344119b752a1SMoni Shoua 344219b752a1SMoni Shoua if (!bind_list) { 344319b752a1SMoni Shoua ret = cma_alloc_port(ps, id_priv, rover); 344419b752a1SMoni Shoua } else { 344519b752a1SMoni Shoua ret = cma_port_is_unique(bind_list, id_priv); 344619b752a1SMoni Shoua if (!ret) 344719b752a1SMoni Shoua cma_bind_port(bind_list, id_priv); 344819b752a1SMoni Shoua } 34495d7220e8STetsuo Handa /* 34505d7220e8STetsuo Handa * Remember previously used port number in order to avoid 34515d7220e8STetsuo Handa * re-using same port immediately after it is closed. 34525d7220e8STetsuo Handa */ 34535d7220e8STetsuo Handa if (!ret) 34545d7220e8STetsuo Handa last_used_port = rover; 34555d7220e8STetsuo Handa if (ret != -EADDRNOTAVAIL) 34565d7220e8STetsuo Handa return ret; 34575d7220e8STetsuo Handa } 34585d7220e8STetsuo Handa if (--remaining) { 34595d7220e8STetsuo Handa rover++; 34605d7220e8STetsuo Handa if ((rover < low) || (rover > high)) 34615d7220e8STetsuo Handa rover = low; 3462aedec080SSean Hefty goto retry; 3463aedec080SSean Hefty } 34645d7220e8STetsuo Handa return -EADDRNOTAVAIL; 3465e51060f0SSean Hefty } 3466e51060f0SSean Hefty 3467a9bb7912SHefty, Sean /* 3468a9bb7912SHefty, Sean * Check that the requested port is available. This is called when trying to 3469a9bb7912SHefty, Sean * bind to a specific port, or when trying to listen on a bound port. In 3470a9bb7912SHefty, Sean * the latter case, the provided id_priv may already be on the bind_list, but 3471a9bb7912SHefty, Sean * we still need to check that it's okay to start listening. 3472a9bb7912SHefty, Sean */ 3473a9bb7912SHefty, Sean static int cma_check_port(struct rdma_bind_list *bind_list, 3474a9bb7912SHefty, Sean struct rdma_id_private *id_priv, uint8_t reuseaddr) 3475e51060f0SSean Hefty { 3476e51060f0SSean Hefty struct rdma_id_private *cur_id; 347743b752daSHefty, Sean struct sockaddr *addr, *cur_addr; 3478e51060f0SSean Hefty 3479f4753834SSean Hefty addr = cma_src_addr(id_priv); 3480b67bfe0dSSasha Levin hlist_for_each_entry(cur_id, &bind_list->owners, node) { 3481a9bb7912SHefty, Sean if (id_priv == cur_id) 3482a9bb7912SHefty, Sean continue; 3483a9bb7912SHefty, Sean 34845b0ec991SSean Hefty if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr && 34855b0ec991SSean Hefty cur_id->reuseaddr) 34865b0ec991SSean Hefty continue; 34875b0ec991SSean Hefty 3488f4753834SSean Hefty cur_addr = cma_src_addr(cur_id); 34895b0ec991SSean Hefty if (id_priv->afonly && cur_id->afonly && 34905b0ec991SSean Hefty (addr->sa_family != cur_addr->sa_family)) 34915b0ec991SSean Hefty continue; 34925b0ec991SSean Hefty 34935b0ec991SSean Hefty if (cma_any_addr(addr) || cma_any_addr(cur_addr)) 3494e51060f0SSean Hefty return -EADDRNOTAVAIL; 3495e51060f0SSean Hefty 349643b752daSHefty, Sean if (!cma_addr_cmp(addr, cur_addr)) 3497e51060f0SSean Hefty return -EADDRINUSE; 3498e51060f0SSean Hefty } 3499e51060f0SSean Hefty return 0; 3500e51060f0SSean Hefty } 3501e51060f0SSean Hefty 35022253fc0cSSteve Wise static int cma_use_port(enum rdma_ucm_port_space ps, 3503aac978e1SHaggai Eran struct rdma_id_private *id_priv) 3504a9bb7912SHefty, Sean { 3505a9bb7912SHefty, Sean struct rdma_bind_list *bind_list; 3506a9bb7912SHefty, Sean unsigned short snum; 3507a9bb7912SHefty, Sean int ret; 3508a9bb7912SHefty, Sean 3509f4753834SSean Hefty snum = ntohs(cma_port(cma_src_addr(id_priv))); 3510a9bb7912SHefty, Sean if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE)) 3511a9bb7912SHefty, Sean return -EACCES; 3512a9bb7912SHefty, Sean 3513fa20105eSGuy Shapiro bind_list = cma_ps_find(id_priv->id.route.addr.dev_addr.net, ps, snum); 3514a9bb7912SHefty, Sean if (!bind_list) { 3515a9bb7912SHefty, Sean ret = cma_alloc_port(ps, id_priv, snum); 3516a9bb7912SHefty, Sean } else { 3517a9bb7912SHefty, Sean ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr); 3518a9bb7912SHefty, Sean if (!ret) 3519a9bb7912SHefty, Sean cma_bind_port(bind_list, id_priv); 3520a9bb7912SHefty, Sean } 3521a9bb7912SHefty, Sean return ret; 3522a9bb7912SHefty, Sean } 3523a9bb7912SHefty, Sean 3524a9bb7912SHefty, Sean static int cma_bind_listen(struct rdma_id_private *id_priv) 3525a9bb7912SHefty, Sean { 3526a9bb7912SHefty, Sean struct rdma_bind_list *bind_list = id_priv->bind_list; 3527a9bb7912SHefty, Sean int ret = 0; 3528a9bb7912SHefty, Sean 3529a9bb7912SHefty, Sean mutex_lock(&lock); 3530a9bb7912SHefty, Sean if (bind_list->owners.first->next) 3531a9bb7912SHefty, Sean ret = cma_check_port(bind_list, id_priv, 0); 3532a9bb7912SHefty, Sean mutex_unlock(&lock); 3533a9bb7912SHefty, Sean return ret; 3534a9bb7912SHefty, Sean } 3535a9bb7912SHefty, Sean 35362253fc0cSSteve Wise static enum rdma_ucm_port_space 35372253fc0cSSteve Wise cma_select_inet_ps(struct rdma_id_private *id_priv) 353858afdcb7SSean Hefty { 353958afdcb7SSean Hefty switch (id_priv->id.ps) { 354058afdcb7SSean Hefty case RDMA_PS_TCP: 354158afdcb7SSean Hefty case RDMA_PS_UDP: 354258afdcb7SSean Hefty case RDMA_PS_IPOIB: 354358afdcb7SSean Hefty case RDMA_PS_IB: 3544aac978e1SHaggai Eran return id_priv->id.ps; 354558afdcb7SSean Hefty default: 3546aac978e1SHaggai Eran 3547aac978e1SHaggai Eran return 0; 354858afdcb7SSean Hefty } 354958afdcb7SSean Hefty } 355058afdcb7SSean Hefty 35512253fc0cSSteve Wise static enum rdma_ucm_port_space 35522253fc0cSSteve Wise cma_select_ib_ps(struct rdma_id_private *id_priv) 355358afdcb7SSean Hefty { 35542253fc0cSSteve Wise enum rdma_ucm_port_space ps = 0; 355558afdcb7SSean Hefty struct sockaddr_ib *sib; 355658afdcb7SSean Hefty u64 sid_ps, mask, sid; 355758afdcb7SSean Hefty 3558f4753834SSean Hefty sib = (struct sockaddr_ib *) cma_src_addr(id_priv); 355958afdcb7SSean Hefty mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK; 356058afdcb7SSean Hefty sid = be64_to_cpu(sib->sib_sid) & mask; 356158afdcb7SSean Hefty 356258afdcb7SSean Hefty if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) { 356358afdcb7SSean Hefty sid_ps = RDMA_IB_IP_PS_IB; 3564aac978e1SHaggai Eran ps = RDMA_PS_IB; 356558afdcb7SSean Hefty } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) && 356658afdcb7SSean Hefty (sid == (RDMA_IB_IP_PS_TCP & mask))) { 356758afdcb7SSean Hefty sid_ps = RDMA_IB_IP_PS_TCP; 3568aac978e1SHaggai Eran ps = RDMA_PS_TCP; 356958afdcb7SSean Hefty } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) && 357058afdcb7SSean Hefty (sid == (RDMA_IB_IP_PS_UDP & mask))) { 357158afdcb7SSean Hefty sid_ps = RDMA_IB_IP_PS_UDP; 3572aac978e1SHaggai Eran ps = RDMA_PS_UDP; 357358afdcb7SSean Hefty } 357458afdcb7SSean Hefty 357558afdcb7SSean Hefty if (ps) { 357658afdcb7SSean Hefty sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib))); 357758afdcb7SSean Hefty sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK | 357858afdcb7SSean Hefty be64_to_cpu(sib->sib_sid_mask)); 357958afdcb7SSean Hefty } 358058afdcb7SSean Hefty return ps; 358158afdcb7SSean Hefty } 358258afdcb7SSean Hefty 3583e51060f0SSean Hefty static int cma_get_port(struct rdma_id_private *id_priv) 3584e51060f0SSean Hefty { 35852253fc0cSSteve Wise enum rdma_ucm_port_space ps; 3586e51060f0SSean Hefty int ret; 3587e51060f0SSean Hefty 3588f4753834SSean Hefty if (cma_family(id_priv) != AF_IB) 358958afdcb7SSean Hefty ps = cma_select_inet_ps(id_priv); 359058afdcb7SSean Hefty else 359158afdcb7SSean Hefty ps = cma_select_ib_ps(id_priv); 359258afdcb7SSean Hefty if (!ps) 3593e51060f0SSean Hefty return -EPROTONOSUPPORT; 3594e51060f0SSean Hefty 3595e51060f0SSean Hefty mutex_lock(&lock); 3596f4753834SSean Hefty if (cma_any_port(cma_src_addr(id_priv))) 3597aedec080SSean Hefty ret = cma_alloc_any_port(ps, id_priv); 3598e51060f0SSean Hefty else 3599e51060f0SSean Hefty ret = cma_use_port(ps, id_priv); 3600e51060f0SSean Hefty mutex_unlock(&lock); 3601e51060f0SSean Hefty 3602e51060f0SSean Hefty return ret; 3603e51060f0SSean Hefty } 3604e51060f0SSean Hefty 3605d14714dfSSean Hefty static int cma_check_linklocal(struct rdma_dev_addr *dev_addr, 3606d14714dfSSean Hefty struct sockaddr *addr) 3607d14714dfSSean Hefty { 3608d90f9b35SRoland Dreier #if IS_ENABLED(CONFIG_IPV6) 3609d14714dfSSean Hefty struct sockaddr_in6 *sin6; 3610d14714dfSSean Hefty 3611d14714dfSSean Hefty if (addr->sa_family != AF_INET6) 3612d14714dfSSean Hefty return 0; 3613d14714dfSSean Hefty 3614d14714dfSSean Hefty sin6 = (struct sockaddr_in6 *) addr; 36155462edddSSomnath Kotur 36165462edddSSomnath Kotur if (!(ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL)) 36175462edddSSomnath Kotur return 0; 36185462edddSSomnath Kotur 36195462edddSSomnath Kotur if (!sin6->sin6_scope_id) 3620d14714dfSSean Hefty return -EINVAL; 3621d14714dfSSean Hefty 3622d14714dfSSean Hefty dev_addr->bound_dev_if = sin6->sin6_scope_id; 3623d14714dfSSean Hefty #endif 3624d14714dfSSean Hefty return 0; 3625d14714dfSSean Hefty } 3626d14714dfSSean Hefty 3627a9bb7912SHefty, Sean int rdma_listen(struct rdma_cm_id *id, int backlog) 3628a9bb7912SHefty, Sean { 3629a9bb7912SHefty, Sean struct rdma_id_private *id_priv; 3630a9bb7912SHefty, Sean int ret; 3631a9bb7912SHefty, Sean 3632a9bb7912SHefty, Sean id_priv = container_of(id, struct rdma_id_private, id); 3633550e5ca7SNir Muchtar if (id_priv->state == RDMA_CM_IDLE) { 3634f4753834SSean Hefty id->route.addr.src_addr.ss_family = AF_INET; 3635f4753834SSean Hefty ret = rdma_bind_addr(id, cma_src_addr(id_priv)); 3636a9bb7912SHefty, Sean if (ret) 3637a9bb7912SHefty, Sean return ret; 3638a9bb7912SHefty, Sean } 3639a9bb7912SHefty, Sean 3640550e5ca7SNir Muchtar if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) 3641a9bb7912SHefty, Sean return -EINVAL; 3642a9bb7912SHefty, Sean 3643a9bb7912SHefty, Sean if (id_priv->reuseaddr) { 3644a9bb7912SHefty, Sean ret = cma_bind_listen(id_priv); 3645a9bb7912SHefty, Sean if (ret) 3646a9bb7912SHefty, Sean goto err; 3647a9bb7912SHefty, Sean } 3648a9bb7912SHefty, Sean 3649a9bb7912SHefty, Sean id_priv->backlog = backlog; 3650a9bb7912SHefty, Sean if (id->device) { 365172219ceaSMichael Wang if (rdma_cap_ib_cm(id->device, 1)) { 3652a9bb7912SHefty, Sean ret = cma_ib_listen(id_priv); 3653a9bb7912SHefty, Sean if (ret) 3654a9bb7912SHefty, Sean goto err; 365504215330SMichael Wang } else if (rdma_cap_iw_cm(id->device, 1)) { 3656a9bb7912SHefty, Sean ret = cma_iw_listen(id_priv, backlog); 3657a9bb7912SHefty, Sean if (ret) 3658a9bb7912SHefty, Sean goto err; 365921655afcSMichael Wang } else { 3660a9bb7912SHefty, Sean ret = -ENOSYS; 3661a9bb7912SHefty, Sean goto err; 3662a9bb7912SHefty, Sean } 3663a9bb7912SHefty, Sean } else 3664a9bb7912SHefty, Sean cma_listen_on_all(id_priv); 3665a9bb7912SHefty, Sean 3666a9bb7912SHefty, Sean return 0; 3667a9bb7912SHefty, Sean err: 3668a9bb7912SHefty, Sean id_priv->backlog = 0; 3669550e5ca7SNir Muchtar cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND); 3670a9bb7912SHefty, Sean return ret; 3671a9bb7912SHefty, Sean } 3672a9bb7912SHefty, Sean EXPORT_SYMBOL(rdma_listen); 3673a9bb7912SHefty, Sean 3674e51060f0SSean Hefty int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr) 3675e51060f0SSean Hefty { 3676e51060f0SSean Hefty struct rdma_id_private *id_priv; 3677e51060f0SSean Hefty int ret; 36786df6b4a9SMoni Shoua struct sockaddr *daddr; 3679e51060f0SSean Hefty 3680680f920aSSean Hefty if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 && 3681680f920aSSean Hefty addr->sa_family != AF_IB) 3682e51060f0SSean Hefty return -EAFNOSUPPORT; 3683e51060f0SSean Hefty 3684e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 3685550e5ca7SNir Muchtar if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND)) 3686e51060f0SSean Hefty return -EINVAL; 3687e51060f0SSean Hefty 3688d14714dfSSean Hefty ret = cma_check_linklocal(&id->route.addr.dev_addr, addr); 3689d14714dfSSean Hefty if (ret) 3690d14714dfSSean Hefty goto err1; 3691d14714dfSSean Hefty 36927b85627bSMoni Shoua memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr)); 36938523c048SSean Hefty if (!cma_any_addr(addr)) { 3694680f920aSSean Hefty ret = cma_translate_addr(addr, &id->route.addr.dev_addr); 3695255d0c14SKrishna Kumar if (ret) 3696255d0c14SKrishna Kumar goto err1; 3697255d0c14SKrishna Kumar 3698ff11c6cdSParav Pandit ret = cma_acquire_dev_by_src_ip(id_priv); 3699e51060f0SSean Hefty if (ret) 3700255d0c14SKrishna Kumar goto err1; 3701e51060f0SSean Hefty } 3702e51060f0SSean Hefty 370368602120SSean Hefty if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) { 37045b0ec991SSean Hefty if (addr->sa_family == AF_INET) 37055b0ec991SSean Hefty id_priv->afonly = 1; 37065b0ec991SSean Hefty #if IS_ENABLED(CONFIG_IPV6) 3707fa20105eSGuy Shapiro else if (addr->sa_family == AF_INET6) { 3708fa20105eSGuy Shapiro struct net *net = id_priv->id.route.addr.dev_addr.net; 3709fa20105eSGuy Shapiro 3710fa20105eSGuy Shapiro id_priv->afonly = net->ipv6.sysctl.bindv6only; 3711fa20105eSGuy Shapiro } 37125b0ec991SSean Hefty #endif 371368602120SSean Hefty } 37149dea9a2fSTatyana Nikolova daddr = cma_dst_addr(id_priv); 37159dea9a2fSTatyana Nikolova daddr->sa_family = addr->sa_family; 37169dea9a2fSTatyana Nikolova 3717e51060f0SSean Hefty ret = cma_get_port(id_priv); 3718e51060f0SSean Hefty if (ret) 3719255d0c14SKrishna Kumar goto err2; 3720e51060f0SSean Hefty 3721e51060f0SSean Hefty return 0; 3722255d0c14SKrishna Kumar err2: 372300313983SSteve Wise rdma_restrack_del(&id_priv->res); 3724ed7a01fdSLeon Romanovsky if (id_priv->cma_dev) 3725a396d43aSSean Hefty cma_release_dev(id_priv); 3726255d0c14SKrishna Kumar err1: 3727550e5ca7SNir Muchtar cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE); 3728e51060f0SSean Hefty return ret; 3729e51060f0SSean Hefty } 3730e51060f0SSean Hefty EXPORT_SYMBOL(rdma_bind_addr); 3731e51060f0SSean Hefty 3732f4753834SSean Hefty static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv) 3733e51060f0SSean Hefty { 3734e51060f0SSean Hefty struct cma_hdr *cma_hdr; 3735e51060f0SSean Hefty 373601602f11SSean Hefty cma_hdr = hdr; 373701602f11SSean Hefty cma_hdr->cma_version = CMA_VERSION; 3738f4753834SSean Hefty if (cma_family(id_priv) == AF_INET) { 37391f5175adSAleksey Senin struct sockaddr_in *src4, *dst4; 37401f5175adSAleksey Senin 3741f4753834SSean Hefty src4 = (struct sockaddr_in *) cma_src_addr(id_priv); 3742f4753834SSean Hefty dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv); 3743e51060f0SSean Hefty 3744e51060f0SSean Hefty cma_set_ip_ver(cma_hdr, 4); 3745e51060f0SSean Hefty cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr; 3746e51060f0SSean Hefty cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr; 3747e51060f0SSean Hefty cma_hdr->port = src4->sin_port; 3748e8160e15SSean Hefty } else if (cma_family(id_priv) == AF_INET6) { 37491f5175adSAleksey Senin struct sockaddr_in6 *src6, *dst6; 37501f5175adSAleksey Senin 3751f4753834SSean Hefty src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv); 3752f4753834SSean Hefty dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv); 37531f5175adSAleksey Senin 37541f5175adSAleksey Senin cma_set_ip_ver(cma_hdr, 6); 37551f5175adSAleksey Senin cma_hdr->src_addr.ip6 = src6->sin6_addr; 37561f5175adSAleksey Senin cma_hdr->dst_addr.ip6 = dst6->sin6_addr; 37571f5175adSAleksey Senin cma_hdr->port = src6->sin6_port; 37581f5175adSAleksey Senin } 3759e51060f0SSean Hefty return 0; 3760e51060f0SSean Hefty } 3761e51060f0SSean Hefty 3762628e5f6dSSean Hefty static int cma_sidr_rep_handler(struct ib_cm_id *cm_id, 3763e7ff98aeSParav Pandit const struct ib_cm_event *ib_event) 3764628e5f6dSSean Hefty { 3765628e5f6dSSean Hefty struct rdma_id_private *id_priv = cm_id->context; 37667582df82SParav Pandit struct rdma_cm_event event = {}; 3767e7ff98aeSParav Pandit const struct ib_cm_sidr_rep_event_param *rep = 3768e7ff98aeSParav Pandit &ib_event->param.sidr_rep_rcvd; 3769628e5f6dSSean Hefty int ret = 0; 3770628e5f6dSSean Hefty 377137e07cdaSBart Van Assche mutex_lock(&id_priv->handler_mutex); 377237e07cdaSBart Van Assche if (id_priv->state != RDMA_CM_CONNECT) 377337e07cdaSBart Van Assche goto out; 3774628e5f6dSSean Hefty 3775628e5f6dSSean Hefty switch (ib_event->event) { 3776628e5f6dSSean Hefty case IB_CM_SIDR_REQ_ERROR: 3777628e5f6dSSean Hefty event.event = RDMA_CM_EVENT_UNREACHABLE; 3778628e5f6dSSean Hefty event.status = -ETIMEDOUT; 3779628e5f6dSSean Hefty break; 3780628e5f6dSSean Hefty case IB_CM_SIDR_REP_RECEIVED: 3781628e5f6dSSean Hefty event.param.ud.private_data = ib_event->private_data; 3782628e5f6dSSean Hefty event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE; 3783628e5f6dSSean Hefty if (rep->status != IB_SIDR_SUCCESS) { 3784628e5f6dSSean Hefty event.event = RDMA_CM_EVENT_UNREACHABLE; 3785628e5f6dSSean Hefty event.status = ib_event->param.sidr_rep_rcvd.status; 3786498683c6SMoni Shoua pr_debug_ratelimited("RDMA CM: UNREACHABLE: bad SIDR reply. status %d\n", 3787498683c6SMoni Shoua event.status); 3788628e5f6dSSean Hefty break; 3789628e5f6dSSean Hefty } 37905c438135SSean Hefty ret = cma_set_qkey(id_priv, rep->qkey); 3791d2ca39f2SYossi Etigin if (ret) { 3792498683c6SMoni Shoua pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to set qkey. status %d\n", ret); 3793d2ca39f2SYossi Etigin event.event = RDMA_CM_EVENT_ADDR_ERROR; 37945c438135SSean Hefty event.status = ret; 3795628e5f6dSSean Hefty break; 3796628e5f6dSSean Hefty } 37974ad6a024SParav Pandit ib_init_ah_attr_from_path(id_priv->id.device, 37984ad6a024SParav Pandit id_priv->id.port_num, 3799628e5f6dSSean Hefty id_priv->id.route.path_rec, 380039839107SParav Pandit &event.param.ud.ah_attr, 380139839107SParav Pandit rep->sgid_attr); 3802628e5f6dSSean Hefty event.param.ud.qp_num = rep->qpn; 3803628e5f6dSSean Hefty event.param.ud.qkey = rep->qkey; 3804628e5f6dSSean Hefty event.event = RDMA_CM_EVENT_ESTABLISHED; 3805628e5f6dSSean Hefty event.status = 0; 3806628e5f6dSSean Hefty break; 3807628e5f6dSSean Hefty default: 3808aba25a3eSParav Pandit pr_err("RDMA CMA: unexpected IB CM event: %d\n", 3809628e5f6dSSean Hefty ib_event->event); 3810628e5f6dSSean Hefty goto out; 3811628e5f6dSSean Hefty } 3812628e5f6dSSean Hefty 3813ed999f82SChuck Lever ret = cma_cm_event_handler(id_priv, &event); 3814aa74f487SParav Pandit 3815aa74f487SParav Pandit rdma_destroy_ah_attr(&event.param.ud.ah_attr); 3816628e5f6dSSean Hefty if (ret) { 3817628e5f6dSSean Hefty /* Destroy the CM ID by returning a non-zero value. */ 3818628e5f6dSSean Hefty id_priv->cm_id.ib = NULL; 3819550e5ca7SNir Muchtar cma_exch(id_priv, RDMA_CM_DESTROYING); 3820de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 3821628e5f6dSSean Hefty rdma_destroy_id(&id_priv->id); 3822628e5f6dSSean Hefty return ret; 3823628e5f6dSSean Hefty } 3824628e5f6dSSean Hefty out: 3825de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 3826628e5f6dSSean Hefty return ret; 3827628e5f6dSSean Hefty } 3828628e5f6dSSean Hefty 3829628e5f6dSSean Hefty static int cma_resolve_ib_udp(struct rdma_id_private *id_priv, 3830628e5f6dSSean Hefty struct rdma_conn_param *conn_param) 3831628e5f6dSSean Hefty { 3832628e5f6dSSean Hefty struct ib_cm_sidr_req_param req; 38330c9361fcSJack Morgenstein struct ib_cm_id *id; 3834e511d1aeSSean Hefty void *private_data; 3835c0b64f58SBart Van Assche u8 offset; 3836c0b64f58SBart Van Assche int ret; 3837628e5f6dSSean Hefty 3838e511d1aeSSean Hefty memset(&req, 0, sizeof req); 3839e8160e15SSean Hefty offset = cma_user_data_offset(id_priv); 3840e8160e15SSean Hefty req.private_data_len = offset + conn_param->private_data_len; 384104ded167SSean Hefty if (req.private_data_len < conn_param->private_data_len) 384204ded167SSean Hefty return -EINVAL; 384304ded167SSean Hefty 3844e8160e15SSean Hefty if (req.private_data_len) { 3845e511d1aeSSean Hefty private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 3846e511d1aeSSean Hefty if (!private_data) 3847628e5f6dSSean Hefty return -ENOMEM; 3848e8160e15SSean Hefty } else { 3849e511d1aeSSean Hefty private_data = NULL; 3850e8160e15SSean Hefty } 3851628e5f6dSSean Hefty 3852628e5f6dSSean Hefty if (conn_param->private_data && conn_param->private_data_len) 3853e511d1aeSSean Hefty memcpy(private_data + offset, conn_param->private_data, 3854e511d1aeSSean Hefty conn_param->private_data_len); 3855628e5f6dSSean Hefty 3856e511d1aeSSean Hefty if (private_data) { 3857e511d1aeSSean Hefty ret = cma_format_hdr(private_data, id_priv); 3858628e5f6dSSean Hefty if (ret) 3859628e5f6dSSean Hefty goto out; 3860e511d1aeSSean Hefty req.private_data = private_data; 3861e8160e15SSean Hefty } 3862628e5f6dSSean Hefty 38630c9361fcSJack Morgenstein id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler, 38640c9361fcSJack Morgenstein id_priv); 38650c9361fcSJack Morgenstein if (IS_ERR(id)) { 38660c9361fcSJack Morgenstein ret = PTR_ERR(id); 3867628e5f6dSSean Hefty goto out; 3868628e5f6dSSean Hefty } 38690c9361fcSJack Morgenstein id_priv->cm_id.ib = id; 3870628e5f6dSSean Hefty 3871f4753834SSean Hefty req.path = id_priv->id.route.path_rec; 3872815d456eSParav Pandit req.sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr; 3873cf53936fSSean Hefty req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv)); 3874628e5f6dSSean Hefty req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8); 3875628e5f6dSSean Hefty req.max_cm_retries = CMA_MAX_CM_RETRIES; 3876628e5f6dSSean Hefty 3877ed999f82SChuck Lever trace_cm_send_sidr_req(id_priv); 3878628e5f6dSSean Hefty ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req); 3879628e5f6dSSean Hefty if (ret) { 3880628e5f6dSSean Hefty ib_destroy_cm_id(id_priv->cm_id.ib); 3881628e5f6dSSean Hefty id_priv->cm_id.ib = NULL; 3882628e5f6dSSean Hefty } 3883628e5f6dSSean Hefty out: 3884e511d1aeSSean Hefty kfree(private_data); 3885628e5f6dSSean Hefty return ret; 3886628e5f6dSSean Hefty } 3887628e5f6dSSean Hefty 3888e51060f0SSean Hefty static int cma_connect_ib(struct rdma_id_private *id_priv, 3889e51060f0SSean Hefty struct rdma_conn_param *conn_param) 3890e51060f0SSean Hefty { 3891e51060f0SSean Hefty struct ib_cm_req_param req; 3892e51060f0SSean Hefty struct rdma_route *route; 3893e51060f0SSean Hefty void *private_data; 38940c9361fcSJack Morgenstein struct ib_cm_id *id; 3895c0b64f58SBart Van Assche u8 offset; 3896c0b64f58SBart Van Assche int ret; 3897e51060f0SSean Hefty 3898e51060f0SSean Hefty memset(&req, 0, sizeof req); 3899e8160e15SSean Hefty offset = cma_user_data_offset(id_priv); 3900e51060f0SSean Hefty req.private_data_len = offset + conn_param->private_data_len; 390104ded167SSean Hefty if (req.private_data_len < conn_param->private_data_len) 390204ded167SSean Hefty return -EINVAL; 390304ded167SSean Hefty 3904e8160e15SSean Hefty if (req.private_data_len) { 3905e51060f0SSean Hefty private_data = kzalloc(req.private_data_len, GFP_ATOMIC); 3906e51060f0SSean Hefty if (!private_data) 3907e51060f0SSean Hefty return -ENOMEM; 3908e8160e15SSean Hefty } else { 3909e8160e15SSean Hefty private_data = NULL; 3910e8160e15SSean Hefty } 3911e51060f0SSean Hefty 3912e51060f0SSean Hefty if (conn_param->private_data && conn_param->private_data_len) 3913e51060f0SSean Hefty memcpy(private_data + offset, conn_param->private_data, 3914e51060f0SSean Hefty conn_param->private_data_len); 3915e51060f0SSean Hefty 39160c9361fcSJack Morgenstein id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv); 39170c9361fcSJack Morgenstein if (IS_ERR(id)) { 39180c9361fcSJack Morgenstein ret = PTR_ERR(id); 3919e51060f0SSean Hefty goto out; 3920e51060f0SSean Hefty } 39210c9361fcSJack Morgenstein id_priv->cm_id.ib = id; 3922e51060f0SSean Hefty 3923e51060f0SSean Hefty route = &id_priv->id.route; 3924e8160e15SSean Hefty if (private_data) { 3925f4753834SSean Hefty ret = cma_format_hdr(private_data, id_priv); 3926e51060f0SSean Hefty if (ret) 3927e51060f0SSean Hefty goto out; 3928e51060f0SSean Hefty req.private_data = private_data; 3929e8160e15SSean Hefty } 3930e51060f0SSean Hefty 3931e51060f0SSean Hefty req.primary_path = &route->path_rec[0]; 3932e51060f0SSean Hefty if (route->num_paths == 2) 3933e51060f0SSean Hefty req.alternate_path = &route->path_rec[1]; 3934e51060f0SSean Hefty 3935815d456eSParav Pandit req.ppath_sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr; 3936815d456eSParav Pandit /* Alternate path SGID attribute currently unsupported */ 3937cf53936fSSean Hefty req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv)); 3938e51060f0SSean Hefty req.qp_num = id_priv->qp_num; 393918c441a6SSean Hefty req.qp_type = id_priv->id.qp_type; 3940e51060f0SSean Hefty req.starting_psn = id_priv->seq_num; 3941e51060f0SSean Hefty req.responder_resources = conn_param->responder_resources; 3942e51060f0SSean Hefty req.initiator_depth = conn_param->initiator_depth; 3943e51060f0SSean Hefty req.flow_control = conn_param->flow_control; 39444ede178aSSean Hefty req.retry_count = min_t(u8, 7, conn_param->retry_count); 39454ede178aSSean Hefty req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count); 3946e51060f0SSean Hefty req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 3947e51060f0SSean Hefty req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT; 3948e51060f0SSean Hefty req.max_cm_retries = CMA_MAX_CM_RETRIES; 3949e51060f0SSean Hefty req.srq = id_priv->srq ? 1 : 0; 3950e51060f0SSean Hefty 3951ed999f82SChuck Lever trace_cm_send_req(id_priv); 3952e51060f0SSean Hefty ret = ib_send_cm_req(id_priv->cm_id.ib, &req); 3953e51060f0SSean Hefty out: 39540c9361fcSJack Morgenstein if (ret && !IS_ERR(id)) { 39550c9361fcSJack Morgenstein ib_destroy_cm_id(id); 3956675a027cSKrishna Kumar id_priv->cm_id.ib = NULL; 3957675a027cSKrishna Kumar } 3958675a027cSKrishna Kumar 3959e51060f0SSean Hefty kfree(private_data); 3960e51060f0SSean Hefty return ret; 3961e51060f0SSean Hefty } 3962e51060f0SSean Hefty 396307ebafbaSTom Tucker static int cma_connect_iw(struct rdma_id_private *id_priv, 396407ebafbaSTom Tucker struct rdma_conn_param *conn_param) 396507ebafbaSTom Tucker { 396607ebafbaSTom Tucker struct iw_cm_id *cm_id; 396707ebafbaSTom Tucker int ret; 396807ebafbaSTom Tucker struct iw_cm_conn_param iw_param; 396907ebafbaSTom Tucker 397007ebafbaSTom Tucker cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv); 39710c9361fcSJack Morgenstein if (IS_ERR(cm_id)) 39720c9361fcSJack Morgenstein return PTR_ERR(cm_id); 397307ebafbaSTom Tucker 397468cdba06SSteve Wise cm_id->tos = id_priv->tos; 3975926ba19bSSteve Wise cm_id->tos_set = id_priv->tos_set; 397607ebafbaSTom Tucker id_priv->cm_id.iw = cm_id; 397707ebafbaSTom Tucker 397824d44a39SSteve Wise memcpy(&cm_id->local_addr, cma_src_addr(id_priv), 397924d44a39SSteve Wise rdma_addr_size(cma_src_addr(id_priv))); 398024d44a39SSteve Wise memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv), 398124d44a39SSteve Wise rdma_addr_size(cma_dst_addr(id_priv))); 398207ebafbaSTom Tucker 39835851bb89SSean Hefty ret = cma_modify_qp_rtr(id_priv, conn_param); 3984675a027cSKrishna Kumar if (ret) 3985675a027cSKrishna Kumar goto out; 398607ebafbaSTom Tucker 3987f45ee80eSHefty, Sean if (conn_param) { 398807ebafbaSTom Tucker iw_param.ord = conn_param->initiator_depth; 398907ebafbaSTom Tucker iw_param.ird = conn_param->responder_resources; 399007ebafbaSTom Tucker iw_param.private_data = conn_param->private_data; 399107ebafbaSTom Tucker iw_param.private_data_len = conn_param->private_data_len; 3992f45ee80eSHefty, Sean iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num; 3993f45ee80eSHefty, Sean } else { 3994f45ee80eSHefty, Sean memset(&iw_param, 0, sizeof iw_param); 399507ebafbaSTom Tucker iw_param.qpn = id_priv->qp_num; 3996f45ee80eSHefty, Sean } 399707ebafbaSTom Tucker ret = iw_cm_connect(cm_id, &iw_param); 399807ebafbaSTom Tucker out: 39990c9361fcSJack Morgenstein if (ret) { 4000675a027cSKrishna Kumar iw_destroy_cm_id(cm_id); 4001675a027cSKrishna Kumar id_priv->cm_id.iw = NULL; 4002675a027cSKrishna Kumar } 400307ebafbaSTom Tucker return ret; 400407ebafbaSTom Tucker } 400507ebafbaSTom Tucker 4006e51060f0SSean Hefty int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param) 4007e51060f0SSean Hefty { 4008e51060f0SSean Hefty struct rdma_id_private *id_priv; 4009e51060f0SSean Hefty int ret; 4010e51060f0SSean Hefty 4011e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 4012550e5ca7SNir Muchtar if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT)) 4013e51060f0SSean Hefty return -EINVAL; 4014e51060f0SSean Hefty 4015e51060f0SSean Hefty if (!id->qp) { 4016e51060f0SSean Hefty id_priv->qp_num = conn_param->qp_num; 4017e51060f0SSean Hefty id_priv->srq = conn_param->srq; 4018e51060f0SSean Hefty } 4019e51060f0SSean Hefty 402072219ceaSMichael Wang if (rdma_cap_ib_cm(id->device, id->port_num)) { 4021b26f9b99SSean Hefty if (id->qp_type == IB_QPT_UD) 4022628e5f6dSSean Hefty ret = cma_resolve_ib_udp(id_priv, conn_param); 4023628e5f6dSSean Hefty else 4024e51060f0SSean Hefty ret = cma_connect_ib(id_priv, conn_param); 402504215330SMichael Wang } else if (rdma_cap_iw_cm(id->device, id->port_num)) 402607ebafbaSTom Tucker ret = cma_connect_iw(id_priv, conn_param); 402721655afcSMichael Wang else 4028e51060f0SSean Hefty ret = -ENOSYS; 4029e51060f0SSean Hefty if (ret) 4030e51060f0SSean Hefty goto err; 4031e51060f0SSean Hefty 4032e51060f0SSean Hefty return 0; 4033e51060f0SSean Hefty err: 4034550e5ca7SNir Muchtar cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED); 4035e51060f0SSean Hefty return ret; 4036e51060f0SSean Hefty } 4037e51060f0SSean Hefty EXPORT_SYMBOL(rdma_connect); 4038e51060f0SSean Hefty 4039e51060f0SSean Hefty static int cma_accept_ib(struct rdma_id_private *id_priv, 4040e51060f0SSean Hefty struct rdma_conn_param *conn_param) 4041e51060f0SSean Hefty { 4042e51060f0SSean Hefty struct ib_cm_rep_param rep; 40435851bb89SSean Hefty int ret; 4044e51060f0SSean Hefty 40455851bb89SSean Hefty ret = cma_modify_qp_rtr(id_priv, conn_param); 4046e51060f0SSean Hefty if (ret) 40470fe313b0SSean Hefty goto out; 40480fe313b0SSean Hefty 40495851bb89SSean Hefty ret = cma_modify_qp_rts(id_priv, conn_param); 40500fe313b0SSean Hefty if (ret) 40510fe313b0SSean Hefty goto out; 40520fe313b0SSean Hefty 4053e51060f0SSean Hefty memset(&rep, 0, sizeof rep); 4054e51060f0SSean Hefty rep.qp_num = id_priv->qp_num; 4055e51060f0SSean Hefty rep.starting_psn = id_priv->seq_num; 4056e51060f0SSean Hefty rep.private_data = conn_param->private_data; 4057e51060f0SSean Hefty rep.private_data_len = conn_param->private_data_len; 4058e51060f0SSean Hefty rep.responder_resources = conn_param->responder_resources; 4059e51060f0SSean Hefty rep.initiator_depth = conn_param->initiator_depth; 4060e51060f0SSean Hefty rep.failover_accepted = 0; 4061e51060f0SSean Hefty rep.flow_control = conn_param->flow_control; 40624ede178aSSean Hefty rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count); 4063e51060f0SSean Hefty rep.srq = id_priv->srq ? 1 : 0; 4064e51060f0SSean Hefty 4065ed999f82SChuck Lever trace_cm_send_rep(id_priv); 40660fe313b0SSean Hefty ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep); 40670fe313b0SSean Hefty out: 40680fe313b0SSean Hefty return ret; 4069e51060f0SSean Hefty } 4070e51060f0SSean Hefty 407107ebafbaSTom Tucker static int cma_accept_iw(struct rdma_id_private *id_priv, 407207ebafbaSTom Tucker struct rdma_conn_param *conn_param) 407307ebafbaSTom Tucker { 407407ebafbaSTom Tucker struct iw_cm_conn_param iw_param; 407507ebafbaSTom Tucker int ret; 407607ebafbaSTom Tucker 4077f2625f7dSSteve Wise if (!conn_param) 4078f2625f7dSSteve Wise return -EINVAL; 4079f2625f7dSSteve Wise 40805851bb89SSean Hefty ret = cma_modify_qp_rtr(id_priv, conn_param); 408107ebafbaSTom Tucker if (ret) 408207ebafbaSTom Tucker return ret; 408307ebafbaSTom Tucker 408407ebafbaSTom Tucker iw_param.ord = conn_param->initiator_depth; 408507ebafbaSTom Tucker iw_param.ird = conn_param->responder_resources; 408607ebafbaSTom Tucker iw_param.private_data = conn_param->private_data; 408707ebafbaSTom Tucker iw_param.private_data_len = conn_param->private_data_len; 408807ebafbaSTom Tucker if (id_priv->id.qp) { 408907ebafbaSTom Tucker iw_param.qpn = id_priv->qp_num; 409007ebafbaSTom Tucker } else 409107ebafbaSTom Tucker iw_param.qpn = conn_param->qp_num; 409207ebafbaSTom Tucker 409307ebafbaSTom Tucker return iw_cm_accept(id_priv->cm_id.iw, &iw_param); 409407ebafbaSTom Tucker } 409507ebafbaSTom Tucker 4096628e5f6dSSean Hefty static int cma_send_sidr_rep(struct rdma_id_private *id_priv, 40975c438135SSean Hefty enum ib_cm_sidr_status status, u32 qkey, 4098628e5f6dSSean Hefty const void *private_data, int private_data_len) 4099628e5f6dSSean Hefty { 4100628e5f6dSSean Hefty struct ib_cm_sidr_rep_param rep; 4101d2ca39f2SYossi Etigin int ret; 4102628e5f6dSSean Hefty 4103628e5f6dSSean Hefty memset(&rep, 0, sizeof rep); 4104628e5f6dSSean Hefty rep.status = status; 4105628e5f6dSSean Hefty if (status == IB_SIDR_SUCCESS) { 41065c438135SSean Hefty ret = cma_set_qkey(id_priv, qkey); 4107d2ca39f2SYossi Etigin if (ret) 4108d2ca39f2SYossi Etigin return ret; 4109628e5f6dSSean Hefty rep.qp_num = id_priv->qp_num; 4110c8f6a362SSean Hefty rep.qkey = id_priv->qkey; 4111628e5f6dSSean Hefty } 4112628e5f6dSSean Hefty rep.private_data = private_data; 4113628e5f6dSSean Hefty rep.private_data_len = private_data_len; 4114628e5f6dSSean Hefty 4115ed999f82SChuck Lever trace_cm_send_sidr_rep(id_priv); 4116628e5f6dSSean Hefty return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep); 4117628e5f6dSSean Hefty } 4118628e5f6dSSean Hefty 411900313983SSteve Wise int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param, 412000313983SSteve Wise const char *caller) 4121e51060f0SSean Hefty { 4122e51060f0SSean Hefty struct rdma_id_private *id_priv; 4123e51060f0SSean Hefty int ret; 4124e51060f0SSean Hefty 4125e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 412683e9502dSNir Muchtar 41272165fc26SLeon Romanovsky rdma_restrack_set_task(&id_priv->res, caller); 412883e9502dSNir Muchtar 4129550e5ca7SNir Muchtar if (!cma_comp(id_priv, RDMA_CM_CONNECT)) 4130e51060f0SSean Hefty return -EINVAL; 4131e51060f0SSean Hefty 4132e51060f0SSean Hefty if (!id->qp && conn_param) { 4133e51060f0SSean Hefty id_priv->qp_num = conn_param->qp_num; 4134e51060f0SSean Hefty id_priv->srq = conn_param->srq; 4135e51060f0SSean Hefty } 4136e51060f0SSean Hefty 413772219ceaSMichael Wang if (rdma_cap_ib_cm(id->device, id->port_num)) { 4138f45ee80eSHefty, Sean if (id->qp_type == IB_QPT_UD) { 4139f45ee80eSHefty, Sean if (conn_param) 4140628e5f6dSSean Hefty ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 41415c438135SSean Hefty conn_param->qkey, 4142628e5f6dSSean Hefty conn_param->private_data, 4143628e5f6dSSean Hefty conn_param->private_data_len); 4144f45ee80eSHefty, Sean else 4145f45ee80eSHefty, Sean ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS, 41465c438135SSean Hefty 0, NULL, 0); 4147f45ee80eSHefty, Sean } else { 4148f45ee80eSHefty, Sean if (conn_param) 4149e51060f0SSean Hefty ret = cma_accept_ib(id_priv, conn_param); 4150e51060f0SSean Hefty else 4151e51060f0SSean Hefty ret = cma_rep_recv(id_priv); 4152f45ee80eSHefty, Sean } 415304215330SMichael Wang } else if (rdma_cap_iw_cm(id->device, id->port_num)) 415407ebafbaSTom Tucker ret = cma_accept_iw(id_priv, conn_param); 415521655afcSMichael Wang else 4156e51060f0SSean Hefty ret = -ENOSYS; 4157e51060f0SSean Hefty 4158e51060f0SSean Hefty if (ret) 4159e51060f0SSean Hefty goto reject; 4160e51060f0SSean Hefty 4161e51060f0SSean Hefty return 0; 4162e51060f0SSean Hefty reject: 4163c5483388SSean Hefty cma_modify_qp_err(id_priv); 4164e51060f0SSean Hefty rdma_reject(id, NULL, 0); 4165e51060f0SSean Hefty return ret; 4166e51060f0SSean Hefty } 416700313983SSteve Wise EXPORT_SYMBOL(__rdma_accept); 4168e51060f0SSean Hefty 41690fe313b0SSean Hefty int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event) 41700fe313b0SSean Hefty { 41710fe313b0SSean Hefty struct rdma_id_private *id_priv; 41720fe313b0SSean Hefty int ret; 41730fe313b0SSean Hefty 41740fe313b0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 41750c9361fcSJack Morgenstein if (!id_priv->cm_id.ib) 41760fe313b0SSean Hefty return -EINVAL; 41770fe313b0SSean Hefty 41780fe313b0SSean Hefty switch (id->device->node_type) { 41790fe313b0SSean Hefty case RDMA_NODE_IB_CA: 41800fe313b0SSean Hefty ret = ib_cm_notify(id_priv->cm_id.ib, event); 41810fe313b0SSean Hefty break; 41820fe313b0SSean Hefty default: 41830fe313b0SSean Hefty ret = 0; 41840fe313b0SSean Hefty break; 41850fe313b0SSean Hefty } 41860fe313b0SSean Hefty return ret; 41870fe313b0SSean Hefty } 41880fe313b0SSean Hefty EXPORT_SYMBOL(rdma_notify); 41890fe313b0SSean Hefty 4190e51060f0SSean Hefty int rdma_reject(struct rdma_cm_id *id, const void *private_data, 4191e51060f0SSean Hefty u8 private_data_len) 4192e51060f0SSean Hefty { 4193e51060f0SSean Hefty struct rdma_id_private *id_priv; 4194e51060f0SSean Hefty int ret; 4195e51060f0SSean Hefty 4196e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 41970c9361fcSJack Morgenstein if (!id_priv->cm_id.ib) 4198e51060f0SSean Hefty return -EINVAL; 4199e51060f0SSean Hefty 420072219ceaSMichael Wang if (rdma_cap_ib_cm(id->device, id->port_num)) { 4201ed999f82SChuck Lever if (id->qp_type == IB_QPT_UD) { 42025c438135SSean Hefty ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0, 4203e51060f0SSean Hefty private_data, private_data_len); 4204ed999f82SChuck Lever } else { 4205ed999f82SChuck Lever trace_cm_send_rej(id_priv); 4206628e5f6dSSean Hefty ret = ib_send_cm_rej(id_priv->cm_id.ib, 4207628e5f6dSSean Hefty IB_CM_REJ_CONSUMER_DEFINED, NULL, 4208628e5f6dSSean Hefty 0, private_data, private_data_len); 4209ed999f82SChuck Lever } 421004215330SMichael Wang } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 421107ebafbaSTom Tucker ret = iw_cm_reject(id_priv->cm_id.iw, 421207ebafbaSTom Tucker private_data, private_data_len); 421321655afcSMichael Wang } else 4214e51060f0SSean Hefty ret = -ENOSYS; 421521655afcSMichael Wang 4216e51060f0SSean Hefty return ret; 4217e51060f0SSean Hefty } 4218e51060f0SSean Hefty EXPORT_SYMBOL(rdma_reject); 4219e51060f0SSean Hefty 4220e51060f0SSean Hefty int rdma_disconnect(struct rdma_cm_id *id) 4221e51060f0SSean Hefty { 4222e51060f0SSean Hefty struct rdma_id_private *id_priv; 4223e51060f0SSean Hefty int ret; 4224e51060f0SSean Hefty 4225e51060f0SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 42260c9361fcSJack Morgenstein if (!id_priv->cm_id.ib) 4227e51060f0SSean Hefty return -EINVAL; 4228e51060f0SSean Hefty 422972219ceaSMichael Wang if (rdma_cap_ib_cm(id->device, id->port_num)) { 4230c5483388SSean Hefty ret = cma_modify_qp_err(id_priv); 4231e51060f0SSean Hefty if (ret) 4232e51060f0SSean Hefty goto out; 4233e51060f0SSean Hefty /* Initiate or respond to a disconnect. */ 4234ed999f82SChuck Lever trace_cm_disconnect(id_priv); 4235ed999f82SChuck Lever if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) { 4236ed999f82SChuck Lever if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0)) 4237ed999f82SChuck Lever trace_cm_sent_drep(id_priv); 4238ed999f82SChuck Lever } else { 4239ed999f82SChuck Lever trace_cm_sent_dreq(id_priv); 4240ed999f82SChuck Lever } 424104215330SMichael Wang } else if (rdma_cap_iw_cm(id->device, id->port_num)) { 424207ebafbaSTom Tucker ret = iw_cm_disconnect(id_priv->cm_id.iw, 0); 424321655afcSMichael Wang } else 424407ebafbaSTom Tucker ret = -EINVAL; 424521655afcSMichael Wang 4246e51060f0SSean Hefty out: 4247e51060f0SSean Hefty return ret; 4248e51060f0SSean Hefty } 4249e51060f0SSean Hefty EXPORT_SYMBOL(rdma_disconnect); 4250e51060f0SSean Hefty 4251c8f6a362SSean Hefty static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast) 4252c8f6a362SSean Hefty { 4253c8f6a362SSean Hefty struct rdma_id_private *id_priv; 4254c8f6a362SSean Hefty struct cma_multicast *mc = multicast->context; 42557582df82SParav Pandit struct rdma_cm_event event = {}; 425637e07cdaSBart Van Assche int ret = 0; 4257c8f6a362SSean Hefty 4258c8f6a362SSean Hefty id_priv = mc->id_priv; 425937e07cdaSBart Van Assche mutex_lock(&id_priv->handler_mutex); 426037e07cdaSBart Van Assche if (id_priv->state != RDMA_CM_ADDR_BOUND && 426137e07cdaSBart Van Assche id_priv->state != RDMA_CM_ADDR_RESOLVED) 426237e07cdaSBart Van Assche goto out; 4263c8f6a362SSean Hefty 42645c438135SSean Hefty if (!status) 42655c438135SSean Hefty status = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey)); 4266498683c6SMoni Shoua else 4267498683c6SMoni Shoua pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n", 4268498683c6SMoni Shoua status); 4269c5483388SSean Hefty mutex_lock(&id_priv->qp_mutex); 4270498683c6SMoni Shoua if (!status && id_priv->id.qp) { 4271c8f6a362SSean Hefty status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid, 427246ea5061SSean Hefty be16_to_cpu(multicast->rec.mlid)); 4273a3dd3a48SChristophe Jaillet if (status) 4274498683c6SMoni Shoua pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to attach QP. status %d\n", 4275498683c6SMoni Shoua status); 4276498683c6SMoni Shoua } 4277c5483388SSean Hefty mutex_unlock(&id_priv->qp_mutex); 4278c8f6a362SSean Hefty 4279c8f6a362SSean Hefty event.status = status; 4280c8f6a362SSean Hefty event.param.ud.private_data = mc->context; 4281c8f6a362SSean Hefty if (!status) { 4282bee3c3c9SMoni Shoua struct rdma_dev_addr *dev_addr = 4283bee3c3c9SMoni Shoua &id_priv->id.route.addr.dev_addr; 4284bee3c3c9SMoni Shoua struct net_device *ndev = 4285052eac6eSParav Pandit dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); 4286bee3c3c9SMoni Shoua enum ib_gid_type gid_type = 4287bee3c3c9SMoni Shoua id_priv->cma_dev->default_gid_type[id_priv->id.port_num - 4288bee3c3c9SMoni Shoua rdma_start_port(id_priv->cma_dev->device)]; 4289bee3c3c9SMoni Shoua 4290c8f6a362SSean Hefty event.event = RDMA_CM_EVENT_MULTICAST_JOIN; 42916d337179SParav Pandit ret = ib_init_ah_from_mcmember(id_priv->id.device, 42926d337179SParav Pandit id_priv->id.port_num, 42936d337179SParav Pandit &multicast->rec, 4294bee3c3c9SMoni Shoua ndev, gid_type, 4295c8f6a362SSean Hefty &event.param.ud.ah_attr); 42966d337179SParav Pandit if (ret) 42976d337179SParav Pandit event.event = RDMA_CM_EVENT_MULTICAST_ERROR; 42986d337179SParav Pandit 4299c8f6a362SSean Hefty event.param.ud.qp_num = 0xFFFFFF; 4300c8f6a362SSean Hefty event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey); 4301bee3c3c9SMoni Shoua if (ndev) 4302bee3c3c9SMoni Shoua dev_put(ndev); 4303c8f6a362SSean Hefty } else 4304c8f6a362SSean Hefty event.event = RDMA_CM_EVENT_MULTICAST_ERROR; 4305c8f6a362SSean Hefty 4306ed999f82SChuck Lever ret = cma_cm_event_handler(id_priv, &event); 4307f685c195SParav Pandit 4308f685c195SParav Pandit rdma_destroy_ah_attr(&event.param.ud.ah_attr); 4309c8f6a362SSean Hefty if (ret) { 4310550e5ca7SNir Muchtar cma_exch(id_priv, RDMA_CM_DESTROYING); 4311de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 4312c8f6a362SSean Hefty rdma_destroy_id(&id_priv->id); 4313c8f6a362SSean Hefty return 0; 4314c8f6a362SSean Hefty } 43158aa08602SSean Hefty 431637e07cdaSBart Van Assche out: 4317de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 4318c8f6a362SSean Hefty return 0; 4319c8f6a362SSean Hefty } 4320c8f6a362SSean Hefty 4321c8f6a362SSean Hefty static void cma_set_mgid(struct rdma_id_private *id_priv, 4322c8f6a362SSean Hefty struct sockaddr *addr, union ib_gid *mgid) 4323c8f6a362SSean Hefty { 4324c8f6a362SSean Hefty unsigned char mc_map[MAX_ADDR_LEN]; 4325c8f6a362SSean Hefty struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 4326c8f6a362SSean Hefty struct sockaddr_in *sin = (struct sockaddr_in *) addr; 4327c8f6a362SSean Hefty struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr; 4328c8f6a362SSean Hefty 4329c8f6a362SSean Hefty if (cma_any_addr(addr)) { 4330c8f6a362SSean Hefty memset(mgid, 0, sizeof *mgid); 4331c8f6a362SSean Hefty } else if ((addr->sa_family == AF_INET6) && 43321c9b2819SJason Gunthorpe ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) == 4333c8f6a362SSean Hefty 0xFF10A01B)) { 4334c8f6a362SSean Hefty /* IPv6 address is an SA assigned MGID. */ 4335c8f6a362SSean Hefty memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 43365bc2b7b3SSean Hefty } else if (addr->sa_family == AF_IB) { 43375bc2b7b3SSean Hefty memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid); 4338076dd53bSVarsha Rao } else if (addr->sa_family == AF_INET6) { 4339e2e62697SJason Gunthorpe ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map); 4340e2e62697SJason Gunthorpe if (id_priv->id.ps == RDMA_PS_UDP) 4341e2e62697SJason Gunthorpe mc_map[7] = 0x01; /* Use RDMA CM signature */ 4342e2e62697SJason Gunthorpe *mgid = *(union ib_gid *) (mc_map + 4); 4343c8f6a362SSean Hefty } else { 4344a9e527e3SRolf Manderscheid ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map); 4345c8f6a362SSean Hefty if (id_priv->id.ps == RDMA_PS_UDP) 4346c8f6a362SSean Hefty mc_map[7] = 0x01; /* Use RDMA CM signature */ 4347c8f6a362SSean Hefty *mgid = *(union ib_gid *) (mc_map + 4); 4348c8f6a362SSean Hefty } 4349c8f6a362SSean Hefty } 4350c8f6a362SSean Hefty 4351c8f6a362SSean Hefty static int cma_join_ib_multicast(struct rdma_id_private *id_priv, 4352c8f6a362SSean Hefty struct cma_multicast *mc) 4353c8f6a362SSean Hefty { 4354c8f6a362SSean Hefty struct ib_sa_mcmember_rec rec; 4355c8f6a362SSean Hefty struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 4356c8f6a362SSean Hefty ib_sa_comp_mask comp_mask; 4357c8f6a362SSean Hefty int ret; 4358c8f6a362SSean Hefty 4359c8f6a362SSean Hefty ib_addr_get_mgid(dev_addr, &rec.mgid); 4360c8f6a362SSean Hefty ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num, 4361c8f6a362SSean Hefty &rec.mgid, &rec); 4362c8f6a362SSean Hefty if (ret) 4363c8f6a362SSean Hefty return ret; 4364c8f6a362SSean Hefty 43655bc2b7b3SSean Hefty ret = cma_set_qkey(id_priv, 0); 43665bc2b7b3SSean Hefty if (ret) 43675bc2b7b3SSean Hefty return ret; 43685bc2b7b3SSean Hefty 43693f446754SRoland Dreier cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid); 43705bc2b7b3SSean Hefty rec.qkey = cpu_to_be32(id_priv->qkey); 43716f8372b6SSean Hefty rdma_addr_get_sgid(dev_addr, &rec.port_gid); 4372c8f6a362SSean Hefty rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 4373ab15c95aSAlex Vesker rec.join_state = mc->join_state; 4374ab15c95aSAlex Vesker 4375ee1c60b1SDasaratharaman Chandramouli if ((rec.join_state == BIT(SENDONLY_FULLMEMBER_JOIN)) && 4376ee1c60b1SDasaratharaman Chandramouli (!ib_sa_sendonly_fullmem_support(&sa_client, 4377ee1c60b1SDasaratharaman Chandramouli id_priv->id.device, 4378ee1c60b1SDasaratharaman Chandramouli id_priv->id.port_num))) { 437943c7c851SJason Gunthorpe dev_warn( 438043c7c851SJason Gunthorpe &id_priv->id.device->dev, 438143c7c851SJason Gunthorpe "RDMA CM: port %u Unable to multicast join: SM doesn't support Send Only Full Member option\n", 438243c7c851SJason Gunthorpe id_priv->id.port_num); 4383ab15c95aSAlex Vesker return -EOPNOTSUPP; 4384ab15c95aSAlex Vesker } 4385c8f6a362SSean Hefty 4386c8f6a362SSean Hefty comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID | 4387c8f6a362SSean Hefty IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE | 4388c8f6a362SSean Hefty IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL | 4389c8f6a362SSean Hefty IB_SA_MCMEMBER_REC_FLOW_LABEL | 4390c8f6a362SSean Hefty IB_SA_MCMEMBER_REC_TRAFFIC_CLASS; 4391c8f6a362SSean Hefty 439284adeee9SYossi Etigin if (id_priv->id.ps == RDMA_PS_IPOIB) 439384adeee9SYossi Etigin comp_mask |= IB_SA_MCMEMBER_REC_RATE | 43942a22fb8cSDotan Barak IB_SA_MCMEMBER_REC_RATE_SELECTOR | 43952a22fb8cSDotan Barak IB_SA_MCMEMBER_REC_MTU_SELECTOR | 43962a22fb8cSDotan Barak IB_SA_MCMEMBER_REC_MTU | 43972a22fb8cSDotan Barak IB_SA_MCMEMBER_REC_HOP_LIMIT; 439884adeee9SYossi Etigin 4399c8f6a362SSean Hefty mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device, 4400c8f6a362SSean Hefty id_priv->id.port_num, &rec, 4401c8f6a362SSean Hefty comp_mask, GFP_KERNEL, 4402c8f6a362SSean Hefty cma_ib_mc_handler, mc); 44038c6ffba0SRusty Russell return PTR_ERR_OR_ZERO(mc->multicast.ib); 4404c8f6a362SSean Hefty } 4405c8f6a362SSean Hefty 44063c86aa70SEli Cohen static void iboe_mcast_work_handler(struct work_struct *work) 44073c86aa70SEli Cohen { 44083c86aa70SEli Cohen struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work); 44093c86aa70SEli Cohen struct cma_multicast *mc = mw->mc; 44103c86aa70SEli Cohen struct ib_sa_multicast *m = mc->multicast.ib; 44113c86aa70SEli Cohen 44123c86aa70SEli Cohen mc->multicast.ib->context = mc; 44133c86aa70SEli Cohen cma_ib_mc_handler(0, m); 44143c86aa70SEli Cohen kref_put(&mc->mcref, release_mc); 44153c86aa70SEli Cohen kfree(mw); 44163c86aa70SEli Cohen } 44173c86aa70SEli Cohen 4418be1d325aSNoa Osherovich static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid, 4419be1d325aSNoa Osherovich enum ib_gid_type gid_type) 44203c86aa70SEli Cohen { 44213c86aa70SEli Cohen struct sockaddr_in *sin = (struct sockaddr_in *)addr; 44223c86aa70SEli Cohen struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; 44233c86aa70SEli Cohen 44243c86aa70SEli Cohen if (cma_any_addr(addr)) { 44253c86aa70SEli Cohen memset(mgid, 0, sizeof *mgid); 44263c86aa70SEli Cohen } else if (addr->sa_family == AF_INET6) { 44273c86aa70SEli Cohen memcpy(mgid, &sin6->sin6_addr, sizeof *mgid); 44283c86aa70SEli Cohen } else { 44295c181bdaSParav Pandit mgid->raw[0] = 44305c181bdaSParav Pandit (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff; 44315c181bdaSParav Pandit mgid->raw[1] = 44325c181bdaSParav Pandit (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e; 44333c86aa70SEli Cohen mgid->raw[2] = 0; 44343c86aa70SEli Cohen mgid->raw[3] = 0; 44353c86aa70SEli Cohen mgid->raw[4] = 0; 44363c86aa70SEli Cohen mgid->raw[5] = 0; 44373c86aa70SEli Cohen mgid->raw[6] = 0; 44383c86aa70SEli Cohen mgid->raw[7] = 0; 44393c86aa70SEli Cohen mgid->raw[8] = 0; 44403c86aa70SEli Cohen mgid->raw[9] = 0; 44413c86aa70SEli Cohen mgid->raw[10] = 0xff; 44423c86aa70SEli Cohen mgid->raw[11] = 0xff; 44433c86aa70SEli Cohen *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr; 44443c86aa70SEli Cohen } 44453c86aa70SEli Cohen } 44463c86aa70SEli Cohen 44473c86aa70SEli Cohen static int cma_iboe_join_multicast(struct rdma_id_private *id_priv, 44483c86aa70SEli Cohen struct cma_multicast *mc) 44493c86aa70SEli Cohen { 44503c86aa70SEli Cohen struct iboe_mcast_work *work; 44513c86aa70SEli Cohen struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr; 4452bee3c3c9SMoni Shoua int err = 0; 44533c86aa70SEli Cohen struct sockaddr *addr = (struct sockaddr *)&mc->addr; 44543c86aa70SEli Cohen struct net_device *ndev = NULL; 4455bee3c3c9SMoni Shoua enum ib_gid_type gid_type; 4456ab15c95aSAlex Vesker bool send_only; 4457ab15c95aSAlex Vesker 4458ab15c95aSAlex Vesker send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN); 44593c86aa70SEli Cohen 44603c86aa70SEli Cohen if (cma_zero_addr((struct sockaddr *)&mc->addr)) 44613c86aa70SEli Cohen return -EINVAL; 44623c86aa70SEli Cohen 44633c86aa70SEli Cohen work = kzalloc(sizeof *work, GFP_KERNEL); 44643c86aa70SEli Cohen if (!work) 44653c86aa70SEli Cohen return -ENOMEM; 44663c86aa70SEli Cohen 44673c86aa70SEli Cohen mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL); 44683c86aa70SEli Cohen if (!mc->multicast.ib) { 44693c86aa70SEli Cohen err = -ENOMEM; 44703c86aa70SEli Cohen goto out1; 44713c86aa70SEli Cohen } 44723c86aa70SEli Cohen 4473be1d325aSNoa Osherovich gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num - 4474be1d325aSNoa Osherovich rdma_start_port(id_priv->cma_dev->device)]; 4475be1d325aSNoa Osherovich cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid, gid_type); 44763c86aa70SEli Cohen 44773c86aa70SEli Cohen mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff); 44783c86aa70SEli Cohen if (id_priv->id.ps == RDMA_PS_UDP) 44793c86aa70SEli Cohen mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY); 44803c86aa70SEli Cohen 44813c86aa70SEli Cohen if (dev_addr->bound_dev_if) 4482052eac6eSParav Pandit ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if); 44833c86aa70SEli Cohen if (!ndev) { 44843c86aa70SEli Cohen err = -ENODEV; 44853c86aa70SEli Cohen goto out2; 44863c86aa70SEli Cohen } 44873c86aa70SEli Cohen mc->multicast.ib->rec.rate = iboe_get_rate(ndev); 44883c86aa70SEli Cohen mc->multicast.ib->rec.hop_limit = 1; 44893c86aa70SEli Cohen mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu); 4490bee3c3c9SMoni Shoua 4491bee3c3c9SMoni Shoua if (addr->sa_family == AF_INET) { 4492c65f6c5aSAlex Vesker if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) { 4493c65f6c5aSAlex Vesker mc->multicast.ib->rec.hop_limit = IPV6_DEFAULT_HOPLIMIT; 4494ab15c95aSAlex Vesker if (!send_only) { 4495bee3c3c9SMoni Shoua err = cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid, 4496bee3c3c9SMoni Shoua true); 4497bee3c3c9SMoni Shoua } 4498bee3c3c9SMoni Shoua } 4499bee3c3c9SMoni Shoua } else { 4500bee3c3c9SMoni Shoua if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) 4501bee3c3c9SMoni Shoua err = -ENOTSUPP; 4502bee3c3c9SMoni Shoua } 45033c86aa70SEli Cohen dev_put(ndev); 4504bee3c3c9SMoni Shoua if (err || !mc->multicast.ib->rec.mtu) { 4505bee3c3c9SMoni Shoua if (!err) 45063c86aa70SEli Cohen err = -EINVAL; 45073c86aa70SEli Cohen goto out2; 45083c86aa70SEli Cohen } 45097b85627bSMoni Shoua rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr, 45107b85627bSMoni Shoua &mc->multicast.ib->rec.port_gid); 45113c86aa70SEli Cohen work->id = id_priv; 45123c86aa70SEli Cohen work->mc = mc; 45133c86aa70SEli Cohen INIT_WORK(&work->work, iboe_mcast_work_handler); 45143c86aa70SEli Cohen kref_get(&mc->mcref); 45153c86aa70SEli Cohen queue_work(cma_wq, &work->work); 45163c86aa70SEli Cohen 45173c86aa70SEli Cohen return 0; 45183c86aa70SEli Cohen 45193c86aa70SEli Cohen out2: 45203c86aa70SEli Cohen kfree(mc->multicast.ib); 45213c86aa70SEli Cohen out1: 45223c86aa70SEli Cohen kfree(work); 45233c86aa70SEli Cohen return err; 45243c86aa70SEli Cohen } 45253c86aa70SEli Cohen 4526c8f6a362SSean Hefty int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr, 4527ab15c95aSAlex Vesker u8 join_state, void *context) 4528c8f6a362SSean Hefty { 4529c8f6a362SSean Hefty struct rdma_id_private *id_priv; 4530c8f6a362SSean Hefty struct cma_multicast *mc; 4531c8f6a362SSean Hefty int ret; 4532c8f6a362SSean Hefty 45337688f2c3SLeon Romanovsky if (!id->device) 45347688f2c3SLeon Romanovsky return -EINVAL; 45357688f2c3SLeon Romanovsky 4536c8f6a362SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 4537550e5ca7SNir Muchtar if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) && 4538550e5ca7SNir Muchtar !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED)) 4539c8f6a362SSean Hefty return -EINVAL; 4540c8f6a362SSean Hefty 4541c8f6a362SSean Hefty mc = kmalloc(sizeof *mc, GFP_KERNEL); 4542c8f6a362SSean Hefty if (!mc) 4543c8f6a362SSean Hefty return -ENOMEM; 4544c8f6a362SSean Hefty 4545ef560861SSean Hefty memcpy(&mc->addr, addr, rdma_addr_size(addr)); 4546c8f6a362SSean Hefty mc->context = context; 4547c8f6a362SSean Hefty mc->id_priv = id_priv; 4548ab15c95aSAlex Vesker mc->join_state = join_state; 4549c8f6a362SSean Hefty 45505d9fb044SIra Weiny if (rdma_protocol_roce(id->device, id->port_num)) { 45513c86aa70SEli Cohen kref_init(&mc->mcref); 45523c86aa70SEli Cohen ret = cma_iboe_join_multicast(id_priv, mc); 4553c0126915SJason Gunthorpe if (ret) 4554c0126915SJason Gunthorpe goto out_err; 4555c0126915SJason Gunthorpe } else if (rdma_cap_ib_mcast(id->device, id->port_num)) { 45565c9a5282SMichael Wang ret = cma_join_ib_multicast(id_priv, mc); 4557c0126915SJason Gunthorpe if (ret) 4558c0126915SJason Gunthorpe goto out_err; 4559c0126915SJason Gunthorpe } else { 4560c8f6a362SSean Hefty ret = -ENOSYS; 4561c0126915SJason Gunthorpe goto out_err; 4562c8f6a362SSean Hefty } 4563c0126915SJason Gunthorpe 4564c0126915SJason Gunthorpe spin_lock(&id_priv->lock); 4565c0126915SJason Gunthorpe list_add(&mc->list, &id_priv->mc_list); 4566c0126915SJason Gunthorpe spin_unlock(&id_priv->lock); 4567c0126915SJason Gunthorpe 4568c0126915SJason Gunthorpe return 0; 4569c0126915SJason Gunthorpe out_err: 4570c0126915SJason Gunthorpe kfree(mc); 4571c8f6a362SSean Hefty return ret; 4572c8f6a362SSean Hefty } 4573c8f6a362SSean Hefty EXPORT_SYMBOL(rdma_join_multicast); 4574c8f6a362SSean Hefty 4575c8f6a362SSean Hefty void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr) 4576c8f6a362SSean Hefty { 4577c8f6a362SSean Hefty struct rdma_id_private *id_priv; 4578c8f6a362SSean Hefty struct cma_multicast *mc; 4579c8f6a362SSean Hefty 4580c8f6a362SSean Hefty id_priv = container_of(id, struct rdma_id_private, id); 4581c8f6a362SSean Hefty spin_lock_irq(&id_priv->lock); 4582c8f6a362SSean Hefty list_for_each_entry(mc, &id_priv->mc_list, list) { 4583ef560861SSean Hefty if (!memcmp(&mc->addr, addr, rdma_addr_size(addr))) { 4584c8f6a362SSean Hefty list_del(&mc->list); 4585c8f6a362SSean Hefty spin_unlock_irq(&id_priv->lock); 4586c8f6a362SSean Hefty 4587c8f6a362SSean Hefty if (id->qp) 4588c8f6a362SSean Hefty ib_detach_mcast(id->qp, 4589c8f6a362SSean Hefty &mc->multicast.ib->rec.mgid, 459046ea5061SSean Hefty be16_to_cpu(mc->multicast.ib->rec.mlid)); 45915c9a5282SMichael Wang 45925c9a5282SMichael Wang BUG_ON(id_priv->cma_dev->device != id->device); 45935c9a5282SMichael Wang 4594a31ad3b0SMichael Wang if (rdma_cap_ib_mcast(id->device, id->port_num)) { 4595c8f6a362SSean Hefty ib_sa_free_multicast(mc->multicast.ib); 4596c8f6a362SSean Hefty kfree(mc); 4597bee3c3c9SMoni Shoua } else if (rdma_protocol_roce(id->device, id->port_num)) { 459888145678SParav Pandit cma_leave_roce_mc_group(id_priv, mc); 4599bee3c3c9SMoni Shoua } 4600c8f6a362SSean Hefty return; 4601c8f6a362SSean Hefty } 4602c8f6a362SSean Hefty } 4603c8f6a362SSean Hefty spin_unlock_irq(&id_priv->lock); 4604c8f6a362SSean Hefty } 4605c8f6a362SSean Hefty EXPORT_SYMBOL(rdma_leave_multicast); 4606c8f6a362SSean Hefty 4607dd5bdff8SOr Gerlitz static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv) 4608dd5bdff8SOr Gerlitz { 4609dd5bdff8SOr Gerlitz struct rdma_dev_addr *dev_addr; 4610dd5bdff8SOr Gerlitz struct cma_ndev_work *work; 4611dd5bdff8SOr Gerlitz 4612dd5bdff8SOr Gerlitz dev_addr = &id_priv->id.route.addr.dev_addr; 4613dd5bdff8SOr Gerlitz 46146266ed6eSSean Hefty if ((dev_addr->bound_dev_if == ndev->ifindex) && 4615fa20105eSGuy Shapiro (net_eq(dev_net(ndev), dev_addr->net)) && 4616dd5bdff8SOr Gerlitz memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) { 4617aba25a3eSParav Pandit pr_info("RDMA CM addr change for ndev %s used by id %p\n", 4618dd5bdff8SOr Gerlitz ndev->name, &id_priv->id); 4619dd5bdff8SOr Gerlitz work = kzalloc(sizeof *work, GFP_KERNEL); 4620dd5bdff8SOr Gerlitz if (!work) 4621dd5bdff8SOr Gerlitz return -ENOMEM; 4622dd5bdff8SOr Gerlitz 4623dd5bdff8SOr Gerlitz INIT_WORK(&work->work, cma_ndev_work_handler); 4624dd5bdff8SOr Gerlitz work->id = id_priv; 4625dd5bdff8SOr Gerlitz work->event.event = RDMA_CM_EVENT_ADDR_CHANGE; 4626e368d23fSParav Pandit cma_id_get(id_priv); 4627dd5bdff8SOr Gerlitz queue_work(cma_wq, &work->work); 4628dd5bdff8SOr Gerlitz } 4629dd5bdff8SOr Gerlitz 4630dd5bdff8SOr Gerlitz return 0; 4631dd5bdff8SOr Gerlitz } 4632dd5bdff8SOr Gerlitz 4633dd5bdff8SOr Gerlitz static int cma_netdev_callback(struct notifier_block *self, unsigned long event, 4634351638e7SJiri Pirko void *ptr) 4635dd5bdff8SOr Gerlitz { 4636351638e7SJiri Pirko struct net_device *ndev = netdev_notifier_info_to_dev(ptr); 4637dd5bdff8SOr Gerlitz struct cma_device *cma_dev; 4638dd5bdff8SOr Gerlitz struct rdma_id_private *id_priv; 4639dd5bdff8SOr Gerlitz int ret = NOTIFY_DONE; 4640dd5bdff8SOr Gerlitz 4641dd5bdff8SOr Gerlitz if (event != NETDEV_BONDING_FAILOVER) 4642dd5bdff8SOr Gerlitz return NOTIFY_DONE; 4643dd5bdff8SOr Gerlitz 46443cd96fddSParav Pandit if (!netif_is_bond_master(ndev)) 4645dd5bdff8SOr Gerlitz return NOTIFY_DONE; 4646dd5bdff8SOr Gerlitz 4647dd5bdff8SOr Gerlitz mutex_lock(&lock); 4648dd5bdff8SOr Gerlitz list_for_each_entry(cma_dev, &dev_list, list) 4649dd5bdff8SOr Gerlitz list_for_each_entry(id_priv, &cma_dev->id_list, list) { 4650dd5bdff8SOr Gerlitz ret = cma_netdev_change(ndev, id_priv); 4651dd5bdff8SOr Gerlitz if (ret) 4652dd5bdff8SOr Gerlitz goto out; 4653dd5bdff8SOr Gerlitz } 4654dd5bdff8SOr Gerlitz 4655dd5bdff8SOr Gerlitz out: 4656dd5bdff8SOr Gerlitz mutex_unlock(&lock); 4657dd5bdff8SOr Gerlitz return ret; 4658dd5bdff8SOr Gerlitz } 4659dd5bdff8SOr Gerlitz 4660dd5bdff8SOr Gerlitz static struct notifier_block cma_nb = { 4661dd5bdff8SOr Gerlitz .notifier_call = cma_netdev_callback 4662dd5bdff8SOr Gerlitz }; 4663dd5bdff8SOr Gerlitz 466411a0ae4cSJason Gunthorpe static int cma_add_one(struct ib_device *device) 4665e51060f0SSean Hefty { 4666e51060f0SSean Hefty struct cma_device *cma_dev; 4667e51060f0SSean Hefty struct rdma_id_private *id_priv; 4668045959dbSMatan Barak unsigned int i; 4669045959dbSMatan Barak unsigned long supported_gids = 0; 467011a0ae4cSJason Gunthorpe int ret; 4671e51060f0SSean Hefty 4672e51060f0SSean Hefty cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL); 4673e51060f0SSean Hefty if (!cma_dev) 467411a0ae4cSJason Gunthorpe return -ENOMEM; 4675e51060f0SSean Hefty 4676e51060f0SSean Hefty cma_dev->device = device; 4677045959dbSMatan Barak cma_dev->default_gid_type = kcalloc(device->phys_port_cnt, 4678045959dbSMatan Barak sizeof(*cma_dev->default_gid_type), 4679045959dbSMatan Barak GFP_KERNEL); 468011a0ae4cSJason Gunthorpe if (!cma_dev->default_gid_type) { 468111a0ae4cSJason Gunthorpe ret = -ENOMEM; 468289052d78SMajd Dibbiny goto free_cma_dev; 468311a0ae4cSJason Gunthorpe } 468489052d78SMajd Dibbiny 468589052d78SMajd Dibbiny cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt, 468689052d78SMajd Dibbiny sizeof(*cma_dev->default_roce_tos), 468789052d78SMajd Dibbiny GFP_KERNEL); 468811a0ae4cSJason Gunthorpe if (!cma_dev->default_roce_tos) { 468911a0ae4cSJason Gunthorpe ret = -ENOMEM; 469089052d78SMajd Dibbiny goto free_gid_type; 469111a0ae4cSJason Gunthorpe } 469289052d78SMajd Dibbiny 4693ea1075edSJason Gunthorpe rdma_for_each_port (device, i) { 4694045959dbSMatan Barak supported_gids = roce_gid_type_mask_support(device, i); 4695045959dbSMatan Barak WARN_ON(!supported_gids); 46965ab2d89bSLeon Romanovsky if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE)) 469763a5f483SMoni Shoua cma_dev->default_gid_type[i - rdma_start_port(device)] = 469863a5f483SMoni Shoua CMA_PREFERRED_ROCE_GID_TYPE; 469963a5f483SMoni Shoua else 4700045959dbSMatan Barak cma_dev->default_gid_type[i - rdma_start_port(device)] = 4701045959dbSMatan Barak find_first_bit(&supported_gids, BITS_PER_LONG); 470289052d78SMajd Dibbiny cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0; 4703045959dbSMatan Barak } 4704e51060f0SSean Hefty 4705e51060f0SSean Hefty init_completion(&cma_dev->comp); 4706be439912SParav Pandit refcount_set(&cma_dev->refcount, 1); 4707e51060f0SSean Hefty INIT_LIST_HEAD(&cma_dev->id_list); 4708e51060f0SSean Hefty ib_set_client_data(device, &cma_client, cma_dev); 4709e51060f0SSean Hefty 4710e51060f0SSean Hefty mutex_lock(&lock); 4711e51060f0SSean Hefty list_add_tail(&cma_dev->list, &dev_list); 4712e51060f0SSean Hefty list_for_each_entry(id_priv, &listen_any_list, list) 4713e51060f0SSean Hefty cma_listen_on_dev(id_priv, cma_dev); 4714e51060f0SSean Hefty mutex_unlock(&lock); 471589052d78SMajd Dibbiny 4716ed999f82SChuck Lever trace_cm_add_one(device); 471711a0ae4cSJason Gunthorpe return 0; 471889052d78SMajd Dibbiny 471989052d78SMajd Dibbiny free_gid_type: 472089052d78SMajd Dibbiny kfree(cma_dev->default_gid_type); 472189052d78SMajd Dibbiny 472289052d78SMajd Dibbiny free_cma_dev: 472389052d78SMajd Dibbiny kfree(cma_dev); 472411a0ae4cSJason Gunthorpe return ret; 4725e51060f0SSean Hefty } 4726e51060f0SSean Hefty 4727e51060f0SSean Hefty static int cma_remove_id_dev(struct rdma_id_private *id_priv) 4728e51060f0SSean Hefty { 47297582df82SParav Pandit struct rdma_cm_event event = {}; 4730550e5ca7SNir Muchtar enum rdma_cm_state state; 4731de910bd9SOr Gerlitz int ret = 0; 4732e51060f0SSean Hefty 4733e51060f0SSean Hefty /* Record that we want to remove the device */ 4734550e5ca7SNir Muchtar state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL); 4735550e5ca7SNir Muchtar if (state == RDMA_CM_DESTROYING) 4736e51060f0SSean Hefty return 0; 4737e51060f0SSean Hefty 4738e51060f0SSean Hefty cma_cancel_operation(id_priv, state); 4739de910bd9SOr Gerlitz mutex_lock(&id_priv->handler_mutex); 4740e51060f0SSean Hefty 4741e51060f0SSean Hefty /* Check for destruction from another callback. */ 4742550e5ca7SNir Muchtar if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL)) 4743de910bd9SOr Gerlitz goto out; 4744e51060f0SSean Hefty 4745a1b1b61fSSean Hefty event.event = RDMA_CM_EVENT_DEVICE_REMOVAL; 4746ed999f82SChuck Lever ret = cma_cm_event_handler(id_priv, &event); 4747de910bd9SOr Gerlitz out: 4748de910bd9SOr Gerlitz mutex_unlock(&id_priv->handler_mutex); 4749de910bd9SOr Gerlitz return ret; 4750e51060f0SSean Hefty } 4751e51060f0SSean Hefty 4752e51060f0SSean Hefty static void cma_process_remove(struct cma_device *cma_dev) 4753e51060f0SSean Hefty { 4754e51060f0SSean Hefty struct rdma_id_private *id_priv; 4755e51060f0SSean Hefty int ret; 4756e51060f0SSean Hefty 4757e51060f0SSean Hefty mutex_lock(&lock); 4758e51060f0SSean Hefty while (!list_empty(&cma_dev->id_list)) { 4759e51060f0SSean Hefty id_priv = list_entry(cma_dev->id_list.next, 4760e51060f0SSean Hefty struct rdma_id_private, list); 4761e51060f0SSean Hefty 4762d02d1f53SSean Hefty list_del(&id_priv->listen_list); 476394de178aSKrishna Kumar list_del_init(&id_priv->list); 4764e368d23fSParav Pandit cma_id_get(id_priv); 4765e51060f0SSean Hefty mutex_unlock(&lock); 4766e51060f0SSean Hefty 4767d02d1f53SSean Hefty ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv); 4768e368d23fSParav Pandit cma_id_put(id_priv); 4769e51060f0SSean Hefty if (ret) 4770e51060f0SSean Hefty rdma_destroy_id(&id_priv->id); 4771e51060f0SSean Hefty 4772e51060f0SSean Hefty mutex_lock(&lock); 4773e51060f0SSean Hefty } 4774e51060f0SSean Hefty mutex_unlock(&lock); 4775e51060f0SSean Hefty 47765ff8c8faSParav Pandit cma_dev_put(cma_dev); 4777e51060f0SSean Hefty wait_for_completion(&cma_dev->comp); 4778e51060f0SSean Hefty } 4779e51060f0SSean Hefty 47807c1eb45aSHaggai Eran static void cma_remove_one(struct ib_device *device, void *client_data) 4781e51060f0SSean Hefty { 47827c1eb45aSHaggai Eran struct cma_device *cma_dev = client_data; 4783e51060f0SSean Hefty 4784ed999f82SChuck Lever trace_cm_remove_one(device); 4785ed999f82SChuck Lever 4786e51060f0SSean Hefty mutex_lock(&lock); 4787e51060f0SSean Hefty list_del(&cma_dev->list); 4788e51060f0SSean Hefty mutex_unlock(&lock); 4789e51060f0SSean Hefty 4790e51060f0SSean Hefty cma_process_remove(cma_dev); 479189052d78SMajd Dibbiny kfree(cma_dev->default_roce_tos); 4792045959dbSMatan Barak kfree(cma_dev->default_gid_type); 4793e51060f0SSean Hefty kfree(cma_dev); 4794e51060f0SSean Hefty } 4795e51060f0SSean Hefty 47964be74b42SHaggai Eran static int cma_init_net(struct net *net) 47974be74b42SHaggai Eran { 47984be74b42SHaggai Eran struct cma_pernet *pernet = cma_pernet(net); 47994be74b42SHaggai Eran 480063826753SMatthew Wilcox xa_init(&pernet->tcp_ps); 480163826753SMatthew Wilcox xa_init(&pernet->udp_ps); 480263826753SMatthew Wilcox xa_init(&pernet->ipoib_ps); 480363826753SMatthew Wilcox xa_init(&pernet->ib_ps); 48044be74b42SHaggai Eran 48054be74b42SHaggai Eran return 0; 48064be74b42SHaggai Eran } 48074be74b42SHaggai Eran 48084be74b42SHaggai Eran static void cma_exit_net(struct net *net) 48094be74b42SHaggai Eran { 48104be74b42SHaggai Eran struct cma_pernet *pernet = cma_pernet(net); 48114be74b42SHaggai Eran 481263826753SMatthew Wilcox WARN_ON(!xa_empty(&pernet->tcp_ps)); 481363826753SMatthew Wilcox WARN_ON(!xa_empty(&pernet->udp_ps)); 481463826753SMatthew Wilcox WARN_ON(!xa_empty(&pernet->ipoib_ps)); 481563826753SMatthew Wilcox WARN_ON(!xa_empty(&pernet->ib_ps)); 48164be74b42SHaggai Eran } 48174be74b42SHaggai Eran 48184be74b42SHaggai Eran static struct pernet_operations cma_pernet_operations = { 48194be74b42SHaggai Eran .init = cma_init_net, 48204be74b42SHaggai Eran .exit = cma_exit_net, 48214be74b42SHaggai Eran .id = &cma_pernet_id, 48224be74b42SHaggai Eran .size = sizeof(struct cma_pernet), 48234be74b42SHaggai Eran }; 48244be74b42SHaggai Eran 4825716abb1fSPeter Huewe static int __init cma_init(void) 4826e51060f0SSean Hefty { 48275d7220e8STetsuo Handa int ret; 4828227b60f5SStephen Hemminger 482932ac9e43SJason Gunthorpe /* 483032ac9e43SJason Gunthorpe * There is a rare lock ordering dependency in cma_netdev_callback() 483132ac9e43SJason Gunthorpe * that only happens when bonding is enabled. Teach lockdep that rtnl 483232ac9e43SJason Gunthorpe * must never be nested under lock so it can find these without having 483332ac9e43SJason Gunthorpe * to test with bonding. 483432ac9e43SJason Gunthorpe */ 483532ac9e43SJason Gunthorpe if (IS_ENABLED(CONFIG_LOCKDEP)) { 483632ac9e43SJason Gunthorpe rtnl_lock(); 483732ac9e43SJason Gunthorpe mutex_lock(&lock); 483832ac9e43SJason Gunthorpe mutex_unlock(&lock); 483932ac9e43SJason Gunthorpe rtnl_unlock(); 484032ac9e43SJason Gunthorpe } 484132ac9e43SJason Gunthorpe 4842dee9acbbSBhaktipriya Shridhar cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM); 4843e51060f0SSean Hefty if (!cma_wq) 4844e51060f0SSean Hefty return -ENOMEM; 4845e51060f0SSean Hefty 48464be74b42SHaggai Eran ret = register_pernet_subsys(&cma_pernet_operations); 48474be74b42SHaggai Eran if (ret) 48484be74b42SHaggai Eran goto err_wq; 48494be74b42SHaggai Eran 4850c1a0b23bSMichael S. Tsirkin ib_sa_register_client(&sa_client); 4851dd5bdff8SOr Gerlitz register_netdevice_notifier(&cma_nb); 4852c1a0b23bSMichael S. Tsirkin 4853e51060f0SSean Hefty ret = ib_register_client(&cma_client); 4854e51060f0SSean Hefty if (ret) 4855e51060f0SSean Hefty goto err; 4856753f618aSNir Muchtar 4857a7bfb93fSzhengbin ret = cma_configfs_init(); 4858a7bfb93fSzhengbin if (ret) 4859a7bfb93fSzhengbin goto err_ib; 4860753f618aSNir Muchtar 4861e51060f0SSean Hefty return 0; 4862e51060f0SSean Hefty 4863a7bfb93fSzhengbin err_ib: 4864a7bfb93fSzhengbin ib_unregister_client(&cma_client); 4865e51060f0SSean Hefty err: 4866dd5bdff8SOr Gerlitz unregister_netdevice_notifier(&cma_nb); 4867c1a0b23bSMichael S. Tsirkin ib_sa_unregister_client(&sa_client); 486844a7b675SChuhong Yuan unregister_pernet_subsys(&cma_pernet_operations); 48694be74b42SHaggai Eran err_wq: 4870e51060f0SSean Hefty destroy_workqueue(cma_wq); 4871e51060f0SSean Hefty return ret; 4872e51060f0SSean Hefty } 4873e51060f0SSean Hefty 4874716abb1fSPeter Huewe static void __exit cma_cleanup(void) 4875e51060f0SSean Hefty { 4876045959dbSMatan Barak cma_configfs_exit(); 4877e51060f0SSean Hefty ib_unregister_client(&cma_client); 4878dd5bdff8SOr Gerlitz unregister_netdevice_notifier(&cma_nb); 4879c1a0b23bSMichael S. Tsirkin ib_sa_unregister_client(&sa_client); 48804be74b42SHaggai Eran unregister_pernet_subsys(&cma_pernet_operations); 4881e51060f0SSean Hefty destroy_workqueue(cma_wq); 4882e51060f0SSean Hefty } 4883e51060f0SSean Hefty 4884e51060f0SSean Hefty module_init(cma_init); 4885e51060f0SSean Hefty module_exit(cma_cleanup); 4886