1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 #include <net/mana/mana_auxiliary.h> 8 #include <net/addrconf.h> 9 10 MODULE_DESCRIPTION("Microsoft Azure Network Adapter IB driver"); 11 MODULE_LICENSE("GPL"); 12 MODULE_IMPORT_NS("NET_MANA"); 13 14 static const struct ib_device_ops mana_ib_dev_ops = { 15 .owner = THIS_MODULE, 16 .driver_id = RDMA_DRIVER_MANA, 17 .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION, 18 .uverbs_robust_udata = true, 19 20 .add_gid = mana_ib_gd_add_gid, 21 .alloc_mw = mana_ib_alloc_mw, 22 .alloc_pd = mana_ib_alloc_pd, 23 .alloc_ucontext = mana_ib_alloc_ucontext, 24 .create_ah = mana_ib_create_ah, 25 .create_cq = mana_ib_create_cq, 26 .create_qp = mana_ib_create_qp, 27 .create_rwq_ind_table = mana_ib_create_rwq_ind_table, 28 .create_wq = mana_ib_create_wq, 29 .dealloc_mw = mana_ib_dealloc_mw, 30 .dealloc_pd = mana_ib_dealloc_pd, 31 .dealloc_ucontext = mana_ib_dealloc_ucontext, 32 .del_gid = mana_ib_gd_del_gid, 33 .dereg_mr = mana_ib_dereg_mr, 34 .destroy_ah = mana_ib_destroy_ah, 35 .destroy_cq = mana_ib_destroy_cq, 36 .destroy_qp = mana_ib_destroy_qp, 37 .destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table, 38 .destroy_wq = mana_ib_destroy_wq, 39 .disassociate_ucontext = mana_ib_disassociate_ucontext, 40 .get_dma_mr = mana_ib_get_dma_mr, 41 .get_link_layer = mana_ib_get_link_layer, 42 .get_port_immutable = mana_ib_get_port_immutable, 43 .mmap = mana_ib_mmap, 44 .modify_qp = mana_ib_modify_qp, 45 .modify_wq = mana_ib_modify_wq, 46 .poll_cq = mana_ib_poll_cq, 47 .post_recv = mana_ib_post_recv, 48 .post_send = mana_ib_post_send, 49 .query_device = mana_ib_query_device, 50 .query_gid = mana_ib_query_gid, 51 .query_pkey = mana_ib_query_pkey, 52 .query_port = mana_ib_query_port, 53 .reg_user_mr = mana_ib_reg_user_mr, 54 .reg_user_mr_dmabuf = mana_ib_reg_user_mr_dmabuf, 55 .req_notify_cq = mana_ib_arm_cq, 56 57 INIT_RDMA_OBJ_SIZE(ib_ah, mana_ib_ah, ibah), 58 INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq), 59 INIT_RDMA_OBJ_SIZE(ib_mw, mana_ib_mw, ibmw), 60 INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd), 61 INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp), 62 INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext), 63 INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mana_ib_rwq_ind_table, 64 ib_ind_table), 65 }; 66 67 static const struct ib_device_ops mana_ib_stats_ops = { 68 .alloc_hw_port_stats = mana_ib_alloc_hw_port_stats, 69 .get_hw_stats = mana_ib_get_hw_stats, 70 }; 71 72 static const struct ib_device_ops mana_ib_device_stats_ops = { 73 .alloc_hw_device_stats = mana_ib_alloc_hw_device_stats, 74 }; 75 76 const struct ib_device_ops mana_ib_dev_dm_ops = { 77 .alloc_dm = mana_ib_alloc_dm, 78 .dealloc_dm = mana_ib_dealloc_dm, 79 .reg_dm_mr = mana_ib_reg_dm_mr, 80 }; 81 82 static int mana_ib_netdev_event(struct notifier_block *this, 83 unsigned long event, void *ptr) 84 { 85 struct mana_ib_dev *dev = container_of(this, struct mana_ib_dev, nb); 86 struct net_device *event_dev = netdev_notifier_info_to_dev(ptr); 87 struct gdma_context *gc = dev->gdma_dev->gdma_context; 88 struct mana_context *mc = gc->mana.driver_data; 89 struct net_device *ndev; 90 int i; 91 92 /* Only process events from our parent device */ 93 for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) 94 if (event_dev == mc->ports[i]) { 95 switch (event) { 96 case NETDEV_CHANGEUPPER: 97 ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker); 98 /* 99 * RDMA core will setup GID based on updated netdev. 100 * It's not possible to race with the core as rtnl lock is being 101 * held. 102 */ 103 ib_device_set_netdev(&dev->ib_dev, ndev, i + 1); 104 105 /* mana_get_primary_netdev() returns ndev with refcount held */ 106 if (ndev) 107 netdev_put(ndev, &dev->dev_tracker); 108 109 return NOTIFY_OK; 110 default: 111 return NOTIFY_DONE; 112 } 113 } 114 return NOTIFY_DONE; 115 } 116 117 static int mana_ib_probe(struct auxiliary_device *adev, 118 const struct auxiliary_device_id *id) 119 { 120 struct mana_adev *madev = container_of(adev, struct mana_adev, adev); 121 struct gdma_context *gc = madev->mdev->gdma_context; 122 struct mana_context *mc = gc->mana.driver_data; 123 struct gdma_dev *mdev = madev->mdev; 124 struct net_device *ndev; 125 struct mana_ib_dev *dev; 126 u8 mac_addr[ETH_ALEN]; 127 int ret, i; 128 129 dev = ib_alloc_device(mana_ib_dev, ib_dev); 130 if (!dev) 131 return -ENOMEM; 132 133 ib_set_device_ops(&dev->ib_dev, &mana_ib_dev_ops); 134 dev->ib_dev.node_type = RDMA_NODE_IB_CA; 135 dev->ib_dev.num_comp_vectors = gc->max_num_queues; 136 dev->ib_dev.dev.parent = gc->dev; 137 dev->gdma_dev = mdev; 138 xa_init_flags(&dev->qp_table_wq, XA_FLAGS_LOCK_IRQ); 139 140 if (mana_ib_is_rnic(dev)) { 141 dev->ib_dev.phys_port_cnt = 1; 142 addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr); 143 ret = mana_ib_gd_query_adapter_caps(dev); 144 if (ret) { 145 ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret); 146 goto free_ib_device; 147 } 148 149 ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops); 150 if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT) 151 ib_set_device_ops(&dev->ib_dev, &mana_ib_device_stats_ops); 152 ib_set_device_ops(&dev->ib_dev, &mana_ib_dev_dm_ops); 153 154 ret = mana_ib_create_eqs(dev); 155 if (ret) { 156 ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret); 157 goto free_ib_device; 158 } 159 160 ret = mana_ib_gd_create_rnic_adapter(dev); 161 if (ret) 162 goto destroy_eqs; 163 164 if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT) 165 dev->ib_dev.phys_port_cnt = mc->num_ports; 166 167 for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) { 168 ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker); 169 if (!ndev) { 170 ret = -ENODEV; 171 ibdev_err(&dev->ib_dev, 172 "Failed to get netdev for IB port %d", i + 1); 173 goto destroy_rnic; 174 } 175 ether_addr_copy(mac_addr, ndev->dev_addr); 176 ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1); 177 /* mana_get_primary_netdev() returns ndev with refcount held */ 178 netdev_put(ndev, &dev->dev_tracker); 179 if (ret) { 180 ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret); 181 goto destroy_rnic; 182 } 183 ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr); 184 if (ret) { 185 ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret); 186 goto destroy_rnic; 187 } 188 } 189 dev->nb.notifier_call = mana_ib_netdev_event; 190 ret = register_netdevice_notifier(&dev->nb); 191 if (ret) { 192 ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret); 193 goto destroy_rnic; 194 } 195 } else { 196 dev->ib_dev.phys_port_cnt = mc->num_ports; 197 ret = mana_eth_query_adapter_caps(dev); 198 if (ret) { 199 ibdev_err(&dev->ib_dev, "Failed to query ETH device caps, ret %d", ret); 200 goto free_ib_device; 201 } 202 } 203 204 dev->av_pool = dma_pool_create("mana_ib_av", gc->dev, MANA_AV_BUFFER_SIZE, 205 MANA_AV_BUFFER_SIZE, 0); 206 if (!dev->av_pool) { 207 ret = -ENOMEM; 208 goto deregister_net_notifier; 209 } 210 211 ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev, 212 mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt); 213 214 ret = ib_register_device(&dev->ib_dev, mana_ib_is_rnic(dev) ? "mana_%d" : "manae_%d", 215 gc->dev); 216 if (ret) 217 goto deallocate_pool; 218 219 dev_set_drvdata(&adev->dev, dev); 220 221 return 0; 222 223 deallocate_pool: 224 dma_pool_destroy(dev->av_pool); 225 deregister_net_notifier: 226 if (mana_ib_is_rnic(dev)) 227 unregister_netdevice_notifier(&dev->nb); 228 destroy_rnic: 229 if (mana_ib_is_rnic(dev)) 230 mana_ib_gd_destroy_rnic_adapter(dev); 231 destroy_eqs: 232 if (mana_ib_is_rnic(dev)) 233 mana_ib_destroy_eqs(dev); 234 free_ib_device: 235 xa_destroy(&dev->qp_table_wq); 236 ib_dealloc_device(&dev->ib_dev); 237 return ret; 238 } 239 240 static void mana_ib_remove(struct auxiliary_device *adev) 241 { 242 struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev); 243 244 if (mana_ib_is_rnic(dev)) 245 mana_drain_gsi_sqs(dev); 246 247 ib_unregister_device(&dev->ib_dev); 248 dma_pool_destroy(dev->av_pool); 249 if (mana_ib_is_rnic(dev)) { 250 unregister_netdevice_notifier(&dev->nb); 251 mana_ib_gd_destroy_rnic_adapter(dev); 252 mana_ib_destroy_eqs(dev); 253 } 254 xa_destroy(&dev->qp_table_wq); 255 ib_dealloc_device(&dev->ib_dev); 256 } 257 258 static const struct auxiliary_device_id mana_id_table[] = { 259 { .name = "mana.rdma", }, 260 { .name = "mana.eth", }, 261 {}, 262 }; 263 264 MODULE_DEVICE_TABLE(auxiliary, mana_id_table); 265 266 static struct auxiliary_driver mana_driver = { 267 .probe = mana_ib_probe, 268 .remove = mana_ib_remove, 269 .id_table = mana_id_table, 270 }; 271 272 module_auxiliary_driver(mana_driver); 273