1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2025, Microsoft Corporation. */ 3 #include <linux/module.h> 4 #include <linux/bitmap.h> 5 #include <linux/cdx/cdx_bus.h> 6 #include <linux/msi.h> 7 #include <linux/xarray.h> 8 9 #include <net/mana/mana.h> 10 11 #define MANA_CDX_GDMA_REGS_SIZE (GDMA_REG_SHM_OFFSET + sizeof(u64)) 12 13 static int mana_cdx_msix_virq(struct gdma_context *gc, int msi) 14 { 15 unsigned int virq = msi_get_virq(gc->dev, msi); 16 17 /* CDX allocates its whole vector pool in mana_cdx_setup_hwc_irqs(), 18 * so a vector with no Linux IRQ was never part of that pool. 19 */ 20 return virq ? virq : -EINVAL; 21 } 22 23 static int mana_cdx_dev_reset(struct gdma_context *gc) 24 { 25 return cdx_dev_reset(gc->dev); 26 } 27 28 static int mana_cdx_setup_hwc_irqs(struct gdma_context *gc) 29 { 30 struct cdx_device *cdx_dev = to_cdx_device(gc->dev); 31 struct gdma_irq_context *gic; 32 unsigned int max_irqs; 33 int nvec; 34 int err, i = 0, j; 35 36 max_irqs = cdx_dev->num_msi; 37 38 /* Need 1 IRQ for HWC and at least 1 for the data path queues */ 39 if (max_irqs < 2) { 40 dev_err(&cdx_dev->dev, "Not enough MSI vectors: %u\n", 41 max_irqs); 42 return -ENOSPC; 43 } 44 45 /* Constrain the device to 32-bit DMA addresses, for both the MSI 46 * doorbells and the data path buffers, and never widen it afterwards. 47 * 48 * The MSI doorbell address has to land within the low 4GB. The data 49 * path carries a second constraint: the device does not perform 50 * writes to the final page of the 48-bit IOVA window, and 51 * iommu_dma_alloc_iova() allocates downwards from the top of that 52 * window, so the first queue created lands on the unusable page and 53 * its event queue stalls permanently. PCI devices are shielded from 54 * this by the low-4GB first attempt in iommu_dma_alloc_iova(), but 55 * that attempt is gated on dev_is_pci() and CDX devices do not 56 * qualify. 57 */ 58 err = dma_set_mask_and_coherent(gc->dev, DMA_BIT_MASK(32)); 59 if (err) { 60 dev_err(&cdx_dev->dev, "Failed to set 32-bit DMA mask\n"); 61 return err; 62 } 63 64 /* The PCI path also calls dma_set_max_seg_size(), but that is 65 * deliberately omitted here: the CDX core does not attach a 66 * dma_parms to the device, so the helper would only WARN, and MANA 67 * never builds scatter-gather lists, so the limit is never read. 68 */ 69 70 err = cdx_enable_msi(cdx_dev); 71 if (err) { 72 dev_err(&cdx_dev->dev, "Failed to enable MSI\n"); 73 return err; 74 } 75 76 err = msi_domain_alloc_irqs(gc->dev, MSI_DEFAULT_DOMAIN, max_irqs); 77 if (err) { 78 dev_err(&cdx_dev->dev, "Failed to alloc MSI IRQs\n"); 79 cdx_disable_msi(cdx_dev); 80 return err; 81 } 82 83 nvec = max_irqs; 84 85 xa_init(&gc->irq_contexts); 86 87 /* No IRQ affinity hint is applied here. The PCI transport spreads its 88 * vectors over the NUMA node local to the device; the CDX platforms 89 * this driver targets are single-node, so there is nothing to spread 90 * over. Affinity management can be added when a multi-node CDX host 91 * exists to validate it against. 92 */ 93 for (i = 0; i < nvec; i++) { 94 int msi = i; 95 96 gic = mana_gd_get_gic(gc, false, &msi); 97 if (IS_ERR(gic)) { 98 err = PTR_ERR(gic); 99 goto free_irq; 100 } 101 } 102 103 gc->max_num_msix = nvec; 104 gc->num_msix_usable = nvec; 105 106 /* CDX sizes its MSI pool at probe time and cannot grow it later, so 107 * never share vectors: vector 0 is reserved for the HWC and each of 108 * the remaining nvec - 1 vectors backs exactly one EQ. That bounds 109 * the queue count at nvec - 1 and guarantees the HWC vector is never 110 * handed out to a data queue. 111 */ 112 gc->msi_sharing = false; 113 114 gc->msi_bitmap = bitmap_zalloc(nvec, GFP_KERNEL); 115 if (!gc->msi_bitmap) { 116 err = -ENOMEM; 117 goto free_irq; 118 } 119 120 /* Reserve the HWC vector */ 121 set_bit(0, gc->msi_bitmap); 122 123 return 0; 124 125 free_irq: 126 for (j = i - 1; j >= 0; j--) 127 mana_gd_put_gic(gc, false, j); 128 129 xa_destroy(&gc->irq_contexts); 130 msi_domain_free_irqs_all(gc->dev, MSI_DEFAULT_DOMAIN); 131 cdx_disable_msi(cdx_dev); 132 return err; 133 } 134 135 static void mana_cdx_remove_irqs(struct gdma_context *gc) 136 { 137 struct cdx_device *cdx_dev = to_cdx_device(gc->dev); 138 int i; 139 140 if (gc->max_num_msix < 1) 141 return; 142 143 for (i = 0; i < gc->max_num_msix; i++) { 144 if (!xa_load(&gc->irq_contexts, i)) 145 continue; 146 147 mana_gd_put_gic(gc, false, i); 148 } 149 150 WARN_ON(!xa_empty(&gc->irq_contexts)); 151 152 xa_destroy(&gc->irq_contexts); 153 msi_domain_free_irqs_all(gc->dev, MSI_DEFAULT_DOMAIN); 154 cdx_disable_msi(cdx_dev); 155 156 bitmap_free(gc->msi_bitmap); 157 gc->msi_bitmap = NULL; 158 gc->max_num_msix = 0; 159 gc->num_msix_usable = 0; 160 } 161 162 static int mana_cdx_setup_remaining_irqs(struct gdma_context *gc) 163 { 164 unsigned int max_queues_vport; 165 166 /* mana_gd_query_max_resources() may turn MSI sharing back on and round 167 * the per-vPort queue count up to MANA_DEF_NUM_QUEUES. CDX sizes its 168 * vector pool at probe time and cannot grow it later, and the HWC 169 * vector must stay private, so give every vPort a private slice of the 170 * remaining vectors instead. gc->max_num_queues is already capped at 171 * num_msix_usable - 1 by the core. 172 */ 173 gc->msi_sharing = false; 174 max_queues_vport = (gc->num_msix_usable - 1) / gc->num_ports; 175 if (!max_queues_vport) { 176 dev_err(gc->dev, "%u MSI vectors cannot serve %u vPorts\n", 177 gc->num_msix_usable, gc->num_ports); 178 return -ENOSPC; 179 } 180 181 gc->max_num_queues_vport = min(gc->max_num_queues, max_queues_vport); 182 183 dev_dbg(gc->dev, "%u MSI vectors, %u vPorts, %u queues, no MSI sharing\n", 184 gc->num_msix_usable, gc->num_ports, gc->max_num_queues_vport); 185 186 return 0; 187 } 188 189 static const struct gdma_bus_ops mana_cdx_bus_ops = { 190 .bus_name = "cdx", 191 .adev_prefix = "mana_cdx", 192 .msix_virq = mana_cdx_msix_virq, 193 .setup_hwc_irqs = mana_cdx_setup_hwc_irqs, 194 .setup_remaining_irqs = mana_cdx_setup_remaining_irqs, 195 .remove_irqs = mana_cdx_remove_irqs, 196 .dev_reset = mana_cdx_dev_reset, 197 }; 198 199 static int mana_cdx_gd_probe(struct cdx_device *cdx_dev) 200 { 201 struct gdma_context *gc; 202 void __iomem *bar0_va; 203 int err; 204 205 /* Each port has 2 CQs, each CQ has at most 1 EQE at a time */ 206 BUILD_BUG_ON(2 * MAX_PORTS_IN_MANA_DEV * GDMA_EQE_SIZE > EQ_SIZE); 207 208 err = cdx_dev_reset(&cdx_dev->dev); 209 if (err) 210 return err; 211 212 err = cdx_set_master(cdx_dev); 213 if (err) 214 return err; 215 216 gc = vzalloc(sizeof(*gc)); 217 if (!gc) { 218 err = -ENOMEM; 219 goto clear_master; 220 } 221 222 mutex_init(&gc->eq_test_event_mutex); 223 mutex_init(&gc->gic_mutex); 224 dev_set_drvdata(&cdx_dev->dev, gc); 225 gc->bar0_pa = cdx_resource_start(cdx_dev, 0); 226 gc->bar0_size = cdx_resource_len(cdx_dev, 0); 227 if (gc->bar0_size < MANA_CDX_GDMA_REGS_SIZE) { 228 dev_err(&cdx_dev->dev, "BAR0 size %#llx is too small\n", 229 (u64)gc->bar0_size); 230 err = -EINVAL; 231 goto free_gc; 232 } 233 234 bar0_va = ioremap(gc->bar0_pa, gc->bar0_size); 235 if (!bar0_va) { 236 err = -ENOMEM; 237 goto free_gc; 238 } 239 240 gc->numa_node = dev_to_node(&cdx_dev->dev); 241 gc->bar0_va = bar0_va; 242 gc->dev = &cdx_dev->dev; 243 gc->bus_ops = &mana_cdx_bus_ops; 244 245 /* gc->is_pf is intentionally left clear: this device uses the VF 246 * register layout, and the vPort and MAC filter registration that 247 * is_pf selects is not required on this bus. 248 */ 249 250 err = mana_gd_setup(gc); 251 if (err) 252 goto unmap_bar; 253 254 err = mana_probe(&gc->mana, false); 255 if (err) 256 goto cleanup_gd; 257 258 err = mana_rdma_probe(&gc->mana_ib); 259 if (err) 260 goto remove_mana; 261 262 /* If a hardware reset event arrived over HWC during probe, roll back: 263 * the device state the rest of probe observed is no longer valid. 264 */ 265 if (test_and_set_bit(GC_PROBE_SUCCEEDED, &gc->flags)) { 266 err = -EPROTO; 267 goto remove_rdma; 268 } 269 270 return 0; 271 272 remove_rdma: 273 mana_rdma_remove(&gc->mana_ib); 274 remove_mana: 275 mana_remove(&gc->mana, false); 276 cleanup_gd: 277 mana_gd_cleanup(gc); 278 unmap_bar: 279 iounmap(bar0_va); 280 free_gc: 281 dev_set_drvdata(&cdx_dev->dev, NULL); 282 vfree(gc); 283 clear_master: 284 cdx_clear_master(cdx_dev); 285 return err; 286 } 287 288 static int mana_cdx_gd_remove(struct cdx_device *cdx_dev) 289 { 290 struct gdma_context *gc = dev_get_drvdata(&cdx_dev->dev); 291 292 mana_rdma_remove(&gc->mana_ib); 293 mana_remove(&gc->mana, false); 294 295 mana_gd_cleanup(gc); 296 cdx_clear_master(cdx_dev); 297 298 iounmap(gc->bar0_va); 299 300 dev_set_drvdata(&cdx_dev->dev, NULL); 301 vfree(gc); 302 return 0; 303 } 304 305 /* Quiesce the device for kexec. This is also called upon reboot/shutdown. */ 306 static void mana_cdx_gd_shutdown(struct cdx_device *cdx_dev) 307 { 308 struct gdma_context *gc = dev_get_drvdata(&cdx_dev->dev); 309 310 mana_rdma_remove(&gc->mana_ib); 311 mana_remove(&gc->mana, true); 312 313 mana_gd_cleanup(gc); 314 cdx_clear_master(cdx_dev); 315 } 316 317 #define CDX_VENDOR_ID_MICROSOFT 0x1414 318 static const struct cdx_device_id mana_cdx_table[] = { 319 { CDX_DEVICE(CDX_VENDOR_ID_MICROSOFT, MANA_CDX_DEVICE_ID) }, 320 {} 321 }; 322 323 MODULE_DEVICE_TABLE(cdx, mana_cdx_table); 324 325 static struct cdx_driver gdma_cdx_driver = { 326 .match_id_table = mana_cdx_table, 327 .probe = mana_cdx_gd_probe, 328 .remove = mana_cdx_gd_remove, 329 .shutdown = mana_cdx_gd_shutdown, 330 .driver = { 331 .name = "mana_cdx", 332 }, 333 }; 334 335 module_driver(gdma_cdx_driver, cdx_driver_register, cdx_driver_unregister); 336 337 MODULE_LICENSE("Dual BSD/GPL"); 338 MODULE_DESCRIPTION("Microsoft Azure Network Adapter driver for the CDX bus"); 339 MODULE_IMPORT_NS("CDX_BUS"); 340 MODULE_IMPORT_NS("NET_MANA"); 341