1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved 4 */ 5 6 #include "mlx5_ib.h" 7 #include "data_direct.h" 8 9 static LIST_HEAD(mlx5_data_direct_dev_list); 10 static LIST_HEAD(mlx5_data_direct_reg_list); 11 12 /* 13 * This mutex should be held when accessing either of the above lists 14 */ 15 static DEFINE_MUTEX(mlx5_data_direct_mutex); 16 17 struct mlx5_data_direct_registration { 18 struct mlx5_ib_dev *ibdev; 19 char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1]; 20 struct list_head list; 21 }; 22 23 static const struct pci_device_id mlx5_data_direct_pci_table[] = { 24 { PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */ 25 { 0, } 26 }; 27 28 static int mlx5_data_direct_vpd_get_vuid(struct mlx5_data_direct_dev *dev) 29 { 30 struct pci_dev *pdev = dev->pdev; 31 unsigned int vpd_size, kw_len; 32 u8 *vpd_data; 33 int start; 34 int ret; 35 36 vpd_data = pci_vpd_alloc(pdev, &vpd_size); 37 if (IS_ERR(vpd_data)) { 38 pci_err(pdev, "Unable to read VPD, err=%ld\n", PTR_ERR(vpd_data)); 39 return PTR_ERR(vpd_data); 40 } 41 42 start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, "VU", &kw_len); 43 if (start < 0) { 44 ret = start; 45 pci_err(pdev, "VU keyword not found, err=%d\n", ret); 46 goto end; 47 } 48 49 dev->vuid = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL); 50 ret = dev->vuid ? 0 : -ENOMEM; 51 52 end: 53 kfree(vpd_data); 54 return ret; 55 } 56 57 static void mlx5_data_direct_shutdown(struct pci_dev *pdev) 58 { 59 pci_disable_device(pdev); 60 } 61 62 static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev) 63 { 64 int err; 65 66 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 67 if (err) { 68 dev_warn(&pdev->dev, 69 "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err); 70 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 71 if (err) { 72 dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err); 73 return err; 74 } 75 } 76 77 dma_set_max_seg_size(&pdev->dev, SZ_2G); 78 return 0; 79 } 80 81 int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid) 82 { 83 struct mlx5_data_direct_registration *reg; 84 struct mlx5_data_direct_dev *dev; 85 86 reg = kzalloc(sizeof(*reg), GFP_KERNEL); 87 if (!reg) 88 return -ENOMEM; 89 90 reg->ibdev = ibdev; 91 strcpy(reg->vuid, vuid); 92 93 mutex_lock(&mlx5_data_direct_mutex); 94 list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) { 95 if (strcmp(dev->vuid, vuid) == 0) { 96 mlx5_ib_data_direct_bind(ibdev, dev); 97 break; 98 } 99 } 100 101 /* Add the registration to its global list, to be used upon bind/unbind 102 * of its affiliated data direct device 103 */ 104 list_add_tail(®->list, &mlx5_data_direct_reg_list); 105 mutex_unlock(&mlx5_data_direct_mutex); 106 return 0; 107 } 108 109 void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev) 110 { 111 struct mlx5_data_direct_registration *reg; 112 113 mutex_lock(&mlx5_data_direct_mutex); 114 list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { 115 if (reg->ibdev == ibdev) { 116 list_del(®->list); 117 kfree(reg); 118 goto end; 119 } 120 } 121 122 WARN_ON(true); 123 end: 124 mutex_unlock(&mlx5_data_direct_mutex); 125 } 126 127 static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev) 128 { 129 struct mlx5_data_direct_registration *reg; 130 131 mutex_lock(&mlx5_data_direct_mutex); 132 list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { 133 if (strcmp(dev->vuid, reg->vuid) == 0) 134 mlx5_ib_data_direct_bind(reg->ibdev, dev); 135 } 136 137 /* Add the data direct device to the global list, further IB devices may 138 * use it later as well 139 */ 140 list_add_tail(&dev->list, &mlx5_data_direct_dev_list); 141 mutex_unlock(&mlx5_data_direct_mutex); 142 } 143 144 static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev) 145 { 146 struct mlx5_data_direct_registration *reg; 147 148 mutex_lock(&mlx5_data_direct_mutex); 149 /* Prevent any further affiliations */ 150 list_del(&dev->list); 151 list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) { 152 if (strcmp(dev->vuid, reg->vuid) == 0) 153 mlx5_ib_data_direct_unbind(reg->ibdev); 154 } 155 mutex_unlock(&mlx5_data_direct_mutex); 156 } 157 158 static int mlx5_data_direct_probe(struct pci_dev *pdev, const struct pci_device_id *id) 159 { 160 struct mlx5_data_direct_dev *dev; 161 int err; 162 163 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 164 if (!dev) 165 return -ENOMEM; 166 167 dev->device = &pdev->dev; 168 dev->pdev = pdev; 169 170 pci_set_drvdata(dev->pdev, dev); 171 err = pci_enable_device(pdev); 172 if (err) { 173 dev_err(dev->device, "Cannot enable PCI device, err=%d\n", err); 174 goto err; 175 } 176 177 pci_set_master(pdev); 178 err = mlx5_data_direct_set_dma_caps(pdev); 179 if (err) 180 goto err_disable; 181 182 if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) && 183 pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) && 184 pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128)) 185 dev_dbg(dev->device, "Enabling pci atomics failed\n"); 186 187 err = mlx5_data_direct_vpd_get_vuid(dev); 188 if (err) 189 goto err_disable; 190 191 mlx5_data_direct_dev_reg(dev); 192 return 0; 193 194 err_disable: 195 pci_disable_device(pdev); 196 err: 197 kfree(dev); 198 return err; 199 } 200 201 static void mlx5_data_direct_remove(struct pci_dev *pdev) 202 { 203 struct mlx5_data_direct_dev *dev = pci_get_drvdata(pdev); 204 205 mlx5_data_direct_dev_unreg(dev); 206 pci_disable_device(pdev); 207 kfree(dev->vuid); 208 kfree(dev); 209 } 210 211 static struct pci_driver mlx5_data_direct_driver = { 212 .name = KBUILD_MODNAME, 213 .id_table = mlx5_data_direct_pci_table, 214 .probe = mlx5_data_direct_probe, 215 .remove = mlx5_data_direct_remove, 216 .shutdown = mlx5_data_direct_shutdown, 217 }; 218 219 int mlx5_data_direct_driver_register(void) 220 { 221 return pci_register_driver(&mlx5_data_direct_driver); 222 } 223 224 void mlx5_data_direct_driver_unregister(void) 225 { 226 pci_unregister_driver(&mlx5_data_direct_driver); 227 } 228