1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/aer.h> 3 #include <linux/delay.h> 4 #include <linux/firmware.h> 5 #include <linux/list.h> 6 #include <linux/module.h> 7 #include <linux/mutex.h> 8 #include <linux/pci.h> 9 #include <linux/pci_ids.h> 10 11 #include "nitrox_dev.h" 12 #include "nitrox_common.h" 13 #include "nitrox_csr.h" 14 #include "nitrox_hal.h" 15 #include "nitrox_isr.h" 16 #include "nitrox_debugfs.h" 17 18 #define CNN55XX_DEV_ID 0x12 19 #define UCODE_HLEN 48 20 #define SE_GROUP 0 21 22 #define DRIVER_VERSION "1.1" 23 #define FW_DIR "cavium/" 24 /* SE microcode */ 25 #define SE_FW FW_DIR "cnn55xx_se.fw" 26 27 static const char nitrox_driver_name[] = "CNN55XX"; 28 29 static LIST_HEAD(ndevlist); 30 static DEFINE_MUTEX(devlist_lock); 31 static unsigned int num_devices; 32 33 /** 34 * nitrox_pci_tbl - PCI Device ID Table 35 */ 36 static const struct pci_device_id nitrox_pci_tbl[] = { 37 {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0}, 38 /* required last entry */ 39 {0, } 40 }; 41 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl); 42 43 static unsigned int qlen = DEFAULT_CMD_QLEN; 44 module_param(qlen, uint, 0644); 45 MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); 46 47 #ifdef CONFIG_PCI_IOV 48 int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs); 49 #else 50 int nitrox_sriov_configure(struct pci_dev *pdev, int num_vfs) 51 { 52 return 0; 53 } 54 #endif 55 56 /** 57 * struct ucode - Firmware Header 58 * @id: microcode ID 59 * @version: firmware version 60 * @code_size: code section size 61 * @raz: alignment 62 * @code: code section 63 */ 64 struct ucode { 65 u8 id; 66 char version[VERSION_LEN - 1]; 67 __be32 code_size; 68 u8 raz[12]; 69 u64 code[0]; 70 }; 71 72 /** 73 * write_to_ucd_unit - Write Firmware to NITROX UCD unit 74 */ 75 static void write_to_ucd_unit(struct nitrox_device *ndev, 76 struct ucode *ucode) 77 { 78 u32 code_size = be32_to_cpu(ucode->code_size) * 2; 79 u64 offset, data; 80 int i = 0; 81 82 /* 83 * UCD structure 84 * 85 * ------------- 86 * | BLK 7 | 87 * ------------- 88 * | BLK 6 | 89 * ------------- 90 * | ... | 91 * ------------- 92 * | BLK 0 | 93 * ------------- 94 * Total of 8 blocks, each size 32KB 95 */ 96 97 /* set the block number */ 98 offset = UCD_UCODE_LOAD_BLOCK_NUM; 99 nitrox_write_csr(ndev, offset, 0); 100 101 code_size = roundup(code_size, 8); 102 while (code_size) { 103 data = ucode->code[i]; 104 /* write 8 bytes at a time */ 105 offset = UCD_UCODE_LOAD_IDX_DATAX(i); 106 nitrox_write_csr(ndev, offset, data); 107 code_size -= 8; 108 i++; 109 } 110 111 /* put all SE cores in group 0 */ 112 offset = POM_GRP_EXECMASKX(SE_GROUP); 113 nitrox_write_csr(ndev, offset, (~0ULL)); 114 115 for (i = 0; i < ndev->hw.se_cores; i++) { 116 /* 117 * write block number and firware length 118 * bit:<2:0> block number 119 * bit:3 is set SE uses 32KB microcode 120 * bit:3 is clear SE uses 64KB microcode 121 */ 122 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i); 123 nitrox_write_csr(ndev, offset, 0x8); 124 } 125 usleep_range(300, 400); 126 } 127 128 static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name) 129 { 130 const struct firmware *fw; 131 struct ucode *ucode; 132 int ret; 133 134 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name); 135 136 ret = request_firmware(&fw, fw_name, DEV(ndev)); 137 if (ret < 0) { 138 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name); 139 return ret; 140 } 141 142 ucode = (struct ucode *)fw->data; 143 /* copy the firmware version */ 144 memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2)); 145 ndev->hw.fw_name[VERSION_LEN - 1] = '\0'; 146 147 write_to_ucd_unit(ndev, ucode); 148 release_firmware(fw); 149 150 return 0; 151 } 152 153 /** 154 * nitrox_add_to_devlist - add NITROX device to global device list 155 * @ndev: NITROX device 156 */ 157 static int nitrox_add_to_devlist(struct nitrox_device *ndev) 158 { 159 struct nitrox_device *dev; 160 int ret = 0; 161 162 INIT_LIST_HEAD(&ndev->list); 163 refcount_set(&ndev->refcnt, 1); 164 165 mutex_lock(&devlist_lock); 166 list_for_each_entry(dev, &ndevlist, list) { 167 if (dev == ndev) { 168 ret = -EEXIST; 169 goto unlock; 170 } 171 } 172 ndev->idx = num_devices++; 173 list_add_tail(&ndev->list, &ndevlist); 174 unlock: 175 mutex_unlock(&devlist_lock); 176 return ret; 177 } 178 179 /** 180 * nitrox_remove_from_devlist - remove NITROX device from 181 * global device list 182 * @ndev: NITROX device 183 */ 184 static void nitrox_remove_from_devlist(struct nitrox_device *ndev) 185 { 186 mutex_lock(&devlist_lock); 187 list_del(&ndev->list); 188 num_devices--; 189 mutex_unlock(&devlist_lock); 190 } 191 192 struct nitrox_device *nitrox_get_first_device(void) 193 { 194 struct nitrox_device *ndev = NULL; 195 196 mutex_lock(&devlist_lock); 197 list_for_each_entry(ndev, &ndevlist, list) { 198 if (nitrox_ready(ndev)) 199 break; 200 } 201 mutex_unlock(&devlist_lock); 202 if (!ndev) 203 return NULL; 204 205 refcount_inc(&ndev->refcnt); 206 /* barrier to sync with other cpus */ 207 smp_mb__after_atomic(); 208 return ndev; 209 } 210 211 void nitrox_put_device(struct nitrox_device *ndev) 212 { 213 if (!ndev) 214 return; 215 216 refcount_dec(&ndev->refcnt); 217 /* barrier to sync with other cpus */ 218 smp_mb__after_atomic(); 219 } 220 221 static int nitrox_device_flr(struct pci_dev *pdev) 222 { 223 int pos = 0; 224 225 pos = pci_save_state(pdev); 226 if (pos) { 227 dev_err(&pdev->dev, "Failed to save pci state\n"); 228 return -ENOMEM; 229 } 230 231 /* check flr support */ 232 if (pcie_has_flr(pdev)) 233 pcie_flr(pdev); 234 235 pci_restore_state(pdev); 236 237 return 0; 238 } 239 240 static int nitrox_pf_sw_init(struct nitrox_device *ndev) 241 { 242 int err; 243 244 err = nitrox_common_sw_init(ndev); 245 if (err) 246 return err; 247 248 err = nitrox_register_interrupts(ndev); 249 if (err) 250 nitrox_common_sw_cleanup(ndev); 251 252 return err; 253 } 254 255 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev) 256 { 257 nitrox_unregister_interrupts(ndev); 258 nitrox_common_sw_cleanup(ndev); 259 } 260 261 /** 262 * nitrox_bist_check - Check NITORX BIST registers status 263 * @ndev: NITROX device 264 */ 265 static int nitrox_bist_check(struct nitrox_device *ndev) 266 { 267 u64 value = 0; 268 int i; 269 270 for (i = 0; i < NR_CLUSTERS; i++) { 271 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i)); 272 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i)); 273 } 274 value += nitrox_read_csr(ndev, UCD_BIST_STATUS); 275 value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG); 276 value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG); 277 value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG); 278 value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG); 279 value += nitrox_read_csr(ndev, POM_BIST_REG); 280 value += nitrox_read_csr(ndev, BMI_BIST_REG); 281 value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT); 282 value += nitrox_read_csr(ndev, BMO_BIST_REG); 283 value += nitrox_read_csr(ndev, LBC_BIST_STATUS); 284 value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0)); 285 if (value) 286 return -EIO; 287 return 0; 288 } 289 290 static int nitrox_pf_hw_init(struct nitrox_device *ndev) 291 { 292 int err; 293 294 err = nitrox_bist_check(ndev); 295 if (err) { 296 dev_err(&ndev->pdev->dev, "BIST check failed\n"); 297 return err; 298 } 299 /* get cores information */ 300 nitrox_get_hwinfo(ndev); 301 302 nitrox_config_nps_unit(ndev); 303 nitrox_config_pom_unit(ndev); 304 nitrox_config_efl_unit(ndev); 305 /* configure IO units */ 306 nitrox_config_bmi_unit(ndev); 307 nitrox_config_bmo_unit(ndev); 308 /* configure Local Buffer Cache */ 309 nitrox_config_lbc_unit(ndev); 310 nitrox_config_rand_unit(ndev); 311 312 /* load firmware on SE cores */ 313 err = nitrox_load_fw(ndev, SE_FW); 314 if (err) 315 return err; 316 317 nitrox_config_emu_unit(ndev); 318 319 return 0; 320 } 321 322 /** 323 * nitrox_probe - NITROX Initialization function. 324 * @pdev: PCI device information struct 325 * @id: entry in nitrox_pci_tbl 326 * 327 * Return: 0, if the driver is bound to the device, or 328 * a negative error if there is failure. 329 */ 330 static int nitrox_probe(struct pci_dev *pdev, 331 const struct pci_device_id *id) 332 { 333 struct nitrox_device *ndev; 334 int err; 335 336 dev_info_once(&pdev->dev, "%s driver version %s\n", 337 nitrox_driver_name, DRIVER_VERSION); 338 339 err = pci_enable_device_mem(pdev); 340 if (err) 341 return err; 342 343 /* do FLR */ 344 err = nitrox_device_flr(pdev); 345 if (err) { 346 dev_err(&pdev->dev, "FLR failed\n"); 347 pci_disable_device(pdev); 348 return err; 349 } 350 351 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 352 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n"); 353 } else { 354 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 355 if (err) { 356 dev_err(&pdev->dev, "DMA configuration failed\n"); 357 pci_disable_device(pdev); 358 return err; 359 } 360 } 361 362 err = pci_request_mem_regions(pdev, nitrox_driver_name); 363 if (err) { 364 pci_disable_device(pdev); 365 return err; 366 } 367 pci_set_master(pdev); 368 369 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL); 370 if (!ndev) { 371 err = -ENOMEM; 372 goto ndev_fail; 373 } 374 375 pci_set_drvdata(pdev, ndev); 376 ndev->pdev = pdev; 377 378 /* add to device list */ 379 nitrox_add_to_devlist(ndev); 380 381 ndev->hw.vendor_id = pdev->vendor; 382 ndev->hw.device_id = pdev->device; 383 ndev->hw.revision_id = pdev->revision; 384 /* command timeout in jiffies */ 385 ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT); 386 ndev->node = dev_to_node(&pdev->dev); 387 if (ndev->node == NUMA_NO_NODE) 388 ndev->node = 0; 389 390 ndev->bar_addr = ioremap(pci_resource_start(pdev, 0), 391 pci_resource_len(pdev, 0)); 392 if (!ndev->bar_addr) { 393 err = -EIO; 394 goto ioremap_err; 395 } 396 /* allocate command queus based on cpus, max queues are 64 */ 397 ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus()); 398 ndev->qlen = qlen; 399 400 err = nitrox_pf_sw_init(ndev); 401 if (err) 402 goto ioremap_err; 403 404 err = nitrox_pf_hw_init(ndev); 405 if (err) 406 goto pf_hw_fail; 407 408 nitrox_debugfs_init(ndev); 409 410 /* clear the statistics */ 411 atomic64_set(&ndev->stats.posted, 0); 412 atomic64_set(&ndev->stats.completed, 0); 413 atomic64_set(&ndev->stats.dropped, 0); 414 415 atomic_set(&ndev->state, __NDEV_READY); 416 /* barrier to sync with other cpus */ 417 smp_mb__after_atomic(); 418 419 err = nitrox_crypto_register(); 420 if (err) 421 goto crypto_fail; 422 423 return 0; 424 425 crypto_fail: 426 nitrox_debugfs_exit(ndev); 427 atomic_set(&ndev->state, __NDEV_NOT_READY); 428 /* barrier to sync with other cpus */ 429 smp_mb__after_atomic(); 430 pf_hw_fail: 431 nitrox_pf_sw_cleanup(ndev); 432 ioremap_err: 433 nitrox_remove_from_devlist(ndev); 434 kfree(ndev); 435 pci_set_drvdata(pdev, NULL); 436 ndev_fail: 437 pci_release_mem_regions(pdev); 438 pci_disable_device(pdev); 439 return err; 440 } 441 442 /** 443 * nitrox_remove - Unbind the driver from the device. 444 * @pdev: PCI device information struct 445 */ 446 static void nitrox_remove(struct pci_dev *pdev) 447 { 448 struct nitrox_device *ndev = pci_get_drvdata(pdev); 449 450 if (!ndev) 451 return; 452 453 if (!refcount_dec_and_test(&ndev->refcnt)) { 454 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n", 455 refcount_read(&ndev->refcnt)); 456 return; 457 } 458 459 dev_info(DEV(ndev), "Removing Device %x:%x\n", 460 ndev->hw.vendor_id, ndev->hw.device_id); 461 462 atomic_set(&ndev->state, __NDEV_NOT_READY); 463 /* barrier to sync with other cpus */ 464 smp_mb__after_atomic(); 465 466 nitrox_remove_from_devlist(ndev); 467 468 #ifdef CONFIG_PCI_IOV 469 /* disable SR-IOV */ 470 nitrox_sriov_configure(pdev, 0); 471 #endif 472 nitrox_crypto_unregister(); 473 nitrox_debugfs_exit(ndev); 474 nitrox_pf_sw_cleanup(ndev); 475 476 iounmap(ndev->bar_addr); 477 kfree(ndev); 478 479 pci_set_drvdata(pdev, NULL); 480 pci_release_mem_regions(pdev); 481 pci_disable_device(pdev); 482 } 483 484 static void nitrox_shutdown(struct pci_dev *pdev) 485 { 486 pci_set_drvdata(pdev, NULL); 487 pci_release_mem_regions(pdev); 488 pci_disable_device(pdev); 489 } 490 491 static struct pci_driver nitrox_driver = { 492 .name = nitrox_driver_name, 493 .id_table = nitrox_pci_tbl, 494 .probe = nitrox_probe, 495 .remove = nitrox_remove, 496 .shutdown = nitrox_shutdown, 497 #ifdef CONFIG_PCI_IOV 498 .sriov_configure = nitrox_sriov_configure, 499 #endif 500 }; 501 502 module_pci_driver(nitrox_driver); 503 504 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>"); 505 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " "); 506 MODULE_LICENSE("GPL"); 507 MODULE_VERSION(DRIVER_VERSION); 508 MODULE_FIRMWARE(SE_FW); 509