1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2018-2020 Broadcom. 4 */ 5 6 #include <linux/delay.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/firmware.h> 9 #include <linux/fs.h> 10 #include <linux/idr.h> 11 #include <linux/kref.h> 12 #include <linux/module.h> 13 #include <linux/mutex.h> 14 #include <linux/pci.h> 15 #include <linux/pci_regs.h> 16 #include <uapi/linux/misc/bcm_vk.h> 17 18 #include "bcm_vk.h" 19 20 #define PCI_DEVICE_ID_VALKYRIE 0x5e87 21 #define PCI_DEVICE_ID_VIPER 0x5e88 22 23 static DEFINE_IDA(bcm_vk_ida); 24 25 enum soc_idx { 26 VALKYRIE_A0 = 0, 27 VALKYRIE_B0, 28 VIPER, 29 VK_IDX_INVALID 30 }; 31 32 enum img_idx { 33 IMG_PRI = 0, 34 IMG_SEC, 35 IMG_PER_TYPE_MAX 36 }; 37 38 struct load_image_entry { 39 const u32 image_type; 40 const char *image_name[IMG_PER_TYPE_MAX]; 41 }; 42 43 #define NUM_BOOT_STAGES 2 44 /* default firmware images names */ 45 static const struct load_image_entry image_tab[][NUM_BOOT_STAGES] = { 46 [VALKYRIE_A0] = { 47 {VK_IMAGE_TYPE_BOOT1, {"vk_a0-boot1.bin", "vk-boot1.bin"}}, 48 {VK_IMAGE_TYPE_BOOT2, {"vk_a0-boot2.bin", "vk-boot2.bin"}} 49 }, 50 [VALKYRIE_B0] = { 51 {VK_IMAGE_TYPE_BOOT1, {"vk_b0-boot1.bin", "vk-boot1.bin"}}, 52 {VK_IMAGE_TYPE_BOOT2, {"vk_b0-boot2.bin", "vk-boot2.bin"}} 53 }, 54 55 [VIPER] = { 56 {VK_IMAGE_TYPE_BOOT1, {"vp-boot1.bin", ""}}, 57 {VK_IMAGE_TYPE_BOOT2, {"vp-boot2.bin", ""}} 58 }, 59 }; 60 61 /* Location of memory base addresses of interest in BAR1 */ 62 /* Load Boot1 to start of ITCM */ 63 #define BAR1_CODEPUSH_BASE_BOOT1 0x100000 64 65 /* Allow minimum 1s for Load Image timeout responses */ 66 #define LOAD_IMAGE_TIMEOUT_MS (1 * MSEC_PER_SEC) 67 68 /* Image startup timeouts */ 69 #define BOOT1_STARTUP_TIMEOUT_MS (5 * MSEC_PER_SEC) 70 #define BOOT2_STARTUP_TIMEOUT_MS (10 * MSEC_PER_SEC) 71 72 /* 1ms wait for checking the transfer complete status */ 73 #define TXFR_COMPLETE_TIMEOUT_MS 1 74 75 /* MSIX usages */ 76 #define VK_MSIX_MSGQ_MAX 3 77 #define VK_MSIX_NOTF_MAX 1 78 #define VK_MSIX_TTY_MAX BCM_VK_NUM_TTY 79 #define VK_MSIX_IRQ_MAX (VK_MSIX_MSGQ_MAX + VK_MSIX_NOTF_MAX + \ 80 VK_MSIX_TTY_MAX) 81 #define VK_MSIX_IRQ_MIN_REQ (VK_MSIX_MSGQ_MAX + VK_MSIX_NOTF_MAX) 82 83 /* Number of bits set in DMA mask*/ 84 #define BCM_VK_DMA_BITS 64 85 86 /* Ucode boot wait time */ 87 #define BCM_VK_UCODE_BOOT_US (100 * USEC_PER_MSEC) 88 /* 50% margin */ 89 #define BCM_VK_UCODE_BOOT_MAX_US ((BCM_VK_UCODE_BOOT_US * 3) >> 1) 90 91 /* deinit time for the card os after receiving doorbell */ 92 #define BCM_VK_DEINIT_TIME_MS (2 * MSEC_PER_SEC) 93 94 /* 95 * module parameters 96 */ 97 static bool auto_load = true; 98 module_param(auto_load, bool, 0444); 99 MODULE_PARM_DESC(auto_load, 100 "Load images automatically at PCIe probe time.\n"); 101 static uint nr_scratch_pages = VK_BAR1_SCRATCH_DEF_NR_PAGES; 102 module_param(nr_scratch_pages, uint, 0444); 103 MODULE_PARM_DESC(nr_scratch_pages, 104 "Number of pre allocated DMAable coherent pages.\n"); 105 106 static int bcm_vk_intf_ver_chk(struct bcm_vk *vk) 107 { 108 struct device *dev = &vk->pdev->dev; 109 u32 reg; 110 u16 major, minor; 111 int ret = 0; 112 113 /* read interface register */ 114 reg = vkread32(vk, BAR_0, BAR_INTF_VER); 115 major = (reg >> BAR_INTF_VER_MAJOR_SHIFT) & BAR_INTF_VER_MASK; 116 minor = reg & BAR_INTF_VER_MASK; 117 118 /* 119 * if major number is 0, it is pre-release and it would be allowed 120 * to continue, else, check versions accordingly 121 */ 122 if (!major) { 123 dev_warn(dev, "Pre-release major.minor=%d.%d - drv %d.%d\n", 124 major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR); 125 } else if (major != SEMANTIC_MAJOR) { 126 dev_err(dev, 127 "Intf major.minor=%d.%d rejected - drv %d.%d\n", 128 major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR); 129 ret = -EPFNOSUPPORT; 130 } else { 131 dev_dbg(dev, 132 "Intf major.minor=%d.%d passed - drv %d.%d\n", 133 major, minor, SEMANTIC_MAJOR, SEMANTIC_MINOR); 134 } 135 return ret; 136 } 137 138 static inline int bcm_vk_wait(struct bcm_vk *vk, enum pci_barno bar, 139 u64 offset, u32 mask, u32 value, 140 unsigned long timeout_ms) 141 { 142 struct device *dev = &vk->pdev->dev; 143 unsigned long start_time; 144 unsigned long timeout; 145 u32 rd_val, boot_status; 146 147 start_time = jiffies; 148 timeout = start_time + msecs_to_jiffies(timeout_ms); 149 150 do { 151 rd_val = vkread32(vk, bar, offset); 152 dev_dbg(dev, "BAR%d Offset=0x%llx: 0x%x\n", 153 bar, offset, rd_val); 154 155 /* check for any boot err condition */ 156 boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); 157 if (boot_status & BOOT_ERR_MASK) { 158 dev_err(dev, "Boot Err 0x%x, progress 0x%x after %d ms\n", 159 (boot_status & BOOT_ERR_MASK) >> BOOT_ERR_SHIFT, 160 boot_status & BOOT_PROG_MASK, 161 jiffies_to_msecs(jiffies - start_time)); 162 return -EFAULT; 163 } 164 165 if (time_after(jiffies, timeout)) 166 return -ETIMEDOUT; 167 168 cpu_relax(); 169 cond_resched(); 170 } while ((rd_val & mask) != value); 171 172 return 0; 173 } 174 175 static void bcm_vk_get_card_info(struct bcm_vk *vk) 176 { 177 struct device *dev = &vk->pdev->dev; 178 u32 offset; 179 int i; 180 u8 *dst; 181 struct bcm_vk_card_info *info = &vk->card_info; 182 183 /* first read the offset from spare register */ 184 offset = vkread32(vk, BAR_0, BAR_CARD_STATIC_INFO); 185 offset &= (pci_resource_len(vk->pdev, BAR_2 * 2) - 1); 186 187 /* based on the offset, read info to internal card info structure */ 188 dst = (u8 *)info; 189 for (i = 0; i < sizeof(*info); i++) 190 *dst++ = vkread8(vk, BAR_2, offset++); 191 192 #define CARD_INFO_LOG_FMT "version : %x\n" \ 193 "os_tag : %s\n" \ 194 "cmpt_tag : %s\n" \ 195 "cpu_freq : %d MHz\n" \ 196 "cpu_scale : %d full, %d lowest\n" \ 197 "ddr_freq : %d MHz\n" \ 198 "ddr_size : %d MB\n" \ 199 "video_freq: %d MHz\n" 200 dev_dbg(dev, CARD_INFO_LOG_FMT, info->version, info->os_tag, 201 info->cmpt_tag, info->cpu_freq_mhz, info->cpu_scale[0], 202 info->cpu_scale[MAX_OPP - 1], info->ddr_freq_mhz, 203 info->ddr_size_MB, info->video_core_freq_mhz); 204 205 /* 206 * get the peer log pointer, only need the offset, and get record 207 * of the log buffer information which would be used for checking 208 * before dump, in case the BAR2 memory has been corrupted. 209 */ 210 vk->peerlog_off = offset; 211 memcpy_fromio(&vk->peerlog_info, vk->bar[BAR_2] + vk->peerlog_off, 212 sizeof(vk->peerlog_info)); 213 214 /* 215 * Do a range checking and if out of bound, the record will be zeroed 216 * which guarantees that nothing would be dumped. In other words, 217 * peer dump is disabled. 218 */ 219 if ((vk->peerlog_info.buf_size > BCM_VK_PEER_LOG_BUF_MAX) || 220 (vk->peerlog_info.mask != (vk->peerlog_info.buf_size - 1)) || 221 (vk->peerlog_info.rd_idx > vk->peerlog_info.mask) || 222 (vk->peerlog_info.wr_idx > vk->peerlog_info.mask)) { 223 dev_err(dev, "Peer log disabled - range error: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n", 224 vk->peerlog_info.buf_size, 225 vk->peerlog_info.mask, 226 vk->peerlog_info.rd_idx, 227 vk->peerlog_info.wr_idx); 228 memset(&vk->peerlog_info, 0, sizeof(vk->peerlog_info)); 229 } else { 230 dev_dbg(dev, "Peer log: Size 0x%x(0x%x), [Rd Wr] = [%d %d]\n", 231 vk->peerlog_info.buf_size, 232 vk->peerlog_info.mask, 233 vk->peerlog_info.rd_idx, 234 vk->peerlog_info.wr_idx); 235 } 236 } 237 238 static void bcm_vk_get_proc_mon_info(struct bcm_vk *vk) 239 { 240 struct device *dev = &vk->pdev->dev; 241 struct bcm_vk_proc_mon_info *mon = &vk->proc_mon_info; 242 u32 num, entry_size, offset, buf_size; 243 u8 *dst; 244 245 /* calculate offset which is based on peerlog offset */ 246 buf_size = vkread32(vk, BAR_2, 247 vk->peerlog_off 248 + offsetof(struct bcm_vk_peer_log, buf_size)); 249 offset = vk->peerlog_off + sizeof(struct bcm_vk_peer_log) 250 + buf_size; 251 252 /* first read the num and entry size */ 253 num = vkread32(vk, BAR_2, offset); 254 entry_size = vkread32(vk, BAR_2, offset + sizeof(num)); 255 256 /* check for max allowed */ 257 if (num > BCM_VK_PROC_MON_MAX) { 258 dev_err(dev, "Processing monitoring entry %d exceeds max %d\n", 259 num, BCM_VK_PROC_MON_MAX); 260 return; 261 } 262 mon->num = num; 263 mon->entry_size = entry_size; 264 265 vk->proc_mon_off = offset; 266 267 /* read it once that will capture those static info */ 268 dst = (u8 *)&mon->entries[0]; 269 offset += sizeof(num) + sizeof(entry_size); 270 memcpy_fromio(dst, vk->bar[BAR_2] + offset, num * entry_size); 271 } 272 273 static int bcm_vk_sync_card_info(struct bcm_vk *vk) 274 { 275 u32 rdy_marker = vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY); 276 277 /* check for marker, but allow diags mode to skip sync */ 278 if (!bcm_vk_msgq_marker_valid(vk)) 279 return (rdy_marker == VK_BAR1_DIAG_RDY_MARKER ? 0 : -EINVAL); 280 281 /* 282 * Write down scratch addr which is used for DMA. For 283 * signed part, BAR1 is accessible only after boot2 has come 284 * up 285 */ 286 if (vk->tdma_addr) { 287 vkwrite32(vk, (u64)vk->tdma_addr >> 32, BAR_1, 288 VK_BAR1_SCRATCH_OFF_HI); 289 vkwrite32(vk, (u32)vk->tdma_addr, BAR_1, 290 VK_BAR1_SCRATCH_OFF_LO); 291 vkwrite32(vk, nr_scratch_pages * PAGE_SIZE, BAR_1, 292 VK_BAR1_SCRATCH_SZ_ADDR); 293 } 294 295 /* get static card info, only need to read once */ 296 bcm_vk_get_card_info(vk); 297 298 /* get the proc mon info once */ 299 bcm_vk_get_proc_mon_info(vk); 300 301 return 0; 302 } 303 304 static void bcm_vk_buf_notify(struct bcm_vk *vk, void *bufp, 305 dma_addr_t host_buf_addr, u32 buf_size) 306 { 307 /* update the dma address to the card */ 308 vkwrite32(vk, (u64)host_buf_addr >> 32, BAR_1, 309 VK_BAR1_DMA_BUF_OFF_HI); 310 vkwrite32(vk, (u32)host_buf_addr, BAR_1, 311 VK_BAR1_DMA_BUF_OFF_LO); 312 vkwrite32(vk, buf_size, BAR_1, VK_BAR1_DMA_BUF_SZ); 313 } 314 315 static int bcm_vk_load_image_by_type(struct bcm_vk *vk, u32 load_type, 316 const char *filename) 317 { 318 struct device *dev = &vk->pdev->dev; 319 const struct firmware *fw = NULL; 320 void *bufp = NULL; 321 size_t max_buf, offset; 322 int ret; 323 u64 offset_codepush; 324 u32 codepush; 325 u32 value; 326 dma_addr_t boot_dma_addr; 327 bool is_stdalone; 328 329 if (load_type == VK_IMAGE_TYPE_BOOT1) { 330 /* 331 * After POR, enable VK soft BOOTSRC so bootrom do not clear 332 * the pushed image (the TCM memories). 333 */ 334 value = vkread32(vk, BAR_0, BAR_BOOTSRC_SELECT); 335 value |= BOOTSRC_SOFT_ENABLE; 336 vkwrite32(vk, value, BAR_0, BAR_BOOTSRC_SELECT); 337 338 codepush = CODEPUSH_BOOTSTART + CODEPUSH_BOOT1_ENTRY; 339 offset_codepush = BAR_CODEPUSH_SBL; 340 341 /* Write a 1 to request SRAM open bit */ 342 vkwrite32(vk, CODEPUSH_BOOTSTART, BAR_0, offset_codepush); 343 344 /* Wait for VK to respond */ 345 ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, SRAM_OPEN, 346 SRAM_OPEN, LOAD_IMAGE_TIMEOUT_MS); 347 if (ret < 0) { 348 dev_err(dev, "boot1 wait SRAM err - ret(%d)\n", ret); 349 goto err_buf_out; 350 } 351 352 max_buf = SZ_256K; 353 bufp = dma_alloc_coherent(dev, 354 max_buf, 355 &boot_dma_addr, GFP_KERNEL); 356 if (!bufp) { 357 dev_err(dev, "Error allocating 0x%zx\n", max_buf); 358 ret = -ENOMEM; 359 goto err_buf_out; 360 } 361 } else if (load_type == VK_IMAGE_TYPE_BOOT2) { 362 codepush = CODEPUSH_BOOT2_ENTRY; 363 offset_codepush = BAR_CODEPUSH_SBI; 364 365 /* Wait for VK to respond */ 366 ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, DDR_OPEN, 367 DDR_OPEN, LOAD_IMAGE_TIMEOUT_MS); 368 if (ret < 0) { 369 dev_err(dev, "boot2 wait DDR open error - ret(%d)\n", 370 ret); 371 goto err_buf_out; 372 } 373 374 max_buf = SZ_4M; 375 bufp = dma_alloc_coherent(dev, 376 max_buf, 377 &boot_dma_addr, GFP_KERNEL); 378 if (!bufp) { 379 dev_err(dev, "Error allocating 0x%zx\n", max_buf); 380 ret = -ENOMEM; 381 goto err_buf_out; 382 } 383 384 bcm_vk_buf_notify(vk, bufp, boot_dma_addr, max_buf); 385 } else { 386 dev_err(dev, "Error invalid image type 0x%x\n", load_type); 387 ret = -EINVAL; 388 goto err_buf_out; 389 } 390 391 offset = 0; 392 ret = request_partial_firmware_into_buf(&fw, filename, dev, 393 bufp, max_buf, offset); 394 if (ret) { 395 dev_err(dev, "Error %d requesting firmware file: %s\n", 396 ret, filename); 397 goto err_firmware_out; 398 } 399 dev_dbg(dev, "size=0x%zx\n", fw->size); 400 if (load_type == VK_IMAGE_TYPE_BOOT1) 401 memcpy_toio(vk->bar[BAR_1] + BAR1_CODEPUSH_BASE_BOOT1, 402 bufp, 403 fw->size); 404 405 dev_dbg(dev, "Signaling 0x%x to 0x%llx\n", codepush, offset_codepush); 406 vkwrite32(vk, codepush, BAR_0, offset_codepush); 407 408 if (load_type == VK_IMAGE_TYPE_BOOT1) { 409 u32 boot_status; 410 411 /* wait until done */ 412 ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, 413 BOOT1_RUNNING, 414 BOOT1_RUNNING, 415 BOOT1_STARTUP_TIMEOUT_MS); 416 417 boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); 418 is_stdalone = !BCM_VK_INTF_IS_DOWN(boot_status) && 419 (boot_status & BOOT_STDALONE_RUNNING); 420 if (ret && !is_stdalone) { 421 dev_err(dev, 422 "Timeout %ld ms waiting for boot1 to come up - ret(%d)\n", 423 BOOT1_STARTUP_TIMEOUT_MS, ret); 424 goto err_firmware_out; 425 } else if (is_stdalone) { 426 u32 reg; 427 428 reg = vkread32(vk, BAR_0, BAR_BOOT1_STDALONE_PROGRESS); 429 if ((reg & BOOT1_STDALONE_PROGRESS_MASK) == 430 BOOT1_STDALONE_SUCCESS) { 431 dev_info(dev, "Boot1 standalone success\n"); 432 ret = 0; 433 } else { 434 dev_err(dev, "Timeout %ld ms - Boot1 standalone failure\n", 435 BOOT1_STARTUP_TIMEOUT_MS); 436 ret = -EINVAL; 437 goto err_firmware_out; 438 } 439 } 440 } else if (load_type == VK_IMAGE_TYPE_BOOT2) { 441 unsigned long timeout; 442 443 timeout = jiffies + msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS); 444 445 /* To send more data to VK than max_buf allowed at a time */ 446 do { 447 /* 448 * Check for ack from card. when Ack is received, 449 * it means all the data is received by card. 450 * Exit the loop after ack is received. 451 */ 452 ret = bcm_vk_wait(vk, BAR_0, BAR_BOOT_STATUS, 453 FW_LOADER_ACK_RCVD_ALL_DATA, 454 FW_LOADER_ACK_RCVD_ALL_DATA, 455 TXFR_COMPLETE_TIMEOUT_MS); 456 if (ret == 0) { 457 dev_dbg(dev, "Exit boot2 download\n"); 458 break; 459 } else if (ret == -EFAULT) { 460 dev_err(dev, "Error detected during ACK waiting"); 461 goto err_firmware_out; 462 } 463 464 /* exit the loop, if there is no response from card */ 465 if (time_after(jiffies, timeout)) { 466 dev_err(dev, "Error. No reply from card\n"); 467 ret = -ETIMEDOUT; 468 goto err_firmware_out; 469 } 470 471 /* Wait for VK to open BAR space to copy new data */ 472 ret = bcm_vk_wait(vk, BAR_0, offset_codepush, 473 codepush, 0, 474 TXFR_COMPLETE_TIMEOUT_MS); 475 if (ret == 0) { 476 offset += max_buf; 477 ret = request_partial_firmware_into_buf 478 (&fw, 479 filename, 480 dev, bufp, 481 max_buf, 482 offset); 483 if (ret) { 484 dev_err(dev, 485 "Error %d requesting firmware file: %s offset: 0x%zx\n", 486 ret, filename, offset); 487 goto err_firmware_out; 488 } 489 dev_dbg(dev, "size=0x%zx\n", fw->size); 490 dev_dbg(dev, "Signaling 0x%x to 0x%llx\n", 491 codepush, offset_codepush); 492 vkwrite32(vk, codepush, BAR_0, offset_codepush); 493 /* reload timeout after every codepush */ 494 timeout = jiffies + 495 msecs_to_jiffies(LOAD_IMAGE_TIMEOUT_MS); 496 } else if (ret == -EFAULT) { 497 dev_err(dev, "Error detected waiting for transfer\n"); 498 goto err_firmware_out; 499 } 500 } while (1); 501 502 /* wait for fw status bits to indicate app ready */ 503 ret = bcm_vk_wait(vk, BAR_0, VK_BAR_FWSTS, 504 VK_FWSTS_READY, 505 VK_FWSTS_READY, 506 BOOT2_STARTUP_TIMEOUT_MS); 507 if (ret < 0) { 508 dev_err(dev, "Boot2 not ready - ret(%d)\n", ret); 509 goto err_firmware_out; 510 } 511 512 is_stdalone = vkread32(vk, BAR_0, BAR_BOOT_STATUS) & 513 BOOT_STDALONE_RUNNING; 514 if (!is_stdalone) { 515 ret = bcm_vk_intf_ver_chk(vk); 516 if (ret) { 517 dev_err(dev, "failure in intf version check\n"); 518 goto err_firmware_out; 519 } 520 521 /* sync & channel other info */ 522 ret = bcm_vk_sync_card_info(vk); 523 if (ret) { 524 dev_err(dev, "Syncing Card Info failure\n"); 525 goto err_firmware_out; 526 } 527 } 528 } 529 530 err_firmware_out: 531 release_firmware(fw); 532 533 err_buf_out: 534 if (bufp) 535 dma_free_coherent(dev, max_buf, bufp, boot_dma_addr); 536 537 return ret; 538 } 539 540 static u32 bcm_vk_next_boot_image(struct bcm_vk *vk) 541 { 542 u32 boot_status; 543 u32 fw_status; 544 u32 load_type = 0; /* default for unknown */ 545 546 boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); 547 fw_status = vkread32(vk, BAR_0, VK_BAR_FWSTS); 548 549 if (!BCM_VK_INTF_IS_DOWN(boot_status) && (boot_status & SRAM_OPEN)) 550 load_type = VK_IMAGE_TYPE_BOOT1; 551 else if (boot_status == BOOT1_RUNNING) 552 load_type = VK_IMAGE_TYPE_BOOT2; 553 554 /* Log status so that we know different stages */ 555 dev_info(&vk->pdev->dev, 556 "boot-status value for next image: 0x%x : fw-status 0x%x\n", 557 boot_status, fw_status); 558 559 return load_type; 560 } 561 562 static enum soc_idx get_soc_idx(struct bcm_vk *vk) 563 { 564 struct pci_dev *pdev = vk->pdev; 565 enum soc_idx idx = VK_IDX_INVALID; 566 u32 rev; 567 static enum soc_idx const vk_soc_tab[] = { VALKYRIE_A0, VALKYRIE_B0 }; 568 569 switch (pdev->device) { 570 case PCI_DEVICE_ID_VALKYRIE: 571 /* get the chip id to decide sub-class */ 572 rev = MAJOR_SOC_REV(vkread32(vk, BAR_0, BAR_CHIP_ID)); 573 if (rev < ARRAY_SIZE(vk_soc_tab)) { 574 idx = vk_soc_tab[rev]; 575 } else { 576 /* Default to A0 firmware for all other chip revs */ 577 idx = VALKYRIE_A0; 578 dev_warn(&pdev->dev, 579 "Rev %d not in image lookup table, default to idx=%d\n", 580 rev, idx); 581 } 582 break; 583 584 case PCI_DEVICE_ID_VIPER: 585 idx = VIPER; 586 break; 587 588 default: 589 dev_err(&pdev->dev, "no images for 0x%x\n", pdev->device); 590 } 591 return idx; 592 } 593 594 static const char *get_load_fw_name(struct bcm_vk *vk, 595 const struct load_image_entry *entry) 596 { 597 const struct firmware *fw; 598 struct device *dev = &vk->pdev->dev; 599 int ret; 600 unsigned long dummy; 601 int i; 602 603 for (i = 0; i < IMG_PER_TYPE_MAX; i++) { 604 fw = NULL; 605 ret = request_partial_firmware_into_buf(&fw, 606 entry->image_name[i], 607 dev, &dummy, 608 sizeof(dummy), 609 0); 610 release_firmware(fw); 611 if (!ret) 612 return entry->image_name[i]; 613 } 614 return NULL; 615 } 616 617 int bcm_vk_auto_load_all_images(struct bcm_vk *vk) 618 { 619 int i, ret = -1; 620 enum soc_idx idx; 621 struct device *dev = &vk->pdev->dev; 622 u32 curr_type; 623 const char *curr_name; 624 625 idx = get_soc_idx(vk); 626 if (idx == VK_IDX_INVALID) 627 goto auto_load_all_exit; 628 629 /* log a message to know the relative loading order */ 630 dev_dbg(dev, "Load All for device %d\n", vk->devid); 631 632 for (i = 0; i < NUM_BOOT_STAGES; i++) { 633 curr_type = image_tab[idx][i].image_type; 634 if (bcm_vk_next_boot_image(vk) == curr_type) { 635 curr_name = get_load_fw_name(vk, &image_tab[idx][i]); 636 if (!curr_name) { 637 dev_err(dev, "No suitable firmware exists for type %d", 638 curr_type); 639 ret = -ENOENT; 640 goto auto_load_all_exit; 641 } 642 ret = bcm_vk_load_image_by_type(vk, curr_type, 643 curr_name); 644 dev_info(dev, "Auto load %s, ret %d\n", 645 curr_name, ret); 646 647 if (ret) { 648 dev_err(dev, "Error loading default %s\n", 649 curr_name); 650 goto auto_load_all_exit; 651 } 652 } 653 } 654 655 auto_load_all_exit: 656 return ret; 657 } 658 659 static int bcm_vk_trigger_autoload(struct bcm_vk *vk) 660 { 661 if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) 662 return -EPERM; 663 664 set_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload); 665 queue_work(vk->wq_thread, &vk->wq_work); 666 667 return 0; 668 } 669 670 /* 671 * deferred work queue for auto download. 672 */ 673 static void bcm_vk_wq_handler(struct work_struct *work) 674 { 675 struct bcm_vk *vk = container_of(work, struct bcm_vk, wq_work); 676 677 if (test_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload)) { 678 bcm_vk_auto_load_all_images(vk); 679 680 /* 681 * at the end of operation, clear AUTO bit and pending 682 * bit 683 */ 684 clear_bit(BCM_VK_WQ_DWNLD_AUTO, vk->wq_offload); 685 clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload); 686 } 687 } 688 689 static long bcm_vk_load_image(struct bcm_vk *vk, 690 const struct vk_image __user *arg) 691 { 692 struct device *dev = &vk->pdev->dev; 693 const char *image_name; 694 struct vk_image image; 695 u32 next_loadable; 696 enum soc_idx idx; 697 int image_idx; 698 int ret = -EPERM; 699 700 if (copy_from_user(&image, arg, sizeof(image))) 701 return -EACCES; 702 703 if ((image.type != VK_IMAGE_TYPE_BOOT1) && 704 (image.type != VK_IMAGE_TYPE_BOOT2)) { 705 dev_err(dev, "invalid image.type %u\n", image.type); 706 return ret; 707 } 708 709 next_loadable = bcm_vk_next_boot_image(vk); 710 if (next_loadable != image.type) { 711 dev_err(dev, "Next expected image %u, Loading %u\n", 712 next_loadable, image.type); 713 return ret; 714 } 715 716 /* 717 * if something is pending download already. This could only happen 718 * for now when the driver is being loaded, or if someone has issued 719 * another download command in another shell. 720 */ 721 if (test_and_set_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload) != 0) { 722 dev_err(dev, "Download operation already pending.\n"); 723 return ret; 724 } 725 726 image_name = image.filename; 727 if (image_name[0] == '\0') { 728 /* Use default image name if NULL */ 729 idx = get_soc_idx(vk); 730 if (idx == VK_IDX_INVALID) 731 goto err_idx; 732 733 /* Image idx starts with boot1 */ 734 image_idx = image.type - VK_IMAGE_TYPE_BOOT1; 735 image_name = get_load_fw_name(vk, &image_tab[idx][image_idx]); 736 if (!image_name) { 737 dev_err(dev, "No suitable image found for type %d", 738 image.type); 739 ret = -ENOENT; 740 goto err_idx; 741 } 742 } else { 743 /* Ensure filename is NULL terminated */ 744 image.filename[sizeof(image.filename) - 1] = '\0'; 745 } 746 ret = bcm_vk_load_image_by_type(vk, image.type, image_name); 747 dev_info(dev, "Load %s, ret %d\n", image_name, ret); 748 err_idx: 749 clear_bit(BCM_VK_WQ_DWNLD_PEND, vk->wq_offload); 750 751 return ret; 752 } 753 754 static void bcm_to_v_reset_doorbell(struct bcm_vk *vk, u32 db_val) 755 { 756 vkwrite32(vk, db_val, BAR_0, VK_BAR0_RESET_DB_BASE); 757 } 758 759 static int bcm_vk_trigger_reset(struct bcm_vk *vk) 760 { 761 u32 i; 762 u32 value, boot_status; 763 static const u32 bar0_reg_clr_list[] = { BAR_OS_UPTIME, 764 BAR_INTF_VER, 765 BAR_CARD_VOLTAGE, 766 BAR_CARD_TEMPERATURE, 767 BAR_CARD_PWR_AND_THRE }; 768 769 /* make tag '\0' terminated */ 770 vkwrite32(vk, 0, BAR_1, VK_BAR1_BOOT1_VER_TAG); 771 772 for (i = 0; i < VK_BAR1_DAUTH_MAX; i++) { 773 vkwrite32(vk, 0, BAR_1, VK_BAR1_DAUTH_STORE_ADDR(i)); 774 vkwrite32(vk, 0, BAR_1, VK_BAR1_DAUTH_VALID_ADDR(i)); 775 } 776 for (i = 0; i < VK_BAR1_SOTP_REVID_MAX; i++) 777 vkwrite32(vk, 0, BAR_1, VK_BAR1_SOTP_REVID_ADDR(i)); 778 779 /* 780 * When boot request fails, the CODE_PUSH_OFFSET stays persistent. 781 * Allowing us to debug the failure. When we call reset, 782 * we should clear CODE_PUSH_OFFSET so ROM does not execute 783 * boot again (and fails again) and instead waits for a new 784 * codepush. And, if previous boot has encountered error, need 785 * to clear the entry values 786 */ 787 boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); 788 if (boot_status & BOOT_ERR_MASK) { 789 dev_info(&vk->pdev->dev, 790 "Card in boot error 0x%x, clear CODEPUSH val\n", 791 boot_status); 792 value = 0; 793 } else { 794 value = vkread32(vk, BAR_0, BAR_CODEPUSH_SBL); 795 value &= CODEPUSH_MASK; 796 } 797 vkwrite32(vk, value, BAR_0, BAR_CODEPUSH_SBL); 798 799 /* reset fw_status with proper reason, and press db */ 800 vkwrite32(vk, VK_FWSTS_RESET_MBOX_DB, BAR_0, VK_BAR_FWSTS); 801 bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_SOFT); 802 803 /* clear other necessary registers records */ 804 for (i = 0; i < ARRAY_SIZE(bar0_reg_clr_list); i++) 805 vkwrite32(vk, 0, BAR_0, bar0_reg_clr_list[i]); 806 807 return 0; 808 } 809 810 static long bcm_vk_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 811 { 812 long ret = -EINVAL; 813 struct bcm_vk_ctx *ctx = file->private_data; 814 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev); 815 void __user *argp = (void __user *)arg; 816 817 dev_dbg(&vk->pdev->dev, 818 "ioctl, cmd=0x%02x, arg=0x%02lx\n", 819 cmd, arg); 820 821 mutex_lock(&vk->mutex); 822 823 switch (cmd) { 824 case VK_IOCTL_LOAD_IMAGE: 825 ret = bcm_vk_load_image(vk, argp); 826 break; 827 828 default: 829 break; 830 } 831 832 mutex_unlock(&vk->mutex); 833 834 return ret; 835 } 836 837 static const struct file_operations bcm_vk_fops = { 838 .owner = THIS_MODULE, 839 .open = bcm_vk_open, 840 .release = bcm_vk_release, 841 .unlocked_ioctl = bcm_vk_ioctl, 842 }; 843 844 static int bcm_vk_on_panic(struct notifier_block *nb, 845 unsigned long e, void *p) 846 { 847 struct bcm_vk *vk = container_of(nb, struct bcm_vk, panic_nb); 848 849 bcm_to_v_reset_doorbell(vk, VK_BAR0_RESET_DB_HARD); 850 851 return 0; 852 } 853 854 static int bcm_vk_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 855 { 856 int err; 857 int i; 858 int id; 859 int irq; 860 char name[20]; 861 struct bcm_vk *vk; 862 struct device *dev = &pdev->dev; 863 struct miscdevice *misc_device; 864 u32 boot_status; 865 866 /* allocate vk structure which is tied to kref for freeing */ 867 vk = kzalloc(sizeof(*vk), GFP_KERNEL); 868 if (!vk) 869 return -ENOMEM; 870 871 kref_init(&vk->kref); 872 mutex_init(&vk->mutex); 873 874 err = pci_enable_device(pdev); 875 if (err) { 876 dev_err(dev, "Cannot enable PCI device\n"); 877 goto err_free_exit; 878 } 879 vk->pdev = pci_dev_get(pdev); 880 881 err = pci_request_regions(pdev, DRV_MODULE_NAME); 882 if (err) { 883 dev_err(dev, "Cannot obtain PCI resources\n"); 884 goto err_disable_pdev; 885 } 886 887 /* make sure DMA is good */ 888 err = dma_set_mask_and_coherent(&pdev->dev, 889 DMA_BIT_MASK(BCM_VK_DMA_BITS)); 890 if (err) { 891 dev_err(dev, "failed to set DMA mask\n"); 892 goto err_disable_pdev; 893 } 894 895 /* The tdma is a scratch area for some DMA testings. */ 896 if (nr_scratch_pages) { 897 vk->tdma_vaddr = dma_alloc_coherent 898 (dev, 899 nr_scratch_pages * PAGE_SIZE, 900 &vk->tdma_addr, GFP_KERNEL); 901 if (!vk->tdma_vaddr) { 902 err = -ENOMEM; 903 goto err_disable_pdev; 904 } 905 } 906 907 pci_set_master(pdev); 908 pci_set_drvdata(pdev, vk); 909 910 irq = pci_alloc_irq_vectors(pdev, 911 1, 912 VK_MSIX_IRQ_MAX, 913 PCI_IRQ_MSI | PCI_IRQ_MSIX); 914 915 if (irq < VK_MSIX_IRQ_MIN_REQ) { 916 dev_err(dev, "failed to get min %d MSIX interrupts, irq(%d)\n", 917 VK_MSIX_IRQ_MIN_REQ, irq); 918 err = (irq >= 0) ? -EINVAL : irq; 919 goto err_disable_pdev; 920 } 921 922 if (irq != VK_MSIX_IRQ_MAX) 923 dev_warn(dev, "Number of IRQs %d allocated - requested(%d).\n", 924 irq, VK_MSIX_IRQ_MAX); 925 926 for (i = 0; i < MAX_BAR; i++) { 927 /* multiple by 2 for 64 bit BAR mapping */ 928 vk->bar[i] = pci_ioremap_bar(pdev, i * 2); 929 if (!vk->bar[i]) { 930 dev_err(dev, "failed to remap BAR%d\n", i); 931 goto err_iounmap; 932 } 933 } 934 935 id = ida_simple_get(&bcm_vk_ida, 0, 0, GFP_KERNEL); 936 if (id < 0) { 937 err = id; 938 dev_err(dev, "unable to get id\n"); 939 goto err_iounmap; 940 } 941 942 vk->devid = id; 943 snprintf(name, sizeof(name), DRV_MODULE_NAME ".%d", id); 944 misc_device = &vk->miscdev; 945 misc_device->minor = MISC_DYNAMIC_MINOR; 946 misc_device->name = kstrdup(name, GFP_KERNEL); 947 if (!misc_device->name) { 948 err = -ENOMEM; 949 goto err_ida_remove; 950 } 951 misc_device->fops = &bcm_vk_fops, 952 953 err = misc_register(misc_device); 954 if (err) { 955 dev_err(dev, "failed to register device\n"); 956 goto err_kfree_name; 957 } 958 959 INIT_WORK(&vk->wq_work, bcm_vk_wq_handler); 960 961 /* create dedicated workqueue */ 962 vk->wq_thread = create_singlethread_workqueue(name); 963 if (!vk->wq_thread) { 964 dev_err(dev, "Fail to create workqueue thread\n"); 965 err = -ENOMEM; 966 goto err_misc_deregister; 967 } 968 969 /* sync other info */ 970 bcm_vk_sync_card_info(vk); 971 972 /* register for panic notifier */ 973 vk->panic_nb.notifier_call = bcm_vk_on_panic; 974 err = atomic_notifier_chain_register(&panic_notifier_list, 975 &vk->panic_nb); 976 if (err) { 977 dev_err(dev, "Fail to register panic notifier\n"); 978 goto err_destroy_workqueue; 979 } 980 981 /* 982 * lets trigger an auto download. We don't want to do it serially here 983 * because at probing time, it is not supposed to block for a long time. 984 */ 985 boot_status = vkread32(vk, BAR_0, BAR_BOOT_STATUS); 986 if (auto_load) { 987 if ((boot_status & BOOT_STATE_MASK) == BROM_RUNNING) { 988 if (bcm_vk_trigger_autoload(vk)) 989 goto err_unregister_panic_notifier; 990 } else { 991 dev_err(dev, 992 "Auto-load skipped - BROM not in proper state (0x%x)\n", 993 boot_status); 994 } 995 } 996 997 dev_dbg(dev, "BCM-VK:%u created\n", id); 998 999 return 0; 1000 1001 err_unregister_panic_notifier: 1002 atomic_notifier_chain_unregister(&panic_notifier_list, 1003 &vk->panic_nb); 1004 1005 err_destroy_workqueue: 1006 destroy_workqueue(vk->wq_thread); 1007 1008 err_misc_deregister: 1009 misc_deregister(misc_device); 1010 1011 err_kfree_name: 1012 kfree(misc_device->name); 1013 misc_device->name = NULL; 1014 1015 err_ida_remove: 1016 ida_simple_remove(&bcm_vk_ida, id); 1017 1018 err_iounmap: 1019 for (i = 0; i < MAX_BAR; i++) { 1020 if (vk->bar[i]) 1021 pci_iounmap(pdev, vk->bar[i]); 1022 } 1023 pci_release_regions(pdev); 1024 1025 err_disable_pdev: 1026 if (vk->tdma_vaddr) 1027 dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE, 1028 vk->tdma_vaddr, vk->tdma_addr); 1029 1030 pci_free_irq_vectors(pdev); 1031 pci_disable_device(pdev); 1032 pci_dev_put(pdev); 1033 1034 err_free_exit: 1035 kfree(vk); 1036 1037 return err; 1038 } 1039 1040 void bcm_vk_release_data(struct kref *kref) 1041 { 1042 struct bcm_vk *vk = container_of(kref, struct bcm_vk, kref); 1043 struct pci_dev *pdev = vk->pdev; 1044 1045 dev_dbg(&pdev->dev, "BCM-VK:%d release data 0x%p\n", vk->devid, vk); 1046 pci_dev_put(pdev); 1047 kfree(vk); 1048 } 1049 1050 static void bcm_vk_remove(struct pci_dev *pdev) 1051 { 1052 int i; 1053 struct bcm_vk *vk = pci_get_drvdata(pdev); 1054 struct miscdevice *misc_device = &vk->miscdev; 1055 1056 /* 1057 * Trigger a reset to card and wait enough time for UCODE to rerun, 1058 * which re-initialize the card into its default state. 1059 * This ensures when driver is re-enumerated it will start from 1060 * a completely clean state. 1061 */ 1062 bcm_vk_trigger_reset(vk); 1063 usleep_range(BCM_VK_UCODE_BOOT_US, BCM_VK_UCODE_BOOT_MAX_US); 1064 1065 /* unregister panic notifier */ 1066 atomic_notifier_chain_unregister(&panic_notifier_list, 1067 &vk->panic_nb); 1068 1069 if (vk->tdma_vaddr) 1070 dma_free_coherent(&pdev->dev, nr_scratch_pages * PAGE_SIZE, 1071 vk->tdma_vaddr, vk->tdma_addr); 1072 1073 /* remove if name is set which means misc dev registered */ 1074 if (misc_device->name) { 1075 misc_deregister(misc_device); 1076 kfree(misc_device->name); 1077 ida_simple_remove(&bcm_vk_ida, vk->devid); 1078 } 1079 1080 cancel_work_sync(&vk->wq_work); 1081 destroy_workqueue(vk->wq_thread); 1082 1083 for (i = 0; i < MAX_BAR; i++) { 1084 if (vk->bar[i]) 1085 pci_iounmap(pdev, vk->bar[i]); 1086 } 1087 1088 dev_dbg(&pdev->dev, "BCM-VK:%d released\n", vk->devid); 1089 1090 pci_release_regions(pdev); 1091 pci_free_irq_vectors(pdev); 1092 pci_disable_device(pdev); 1093 1094 kref_put(&vk->kref, bcm_vk_release_data); 1095 } 1096 1097 static void bcm_vk_shutdown(struct pci_dev *pdev) 1098 { 1099 struct bcm_vk *vk = pci_get_drvdata(pdev); 1100 u32 reg, boot_stat; 1101 1102 reg = vkread32(vk, BAR_0, BAR_BOOT_STATUS); 1103 boot_stat = reg & BOOT_STATE_MASK; 1104 1105 if (boot_stat == BOOT1_RUNNING) { 1106 /* simply trigger a reset interrupt to park it */ 1107 bcm_vk_trigger_reset(vk); 1108 } else if (boot_stat == BROM_NOT_RUN) { 1109 int err; 1110 u16 lnksta; 1111 1112 /* 1113 * The boot status only reflects boot condition since last reset 1114 * As ucode will run only once to configure pcie, if multiple 1115 * resets happen, we lost track if ucode has run or not. 1116 * Here, read the current link speed and use that to 1117 * sync up the bootstatus properly so that on reboot-back-up, 1118 * it has the proper state to start with autoload 1119 */ 1120 err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); 1121 if (!err && 1122 (lnksta & PCI_EXP_LNKSTA_CLS) != PCI_EXP_LNKSTA_CLS_2_5GB) { 1123 reg |= BROM_STATUS_COMPLETE; 1124 vkwrite32(vk, reg, BAR_0, BAR_BOOT_STATUS); 1125 } 1126 } 1127 } 1128 1129 static const struct pci_device_id bcm_vk_ids[] = { 1130 { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VALKYRIE), }, 1131 { PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_VIPER), }, 1132 { } 1133 }; 1134 MODULE_DEVICE_TABLE(pci, bcm_vk_ids); 1135 1136 static struct pci_driver pci_driver = { 1137 .name = DRV_MODULE_NAME, 1138 .id_table = bcm_vk_ids, 1139 .probe = bcm_vk_probe, 1140 .remove = bcm_vk_remove, 1141 .shutdown = bcm_vk_shutdown, 1142 }; 1143 module_pci_driver(pci_driver); 1144 1145 MODULE_DESCRIPTION("Broadcom VK Host Driver"); 1146 MODULE_AUTHOR("Scott Branden <scott.branden@broadcom.com>"); 1147 MODULE_LICENSE("GPL v2"); 1148 MODULE_VERSION("1.0"); 1149