1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * 4 * Bluetooth support for Intel PCIe devices 5 * 6 * Copyright (C) 2024 Intel Corporation 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/firmware.h> 12 #include <linux/overflow.h> 13 #include <linux/pci.h> 14 #include <linux/string.h> 15 #include <linux/wait.h> 16 #include <linux/delay.h> 17 #include <linux/interrupt.h> 18 #include <linux/acpi.h> 19 #include <linux/suspend.h> 20 21 #include <linux/unaligned.h> 22 #include <linux/devcoredump.h> 23 #include <linux/scatterlist.h> 24 25 #include <net/bluetooth/bluetooth.h> 26 #include <net/bluetooth/hci_core.h> 27 #include <net/bluetooth/hci_drv.h> 28 29 #include "btintel.h" 30 #include "btintel_pcie.h" 31 32 #define VERSION "0.1" 33 34 #define BTINTEL_PCI_DEVICE(dev, subdev) \ 35 .vendor = PCI_VENDOR_ID_INTEL, \ 36 .device = (dev), \ 37 .subvendor = PCI_ANY_ID, \ 38 .subdevice = (subdev), \ 39 .driver_data = 0 40 41 #define POLL_INTERVAL_US 10 42 43 #define BTINTEL_PCIE_DMA_ALIGN_128B 128 /* 128 byte aligned */ 44 45 /* Intel Bluetooth PCIe device id table */ 46 static const struct pci_device_id btintel_pcie_table[] = { 47 /* BlazarI, Wildcat Lake */ 48 { BTINTEL_PCI_DEVICE(0x4D76, PCI_ANY_ID) }, 49 /* BlazarI, Lunar Lake */ 50 { BTINTEL_PCI_DEVICE(0xA876, PCI_ANY_ID) }, 51 /* Scorpious, Panther Lake-H484 */ 52 { BTINTEL_PCI_DEVICE(0xE376, PCI_ANY_ID) }, 53 /* Scorpious, Panther Lake-H404 */ 54 { BTINTEL_PCI_DEVICE(0xE476, PCI_ANY_ID) }, 55 /* Scorpious2, Nova Lake-PCD-H */ 56 { BTINTEL_PCI_DEVICE(0xD346, PCI_ANY_ID) }, 57 /* Scorpious2, Nova Lake-PCD-S */ 58 { BTINTEL_PCI_DEVICE(0x6E74, PCI_ANY_ID) }, 59 /* Draco */ 60 { BTINTEL_PCI_DEVICE(0x2731, PCI_ANY_ID) }, 61 { BTINTEL_PCI_DEVICE(0x2732, PCI_ANY_ID) }, 62 /* Scorpious Peak2 C0, Titan Lake */ 63 { BTINTEL_PCI_DEVICE(0x9329, PCI_ANY_ID) }, 64 { 0 } 65 }; 66 MODULE_DEVICE_TABLE(pci, btintel_pcie_table); 67 68 struct btintel_pcie_dev_recovery { 69 struct list_head list; 70 u8 count; 71 time64_t last_error; 72 char name[]; 73 }; 74 75 /* Intel PCIe uses 4 bytes of HCI type instead of 1 byte BT SIG HCI type */ 76 #define BTINTEL_PCIE_HCI_TYPE_LEN 4 77 #define BTINTEL_PCIE_HCI_CMD_PKT 0x00000001 78 #define BTINTEL_PCIE_HCI_ACL_PKT 0x00000002 79 #define BTINTEL_PCIE_HCI_SCO_PKT 0x00000003 80 #define BTINTEL_PCIE_HCI_EVT_PKT 0x00000004 81 #define BTINTEL_PCIE_HCI_ISO_PKT 0x00000005 82 83 #define BTINTEL_PCIE_MAGIC_NUM 0xA5A5A5A5 84 85 #define BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER 0x17A2 86 #define BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT 0x1E61 87 88 #define BTINTEL_PCIE_RESET_WINDOW_SECS 5 89 #define BTINTEL_PCIE_FLR_MAX_RETRY 1 90 91 /* Alive interrupt context */ 92 enum { 93 BTINTEL_PCIE_ROM, 94 BTINTEL_PCIE_FW_DL, 95 BTINTEL_PCIE_HCI_RESET, 96 BTINTEL_PCIE_INTEL_HCI_RESET1, 97 BTINTEL_PCIE_INTEL_HCI_RESET2, 98 BTINTEL_PCIE_D0, 99 BTINTEL_PCIE_D3 100 }; 101 102 enum { 103 BTINTEL_PCIE_DSM_SET_RESET_TIMING = 1, 104 BTINTEL_PCIE_DSM_GET_RESET_TIMING = 2, 105 BTINTEL_PCIE_DSM_BT_PLDR_CONFIG = 3, 106 BTINTEL_PCIE_DSM_GET_RESET_TYPE = 4, 107 BTINTEL_PCIE_DSM_DYNAMIC_PLDR = 5, 108 BTINTEL_PCIE_DSM_GET_RESET_METHOD = 6, 109 BTINTEL_PCIE_DSM_SET_PLDR_DELAY = 7, 110 }; 111 112 enum btintel_dsm_internal_product_reset_mode { 113 BTINTEL_PCIE_DSM_PLDR_MODE_EN_PROD_RESET = BIT(0), 114 BTINTEL_PCIE_DSM_PLDR_MODE_EN_WIFI_FLR = BIT(1), 115 BTINTEL_PCIE_DSM_PLDR_MODE_EN_BT_OFF_ON = BIT(2), 116 }; 117 118 /* Structure for dbgc fragment buffer 119 * @buf_addr_lsb: LSB of the buffer's physical address 120 * @buf_addr_msb: MSB of the buffer's physical address 121 * @buf_size: Total size of the buffer 122 */ 123 struct btintel_pcie_dbgc_ctxt_buf { 124 u32 buf_addr_lsb; 125 u32 buf_addr_msb; 126 u32 buf_size; 127 }; 128 129 /* Structure for dbgc fragment 130 * @magic_num: 0XA5A5A5A5 131 * @ver: For Driver-FW compatibility 132 * @total_size: Total size of the payload debug info 133 * @num_buf: Num of allocated debug bufs 134 * @bufs: All buffer's addresses and sizes 135 */ 136 struct btintel_pcie_dbgc_ctxt { 137 u32 magic_num; 138 u32 ver; 139 u32 total_size; 140 u32 num_buf; 141 struct btintel_pcie_dbgc_ctxt_buf bufs[BTINTEL_PCIE_DBGC_BUFFER_COUNT]; 142 }; 143 144 struct btintel_pcie_mdbgc_ctxt { 145 u32 magic_num; 146 u32 ver; 147 u32 buf1_index; 148 u32 buf1_count; 149 struct btintel_pcie_dbgc_ctxt_buf buf1[BTINTEL_PCIE_DBGC_BUFFER_COUNT]; 150 u32 buf2_index; 151 u32 buf2_count; 152 struct btintel_pcie_dbgc_ctxt_buf buf2[BTINTEL_PCIE_DBGC_BUFFER_COUNT]; 153 u32 buf3_index; 154 u32 buf3_count; 155 struct btintel_pcie_dbgc_ctxt_buf buf3[BTINTEL_PCIE_DBGC_BUFFER_COUNT]; 156 }; 157 158 struct btintel_pcie_trigger_evt { 159 u8 type; 160 u8 len; 161 __le32 addr; 162 __le32 size; 163 } __packed; 164 165 struct btintel_pcie_fwtrigger_evt { 166 __le32 reserved; 167 u8 type; /* Debug Trigger event */ 168 __le16 len; 169 u8 event_type; 170 __le16 event_id; 171 __le16 reserved2; 172 } __packed; 173 174 static LIST_HEAD(btintel_pcie_recovery_list); 175 static DEFINE_SPINLOCK(btintel_pcie_recovery_lock); 176 177 static inline char *btintel_pcie_alivectxt_state2str(u32 alive_intr_ctxt) 178 { 179 switch (alive_intr_ctxt) { 180 case BTINTEL_PCIE_ROM: 181 return "rom"; 182 case BTINTEL_PCIE_FW_DL: 183 return "fw_dl"; 184 case BTINTEL_PCIE_D0: 185 return "d0"; 186 case BTINTEL_PCIE_D3: 187 return "d3"; 188 case BTINTEL_PCIE_HCI_RESET: 189 return "hci_reset"; 190 case BTINTEL_PCIE_INTEL_HCI_RESET1: 191 return "intel_reset1"; 192 case BTINTEL_PCIE_INTEL_HCI_RESET2: 193 return "intel_reset2"; 194 default: 195 return "unknown"; 196 } 197 } 198 199 /* Returns true when firmware traces are routed to the WiFi DBGC. In that 200 * mode the host must not allocate DBGC buffers and must not publish their 201 * addresses in the context info. 202 */ 203 static inline bool btintel_pcie_dbg_to_wifi(struct btintel_pcie_data *data) 204 { 205 return data->dbg_path_cache != BTINTEL_PCIE_DRAM; 206 } 207 208 /* Helper function to allocate and setup a debug buffer group 209 * @data: driver data structure 210 * @buf: pointer to data_buf array pointer 211 * @p_addr: pointer to physical DMA address 212 * @v_addr: pointer to virtual address 213 * @frag: pointer to fragment buffer array 214 * @buf_index: buffer index (for error messages) 215 * @buf_count: number of buffers to allocate 216 */ 217 static int btintel_pcie_alloc_dbgc_buf(struct btintel_pcie_data *data, 218 struct data_buf **buf, 219 dma_addr_t *p_addr, 220 void **v_addr, 221 struct btintel_pcie_dbgc_ctxt_buf *frag, 222 u32 buf_index, 223 u32 buf_count) 224 { 225 struct data_buf *b; 226 int i; 227 228 *buf = devm_kcalloc(&data->pdev->dev, buf_count, 229 sizeof(**buf), GFP_KERNEL); 230 if (!*buf) { 231 BT_ERR("Failed to allocate dbgc buf: %u", buf_index + 1); 232 return -ENOMEM; 233 } 234 235 *v_addr = dmam_alloc_coherent(&data->pdev->dev, 236 buf_count * 237 BTINTEL_PCIE_DBGC_BUFFER_SIZE, 238 p_addr, 239 GFP_KERNEL | __GFP_NOWARN); 240 if (!*v_addr) { 241 BT_ERR("Failed to allocate dbgc buf: %u DMA", buf_index + 1); 242 return -ENOMEM; 243 } 244 245 for (i = 0; i < buf_count; i++) { 246 b = &(*buf)[i]; 247 b->data_p_addr = *p_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE; 248 b->data = *v_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE; 249 frag[i].buf_addr_lsb = lower_32_bits(b->data_p_addr); 250 frag[i].buf_addr_msb = upper_32_bits(b->data_p_addr); 251 frag[i].buf_size = BTINTEL_PCIE_DBGC_BUFFER_SIZE; 252 } 253 254 return 0; 255 } 256 257 /* This function initializes the memory for MDBGC buffers and formats the 258 * DBGC fragment which consists header info and DBGC buffer's LSB, MSB and 259 * size as the payload 260 */ 261 static int btintel_pcie_setup_mdbgc(struct btintel_pcie_data *data) 262 { 263 struct btintel_pcie_mdbgc_ctxt db_frag; 264 u32 frag_size = sizeof(db_frag); 265 void *frag_v_addr; 266 int err; 267 268 data->mdbgc.count = BTINTEL_PCIE_DBGC_BUFFER_COUNT; 269 270 /* Allocate fragment context structure */ 271 frag_v_addr = dmam_alloc_coherent(&data->pdev->dev, frag_size, 272 &data->mdbgc.frag_p_addr, 273 GFP_KERNEL | __GFP_NOWARN); 274 if (!frag_v_addr) { 275 BT_ERR("Failed to allocate mdbgc context"); 276 return -ENOMEM; 277 } 278 279 data->mdbgc.frag_v_addr = frag_v_addr; 280 data->mdbgc.frag_size = frag_size; 281 282 /* Initialize fragment header */ 283 memset(&db_frag, 0, sizeof(db_frag)); 284 db_frag.magic_num = BTINTEL_PCIE_MAGIC_NUM; 285 db_frag.ver = BTINTEL_PCIE_MDBGC_FRAG_VERSION; 286 287 /* Allocate DBGC buffer 1 */ 288 db_frag.buf1_index = BTINTEL_PCIE_MDBGC_ALLOCATIONID_1; 289 db_frag.buf1_count = data->mdbgc.count; 290 err = btintel_pcie_alloc_dbgc_buf(data, 291 &data->mdbgc.buf1, 292 &data->mdbgc.buf1_p_addr, 293 &data->mdbgc.buf1_v_addr, 294 db_frag.buf1, 295 0, 296 data->mdbgc.count); 297 if (err) 298 return err; 299 300 /* Allocate DBGC buffer 2 */ 301 db_frag.buf2_index = BTINTEL_PCIE_MDBGC_ALLOCATIONID_2; 302 db_frag.buf2_count = data->mdbgc.count; 303 err = btintel_pcie_alloc_dbgc_buf(data, 304 &data->mdbgc.buf2, 305 &data->mdbgc.buf2_p_addr, 306 &data->mdbgc.buf2_v_addr, 307 db_frag.buf2, 308 1, 309 data->mdbgc.count); 310 if (err) 311 return err; 312 313 /* Allocate DBGC buffer 3 */ 314 db_frag.buf3_index = BTINTEL_PCIE_MDBGC_ALLOCATIONID_3; 315 db_frag.buf3_count = data->mdbgc.count; 316 err = btintel_pcie_alloc_dbgc_buf(data, 317 &data->mdbgc.buf3, 318 &data->mdbgc.buf3_p_addr, 319 &data->mdbgc.buf3_v_addr, 320 db_frag.buf3, 321 2, 322 data->mdbgc.count); 323 if (err) 324 return err; 325 326 /* Copy fragment to DMA coherent memory */ 327 memcpy(data->mdbgc.frag_v_addr, &db_frag, sizeof(db_frag)); 328 return 0; 329 } 330 331 /* This function initializes the memory for DBGC buffers and formats the 332 * DBGC fragment which consists header info and DBGC buffer's LSB, MSB and 333 * size as the payload 334 */ 335 static int btintel_pcie_setup_dbgc(struct btintel_pcie_data *data) 336 { 337 struct btintel_pcie_dbgc_ctxt db_frag; 338 u32 frag_size = sizeof(db_frag); 339 int err; 340 341 data->dbgc.count = BTINTEL_PCIE_DBGC_BUFFER_COUNT; 342 343 /* Allocate fragment context structure */ 344 data->dbgc.frag_v_addr = dmam_alloc_coherent(&data->pdev->dev, 345 frag_size, 346 &data->dbgc.frag_p_addr, 347 GFP_KERNEL | __GFP_NOWARN); 348 if (!data->dbgc.frag_v_addr) { 349 BT_ERR("Failed to allocate dbgc context"); 350 return -ENOMEM; 351 } 352 353 data->dbgc.frag_size = sizeof(struct btintel_pcie_dbgc_ctxt); 354 355 /* Initialize fragment header */ 356 memset(&db_frag, 0, sizeof(db_frag)); 357 db_frag.magic_num = BTINTEL_PCIE_MAGIC_NUM; 358 db_frag.ver = BTINTEL_PCIE_DBGC_FRAG_VERSION; 359 db_frag.total_size = BTINTEL_PCIE_DBGC_FRAG_PAYLOAD_SIZE; 360 db_frag.num_buf = BTINTEL_PCIE_DBGC_FRAG_BUFFER_COUNT; 361 362 /* Allocate DBGC buffers */ 363 err = btintel_pcie_alloc_dbgc_buf(data, 364 &data->dbgc.bufs, 365 &data->dbgc.buf_p_addr, 366 &data->dbgc.buf_v_addr, 367 db_frag.bufs, 368 0, 369 data->dbgc.count); 370 if (err) 371 return err; 372 373 /* Copy fragment to DMA coherent memory */ 374 memcpy(data->dbgc.frag_v_addr, &db_frag, sizeof(db_frag)); 375 return 0; 376 } 377 378 static inline void ipc_print_ia_ring(struct hci_dev *hdev, struct ia *ia, 379 u16 queue_num) 380 { 381 bt_dev_dbg(hdev, "IA: %s: tr-h:%02u tr-t:%02u cr-h:%02u cr-t:%02u", 382 queue_num == BTINTEL_PCIE_TXQ_NUM ? "TXQ" : "RXQ", 383 ia->tr_hia[queue_num], ia->tr_tia[queue_num], 384 ia->cr_hia[queue_num], ia->cr_tia[queue_num]); 385 } 386 387 static inline void ipc_print_urbd1(struct hci_dev *hdev, struct urbd1 *urbd1, 388 u16 index) 389 { 390 bt_dev_dbg(hdev, "RXQ:urbd1(%u) frbd_tag:%u status: 0x%x fixed:0x%x", 391 index, urbd1->frbd_tag, urbd1->status, urbd1->fixed); 392 } 393 394 static struct btintel_pcie_data *btintel_pcie_get_data(struct msix_entry *entry) 395 { 396 u8 queue = entry->entry; 397 struct msix_entry *entries = entry - queue; 398 399 return container_of(entries, struct btintel_pcie_data, msix_entries[0]); 400 } 401 402 /* Set the doorbell for TXQ to notify the device that @index (actually index-1) 403 * of the TFD is updated and ready to transmit. 404 */ 405 static void btintel_pcie_set_tx_db(struct btintel_pcie_data *data, u16 index) 406 { 407 u32 val; 408 409 val = index; 410 val |= (BTINTEL_PCIE_TX_DB_VEC << 16); 411 412 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val); 413 } 414 415 /* Copy the data to next(@tfd_index) data buffer and update the TFD(transfer 416 * descriptor) with the data length and the DMA address of the data buffer. 417 */ 418 static void btintel_pcie_prepare_tx(struct txq *txq, u16 tfd_index, 419 struct sk_buff *skb) 420 { 421 struct data_buf *buf; 422 struct tfd *tfd; 423 424 tfd = &txq->tfds[tfd_index]; 425 memset(tfd, 0, sizeof(*tfd)); 426 427 buf = &txq->bufs[tfd_index]; 428 429 tfd->size = skb->len; 430 tfd->addr = buf->data_p_addr; 431 432 /* Copy the outgoing data to DMA buffer */ 433 memcpy(buf->data, skb->data, tfd->size); 434 } 435 436 static inline void btintel_pcie_dump_debug_registers(struct hci_dev *hdev) 437 { 438 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 439 u16 cr_hia, cr_tia; 440 u32 reg, mbox_reg; 441 struct sk_buff *skb; 442 u8 buf[80]; 443 444 skb = alloc_skb(1024, GFP_ATOMIC); 445 if (!skb) 446 return; 447 448 strscpy(buf, "---- Dump of debug registers ---"); 449 bt_dev_dbg(hdev, "%s", buf); 450 skb_put_data(skb, buf, strlen(buf)); 451 452 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 453 snprintf(buf, sizeof(buf), "boot stage: 0x%8.8x", reg); 454 bt_dev_dbg(hdev, "%s", buf); 455 skb_put_data(skb, buf, strlen(buf)); 456 data->boot_stage_cache = reg; 457 458 if (reg & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_WARNING) 459 bt_dev_warn(hdev, "Controller device warning (boot_stage: 0x%8.8x)", reg); 460 461 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_STATUS_REG); 462 snprintf(buf, sizeof(buf), "ipc status: 0x%8.8x", reg); 463 skb_put_data(skb, buf, strlen(buf)); 464 bt_dev_dbg(hdev, "%s", buf); 465 466 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_CONTROL_REG); 467 snprintf(buf, sizeof(buf), "ipc control: 0x%8.8x", reg); 468 skb_put_data(skb, buf, strlen(buf)); 469 bt_dev_dbg(hdev, "%s", buf); 470 471 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_SLEEP_CTL_REG); 472 snprintf(buf, sizeof(buf), "ipc sleep control: 0x%8.8x", reg); 473 skb_put_data(skb, buf, strlen(buf)); 474 bt_dev_dbg(hdev, "%s", buf); 475 476 /*Read the Mail box status and registers*/ 477 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_STATUS_REG); 478 snprintf(buf, sizeof(buf), "mbox status: 0x%8.8x", reg); 479 skb_put_data(skb, buf, strlen(buf)); 480 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX1) { 481 mbox_reg = btintel_pcie_rd_reg32(data, 482 BTINTEL_PCIE_CSR_MBOX_1_REG); 483 snprintf(buf, sizeof(buf), "mbox_1: 0x%8.8x", mbox_reg); 484 skb_put_data(skb, buf, strlen(buf)); 485 bt_dev_dbg(hdev, "%s", buf); 486 } 487 488 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX2) { 489 mbox_reg = btintel_pcie_rd_reg32(data, 490 BTINTEL_PCIE_CSR_MBOX_2_REG); 491 snprintf(buf, sizeof(buf), "mbox_2: 0x%8.8x", mbox_reg); 492 skb_put_data(skb, buf, strlen(buf)); 493 bt_dev_dbg(hdev, "%s", buf); 494 } 495 496 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX3) { 497 mbox_reg = btintel_pcie_rd_reg32(data, 498 BTINTEL_PCIE_CSR_MBOX_3_REG); 499 snprintf(buf, sizeof(buf), "mbox_3: 0x%8.8x", mbox_reg); 500 skb_put_data(skb, buf, strlen(buf)); 501 bt_dev_dbg(hdev, "%s", buf); 502 } 503 504 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX4) { 505 mbox_reg = btintel_pcie_rd_reg32(data, 506 BTINTEL_PCIE_CSR_MBOX_4_REG); 507 snprintf(buf, sizeof(buf), "mbox_4: 0x%8.8x", mbox_reg); 508 skb_put_data(skb, buf, strlen(buf)); 509 bt_dev_dbg(hdev, "%s", buf); 510 } 511 512 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM]; 513 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; 514 snprintf(buf, sizeof(buf), "rxq: cr_tia: %u cr_hia: %u", cr_tia, cr_hia); 515 skb_put_data(skb, buf, strlen(buf)); 516 bt_dev_dbg(hdev, "%s", buf); 517 518 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM]; 519 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM]; 520 snprintf(buf, sizeof(buf), "txq: cr_tia: %u cr_hia: %u", cr_tia, cr_hia); 521 skb_put_data(skb, buf, strlen(buf)); 522 bt_dev_dbg(hdev, "%s", buf); 523 strscpy(buf, "--------------------------------"); 524 bt_dev_dbg(hdev, "%s", buf); 525 526 hci_recv_diag(hdev, skb); 527 } 528 529 static int btintel_pcie_send_sync(struct btintel_pcie_data *data, 530 struct sk_buff *skb, u32 pkt_type, u16 opcode) 531 { 532 int ret; 533 u16 tfd_index; 534 u32 old_ctxt; 535 bool wait_on_alive = false; 536 struct hci_dev *hdev = data->hdev; 537 538 struct txq *txq = &data->txq; 539 540 tfd_index = data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM]; 541 542 if (tfd_index >= txq->count) 543 return -ERANGE; 544 545 if (skb->len > BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN) { 546 bt_dev_err(hdev, "TX skb too large (%u > %u)", skb->len, 547 BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN); 548 return -EMSGSIZE; 549 } 550 551 /* Firmware raises alive interrupt on HCI_OP_RESET or 552 * BTINTEL_HCI_OP_RESET 553 */ 554 wait_on_alive = (pkt_type == BTINTEL_PCIE_HCI_CMD_PKT && 555 (opcode == BTINTEL_HCI_OP_RESET || opcode == HCI_OP_RESET)); 556 557 if (wait_on_alive) { 558 data->gp0_received = false; 559 old_ctxt = data->alive_intr_ctxt; 560 data->alive_intr_ctxt = 561 (opcode == BTINTEL_HCI_OP_RESET ? BTINTEL_PCIE_INTEL_HCI_RESET1 : 562 BTINTEL_PCIE_HCI_RESET); 563 bt_dev_dbg(data->hdev, "sending cmd: 0x%4.4x alive context changed: %s -> %s", 564 opcode, btintel_pcie_alivectxt_state2str(old_ctxt), 565 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt)); 566 } 567 568 memcpy(skb_push(skb, BTINTEL_PCIE_HCI_TYPE_LEN), &pkt_type, 569 BTINTEL_PCIE_HCI_TYPE_LEN); 570 571 /* Prepare for TX. It updates the TFD with the length of data and 572 * address of the DMA buffer, and copy the data to the DMA buffer 573 */ 574 btintel_pcie_prepare_tx(txq, tfd_index, skb); 575 576 tfd_index = (tfd_index + 1) % txq->count; 577 data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM] = tfd_index; 578 579 /* Arm wait event condition */ 580 data->tx_wait_done = false; 581 582 /* Set the doorbell to notify the device */ 583 btintel_pcie_set_tx_db(data, tfd_index); 584 585 /* Wait for the complete interrupt - URBD0 */ 586 ret = wait_event_timeout(data->tx_wait_q, data->tx_wait_done, 587 msecs_to_jiffies(BTINTEL_PCIE_TX_WAIT_TIMEOUT_MS)); 588 if (!ret) { 589 bt_dev_err(data->hdev, "Timeout (%u ms) on tx completion", 590 BTINTEL_PCIE_TX_WAIT_TIMEOUT_MS); 591 btintel_pcie_dump_debug_registers(data->hdev); 592 return -ETIME; 593 } 594 595 if (wait_on_alive) { 596 ret = wait_event_timeout(data->gp0_wait_q, 597 data->gp0_received, 598 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT_MS)); 599 if (!ret) { 600 hdev->stat.err_tx++; 601 bt_dev_err(hdev, "Timeout (%u ms) on alive interrupt, alive context: %s", 602 BTINTEL_DEFAULT_INTR_TIMEOUT_MS, 603 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt)); 604 return -ETIME; 605 } 606 } 607 return 0; 608 } 609 610 /* Set the doorbell for RXQ to notify the device that @index (actually index-1) 611 * is available to receive the data 612 */ 613 static void btintel_pcie_set_rx_db(struct btintel_pcie_data *data, u16 index) 614 { 615 u32 val; 616 617 val = index; 618 val |= (BTINTEL_PCIE_RX_DB_VEC << 16); 619 620 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val); 621 } 622 623 /* Update the FRBD (free buffer descriptor) with the @frbd_index and the 624 * DMA address of the free buffer. 625 */ 626 static void btintel_pcie_prepare_rx(struct rxq *rxq, u16 frbd_index) 627 { 628 struct data_buf *buf; 629 struct frbd *frbd; 630 631 /* Get the buffer of the FRBD for DMA */ 632 buf = &rxq->bufs[frbd_index]; 633 634 frbd = &rxq->frbds[frbd_index]; 635 memset(frbd, 0, sizeof(*frbd)); 636 637 /* Update FRBD */ 638 frbd->tag = frbd_index; 639 frbd->addr = buf->data_p_addr; 640 } 641 642 static int btintel_pcie_submit_rx(struct btintel_pcie_data *data) 643 { 644 u16 frbd_index; 645 struct rxq *rxq = &data->rxq; 646 647 frbd_index = data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM]; 648 649 if (frbd_index >= rxq->count) 650 return -ERANGE; 651 652 /* Prepare for RX submit. It updates the FRBD with the address of DMA 653 * buffer 654 */ 655 btintel_pcie_prepare_rx(rxq, frbd_index); 656 657 frbd_index = (frbd_index + 1) % rxq->count; 658 data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM] = frbd_index; 659 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM); 660 661 /* Set the doorbell to notify the device */ 662 btintel_pcie_set_rx_db(data, frbd_index); 663 664 return 0; 665 } 666 667 static int btintel_pcie_start_rx(struct btintel_pcie_data *data) 668 { 669 int i, ret; 670 struct rxq *rxq = &data->rxq; 671 672 /* Post (BTINTEL_PCIE_RX_DESCS_COUNT - 3) buffers to overcome the 673 * hardware issues leading to race condition at the firmware. 674 */ 675 676 for (i = 0; i < rxq->count - 3; i++) { 677 ret = btintel_pcie_submit_rx(data); 678 if (ret) 679 return ret; 680 } 681 682 return 0; 683 } 684 685 static void btintel_pcie_reset_ia(struct btintel_pcie_data *data) 686 { 687 memset(data->ia.tr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 688 memset(data->ia.tr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 689 memset(data->ia.cr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 690 memset(data->ia.cr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 691 } 692 693 static int btintel_pcie_reset_bt(struct btintel_pcie_data *data) 694 { 695 u32 reg; 696 int retry = 3; 697 698 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 699 700 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 701 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT | 702 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT); 703 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON; 704 705 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 706 707 do { 708 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 709 if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_STS) 710 break; 711 usleep_range(10000, 12000); 712 713 } while (--retry > 0); 714 usleep_range(10000, 12000); 715 716 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 717 718 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 719 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT | 720 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT); 721 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET; 722 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 723 usleep_range(10000, 12000); 724 725 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 726 bt_dev_dbg(data->hdev, "csr register after reset: 0x%8.8x", reg); 727 728 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 729 730 /* If shared hardware reset is success then boot stage register shall be 731 * set to 0 732 */ 733 return reg == 0 ? 0 : -ENODEV; 734 } 735 736 static void btintel_pcie_mac_init(struct btintel_pcie_data *data) 737 { 738 u32 reg; 739 740 /* Set MAC_INIT bit to start primary bootloader */ 741 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 742 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT | 743 BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON | 744 BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET); 745 reg |= (BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 746 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT); 747 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 748 } 749 750 static int btintel_pcie_get_mac_access(struct btintel_pcie_data *data) 751 { 752 u32 reg; 753 int retry = 15; 754 755 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 756 757 if (!(reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ)) { 758 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ; 759 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 760 } 761 762 do { 763 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 764 if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS) 765 return 0; 766 /* Need delay here for Target Access harwdware to settle down*/ 767 usleep_range(1000, 1200); 768 769 } while (--retry > 0); 770 771 return -ETIME; 772 } 773 774 static void btintel_pcie_release_mac_access(struct btintel_pcie_data *data) 775 { 776 u32 reg; 777 778 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 779 780 if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ) { 781 reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ; 782 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 783 } 784 } 785 786 static struct scatterlist *btintel_pcie_alloc_sgtable(ssize_t size) 787 { 788 int nents; 789 struct page *page; 790 struct scatterlist *sg, *result; 791 792 if (size <= 0) 793 return NULL; 794 795 nents = DIV_ROUND_UP(size, PAGE_SIZE); 796 result = kcalloc(nents, sizeof(*result), GFP_KERNEL); 797 if (!result) 798 return NULL; 799 800 sg_init_table(result, nents); 801 sg = result; 802 803 while (size > 0) { 804 ssize_t bytes = min_t(ssize_t, size, PAGE_SIZE); 805 806 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 807 if (!page) 808 goto err_free; 809 810 sg_set_page(sg, page, bytes, 0); 811 sg = sg_next(sg); 812 size -= bytes; 813 } 814 return result; 815 816 err_free: 817 for (sg = result; sg; sg = sg_next(sg)) { 818 page = sg_page(sg); 819 if (page) 820 __free_page(page); 821 } 822 kfree(result); 823 return NULL; 824 } 825 826 static struct btintel_pcie_dump_entry * 827 btintel_pcie_dump_entry_alloc(u32 data_size) 828 { 829 struct btintel_pcie_dump_entry *entry; 830 831 entry = vzalloc(sizeof(*entry) + data_size); 832 if (!entry) 833 return NULL; 834 835 entry->size = data_size; 836 INIT_LIST_HEAD(&entry->list); 837 return entry; 838 } 839 840 static void btintel_pcie_dump_list_free(struct list_head *list) 841 { 842 struct btintel_pcie_dump_entry *entry, *tmp; 843 844 list_for_each_entry_safe(entry, tmp, list, list) { 845 list_del(&entry->list); 846 vfree(entry); 847 } 848 } 849 850 static u32 btintel_pcie_dump_list_total_size(struct list_head *list) 851 { 852 struct btintel_pcie_dump_entry *entry; 853 u32 total = 0; 854 855 list_for_each_entry(entry, list, list) 856 total += entry->size; 857 858 return total; 859 } 860 861 static int btintel_pcie_dump_dram(struct list_head *list, 862 u8 count, struct data_buf *bufs, 863 u32 write_ptr, u32 wrap_ctr, u32 cur_frag, 864 u32 region_id, const char *name) 865 { 866 struct btintel_pcie_dump_entry *entry; 867 struct btintel_pcie_ini_dump_data *dump_data; 868 struct btintel_pcie_ini_monitor_dump *mon_dump; 869 struct btintel_pcie_dump_range *range; 870 u32 mon_hdr_size, ranges_size, payload_size, total_size; 871 int i; 872 873 mon_hdr_size = sizeof(*mon_dump); 874 ranges_size = count * (sizeof(*range) + BTINTEL_PCIE_DBGC_BUFFER_SIZE); 875 payload_size = mon_hdr_size + ranges_size; 876 total_size = sizeof(*dump_data) + payload_size; 877 878 entry = btintel_pcie_dump_entry_alloc(total_size); 879 if (!entry) 880 return -ENOMEM; 881 882 dump_data = (void *)entry->data; 883 dump_data->type = BTINTEL_PCIE_INI_REGION_DRAM_BUFFER; 884 dump_data->sub_type = 0; 885 dump_data->sub_type_ver = 0; 886 dump_data->reserved = 0; 887 dump_data->len = cpu_to_le32(payload_size); 888 889 mon_dump = (void *)dump_data->data; 890 mon_dump->header.version = cpu_to_le32(BTINTEL_PCIE_INI_DUMP_VER); 891 mon_dump->header.region_id = cpu_to_le32(region_id); 892 mon_dump->header.num_of_ranges = cpu_to_le32(count); 893 mon_dump->header.name_len = cpu_to_le32(BTINTEL_PCIE_INI_MAX_NAME); 894 memset(mon_dump->header.name, 0, BTINTEL_PCIE_INI_MAX_NAME); 895 strscpy(mon_dump->header.name, name, BTINTEL_PCIE_INI_MAX_NAME); 896 897 mon_dump->write_ptr = cpu_to_le32(write_ptr); 898 mon_dump->cycle_cnt = cpu_to_le32(wrap_ctr); 899 mon_dump->cur_frag = cpu_to_le32(cur_frag); 900 901 range = (void *)mon_dump->data; 902 for (i = 0; i < count; i++) { 903 range->range_data_size = 904 cpu_to_le32(BTINTEL_PCIE_DBGC_BUFFER_SIZE); 905 range->dram_base_addr = cpu_to_le64(bufs[i].data_p_addr); 906 memcpy(range->data, bufs[i].data, 907 BTINTEL_PCIE_DBGC_BUFFER_SIZE); 908 range = (void *)range->data + BTINTEL_PCIE_DBGC_BUFFER_SIZE; 909 } 910 911 list_add_tail(&entry->list, list); 912 return 0; 913 } 914 915 static int 916 btintel_pcie_dump_dram_monitor(struct btintel_pcie_data *data, 917 struct list_head *list, u8 count, 918 struct data_buf *bufs, u32 status_reg, 919 u32 wrap_reg, u8 allocation_id, u32 region_id, 920 const char *name) 921 { 922 u32 allocation_offset, write_ptr, wrap_ctr, wr_ptr_status; 923 u8 cur_frag; 924 925 allocation_offset = allocation_id * 926 BTINTEL_PCIE_DBGC_ALLOCATION_OFFSET; 927 wr_ptr_status = btintel_pcie_rd_dev_mem(data, status_reg + 928 allocation_offset); 929 wrap_ctr = btintel_pcie_rd_dev_mem(data, wrap_reg + 930 allocation_offset); 931 932 write_ptr = wr_ptr_status & BTINTEL_PCIE_DBG_OFFSET_BIT_MASK; 933 write_ptr >>= 2; 934 cur_frag = BTINTEL_PCIE_DBGC_DBG_BUF_IDX(wr_ptr_status); 935 if (cur_frag >= count) { 936 bt_dev_warn(data->hdev, 937 "Invalid DRAM monitor fragment %u for allocation %u", 938 cur_frag, allocation_id); 939 return -EINVAL; 940 } 941 942 bt_dev_dbg(data->hdev, 943 "allocation=%u wr_ptr_status=0x%08x write_ptr=0x%06x cur_frag=%u wrap_ctr=0x%08x", 944 allocation_id, wr_ptr_status, write_ptr, cur_frag, wrap_ctr); 945 946 return btintel_pcie_dump_dram(list, count, bufs, write_ptr, wrap_ctr, 947 cur_frag, region_id, name); 948 } 949 950 static int btintel_pcie_dump_target_region(struct btintel_pcie_data *data, 951 struct list_head *list, 952 u32 region_id, const char *name, 953 u32 addr_start, u32 size) 954 { 955 struct btintel_pcie_dump_entry *entry; 956 struct btintel_pcie_ini_dump_data *dump_data; 957 struct btintel_pcie_ini_dump_header *hdr; 958 struct btintel_pcie_dump_range *range; 959 u32 payload_size, total_size, target_mem_offset, tempdata; 960 u8 *dest; 961 int i; 962 963 if (!size) { 964 bt_dev_warn(data->hdev, "Skipping empty dump region: %s", name); 965 return 1; 966 } 967 968 if (!addr_start) { 969 bt_dev_warn(data->hdev, 970 "Skipping dump region with zero address: %s", 971 name); 972 return 1; 973 } 974 975 if (addr_start > U32_MAX - size) { 976 bt_dev_warn(data->hdev, 977 "Skipping dump region %s: addr_start 0x%08x + size %u would overflow", 978 name, addr_start, size); 979 return 1; 980 } 981 982 /* Align to 4 bytes - target access reads 32-bit words */ 983 size = round_down(size, sizeof(u32)); 984 if (!size) { 985 bt_dev_warn(data->hdev, 986 "Skipping dump region %s: size is smaller than 4-byte access", 987 name); 988 return 1; 989 } 990 991 bt_dev_dbg(data->hdev, 992 "Target access: region=%s start=0x%08x size=%u", 993 name, addr_start, size); 994 995 payload_size = sizeof(*hdr) + sizeof(*range) + size; 996 total_size = sizeof(*dump_data) + payload_size; 997 entry = btintel_pcie_dump_entry_alloc(total_size); 998 if (!entry) 999 return -ENOMEM; 1000 1001 dump_data = (void *)entry->data; 1002 dump_data->type = BTINTEL_PCIE_INI_REGION_DEVICE_MEMORY; 1003 dump_data->sub_type = 0; 1004 dump_data->sub_type_ver = 0; 1005 dump_data->reserved = 0; 1006 dump_data->len = cpu_to_le32(payload_size); 1007 1008 hdr = (void *)dump_data->data; 1009 hdr->version = cpu_to_le32(BTINTEL_PCIE_INI_DUMP_VER); 1010 hdr->region_id = cpu_to_le32(region_id); 1011 hdr->num_of_ranges = cpu_to_le32(1); 1012 hdr->name_len = cpu_to_le32(BTINTEL_PCIE_INI_MAX_NAME); 1013 memset(hdr->name, 0, BTINTEL_PCIE_INI_MAX_NAME); 1014 strscpy(hdr->name, name, BTINTEL_PCIE_INI_MAX_NAME); 1015 1016 range = (void *)(hdr + 1); 1017 range->range_data_size = cpu_to_le32(size); 1018 range->internal_base_addr = cpu_to_le32(addr_start); 1019 1020 dest = (u8 *)range->data; 1021 target_mem_offset = size / sizeof(u32); 1022 for (i = 0; i < target_mem_offset; i++) { 1023 u32 offset = BTINTEL_PCIE_TARGET_ACCESS_FRAG_OFFSET * i; 1024 1025 tempdata = btintel_pcie_rd_dev_mem(data, 1026 addr_start + offset); 1027 memcpy(dest, &tempdata, sizeof(tempdata)); 1028 dest += sizeof(tempdata); 1029 } 1030 1031 list_add_tail(&entry->list, list); 1032 return 0; 1033 } 1034 1035 static int btintel_pcie_dump_smem_monitor_region(struct btintel_pcie_data *data, 1036 struct list_head *list, 1037 u32 region_id, 1038 const char *name, 1039 u32 addr_start, u32 size) 1040 { 1041 struct btintel_pcie_dump_entry *entry; 1042 struct btintel_pcie_ini_dump_data *dump_data; 1043 struct btintel_pcie_ini_monitor_dump *mon; 1044 struct btintel_pcie_dump_range *range; 1045 u32 payload_size, total_size, target_mem_offset, tempdata; 1046 u8 *dest; 1047 int i; 1048 1049 if (!size || !addr_start) { 1050 bt_dev_err(data->hdev, "Skipping smem dump: size = %u addr = %8.8x", 1051 size, addr_start); 1052 return -EINVAL; 1053 } 1054 1055 payload_size = sizeof(*mon) + sizeof(*range) + size; 1056 total_size = sizeof(*dump_data) + payload_size; 1057 entry = btintel_pcie_dump_entry_alloc(total_size); 1058 if (!entry) 1059 return -ENOMEM; 1060 1061 dump_data = (void *)entry->data; 1062 dump_data->type = BTINTEL_PCIE_INI_REGION_INTERNAL_BUFFER; 1063 dump_data->sub_type = 0; 1064 dump_data->sub_type_ver = 0; 1065 dump_data->reserved = 0; 1066 dump_data->len = cpu_to_le32(payload_size); 1067 1068 mon = (void *)dump_data->data; 1069 mon->header.version = cpu_to_le32(BTINTEL_PCIE_INI_DUMP_VER); 1070 mon->header.region_id = cpu_to_le32(region_id); 1071 mon->header.num_of_ranges = cpu_to_le32(1); 1072 mon->header.name_len = cpu_to_le32(BTINTEL_PCIE_INI_MAX_NAME); 1073 memset(mon->header.name, 0, BTINTEL_PCIE_INI_MAX_NAME); 1074 strscpy(mon->header.name, name, BTINTEL_PCIE_INI_MAX_NAME); 1075 1076 mon->write_ptr = cpu_to_le32(0); 1077 mon->cycle_cnt = cpu_to_le32(0); 1078 mon->cur_frag = cpu_to_le32(0); 1079 1080 range = (void *)mon->data; 1081 range->range_data_size = cpu_to_le32(size); 1082 range->internal_base_addr = cpu_to_le32(addr_start); 1083 1084 dest = (u8 *)range->data; 1085 target_mem_offset = size / sizeof(u32); 1086 for (i = 0; i < target_mem_offset; i++) { 1087 u32 offset = BTINTEL_PCIE_TARGET_ACCESS_FRAG_OFFSET * i; 1088 1089 tempdata = btintel_pcie_rd_dev_mem(data, 1090 addr_start + offset); 1091 memcpy(dest, &tempdata, sizeof(tempdata)); 1092 dest += sizeof(tempdata); 1093 } 1094 1095 list_add_tail(&entry->list, list); 1096 return 0; 1097 } 1098 1099 static int btintel_pcie_dump_info(struct btintel_pcie_data *data, 1100 struct list_head *list, 1101 u64 regions_mask) 1102 { 1103 struct btintel_pcie_dump_entry *entry; 1104 struct btintel_pcie_error_dump_data *tlv; 1105 struct btintel_pcie_ini_dump_info *dump; 1106 u32 size = sizeof(*tlv) + sizeof(*dump); 1107 char build_tag[64]; 1108 1109 entry = btintel_pcie_dump_entry_alloc(size); 1110 if (!entry) 1111 return -ENOMEM; 1112 1113 tlv = (void *)entry->data; 1114 tlv->type = cpu_to_le32(BTINTEL_PCIE_INI_DUMP_INFO_TYPE); 1115 tlv->len = cpu_to_le32(sizeof(*dump)); 1116 1117 dump = (void *)tlv->data; 1118 memset(dump, 0, sizeof(*dump)); 1119 1120 dump->version = cpu_to_le32(BTINTEL_PCIE_INI_DUMP_VER); 1121 dump->trigger_reason = cpu_to_le32(data->dmp_hdr.trigger_reason); 1122 1123 if (data->dmp_hdr.trigger_reason == 1124 BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT) 1125 dump->time_point = 1126 cpu_to_le32(BTINTEL_PCIE_TIME_POINT_FW_ASSERT); 1127 else if (data->dmp_hdr.trigger_reason == 1128 BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER) 1129 dump->time_point = 1130 cpu_to_le32(BTINTEL_PCIE_TIME_POINT_USER_TRIGGER); 1131 1132 dump->hw_type = cpu_to_le32(INTEL_CNVX_TOP_TYPE(data->dmp_hdr.cnvi_top)); 1133 dump->hw_step = cpu_to_le32(INTEL_CNVX_TOP_STEP(data->dmp_hdr.cnvi_top)); 1134 bt_dev_dbg(data->hdev, "hw_type=0x%x hw_step=0x%x (cnvi_top=0x%x)", 1135 le32_to_cpu(dump->hw_type), le32_to_cpu(dump->hw_step), 1136 data->dmp_hdr.cnvi_top); 1137 1138 dump->ver_type = cpu_to_le32(data->dmp_hdr.cnvi_bt); 1139 dump->ver_subtype = cpu_to_le32(data->dmp_hdr.fw_sha); 1140 dump->rf_id_type = cpu_to_le32(INTEL_CNVX_TOP_TYPE(data->dmp_hdr.cnvr_top)); 1141 dump->rf_id_dash = cpu_to_le32(INTEL_CNVX_TOP_DASH(data->dmp_hdr.cnvr_top)); 1142 dump->rf_id_step = cpu_to_le32(INTEL_CNVX_TOP_STEP(data->dmp_hdr.cnvr_top)); 1143 dump->rf_id_flavor = cpu_to_le32(INTEL_CNVX_TOP_FLAVOR(data->dmp_hdr.cnvr_top)); 1144 bt_dev_dbg(data->hdev, 1145 "rf_id_type=0x%x rf_id_dash=0x%x rf_id_step=0x%x rf_id_flavor=0x%x (cnvr_top=0x%x)", 1146 le32_to_cpu(dump->rf_id_type), 1147 le32_to_cpu(dump->rf_id_dash), 1148 le32_to_cpu(dump->rf_id_step), 1149 le32_to_cpu(dump->rf_id_flavor), 1150 data->dmp_hdr.cnvr_top); 1151 dump->lmac_major = cpu_to_le32(0); 1152 dump->lmac_minor = cpu_to_le32(0); 1153 dump->umac_major = cpu_to_le32(0); 1154 dump->umac_minor = cpu_to_le32(0); 1155 dump->fw_mon_mode = cpu_to_le32(BTINTEL_PCIE_FW_MON_MODE_DRAM); 1156 1157 dump->regions_mask = cpu_to_le64(regions_mask); 1158 1159 bt_dev_dbg(data->hdev, "ExpectedRegionIDs regions_mask=0x%016llx", 1160 le64_to_cpu(dump->regions_mask)); 1161 1162 snprintf(build_tag, sizeof(build_tag), "%08X", data->dmp_hdr.fw_sha); 1163 1164 dump->build_tag_len = cpu_to_le32(strlen(build_tag)); 1165 memcpy(dump->build_tag, build_tag, min(strlen(build_tag), 1166 sizeof(dump->build_tag))); 1167 1168 dump->num_of_cfg_names = cpu_to_le32(0); 1169 1170 bt_dev_dbg(data->hdev, 1171 "HwType=0x%08x HwStep=0x%08x RfIdType=0x%08x RfIdDash=0x%08x RfIdStep=0x%08x RfIdFlavor=0x%08x", 1172 le32_to_cpu(dump->hw_type), le32_to_cpu(dump->hw_step), 1173 le32_to_cpu(dump->rf_id_type), 1174 le32_to_cpu(dump->rf_id_dash), 1175 le32_to_cpu(dump->rf_id_step), 1176 le32_to_cpu(dump->rf_id_flavor)); 1177 bt_dev_dbg(data->hdev, "VerType=0x%08x VerSubType=0x%08x", 1178 le32_to_cpu(dump->ver_type), le32_to_cpu(dump->ver_subtype)); 1179 bt_dev_dbg(data->hdev, 1180 "LmacMajor=0x%08x LmacMinor=0x%08x UmacMajor=0x%08x UmacMinor=0x%08x", 1181 le32_to_cpu(dump->lmac_major), le32_to_cpu(dump->lmac_minor), 1182 le32_to_cpu(dump->umac_major), 1183 le32_to_cpu(dump->umac_minor)); 1184 bt_dev_dbg(data->hdev, "TriggerReason=0x%04x MonMode=%u BuildTag=%.64s", 1185 le32_to_cpu(dump->trigger_reason), 1186 le32_to_cpu(dump->fw_mon_mode), 1187 dump->build_tag); 1188 1189 list_add(&entry->list, list); 1190 1191 return 0; 1192 } 1193 1194 static bool btintel_pcie_is_mdbgc_supported(struct btintel_pcie_data *data) 1195 { 1196 return data->pdev->device == BTINTEL_PCIE_DEVICE_ID_NVL_S_SCP2 || 1197 data->pdev->device == BTINTEL_PCIE_DEVICE_ID_NVL_Hx_SCP2 || 1198 data->pdev->device == BTINTEL_PCIE_DEVICE_ID_PTL_FMP2; 1199 } 1200 1201 static void btintel_pcie_dump_mem_range(struct btintel_pcie_data *data, 1202 struct list_head *list, u32 region_id, 1203 const char *name, u32 addr_start, 1204 u32 region_size, u64 *regions_mask) 1205 { 1206 if (!region_size) 1207 return; 1208 1209 if (!btintel_pcie_dump_target_region(data, list, region_id, name, 1210 addr_start, region_size)) 1211 *regions_mask |= BIT_ULL(region_id); 1212 } 1213 1214 static int btintel_pcie_read_debug_regions(struct btintel_pcie_data *data) 1215 { 1216 struct btintel_pcie_dbgc *dbgc = NULL; 1217 struct btintel_pcie_mdbgc *mdbgc = NULL; 1218 struct btintel_pcie_dump_mem_info *dump_info = &data->dump_info; 1219 struct hci_dev *hdev = data->hdev; 1220 struct btintel_pcie_dump_entry *entry; 1221 struct btintel_pcie_dump_file_hdr *file_hdr; 1222 struct scatterlist *sg_dump_data; 1223 u32 status_reg, wrap_reg; 1224 u64 regions_mask = 0; 1225 u8 hw_variant; 1226 u32 file_len; 1227 u8 count; 1228 int ret; 1229 LIST_HEAD(dump_list); 1230 1231 if (!IS_ENABLED(CONFIG_DEV_COREDUMP)) 1232 return -EOPNOTSUPP; 1233 1234 if (btintel_pcie_is_mdbgc_supported(data)) { 1235 mdbgc = &data->mdbgc; 1236 count = mdbgc->count; 1237 } else { 1238 dbgc = &data->dbgc; 1239 count = dbgc->count; 1240 } 1241 1242 hw_variant = INTEL_HW_VARIANT(data->cnvi); 1243 1244 if (hw_variant == BTINTEL_HWID_BZRI || 1245 hw_variant == BTINTEL_HWID_BZRIW) { 1246 status_reg = BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS; 1247 wrap_reg = BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND; 1248 } else if (hw_variant >= BTINTEL_HWID_SCP) { 1249 status_reg = BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS_SCP; 1250 wrap_reg = BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND_SCP; 1251 } else { 1252 bt_dev_err(hdev, 1253 "Unsupported Intel hardware variant (0x%2.2x)", 1254 hw_variant); 1255 return -EINVAL; 1256 } 1257 1258 if (btintel_pcie_is_mdbgc_supported(data)) { 1259 ret = btintel_pcie_dump_dram_monitor(data, &dump_list, count, 1260 mdbgc->buf1, status_reg, 1261 wrap_reg, 1262 BTINTEL_PCIE_MDBGC_ALLOCATIONID_1, 1263 BTINTEL_PCIE_INI_ID_DRAM_MONITOR1, 1264 "monitor"); 1265 if (!ret) 1266 regions_mask |= 1267 BIT_ULL(BTINTEL_PCIE_INI_ID_DRAM_MONITOR1); 1268 else 1269 bt_dev_warn(hdev, "Failed to dump DRAM buf1: %d", ret); 1270 1271 ret = btintel_pcie_dump_dram_monitor(data, &dump_list, count, 1272 mdbgc->buf2, status_reg, 1273 wrap_reg, 1274 BTINTEL_PCIE_MDBGC_ALLOCATIONID_2, 1275 BTINTEL_PCIE_INI_ID_DRAM_MONITOR2, 1276 "monitor2"); 1277 if (!ret) 1278 regions_mask |= 1279 BIT_ULL(BTINTEL_PCIE_INI_ID_DRAM_MONITOR2); 1280 else 1281 bt_dev_warn(hdev, "Failed to dump DRAM buf2: %d", ret); 1282 1283 ret = btintel_pcie_dump_dram_monitor(data, &dump_list, count, 1284 mdbgc->buf3, status_reg, 1285 wrap_reg, 1286 BTINTEL_PCIE_MDBGC_ALLOCATIONID_3, 1287 BTINTEL_PCIE_INI_ID_DRAM_MONITOR3, 1288 "monitor3"); 1289 if (!ret) 1290 regions_mask |= 1291 BIT_ULL(BTINTEL_PCIE_INI_ID_DRAM_MONITOR3); 1292 else 1293 bt_dev_warn(hdev, "Failed to dump DRAM buf3: %d", ret); 1294 } else { 1295 ret = btintel_pcie_dump_dram_monitor(data, &dump_list, count, 1296 dbgc->bufs, status_reg, 1297 wrap_reg, 1298 BTINTEL_PCIE_MDBGC_ALLOCATIONID_1, 1299 BTINTEL_PCIE_INI_ID_DRAM_MONITOR1, 1300 "monitor"); 1301 if (!ret) 1302 regions_mask |= 1303 BIT_ULL(BTINTEL_PCIE_INI_ID_DRAM_MONITOR1); 1304 else 1305 bt_dev_warn(hdev, 1306 "Failed to dump DRAM region: %d", ret); 1307 } 1308 1309 if (dump_info->smem_size && 1310 !btintel_pcie_dump_smem_monitor_region(data, &dump_list, 1311 BTINTEL_PCIE_INI_ID_SMEM, 1312 "monitor_smem", 1313 dump_info->smem_addr_start, 1314 dump_info->smem_size)) 1315 regions_mask |= BIT_ULL(BTINTEL_PCIE_INI_ID_SMEM); 1316 1317 if (dump_info->exception_dump_addr && dump_info->exception_dump_len) { 1318 ret = btintel_pcie_dump_target_region(data, &dump_list, 1319 BTINTEL_PCIE_INI_ID_EXCEPTION_EVT, 1320 "EXCEPTION_EVT_BUFFER", 1321 dump_info->exception_dump_addr, 1322 dump_info->exception_dump_len); 1323 if (!ret) 1324 regions_mask |= BIT_ULL(BTINTEL_PCIE_INI_ID_EXCEPTION_EVT); 1325 } 1326 1327 btintel_pcie_dump_mem_range(data, &dump_list, 1328 BTINTEL_PCIE_INI_ID_DCCM, "DCCM", 1329 dump_info->dccm_addr_start, 1330 dump_info->dccm_size, 1331 ®ions_mask); 1332 1333 btintel_pcie_dump_mem_range(data, &dump_list, 1334 BTINTEL_PCIE_INI_ID_SDS, "SDS", 1335 dump_info->sds_start_addr_start, 1336 dump_info->sds_start_size, 1337 ®ions_mask); 1338 1339 btintel_pcie_dump_mem_range(data, &dump_list, 1340 BTINTEL_PCIE_INI_ID_SDS_IOSF, "SDS_IOSF", 1341 dump_info->sds_iosf_data_addr_start, 1342 dump_info->sds_iosf_data_size, 1343 ®ions_mask); 1344 1345 btintel_pcie_dump_mem_range(data, &dump_list, 1346 BTINTEL_PCIE_INI_ID_ECL, "ECL_REGION", 1347 dump_info->ecl_addr_start, 1348 dump_info->ecl_size, 1349 ®ions_mask); 1350 1351 ret = btintel_pcie_dump_info(data, &dump_list, regions_mask); 1352 if (ret) { 1353 btintel_pcie_dump_list_free(&dump_list); 1354 return ret; 1355 } 1356 1357 file_len = sizeof(*file_hdr) + 1358 btintel_pcie_dump_list_total_size(&dump_list); 1359 entry = btintel_pcie_dump_entry_alloc(sizeof(*file_hdr)); 1360 if (!entry) { 1361 btintel_pcie_dump_list_free(&dump_list); 1362 return -ENOMEM; 1363 } 1364 1365 file_hdr = (void *)entry->data; 1366 file_hdr->barker = cpu_to_le32(BTINTEL_PCIE_INI_ERROR_DUMP_BARKER); 1367 file_hdr->file_len = cpu_to_le32(file_len); 1368 list_add(&entry->list, &dump_list); 1369 1370 sg_dump_data = btintel_pcie_alloc_sgtable(file_len); 1371 if (sg_dump_data) { 1372 int sg_entries = sg_nents(sg_dump_data); 1373 u32 offs = 0; 1374 1375 list_for_each_entry(entry, &dump_list, list) { 1376 sg_pcopy_from_buffer(sg_dump_data, sg_entries, 1377 entry->data, entry->size, offs); 1378 offs += entry->size; 1379 } 1380 1381 bt_dev_dbg(hdev, "triggering dev_coredumpsg()"); 1382 dev_coredumpsg(&hdev->dev, sg_dump_data, file_len, GFP_KERNEL); 1383 } else { 1384 bt_dev_err(hdev, "Failed to allocate scatter-gather table for coredump"); 1385 ret = -ENOMEM; 1386 } 1387 1388 btintel_pcie_dump_list_free(&dump_list); 1389 return ret; 1390 } 1391 1392 static void btintel_pcie_dump_traces(struct hci_dev *hdev) 1393 { 1394 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 1395 int ret; 1396 1397 ret = btintel_pcie_get_mac_access(data); 1398 if (ret) { 1399 bt_dev_err(hdev, "Failed to get mac access: (%d)", ret); 1400 return; 1401 } 1402 1403 ret = btintel_pcie_read_debug_regions(data); 1404 1405 btintel_pcie_release_mac_access(data); 1406 1407 if (ret) 1408 bt_dev_err(hdev, "Failed to dump traces: (%d)", ret); 1409 } 1410 1411 static bool btintel_pcie_is_blazariw(struct pci_dev *pdev) 1412 { 1413 return pdev->device == 0x4D76; 1414 } 1415 1416 /* This function enables BT function by setting BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT bit in 1417 * BTINTEL_PCIE_CSR_FUNC_CTRL_REG register and wait for MSI-X with 1418 * BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0. 1419 * Then the host reads firmware version from BTINTEL_CSR_F2D_MBX and the boot stage 1420 * from BTINTEL_PCIE_CSR_BOOT_STAGE_REG. 1421 */ 1422 static int btintel_pcie_enable_bt(struct btintel_pcie_data *data) 1423 { 1424 int err; 1425 u32 reg; 1426 1427 data->gp0_received = false; 1428 1429 /* Update the DMA address of CI struct to CSR */ 1430 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_LSB_REG, 1431 data->ci_p_addr & 0xffffffff); 1432 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_MSB_REG, 1433 (u64)data->ci_p_addr >> 32); 1434 1435 /* On BlazarIW, the D0 entry to MAC init does not complete in 1436 * time. Wait 50 ms (worst case as per HW analysis) for the 1437 * shared hardware reset flow to complete before proceeding with 1438 * MAC init. 1439 */ 1440 if (btintel_pcie_is_blazariw(data->pdev)) 1441 msleep(50); 1442 1443 /* Reset the cached value of boot stage. it is updated by the MSI-X 1444 * gp0 interrupt handler. 1445 */ 1446 data->boot_stage_cache = 0x0; 1447 1448 /* Set MAC_INIT bit to start primary bootloader */ 1449 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 1450 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT | 1451 BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON | 1452 BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET); 1453 reg |= (BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 1454 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT); 1455 1456 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 1457 1458 /* MAC is ready. Enable BT FUNC */ 1459 btintel_pcie_set_reg_bits(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, 1460 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT); 1461 1462 btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 1463 1464 /* wait for interrupt from the device after booting up to primary 1465 * bootloader. 1466 */ 1467 data->alive_intr_ctxt = BTINTEL_PCIE_ROM; 1468 err = wait_event_timeout(data->gp0_wait_q, data->gp0_received, 1469 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT_MS)); 1470 if (!err) 1471 return -ETIME; 1472 1473 /* Check cached boot stage is BTINTEL_PCIE_CSR_BOOT_STAGE_ROM(BIT(0)) */ 1474 if (~data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ROM) 1475 return -ENODEV; 1476 1477 return 0; 1478 } 1479 1480 static inline bool btintel_pcie_in_op(struct btintel_pcie_data *data) 1481 { 1482 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW; 1483 } 1484 1485 static inline bool btintel_pcie_in_iml(struct btintel_pcie_data *data) 1486 { 1487 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_IML && 1488 !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW); 1489 } 1490 1491 static inline bool btintel_pcie_in_d3(struct btintel_pcie_data *data) 1492 { 1493 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY; 1494 } 1495 1496 static inline bool btintel_pcie_in_d0(struct btintel_pcie_data *data) 1497 { 1498 return !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY); 1499 } 1500 1501 static inline bool btintel_pcie_in_device_halt(struct btintel_pcie_data *data) 1502 { 1503 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_HALTED; 1504 } 1505 1506 static void btintel_pcie_wr_sleep_cntrl(struct btintel_pcie_data *data, 1507 u32 dxstate) 1508 { 1509 bt_dev_dbg(data->hdev, "writing sleep_ctl_reg: 0x%8.8x", dxstate); 1510 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_IPC_SLEEP_CTL_REG, dxstate); 1511 } 1512 1513 static int btintel_pcie_read_device_mem(struct btintel_pcie_data *data, 1514 void *buf, u32 dev_addr, int len) 1515 { 1516 int err; 1517 u32 *val = buf; 1518 1519 /* Get device mac access */ 1520 err = btintel_pcie_get_mac_access(data); 1521 if (err) { 1522 bt_dev_err(data->hdev, "Failed to get mac access %d", err); 1523 return err; 1524 } 1525 1526 for (; len > 0; len -= 4, dev_addr += 4, val++) 1527 *val = btintel_pcie_rd_dev_mem(data, dev_addr); 1528 1529 btintel_pcie_release_mac_access(data); 1530 1531 return 0; 1532 } 1533 1534 static inline bool btintel_pcie_in_lockdown(struct btintel_pcie_data *data) 1535 { 1536 return (data->boot_stage_cache & 1537 BTINTEL_PCIE_CSR_BOOT_STAGE_ROM_LOCKDOWN) || 1538 (data->boot_stage_cache & 1539 BTINTEL_PCIE_CSR_BOOT_STAGE_IML_LOCKDOWN); 1540 } 1541 1542 static inline bool btintel_pcie_in_error(struct btintel_pcie_data *data) 1543 { 1544 if (data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_WARNING) 1545 bt_dev_warn(data->hdev, "Controller device warning (boot_stage: 0x%8.8x)", 1546 data->boot_stage_cache); 1547 1548 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ABORT_HANDLER; 1549 } 1550 1551 static u32 btintel_pcie_region_size(struct hci_dev *hdev, const char *name, 1552 u32 start, u32 end) 1553 { 1554 u64 size; 1555 1556 if (!start || !end) 1557 return 0; 1558 1559 if (end < start) { 1560 bt_dev_warn(hdev, "%s region invalid: start=0x%08x end=0x%08x", 1561 name, start, end); 1562 return 0; 1563 } 1564 1565 size = (u64)end + 0x04 - start; 1566 if (size % sizeof(u32)) 1567 bt_dev_warn(hdev, "%s region size %llu is not 4-byte aligned", 1568 name, size); 1569 1570 if (size > BTINTEL_PCIE_REGION_MAX_SIZE) { 1571 bt_dev_warn(hdev, "%s region size %llu exceeds cap %u, ignoring", 1572 name, size, BTINTEL_PCIE_REGION_MAX_SIZE); 1573 return 0; 1574 } 1575 1576 return (u32)size; 1577 } 1578 1579 static const char *btintel_pcie_tlv_str(u8 tlv_type) 1580 { 1581 switch (tlv_type) { 1582 case BTINTEL_PCIE_TLV_TYPE_EXCEPTION_DUMP_ADDRESS: 1583 return "EXCEPTION_DUMP_ADDRESS"; 1584 case BTINTEL_PCIE_TLV_TYPE_DCCM_MEM_ADDRESS: 1585 return "DCCM_MEM_ADDRESS"; 1586 case BTINTEL_PCIE_TLV_TYPE_SDS_MEM_ADDRESS: 1587 return "SDS_MEM_ADDRESS"; 1588 case BTINTEL_PCIE_TLV_TYPE_ECL_MEM_ADDRESS: 1589 return "ECL_MEM_ADDRESS"; 1590 case BTINTEL_PCIE_TLV_TYPE_SMEM_ADDRESS: 1591 return "SMEM_ADDRESS"; 1592 default: 1593 return "UNKNOWN"; 1594 } 1595 } 1596 1597 static int btintel_parse_mbox_tlv(struct btintel_pcie_data *data) 1598 { 1599 /* Custom TLV structure for mailbox parsing 1600 * len is __le16 as per agreement with FW 1601 */ 1602 struct mbox_tlv { 1603 u8 type; 1604 __le16 len; 1605 u8 val[]; 1606 } __packed; 1607 1608 u8 *buffer, *ptr; 1609 u32 buffer_len, remaining; 1610 int err; 1611 u32 tbl_addr, tbl_size; 1612 struct mbox_tlv *tlv; 1613 struct btintel_data *cnvi_data = hci_get_priv(data->hdev); 1614 u8 hw_variant = INTEL_HW_VARIANT(cnvi_data->cnvi_bt); 1615 long t; 1616 1617 /* Wait for GP0 alive interrupt to post RX buffers */ 1618 t = wait_event_timeout(data->mbox_parse_wait_q, 1619 test_bit(BTINTEL_PCIE_MBOX_PARSE_READY, &data->flags), 1620 msecs_to_jiffies(BTINTEL_PCIE_MBOX_INTR_TIMEOUT_MS)); 1621 if (!t) { 1622 bt_dev_warn(data->hdev, 1623 "Timeout (%u ms) waiting for alive interrupt before mbox TLV parse; skipping", 1624 BTINTEL_PCIE_MBOX_INTR_TIMEOUT_MS); 1625 return 0; 1626 } 1627 clear_bit(BTINTEL_PCIE_MBOX_PARSE_READY, &data->flags); 1628 1629 bt_dev_info(data->hdev, 1630 "mbox TLV parse started at %lld ns (%lld us after mbox interrupt)", 1631 ktime_to_ns(ktime_get()), 1632 ktime_to_us(ktime_sub(ktime_get(), data->mbox_intr_ts))); 1633 1634 memset(&data->dump_info, 0, sizeof(data->dump_info)); 1635 1636 /* Snapshot to avoid TOCTOU with the GP1 IRQ handler */ 1637 tbl_size = READ_ONCE(data->debug_table_size); 1638 tbl_addr = READ_ONCE(data->debug_table_addr); 1639 1640 if (!tbl_size || !tbl_addr) 1641 return -EINVAL; 1642 1643 /* Ensure size is 4-byte aligned; btintel_pcie_read_device_mem() 1644 * reads device memory in 4-byte units. 1645 */ 1646 tbl_size = ALIGN_DOWN(tbl_size, 4); 1647 1648 if (!tbl_size) 1649 return -EINVAL; 1650 1651 if (tbl_size > SZ_1M) { 1652 bt_dev_err(data->hdev, "Debug table size too large: %u", 1653 tbl_size); 1654 return -EINVAL; 1655 } 1656 1657 buffer_len = tbl_size; 1658 1659 buffer = vmalloc(buffer_len); 1660 if (!buffer) 1661 return -ENOMEM; 1662 1663 btintel_pcie_mac_init(data); 1664 1665 err = btintel_pcie_read_device_mem(data, buffer, tbl_addr, 1666 buffer_len); 1667 if (err) 1668 goto exit_on_error; 1669 1670 print_hex_dump(KERN_INFO, "Bluetooth: mbox_tlv: ", DUMP_PREFIX_OFFSET, 16, 1, 1671 buffer, buffer_len, false); 1672 1673 ptr = buffer; 1674 remaining = buffer_len; 1675 1676 /* Parse TLV structures: 1 byte type + 2 bytes length + 1677 * variable value 1678 */ 1679 while (remaining >= sizeof(struct mbox_tlv)) { 1680 u16 tlv_len; 1681 u32 tlv_total; 1682 1683 tlv = (struct mbox_tlv *)ptr; 1684 tlv_len = le16_to_cpu(tlv->len); 1685 tlv_total = sizeof(tlv->type) + sizeof(tlv->len) + tlv_len; 1686 1687 if (tlv_total > remaining) { 1688 bt_dev_err(data->hdev, "TLV parse error: not enough data for TLV value (type=%u, len=%u)", 1689 tlv->type, tlv_len); 1690 break; 1691 } 1692 1693 switch (tlv->type) { 1694 case BTINTEL_PCIE_TLV_TYPE_EXCEPTION_DUMP_ADDRESS: 1695 if (tlv_len < 8) { 1696 bt_dev_err(data->hdev, 1697 "TLV %s too short: len=%u (need 8)", 1698 btintel_pcie_tlv_str(tlv->type), 1699 tlv_len); 1700 break; 1701 } 1702 data->dump_info.exception_dump_addr = 1703 get_unaligned_le32(&tlv->val[0]); 1704 data->dump_info.exception_dump_len = 1705 get_unaligned_le32(&tlv->val[4]); 1706 if (data->dump_info.exception_dump_len > 1707 BTINTEL_PCIE_REGION_MAX_SIZE) { 1708 bt_dev_warn(data->hdev, 1709 "EXCEPTION_EVT_BUFFER size %u exceeds cap %u, ignoring", 1710 data->dump_info.exception_dump_len, 1711 BTINTEL_PCIE_REGION_MAX_SIZE); 1712 data->dump_info.exception_dump_addr = 0; 1713 data->dump_info.exception_dump_len = 0; 1714 } 1715 break; 1716 case BTINTEL_PCIE_TLV_TYPE_DCCM_MEM_ADDRESS: 1717 if (tlv_len < 8) { 1718 bt_dev_err(data->hdev, 1719 "TLV %s too short: len=%u (need 8)", 1720 btintel_pcie_tlv_str(tlv->type), 1721 tlv_len); 1722 break; 1723 } 1724 data->dump_info.dccm_addr_start = 1725 get_unaligned_le32(&tlv->val[0]); 1726 data->dump_info.dccm_size = 1727 btintel_pcie_region_size(data->hdev, "DCCM", 1728 data->dump_info.dccm_addr_start, 1729 get_unaligned_le32(&tlv->val[4])); 1730 if (!data->dump_info.dccm_size) 1731 data->dump_info.dccm_addr_start = 0; 1732 break; 1733 case BTINTEL_PCIE_TLV_TYPE_SDS_MEM_ADDRESS: { 1734 struct btintel_pcie_dump_mem_info *di = &data->dump_info; 1735 1736 /* hw_variant comes from cnvi_bt which is set during 1737 * setup. If mailbox fires before setup completes, 1738 * hw_variant is 0. Skip SDS parsing in that case. 1739 */ 1740 if (!hw_variant) { 1741 bt_dev_dbg(data->hdev, "SDS TLV: skipped, hw_variant not yet known"); 1742 break; 1743 } 1744 if (tlv_len == 16 && hw_variant > BTINTEL_HWID_BZRI) { 1745 di->sds_start_addr_start = 1746 get_unaligned_le32(&tlv->val[0]); 1747 di->sds_start_size = 1748 btintel_pcie_region_size(data->hdev, "SDS", 1749 di->sds_start_addr_start, 1750 get_unaligned_le32(&tlv->val[4])); 1751 di->sds_iosf_data_addr_start = 1752 get_unaligned_le32(&tlv->val[8]); 1753 di->sds_iosf_data_size = 1754 btintel_pcie_region_size(data->hdev, "SDS_IOSF", 1755 di->sds_iosf_data_addr_start, 1756 get_unaligned_le32(&tlv->val[12])); 1757 } else if (tlv_len == 24 && 1758 (hw_variant == BTINTEL_HWID_BZRI || 1759 hw_variant == BTINTEL_HWID_BZRIW)) { 1760 di->sds_fixed_rom_addr_start = 1761 get_unaligned_le32(&tlv->val[0]); 1762 di->sds_fixed_rom_size = 1763 btintel_pcie_region_size(data->hdev, "SDS_FIXED_ROM", 1764 di->sds_fixed_rom_addr_start, 1765 get_unaligned_le32(&tlv->val[4])); 1766 di->sds_start_addr_start = 1767 get_unaligned_le32(&tlv->val[8]); 1768 di->sds_start_size = 1769 btintel_pcie_region_size(data->hdev, "SDS", 1770 di->sds_start_addr_start, 1771 get_unaligned_le32(&tlv->val[12])); 1772 di->sds_iosf_data_addr_start = 1773 get_unaligned_le32(&tlv->val[16]); 1774 di->sds_iosf_data_size = 1775 btintel_pcie_region_size(data->hdev, "SDS_IOSF", 1776 di->sds_iosf_data_addr_start, 1777 get_unaligned_le32(&tlv->val[20])); 1778 } else { 1779 bt_dev_err(data->hdev, 1780 "SDS TLV: hw=0x%2.2x len=%u", 1781 hw_variant, tlv_len); 1782 } 1783 if (!di->sds_start_size) 1784 di->sds_start_addr_start = 0; 1785 if (!di->sds_iosf_data_size) 1786 di->sds_iosf_data_addr_start = 0; 1787 if (!di->sds_fixed_rom_size) 1788 di->sds_fixed_rom_addr_start = 0; 1789 break; 1790 } 1791 case BTINTEL_PCIE_TLV_TYPE_ECL_MEM_ADDRESS: 1792 if (tlv_len < 8) { 1793 bt_dev_err(data->hdev, 1794 "TLV %s too short: len=%u (need 8)", 1795 btintel_pcie_tlv_str(tlv->type), 1796 tlv_len); 1797 break; 1798 } 1799 data->dump_info.ecl_addr_start = 1800 get_unaligned_le32(&tlv->val[0]); 1801 data->dump_info.ecl_size = 1802 btintel_pcie_region_size(data->hdev, "ECL", 1803 data->dump_info.ecl_addr_start, 1804 get_unaligned_le32(&tlv->val[4])); 1805 if (!data->dump_info.ecl_size) 1806 data->dump_info.ecl_addr_start = 0; 1807 break; 1808 case BTINTEL_PCIE_TLV_TYPE_SMEM_ADDRESS: 1809 if (tlv_len < 8) { 1810 bt_dev_err(data->hdev, 1811 "TLV %s too short: len=%u (need 8)", 1812 btintel_pcie_tlv_str(tlv->type), 1813 tlv_len); 1814 break; 1815 } 1816 data->dump_info.smem_addr_start = 1817 get_unaligned_le32(&tlv->val[0]); 1818 data->dump_info.smem_size = 1819 btintel_pcie_region_size(data->hdev, "SMEM", 1820 data->dump_info.smem_addr_start, 1821 get_unaligned_le32(&tlv->val[4])); 1822 if (!data->dump_info.smem_size) 1823 data->dump_info.smem_addr_start = 0; 1824 break; 1825 default: 1826 bt_dev_dbg(data->hdev, "Unknown TLV type: %u length: %u", 1827 tlv->type, tlv_len); 1828 break; 1829 } 1830 1831 /* Move to next TLV */ 1832 ptr += tlv_total; 1833 remaining -= tlv_total; 1834 } 1835 1836 bt_dev_info(data->hdev, 1837 "exception_dump: addr:0x%08x len:0x%08x", 1838 data->dump_info.exception_dump_addr, 1839 data->dump_info.exception_dump_len); 1840 bt_dev_info(data->hdev, 1841 "dccm: start:0x%8.8x size:%u", 1842 data->dump_info.dccm_addr_start, 1843 data->dump_info.dccm_size); 1844 bt_dev_info(data->hdev, 1845 "sds_fixed_rom: start:0x%8.8x size:%u", 1846 data->dump_info.sds_fixed_rom_addr_start, 1847 data->dump_info.sds_fixed_rom_size); 1848 bt_dev_info(data->hdev, 1849 "sds: start:0x%8.8x size:%u", 1850 data->dump_info.sds_start_addr_start, 1851 data->dump_info.sds_start_size); 1852 bt_dev_info(data->hdev, 1853 "sds_iosf: start:0x%8.8x size:%u", 1854 data->dump_info.sds_iosf_data_addr_start, 1855 data->dump_info.sds_iosf_data_size); 1856 bt_dev_info(data->hdev, 1857 "ecl: start:0x%8.8x size:%u", 1858 data->dump_info.ecl_addr_start, 1859 data->dump_info.ecl_size); 1860 bt_dev_info(data->hdev, 1861 "smem: start:0x%8.8x size:%u", 1862 data->dump_info.smem_addr_start, 1863 data->dump_info.smem_size); 1864 1865 vfree(buffer); 1866 return 0; 1867 1868 exit_on_error: 1869 vfree(buffer); 1870 return err; 1871 } 1872 1873 static void btintel_pcie_msix_gp1_handler(struct btintel_pcie_data *data) 1874 { 1875 bool target_access = false; 1876 u32 addr = 0, size = 0; 1877 1878 /* Read the Mail box status and registers */ 1879 data->mbox.mbox_status = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_STATUS_REG); 1880 if (data->mbox.mbox_status & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX1) { 1881 data->mbox.mbox1 = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_1_REG); 1882 if (data->mbox.mbox1 == 1883 BTINTEL_PCIE_BUILD_SPECIFIC_RESOURCES_MAPPING) { 1884 bt_dev_info(data->hdev, 1885 "mailbox for target access"); 1886 target_access = true; 1887 } 1888 } 1889 1890 if (data->mbox.mbox_status & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX2) { 1891 data->mbox.mbox2 = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_2_REG); 1892 if (target_access) 1893 addr = data->mbox.mbox2; 1894 } 1895 1896 if (data->mbox.mbox_status & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX3) { 1897 data->mbox.mbox3 = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_3_REG); 1898 if (target_access) 1899 size = data->mbox.mbox3; 1900 } 1901 1902 if (data->mbox.mbox_status & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX4) 1903 data->mbox.mbox4 = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_4_REG); 1904 1905 bt_dev_dbg(data->hdev, 1906 "GP1: sts:0x%08x mb1:0x%08x mb2:0x%08x mb3:0x%08x mb4:0x%08x", 1907 data->mbox.mbox_status, data->mbox.mbox1, 1908 data->mbox.mbox2, data->mbox.mbox3, 1909 data->mbox.mbox4); 1910 1911 if (target_access && 1912 !test_and_set_bit(BTINTEL_PCIE_MAIL_BOX_INTR, 1913 &data->flags)) { 1914 /* Arm the mbox<->alive handshake */ 1915 clear_bit(BTINTEL_PCIE_MBOX_PARSE_READY, &data->flags); 1916 set_bit(BTINTEL_PCIE_MBOX_PARSE_PENDING, &data->flags); 1917 data->mbox_intr_ts = ktime_get(); 1918 1919 bt_dev_info(data->hdev, 1920 "mbox interrupt received at %lld ns; queuing mbox_work", 1921 ktime_to_ns(data->mbox_intr_ts)); 1922 1923 WRITE_ONCE(data->debug_table_addr, addr); 1924 WRITE_ONCE(data->debug_table_size, size); 1925 if (!queue_work(data->dump_workqueue, 1926 &data->mbox_work)) { 1927 clear_bit(BTINTEL_PCIE_MAIL_BOX_INTR, &data->flags); 1928 clear_bit(BTINTEL_PCIE_MBOX_PARSE_PENDING, &data->flags); 1929 } 1930 } 1931 1932 /* Mailbox is read, ack to FW */ 1933 btintel_pcie_set_reg_bits(data, 1934 BTINTEL_PCIE_CSR_IPC_DOORBELL_VEC_REG, 1935 BTINTEL_PCIE_CSR_DOORBELL_MBOX_READ_CONFIRM); 1936 } 1937 1938 /* This function handles the MSI-X interrupt for gp0 cause (bit 0 in 1939 * BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES) which is sent for boot stage and image response. 1940 */ 1941 static void btintel_pcie_msix_gp0_handler(struct btintel_pcie_data *data) 1942 { 1943 bool submit_rx, signal_waitq; 1944 u32 reg, old_ctxt; 1945 1946 /* This interrupt is for three different causes and it is not easy to 1947 * know what causes the interrupt. So, it compares each register value 1948 * with cached value and update it before it wake up the queue. 1949 */ 1950 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 1951 if (reg != data->boot_stage_cache) 1952 data->boot_stage_cache = reg; 1953 1954 bt_dev_dbg(data->hdev, "Alive context: %s old_boot_stage: 0x%8.8x new_boot_stage: 0x%8.8x", 1955 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt), 1956 data->boot_stage_cache, reg); 1957 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IMG_RESPONSE_REG); 1958 if (reg != data->img_resp_cache) 1959 data->img_resp_cache = reg; 1960 1961 if (btintel_pcie_in_error(data)) { 1962 bt_dev_err(data->hdev, "Controller in error state (boot_stage: 0x%8.8x)", 1963 data->boot_stage_cache); 1964 btintel_pcie_dump_debug_registers(data->hdev); 1965 return; 1966 } 1967 1968 if (btintel_pcie_in_lockdown(data)) { 1969 bt_dev_err(data->hdev, "Controller in lockdown state"); 1970 btintel_pcie_dump_debug_registers(data->hdev); 1971 return; 1972 } 1973 1974 data->gp0_received = true; 1975 1976 old_ctxt = data->alive_intr_ctxt; 1977 submit_rx = false; 1978 signal_waitq = false; 1979 1980 /* Serialize the alive_intr_ctxt claim against set_dxstate()'s 1981 * register-based fallback, which can run concurrently. 1982 */ 1983 spin_lock(&data->irq_lock); 1984 switch (data->alive_intr_ctxt) { 1985 case BTINTEL_PCIE_ROM: 1986 data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL; 1987 signal_waitq = true; 1988 break; 1989 case BTINTEL_PCIE_FW_DL: 1990 /* Error case is already handled. Ideally control shall not 1991 * reach here 1992 */ 1993 break; 1994 case BTINTEL_PCIE_INTEL_HCI_RESET1: 1995 if (btintel_pcie_in_op(data)) { 1996 submit_rx = true; 1997 signal_waitq = true; 1998 break; 1999 } 2000 2001 if (btintel_pcie_in_iml(data)) { 2002 submit_rx = true; 2003 signal_waitq = true; 2004 data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL; 2005 break; 2006 } 2007 break; 2008 case BTINTEL_PCIE_INTEL_HCI_RESET2: 2009 if (btintel_test_and_clear_flag(data->hdev, INTEL_WAIT_FOR_D0)) { 2010 btintel_wake_up_flag(data->hdev, INTEL_WAIT_FOR_D0); 2011 data->alive_intr_ctxt = BTINTEL_PCIE_D0; 2012 } 2013 break; 2014 case BTINTEL_PCIE_D0: 2015 if (btintel_pcie_in_d3(data)) { 2016 data->alive_intr_ctxt = BTINTEL_PCIE_D3; 2017 signal_waitq = true; 2018 break; 2019 } 2020 break; 2021 case BTINTEL_PCIE_D3: 2022 if (btintel_pcie_in_d0(data)) { 2023 data->alive_intr_ctxt = BTINTEL_PCIE_D0; 2024 submit_rx = true; 2025 signal_waitq = true; 2026 break; 2027 } 2028 break; 2029 case BTINTEL_PCIE_HCI_RESET: 2030 data->alive_intr_ctxt = BTINTEL_PCIE_D0; 2031 submit_rx = true; 2032 signal_waitq = true; 2033 break; 2034 default: 2035 bt_dev_err(data->hdev, "Unknown state: 0x%2.2x", 2036 data->alive_intr_ctxt); 2037 break; 2038 } 2039 spin_unlock(&data->irq_lock); 2040 2041 if (submit_rx) { 2042 btintel_pcie_reset_ia(data); 2043 btintel_pcie_start_rx(data); 2044 2045 /* Complete the mbox<->alive handshake */ 2046 if (test_and_clear_bit(BTINTEL_PCIE_MBOX_PARSE_PENDING, &data->flags)) { 2047 set_bit(BTINTEL_PCIE_MBOX_PARSE_READY, &data->flags); 2048 wake_up(&data->mbox_parse_wait_q); 2049 } 2050 } 2051 2052 if (signal_waitq) { 2053 bt_dev_dbg(data->hdev, "wake up gp0 wait_q"); 2054 wake_up(&data->gp0_wait_q); 2055 } 2056 2057 if (old_ctxt != data->alive_intr_ctxt) 2058 bt_dev_dbg(data->hdev, "alive context changed: %s -> %s", 2059 btintel_pcie_alivectxt_state2str(old_ctxt), 2060 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt)); 2061 } 2062 2063 /* This function handles the MSX-X interrupt for rx queue 0 which is for TX 2064 */ 2065 static void btintel_pcie_msix_tx_handle(struct btintel_pcie_data *data) 2066 { 2067 u16 cr_tia, cr_hia; 2068 struct txq *txq; 2069 struct urbd0 *urbd0; 2070 2071 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM]; 2072 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM]; 2073 2074 if (cr_tia == cr_hia) 2075 return; 2076 2077 txq = &data->txq; 2078 2079 if (cr_hia >= txq->count) { 2080 bt_dev_err(data->hdev, "TXQ: invalid cr_hia %u", cr_hia); 2081 return; 2082 } 2083 2084 while (cr_tia != cr_hia) { 2085 data->tx_wait_done = true; 2086 wake_up(&data->tx_wait_q); 2087 2088 urbd0 = &txq->urbd0s[cr_tia]; 2089 2090 if (urbd0->tfd_index >= txq->count) 2091 return; 2092 2093 cr_tia = (cr_tia + 1) % txq->count; 2094 data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM] = cr_tia; 2095 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_TXQ_NUM); 2096 } 2097 } 2098 2099 static int btintel_pcie_recv_event(struct hci_dev *hdev, struct sk_buff *skb) 2100 { 2101 struct hci_event_hdr *hdr = (void *)skb->data; 2102 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 2103 2104 if (skb->len > HCI_EVENT_HDR_SIZE && hdr->evt == 0xff && 2105 hdr->plen > 0) { 2106 const void *ptr = skb->data + HCI_EVENT_HDR_SIZE + 1; 2107 unsigned int len = skb->len - HCI_EVENT_HDR_SIZE - 1; 2108 2109 if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) { 2110 switch (skb->data[2]) { 2111 case 0x02: 2112 /* When switching to the operational firmware 2113 * the device sends a vendor specific event 2114 * indicating that the bootup completed. 2115 */ 2116 btintel_bootup(hdev, ptr, len); 2117 2118 /* If bootup event is from operational image, 2119 * driver needs to write sleep control register to 2120 * move into D0 state 2121 */ 2122 if (btintel_pcie_in_op(data)) { 2123 btintel_pcie_wr_sleep_cntrl(data, BTINTEL_PCIE_STATE_D0); 2124 data->alive_intr_ctxt = BTINTEL_PCIE_INTEL_HCI_RESET2; 2125 kfree_skb(skb); 2126 return 0; 2127 } 2128 2129 if (btintel_pcie_in_iml(data)) { 2130 /* In case of IML, there is no concept 2131 * of D0 transition. Just mimic as if 2132 * IML moved to D0 by clearing INTEL_WAIT_FOR_D0 2133 * bit and waking up the task waiting on 2134 * INTEL_WAIT_FOR_D0. This is required 2135 * as intel_boot() is common function for 2136 * both IML and OP image loading. 2137 */ 2138 if (btintel_test_and_clear_flag(data->hdev, 2139 INTEL_WAIT_FOR_D0)) 2140 btintel_wake_up_flag(data->hdev, 2141 INTEL_WAIT_FOR_D0); 2142 } 2143 kfree_skb(skb); 2144 return 0; 2145 case 0x06: 2146 /* When the firmware loading completes the 2147 * device sends out a vendor specific event 2148 * indicating the result of the firmware 2149 * loading. 2150 */ 2151 btintel_secure_send_result(hdev, ptr, len); 2152 kfree_skb(skb); 2153 return 0; 2154 } 2155 } 2156 2157 /* This is a debug event that comes from IML and OP image when it 2158 * starts execution. There is no need pass this event to stack. 2159 */ 2160 if (skb->data[2] == 0x97) { 2161 hci_recv_diag(hdev, skb); 2162 return 0; 2163 } 2164 } 2165 2166 return hci_recv_frame(hdev, skb); 2167 } 2168 /* Process the received rx data 2169 * It check the frame header to identify the data type and create skb 2170 * and calling HCI API 2171 */ 2172 static int btintel_pcie_recv_frame(struct btintel_pcie_data *data, 2173 struct sk_buff *skb) 2174 { 2175 int ret; 2176 u8 pkt_type; 2177 u32 plen; 2178 u32 pcie_pkt_type; 2179 void *pdata; 2180 struct hci_dev *hdev = data->hdev; 2181 2182 spin_lock(&data->hci_rx_lock); 2183 2184 /* The first 4 bytes indicates the Intel PCIe specific packet type */ 2185 pdata = skb_pull_data(skb, BTINTEL_PCIE_HCI_TYPE_LEN); 2186 if (!pdata) { 2187 bt_dev_err(hdev, "Corrupted packet received"); 2188 ret = -EILSEQ; 2189 goto exit_error; 2190 } 2191 2192 pcie_pkt_type = get_unaligned_le32(pdata); 2193 2194 switch (pcie_pkt_type) { 2195 case BTINTEL_PCIE_HCI_ACL_PKT: 2196 if (skb->len >= HCI_ACL_HDR_SIZE) { 2197 plen = HCI_ACL_HDR_SIZE + __le16_to_cpu(hci_acl_hdr(skb)->dlen); 2198 pkt_type = HCI_ACLDATA_PKT; 2199 } else { 2200 bt_dev_err(hdev, "ACL packet is too short"); 2201 ret = -EILSEQ; 2202 goto exit_error; 2203 } 2204 break; 2205 2206 case BTINTEL_PCIE_HCI_SCO_PKT: 2207 if (skb->len >= HCI_SCO_HDR_SIZE) { 2208 plen = HCI_SCO_HDR_SIZE + hci_sco_hdr(skb)->dlen; 2209 pkt_type = HCI_SCODATA_PKT; 2210 } else { 2211 bt_dev_err(hdev, "SCO packet is too short"); 2212 ret = -EILSEQ; 2213 goto exit_error; 2214 } 2215 break; 2216 2217 case BTINTEL_PCIE_HCI_EVT_PKT: 2218 if (skb->len >= HCI_EVENT_HDR_SIZE) { 2219 plen = HCI_EVENT_HDR_SIZE + hci_event_hdr(skb)->plen; 2220 pkt_type = HCI_EVENT_PKT; 2221 } else { 2222 bt_dev_err(hdev, "Event packet is too short"); 2223 ret = -EILSEQ; 2224 goto exit_error; 2225 } 2226 break; 2227 2228 case BTINTEL_PCIE_HCI_ISO_PKT: 2229 if (skb->len >= HCI_ISO_HDR_SIZE) { 2230 plen = HCI_ISO_HDR_SIZE + __le16_to_cpu(hci_iso_hdr(skb)->dlen); 2231 pkt_type = HCI_ISODATA_PKT; 2232 } else { 2233 bt_dev_err(hdev, "ISO packet is too short"); 2234 ret = -EILSEQ; 2235 goto exit_error; 2236 } 2237 break; 2238 2239 default: 2240 bt_dev_err(hdev, "Invalid packet type received: 0x%4.4x", 2241 pcie_pkt_type); 2242 ret = -EINVAL; 2243 goto exit_error; 2244 } 2245 2246 if (skb->len < plen) { 2247 bt_dev_err(hdev, "Received corrupted packet. type: 0x%2.2x", 2248 pkt_type); 2249 ret = -EILSEQ; 2250 goto exit_error; 2251 } 2252 2253 bt_dev_dbg(hdev, "pkt_type: 0x%2.2x len: %u", pkt_type, plen); 2254 2255 hci_skb_pkt_type(skb) = pkt_type; 2256 hdev->stat.byte_rx += plen; 2257 skb_trim(skb, plen); 2258 2259 if (pcie_pkt_type == BTINTEL_PCIE_HCI_EVT_PKT) 2260 ret = btintel_pcie_recv_event(hdev, skb); 2261 else 2262 ret = hci_recv_frame(hdev, skb); 2263 skb = NULL; /* skb is freed in the callee */ 2264 2265 exit_error: 2266 kfree_skb(skb); 2267 2268 if (ret) 2269 hdev->stat.err_rx++; 2270 2271 spin_unlock(&data->hci_rx_lock); 2272 2273 return ret; 2274 } 2275 2276 static void btintel_pcie_read_hwexp(struct btintel_pcie_data *data) 2277 { 2278 int err, offset, pending; 2279 u32 len; 2280 struct sk_buff *skb; 2281 u8 *buf, prefix[64]; 2282 u32 addr, val; 2283 u16 pkt_len; 2284 2285 struct tlv { 2286 u8 type; 2287 __le16 len; 2288 u8 val[]; 2289 } __packed; 2290 2291 struct tlv *tlv; 2292 2293 switch (data->dmp_hdr.cnvi_top & 0xfff) { 2294 case BTINTEL_CNVI_BLAZARI: 2295 case BTINTEL_CNVI_BLAZARIW: 2296 /* only from step B0 onwards */ 2297 if (INTEL_CNVX_TOP_STEP(data->dmp_hdr.cnvi_top) != 0x01) 2298 return; 2299 break; 2300 case BTINTEL_CNVI_SCP: 2301 case BTINTEL_CNVI_SCP2: 2302 case BTINTEL_CNVI_SCP2_C0: 2303 case BTINTEL_CNVI_SCP2F: 2304 case BTINTEL_CNVI_DRACO: 2305 break; 2306 default: 2307 bt_dev_err(data->hdev, "Unsupported cnvi 0x%8.8x", data->dmp_hdr.cnvi_top); 2308 return; 2309 } 2310 2311 len = data->dump_info.exception_dump_len; 2312 addr = data->dump_info.exception_dump_addr; 2313 2314 if (!addr || len < sizeof(__le32) || len > SZ_4K) { 2315 bt_dev_err(data->hdev, "Invalid exception address: 0x%8.8x or length: %u", 2316 addr, len); 2317 return; 2318 } 2319 2320 /* Ensure size is 4-byte aligned; btintel_pcie_read_device_mem() 2321 * reads device memory in 4-byte units. 2322 */ 2323 len = ALIGN_DOWN(len, 4); 2324 2325 buf = kzalloc(len, GFP_KERNEL); 2326 if (!buf) 2327 goto exit_on_error; 2328 2329 btintel_pcie_mac_init(data); 2330 2331 err = btintel_pcie_read_device_mem(data, buf, addr, len); 2332 if (err) 2333 goto exit_on_error; 2334 2335 val = get_unaligned_le32(buf); 2336 if (val != BTINTEL_PCIE_MAGIC_NUM) { 2337 bt_dev_err(data->hdev, "Invalid exception dump signature: 0x%8.8x", 2338 val); 2339 goto exit_on_error; 2340 } 2341 2342 snprintf(prefix, sizeof(prefix), "Bluetooth: %s: ", bt_dev_name(data->hdev)); 2343 2344 offset = 4; 2345 do { 2346 pending = len - offset; 2347 if (pending < sizeof(*tlv)) 2348 break; 2349 tlv = (struct tlv *)(buf + offset); 2350 2351 /* If type == 0, then there are no more TLVs to be parsed */ 2352 if (!tlv->type) { 2353 bt_dev_dbg(data->hdev, "Invalid TLV type 0"); 2354 break; 2355 } 2356 pkt_len = le16_to_cpu(tlv->len); 2357 offset += sizeof(*tlv); 2358 pending = len - offset; 2359 if (pkt_len > pending) 2360 break; 2361 2362 offset += pkt_len; 2363 2364 /* Only TLVs of type == 1 are HCI events, no need to process other 2365 * TLVs 2366 */ 2367 if (tlv->type != 1) 2368 continue; 2369 2370 bt_dev_dbg(data->hdev, "TLV packet length: %u", pkt_len); 2371 if (pkt_len > HCI_MAX_EVENT_SIZE) 2372 break; 2373 skb = bt_skb_alloc(pkt_len, GFP_KERNEL); 2374 if (!skb) 2375 goto exit_on_error; 2376 hci_skb_pkt_type(skb) = HCI_EVENT_PKT; 2377 skb_put_data(skb, tlv->val, pkt_len); 2378 2379 /* copy Intel specific pcie packet type */ 2380 val = BTINTEL_PCIE_HCI_EVT_PKT; 2381 memcpy(skb_push(skb, BTINTEL_PCIE_HCI_TYPE_LEN), &val, 2382 BTINTEL_PCIE_HCI_TYPE_LEN); 2383 2384 print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_OFFSET, 16, 1, 2385 tlv->val, pkt_len, false); 2386 2387 btintel_pcie_recv_frame(data, skb); 2388 } while (offset < len); 2389 2390 exit_on_error: 2391 kfree(buf); 2392 } 2393 2394 static int btintel_pcie_dump_fwtrigger_event(struct btintel_pcie_data *data) 2395 { 2396 struct btintel_pcie_fwtrigger_evt *evt; 2397 struct sk_buff *skb; 2398 unsigned int len; 2399 int err; 2400 u8 *buf; 2401 2402 if (!data->debug_evt_size || !data->debug_evt_addr) 2403 return -EINVAL; 2404 2405 len = data->debug_evt_size; 2406 2407 len = ALIGN_DOWN(len, 4); 2408 2409 if (len < sizeof(*evt) || len > HCI_MAX_EVENT_SIZE) { 2410 bt_dev_err(data->hdev, "Invalid FW trigger data size (%u bytes)", len); 2411 return -EINVAL; 2412 } 2413 2414 buf = kzalloc(len, GFP_KERNEL); 2415 if (!buf) 2416 return -ENOMEM; 2417 2418 btintel_pcie_mac_init(data); 2419 2420 err = btintel_pcie_read_device_mem(data, buf, data->debug_evt_addr, 2421 len); 2422 if (err) 2423 goto exit_on_error; 2424 2425 evt = (void *)buf; 2426 bt_dev_dbg(data->hdev, "event type: 0x%2.2x event id: 0x%4.4x len: %u", 2427 evt->event_type, le16_to_cpu(evt->event_id), len); 2428 2429 skb = bt_skb_alloc(len, GFP_KERNEL); 2430 if (!skb) { 2431 err = -ENOMEM; 2432 goto exit_on_error; 2433 } 2434 skb_put_data(skb, buf, len); 2435 2436 hci_recv_diag(data->hdev, skb); 2437 err = 0; 2438 2439 exit_on_error: 2440 kfree(buf); 2441 return err; 2442 } 2443 2444 /* Queue a coredump dump_traces() pass. 2445 * 2446 * Returns true if a new coredump was queued, false if one was already 2447 * in-flight (the BTINTEL_PCIE_COREDUMP_INPROGRESS bit serves as the 2448 * single-writer guard for the @coredump_work item) or the workqueue is 2449 * disabled (reset / remove in progress). 2450 * 2451 * Always queue this AFTER any companion event-reader work (hwexp / 2452 * fwtrigger) so that, on the ordered @dump_workqueue, the event reader 2453 * runs first and the trigger metadata is populated before dump_traces 2454 * consumes it. 2455 */ 2456 static bool btintel_pcie_queue_coredump(struct btintel_pcie_data *data, 2457 u16 trigger_reason) 2458 { 2459 if (test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags)) 2460 return false; 2461 2462 data->dmp_hdr.trigger_reason = trigger_reason; 2463 2464 if (queue_work(data->dump_workqueue, &data->coredump_work)) 2465 return true; 2466 2467 /* Workqueue is disabled (reset/remove drained it). Release the 2468 * guard so a later trigger, after re-probe, can succeed. 2469 */ 2470 clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags); 2471 return false; 2472 } 2473 2474 static void btintel_pcie_msix_fw_trigger_handler(struct btintel_pcie_data *data) 2475 { 2476 bt_dev_dbg(data->hdev, "Received firmware smart trigger cause"); 2477 2478 /* Per-work guard: deduplicate concurrent FW-trigger interrupts. 2479 * Cleared at the tail of btintel_pcie_fwtrigger_worker(). 2480 */ 2481 if (test_and_set_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, 2482 &data->flags)) 2483 return; 2484 2485 if (!queue_work(data->dump_workqueue, &data->fwtrigger_work)) { 2486 clear_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags); 2487 return; 2488 } 2489 2490 /* Queue coredump after the fwtrigger event reader so dmp_hdr.event_* 2491 * is populated before dump_traces consumes it. 2492 */ 2493 btintel_pcie_queue_coredump(data, BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT); 2494 } 2495 2496 static void btintel_pcie_msix_hw_exp_handler(struct btintel_pcie_data *data) 2497 { 2498 bt_dev_err(data->hdev, "Received hw exception interrupt"); 2499 2500 /* CORE_HALTED is the single-writer guard for this handler. It is 2501 * set once on first HW exception and cleared only by re-probe 2502 * (data is reallocated), so it also serializes hwexp_work 2503 * scheduling without needing a separate bit. 2504 */ 2505 if (test_and_set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags)) 2506 return; 2507 2508 /* Queue companion coredump first so it is appended after hwexp_work 2509 * on the ordered @dump_workqueue (preserves the original 2510 * coredump-then-hwexp ordering). 2511 */ 2512 btintel_pcie_queue_coredump(data, BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT); 2513 2514 queue_work(data->dump_workqueue, &data->hwexp_work); 2515 } 2516 2517 static void btintel_pcie_coredump_worker(struct work_struct *work) 2518 { 2519 struct btintel_pcie_data *data = container_of(work, 2520 struct btintel_pcie_data, coredump_work); 2521 2522 /* hdev is NULL until setup_hdev() succeeds, and is cleared on 2523 * teardown after disable_work_sync() drains us; bail in that case. 2524 */ 2525 if (!data->hdev) 2526 goto out; 2527 2528 /* When firmware routes debug traces to the WiFi DBGC, no host 2529 * DBGC buffers were allocated, so there is nothing to dump here. 2530 */ 2531 if (btintel_pcie_dbg_to_wifi(data)) { 2532 bt_dev_info(data->hdev, 2533 "Skipping coredump: debug traces routed to WiFi DBGC"); 2534 goto out; 2535 } 2536 2537 btintel_pcie_dump_traces(data->hdev); 2538 out: 2539 /* Release guard last so a new trigger can run only after this 2540 * pass has fully completed (including dev_coredumpv()). 2541 */ 2542 clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags); 2543 } 2544 2545 static void btintel_pcie_hwexp_worker(struct work_struct *work) 2546 { 2547 struct btintel_pcie_data *data = container_of(work, 2548 struct btintel_pcie_data, hwexp_work); 2549 2550 if (!data->hdev) 2551 return; 2552 2553 /* Unlike usb products, controller will not send hardware exception 2554 * event on exception. Instead controller writes the hardware event 2555 * to device memory along with optional debug events, raises MSIX 2556 * and halts. Driver shall read the exception event from device 2557 * memory and passes it to the stack for further processing. 2558 * 2559 * Re-entry is gated by BTINTEL_PCIE_CORE_HALTED in the IRQ 2560 * handler, which is only cleared by re-probe; no per-work bit 2561 * is needed here. 2562 */ 2563 btintel_pcie_read_hwexp(data); 2564 } 2565 2566 static void btintel_pcie_fwtrigger_worker(struct work_struct *work) 2567 { 2568 struct btintel_pcie_data *data = container_of(work, 2569 struct btintel_pcie_data, fwtrigger_work); 2570 int err; 2571 2572 if (!data->hdev) 2573 goto out; 2574 2575 err = btintel_pcie_dump_fwtrigger_event(data); 2576 if (err) 2577 bt_dev_warn(data->hdev, "failed to log fwtrigger event"); 2578 out: 2579 /* Release guard last; matches set in fw_trigger handler. */ 2580 clear_bit(BTINTEL_PCIE_FWTRIGGER_DUMP_INPROGRESS, &data->flags); 2581 } 2582 2583 static void btintel_pcie_mbox_worker(struct work_struct *work) 2584 { 2585 struct btintel_pcie_data *data = container_of(work, 2586 struct btintel_pcie_data, mbox_work); 2587 2588 if (!data->hdev) 2589 goto out; 2590 2591 btintel_parse_mbox_tlv(data); 2592 out: 2593 /* Release guard last; matches set in gp1 handler. */ 2594 clear_bit(BTINTEL_PCIE_MAIL_BOX_INTR, &data->flags); 2595 } 2596 2597 static void btintel_pcie_rx_work(struct work_struct *work) 2598 { 2599 struct btintel_pcie_data *data = container_of(work, 2600 struct btintel_pcie_data, rx_work); 2601 struct sk_buff *skb; 2602 2603 /* Process the sk_buf in queue and send to the HCI layer */ 2604 while ((skb = skb_dequeue(&data->rx_skb_q))) { 2605 btintel_pcie_recv_frame(data, skb); 2606 } 2607 } 2608 2609 /* create sk_buff with data and save it to queue and start RX work */ 2610 static int btintel_pcie_submit_rx_work(struct btintel_pcie_data *data, u8 status, 2611 void *buf) 2612 { 2613 int ret, len; 2614 struct rfh_hdr *rfh_hdr; 2615 struct sk_buff *skb; 2616 2617 rfh_hdr = buf; 2618 2619 len = rfh_hdr->packet_len; 2620 if (len == 0 || len > BTINTEL_PCIE_BUFFER_SIZE - sizeof(*rfh_hdr)) { 2621 bt_dev_err(data->hdev, "Invalid packet_len %d (max %zu)", len, 2622 BTINTEL_PCIE_BUFFER_SIZE - sizeof(*rfh_hdr)); 2623 ret = -EINVAL; 2624 goto resubmit; 2625 } 2626 2627 /* Remove RFH header */ 2628 buf += sizeof(*rfh_hdr); 2629 2630 skb = alloc_skb(len, GFP_ATOMIC); 2631 if (!skb) 2632 goto resubmit; 2633 2634 skb_put_data(skb, buf, len); 2635 skb_queue_tail(&data->rx_skb_q, skb); 2636 queue_work(data->workqueue, &data->rx_work); 2637 2638 resubmit: 2639 ret = btintel_pcie_submit_rx(data); 2640 2641 return ret; 2642 } 2643 2644 /* Handles the MSI-X interrupt for rx queue 1 which is for RX */ 2645 static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) 2646 { 2647 u16 cr_hia, cr_tia; 2648 struct rxq *rxq; 2649 struct urbd1 *urbd1; 2650 struct data_buf *buf; 2651 int ret; 2652 struct hci_dev *hdev = data->hdev; 2653 2654 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM]; 2655 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; 2656 2657 bt_dev_dbg(hdev, "RXQ: cr_hia: %u cr_tia: %u", cr_hia, cr_tia); 2658 2659 /* Check CR_TIA and CR_HIA for change */ 2660 if (cr_tia == cr_hia) 2661 return; 2662 2663 rxq = &data->rxq; 2664 2665 if (cr_hia >= rxq->count) { 2666 bt_dev_err(hdev, "RXQ: invalid cr_hia %u", cr_hia); 2667 return; 2668 } 2669 2670 /* The firmware sends multiple CD in a single MSI-X and it needs to 2671 * process all received CDs in this interrupt. 2672 */ 2673 while (cr_tia != cr_hia) { 2674 urbd1 = &rxq->urbd1s[cr_tia]; 2675 ipc_print_urbd1(data->hdev, urbd1, cr_tia); 2676 2677 if (urbd1->frbd_tag >= rxq->count) { 2678 bt_dev_err(hdev, "RXQ: invalid frbd_tag %u", 2679 urbd1->frbd_tag); 2680 return; 2681 } 2682 2683 buf = &rxq->bufs[urbd1->frbd_tag]; 2684 if (!buf) { 2685 bt_dev_err(hdev, "RXQ: failed to get the DMA buffer for %d", 2686 urbd1->frbd_tag); 2687 return; 2688 } 2689 2690 ret = btintel_pcie_submit_rx_work(data, urbd1->status, 2691 buf->data); 2692 if (ret) { 2693 bt_dev_err(hdev, "RXQ: failed to submit rx request"); 2694 return; 2695 } 2696 2697 cr_tia = (cr_tia + 1) % rxq->count; 2698 data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM] = cr_tia; 2699 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM); 2700 } 2701 } 2702 2703 static inline bool btintel_pcie_is_rxq_empty(struct btintel_pcie_data *data) 2704 { 2705 return data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM] == data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; 2706 } 2707 2708 static inline bool btintel_pcie_is_txackq_empty(struct btintel_pcie_data *data) 2709 { 2710 return data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM] == data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM]; 2711 } 2712 2713 static irqreturn_t btintel_pcie_irq_msix_handler(int irq, void *dev_id) 2714 { 2715 struct msix_entry *entry = dev_id; 2716 struct btintel_pcie_data *data = btintel_pcie_get_data(entry); 2717 u32 intr_fh, intr_hw; 2718 2719 spin_lock(&data->irq_lock); 2720 intr_fh = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES); 2721 intr_hw = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES); 2722 2723 /* Clear causes registers to avoid being handling the same cause */ 2724 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES, intr_fh); 2725 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES, intr_hw); 2726 spin_unlock(&data->irq_lock); 2727 2728 if (unlikely(!(intr_fh | intr_hw))) { 2729 /* Ignore interrupt, inta == 0 */ 2730 bt_warn_ratelimited("Bluetooth: btintel_pcie: Received spurious interrupt\n"); 2731 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_AUTOMASK_ST, 2732 BIT(entry->entry)); 2733 return IRQ_NONE; 2734 } 2735 2736 /* This interrupt is raised when there is an hardware exception */ 2737 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_HWEXP) 2738 btintel_pcie_msix_hw_exp_handler(data); 2739 2740 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP1) 2741 btintel_pcie_msix_gp1_handler(data); 2742 2743 2744 /* For TX */ 2745 if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0) { 2746 btintel_pcie_msix_tx_handle(data); 2747 if (!btintel_pcie_is_rxq_empty(data)) 2748 btintel_pcie_msix_rx_handle(data); 2749 } 2750 2751 /* For RX */ 2752 if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1) { 2753 btintel_pcie_msix_rx_handle(data); 2754 if (!btintel_pcie_is_txackq_empty(data)) 2755 btintel_pcie_msix_tx_handle(data); 2756 } 2757 2758 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_FWTRIG) 2759 btintel_pcie_msix_fw_trigger_handler(data); 2760 2761 /* This interrupt is triggered by the firmware after updating 2762 * boot_stage register and image_response register 2763 */ 2764 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0) 2765 btintel_pcie_msix_gp0_handler(data); 2766 2767 /* 2768 * Before sending the interrupt the HW disables it to prevent a nested 2769 * interrupt. This is done by writing 1 to the corresponding bit in 2770 * the mask register. After handling the interrupt, it should be 2771 * re-enabled by clearing this bit. This register is defined as write 1 2772 * clear (W1C) register, meaning that it's cleared by writing 1 2773 * to the bit. 2774 */ 2775 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_AUTOMASK_ST, 2776 BIT(entry->entry)); 2777 2778 return IRQ_HANDLED; 2779 } 2780 2781 /* This function requests the irq for MSI-X and registers the handlers per irq. 2782 * Currently, it requests only 1 irq for all interrupt causes. 2783 */ 2784 static int btintel_pcie_setup_irq(struct btintel_pcie_data *data) 2785 { 2786 int err; 2787 int num_irqs, i, j; 2788 2789 for (i = 0; i < BTINTEL_PCIE_MSIX_VEC_MAX; i++) 2790 data->msix_entries[i].entry = i; 2791 2792 num_irqs = pci_alloc_irq_vectors(data->pdev, BTINTEL_PCIE_MSIX_VEC_MIN, 2793 BTINTEL_PCIE_MSIX_VEC_MAX, PCI_IRQ_MSIX); 2794 if (num_irqs < 0) 2795 return num_irqs; 2796 2797 data->alloc_vecs = num_irqs; 2798 data->msix_enabled = 1; 2799 data->def_irq = 0; 2800 2801 /* setup irq handler */ 2802 for (i = 0; i < data->alloc_vecs; i++) { 2803 struct msix_entry *msix_entry; 2804 2805 msix_entry = &data->msix_entries[i]; 2806 msix_entry->vector = pci_irq_vector(data->pdev, i); 2807 2808 err = devm_request_threaded_irq(&data->pdev->dev, 2809 msix_entry->vector, 2810 NULL, 2811 btintel_pcie_irq_msix_handler, 2812 IRQF_ONESHOT | IRQF_SHARED, 2813 KBUILD_MODNAME, 2814 msix_entry); 2815 if (err) { 2816 for (j = 0; j < i; j++) 2817 devm_free_irq(&data->pdev->dev, 2818 data->msix_entries[j].vector, 2819 &data->msix_entries[j]); 2820 pci_free_irq_vectors(data->pdev); 2821 data->alloc_vecs = 0; 2822 return err; 2823 } 2824 } 2825 return 0; 2826 } 2827 2828 struct btintel_pcie_causes_list { 2829 u32 cause; 2830 u32 mask_reg; 2831 u8 cause_num; 2832 }; 2833 2834 static struct btintel_pcie_causes_list causes_list[] = { 2835 { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x00 }, 2836 { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x01 }, 2837 { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x20 }, 2838 { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_HWEXP, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x23 }, 2839 { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_FWTRIG, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x25 }, 2840 }; 2841 2842 /* This function configures the interrupt masks for both HW_INT_CAUSES and 2843 * FH_INT_CAUSES which are meaningful to us. 2844 * 2845 * After resetting BT function via PCIE FLR or FUNC_CTRL reset, the driver 2846 * need to call this function again to configure since the masks 2847 * are reset to 0xFFFFFFFF after reset. 2848 */ 2849 static void btintel_pcie_config_msix(struct btintel_pcie_data *data) 2850 { 2851 int i; 2852 int val = data->def_irq | BTINTEL_PCIE_MSIX_NON_AUTO_CLEAR_CAUSE; 2853 2854 /* Set Non Auto Clear Cause */ 2855 for (i = 0; i < ARRAY_SIZE(causes_list); i++) { 2856 btintel_pcie_wr_reg8(data, 2857 BTINTEL_PCIE_CSR_MSIX_IVAR(causes_list[i].cause_num), 2858 val); 2859 btintel_pcie_clr_reg_bits(data, 2860 causes_list[i].mask_reg, 2861 causes_list[i].cause); 2862 } 2863 2864 /* Save the initial interrupt mask */ 2865 data->fh_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK); 2866 data->hw_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK); 2867 } 2868 2869 static int btintel_pcie_config_pcie(struct pci_dev *pdev, 2870 struct btintel_pcie_data *data) 2871 { 2872 int err; 2873 2874 err = pcim_enable_device(pdev); 2875 if (err) 2876 return err; 2877 2878 pci_set_master(pdev); 2879 2880 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 2881 if (err) { 2882 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 2883 if (err) 2884 return err; 2885 } 2886 2887 data->base_addr = pcim_iomap_region(pdev, 0, KBUILD_MODNAME); 2888 if (IS_ERR(data->base_addr)) 2889 return PTR_ERR(data->base_addr); 2890 2891 /* Do shared hardware reset to ensure a clean start before 2892 * configuring interrupts. 2893 */ 2894 err = btintel_pcie_reset_bt(data); 2895 if (err) 2896 return err; 2897 2898 err = btintel_pcie_setup_irq(data); 2899 if (err) 2900 return err; 2901 2902 /* Configure MSI-X with causes list */ 2903 btintel_pcie_config_msix(data); 2904 2905 return 0; 2906 } 2907 2908 static void btintel_pcie_init_ci(struct btintel_pcie_data *data, 2909 struct ctx_info *ci) 2910 { 2911 ci->version = 0x1; 2912 ci->size = sizeof(*ci); 2913 ci->config = 0x0000; 2914 ci->addr_cr_hia = data->ia.cr_hia_p_addr; 2915 ci->addr_tr_tia = data->ia.tr_tia_p_addr; 2916 ci->addr_cr_tia = data->ia.cr_tia_p_addr; 2917 ci->addr_tr_hia = data->ia.tr_hia_p_addr; 2918 ci->num_cr_ia = BTINTEL_PCIE_NUM_QUEUES; 2919 ci->num_tr_ia = BTINTEL_PCIE_NUM_QUEUES; 2920 ci->addr_urbdq0 = data->txq.urbd0s_p_addr; 2921 ci->addr_tfdq = data->txq.tfds_p_addr; 2922 ci->num_tfdq = data->txq.count; 2923 ci->num_urbdq0 = data->txq.count; 2924 ci->tfdq_db_vec = BTINTEL_PCIE_TXQ_NUM; 2925 ci->urbdq0_db_vec = BTINTEL_PCIE_TXQ_NUM; 2926 ci->rbd_size = BTINTEL_PCIE_RBD_SIZE_4K; 2927 ci->addr_frbdq = data->rxq.frbds_p_addr; 2928 ci->num_frbdq = data->rxq.count; 2929 ci->frbdq_db_vec = BTINTEL_PCIE_RXQ_NUM; 2930 ci->addr_urbdq1 = data->rxq.urbd1s_p_addr; 2931 ci->num_urbdq1 = data->rxq.count; 2932 ci->urbdq_db_vec = BTINTEL_PCIE_RXQ_NUM; 2933 2934 ci->dbg_output_mode = btintel_pcie_dbg_to_wifi(data) ? 2935 BTINTEL_PCIE_WIFI_DBGC : BTINTEL_PCIE_DRAM; 2936 if (btintel_pcie_dbg_to_wifi(data)) { 2937 /* Firmware forwards debug traces to the WiFi DBGC, so no 2938 * host DBGC buffer is needed; leave dbgc_addr/size as 0. 2939 */ 2940 ci->dbgc_addr = 0; 2941 ci->dbgc_size = 0; 2942 } else if (btintel_pcie_is_mdbgc_supported(data)) { 2943 ci->dbgc_addr = data->mdbgc.frag_p_addr; 2944 ci->dbgc_size = data->mdbgc.frag_size; 2945 } else { 2946 ci->dbgc_addr = data->dbgc.frag_p_addr; 2947 ci->dbgc_size = data->dbgc.frag_size; 2948 } 2949 2950 ci->dbg_preset = 0x00; 2951 } 2952 2953 static void btintel_pcie_free_txq_bufs(struct btintel_pcie_data *data, 2954 struct txq *txq) 2955 { 2956 /* Free data buffers first */ 2957 dma_free_coherent(&data->pdev->dev, txq->count * BTINTEL_PCIE_BUFFER_SIZE, 2958 txq->buf_v_addr, txq->buf_p_addr); 2959 kfree(txq->bufs); 2960 } 2961 2962 static int btintel_pcie_setup_txq_bufs(struct btintel_pcie_data *data, 2963 struct txq *txq) 2964 { 2965 int i; 2966 struct data_buf *buf; 2967 2968 /* Allocate the same number of buffers as the descriptor */ 2969 txq->bufs = kmalloc_objs(*buf, txq->count); 2970 if (!txq->bufs) 2971 return -ENOMEM; 2972 2973 /* Allocate full chunk of data buffer for DMA first and do indexing and 2974 * initialization next, so it can be freed easily 2975 */ 2976 txq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev, 2977 txq->count * BTINTEL_PCIE_BUFFER_SIZE, 2978 &txq->buf_p_addr, 2979 GFP_KERNEL | __GFP_NOWARN); 2980 if (!txq->buf_v_addr) { 2981 kfree(txq->bufs); 2982 return -ENOMEM; 2983 } 2984 2985 /* Setup the allocated DMA buffer to bufs. Each data_buf should 2986 * have virtual address and physical address 2987 */ 2988 for (i = 0; i < txq->count; i++) { 2989 buf = &txq->bufs[i]; 2990 buf->data_p_addr = txq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 2991 buf->data = txq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 2992 } 2993 2994 return 0; 2995 } 2996 2997 static void btintel_pcie_free_rxq_bufs(struct btintel_pcie_data *data, 2998 struct rxq *rxq) 2999 { 3000 /* Free data buffers first */ 3001 dma_free_coherent(&data->pdev->dev, rxq->count * BTINTEL_PCIE_BUFFER_SIZE, 3002 rxq->buf_v_addr, rxq->buf_p_addr); 3003 kfree(rxq->bufs); 3004 } 3005 3006 static int btintel_pcie_setup_rxq_bufs(struct btintel_pcie_data *data, 3007 struct rxq *rxq) 3008 { 3009 int i; 3010 struct data_buf *buf; 3011 3012 /* Allocate the same number of buffers as the descriptor */ 3013 rxq->bufs = kmalloc_objs(*buf, rxq->count); 3014 if (!rxq->bufs) 3015 return -ENOMEM; 3016 3017 /* Allocate full chunk of data buffer for DMA first and do indexing and 3018 * initialization next, so it can be freed easily 3019 */ 3020 rxq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev, 3021 rxq->count * BTINTEL_PCIE_BUFFER_SIZE, 3022 &rxq->buf_p_addr, 3023 GFP_KERNEL | __GFP_NOWARN); 3024 if (!rxq->buf_v_addr) { 3025 kfree(rxq->bufs); 3026 return -ENOMEM; 3027 } 3028 3029 /* Setup the allocated DMA buffer to bufs. Each data_buf should 3030 * have virtual address and physical address 3031 */ 3032 for (i = 0; i < rxq->count; i++) { 3033 buf = &rxq->bufs[i]; 3034 buf->data_p_addr = rxq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 3035 buf->data = rxq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 3036 } 3037 3038 return 0; 3039 } 3040 3041 static void btintel_pcie_free(struct btintel_pcie_data *data) 3042 { 3043 btintel_pcie_free_rxq_bufs(data, &data->rxq); 3044 btintel_pcie_free_txq_bufs(data, &data->txq); 3045 3046 dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr); 3047 dma_pool_destroy(data->dma_pool); 3048 } 3049 3050 /* Allocate tx and rx queues, any related data structures and buffers. 3051 */ 3052 static int btintel_pcie_alloc(struct btintel_pcie_data *data) 3053 { 3054 int err = 0; 3055 size_t total; 3056 dma_addr_t p_addr; 3057 void *v_addr; 3058 size_t tfd_size, frbd_size, ctx_size, ci_size, urbd0_size, urbd1_size; 3059 3060 /* Allocate the chunk of DMA memory for descriptors, index array, and 3061 * context information, instead of allocating individually. 3062 * The DMA memory for data buffer is allocated while setting up the 3063 * each queue. 3064 * 3065 * Total size is sum of the following and each of the individual sizes 3066 * are aligned to 128 bytes before adding up. 3067 * 3068 * + size of TFD * Number of descriptors in queue 3069 * + size of URBD0 * Number of descriptors in queue 3070 * + size of FRBD * Number of descriptors in queue 3071 * + size of URBD1 * Number of descriptors in queue 3072 * + size of index * Number of queues(2) * type of index array(4) 3073 * + size of context information 3074 */ 3075 tfd_size = ALIGN(sizeof(struct tfd) * BTINTEL_PCIE_TX_DESCS_COUNT, 3076 BTINTEL_PCIE_DMA_ALIGN_128B); 3077 urbd0_size = ALIGN(sizeof(struct urbd0) * BTINTEL_PCIE_TX_DESCS_COUNT, 3078 BTINTEL_PCIE_DMA_ALIGN_128B); 3079 3080 frbd_size = ALIGN(sizeof(struct frbd) * BTINTEL_PCIE_RX_DESCS_COUNT, 3081 BTINTEL_PCIE_DMA_ALIGN_128B); 3082 urbd1_size = ALIGN(sizeof(struct urbd1) * BTINTEL_PCIE_RX_DESCS_COUNT, 3083 BTINTEL_PCIE_DMA_ALIGN_128B); 3084 3085 ci_size = ALIGN(sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES, 3086 BTINTEL_PCIE_DMA_ALIGN_128B); 3087 3088 ctx_size = ALIGN(sizeof(struct ctx_info), BTINTEL_PCIE_DMA_ALIGN_128B); 3089 3090 total = tfd_size + urbd0_size + frbd_size + urbd1_size + ctx_size + ci_size * 4; 3091 3092 data->dma_pool = dma_pool_create(KBUILD_MODNAME, &data->pdev->dev, 3093 total, BTINTEL_PCIE_DMA_ALIGN_128B, 0); 3094 if (!data->dma_pool) { 3095 err = -ENOMEM; 3096 goto exit_error; 3097 } 3098 3099 v_addr = dma_pool_zalloc(data->dma_pool, GFP_KERNEL | __GFP_NOWARN, 3100 &p_addr); 3101 if (!v_addr) { 3102 dma_pool_destroy(data->dma_pool); 3103 err = -ENOMEM; 3104 goto exit_error; 3105 } 3106 3107 data->dma_p_addr = p_addr; 3108 data->dma_v_addr = v_addr; 3109 3110 /* Setup descriptor count */ 3111 data->txq.count = BTINTEL_PCIE_TX_DESCS_COUNT; 3112 data->rxq.count = BTINTEL_PCIE_RX_DESCS_COUNT; 3113 3114 /* Setup tfds */ 3115 data->txq.tfds_p_addr = p_addr; 3116 data->txq.tfds = v_addr; 3117 3118 p_addr += tfd_size; 3119 v_addr += tfd_size; 3120 3121 /* Setup urbd0 */ 3122 data->txq.urbd0s_p_addr = p_addr; 3123 data->txq.urbd0s = v_addr; 3124 3125 p_addr += urbd0_size; 3126 v_addr += urbd0_size; 3127 3128 /* Setup FRBD*/ 3129 data->rxq.frbds_p_addr = p_addr; 3130 data->rxq.frbds = v_addr; 3131 3132 p_addr += frbd_size; 3133 v_addr += frbd_size; 3134 3135 /* Setup urbd1 */ 3136 data->rxq.urbd1s_p_addr = p_addr; 3137 data->rxq.urbd1s = v_addr; 3138 3139 p_addr += urbd1_size; 3140 v_addr += urbd1_size; 3141 3142 /* Setup data buffers for txq */ 3143 err = btintel_pcie_setup_txq_bufs(data, &data->txq); 3144 if (err) 3145 goto exit_error_pool; 3146 3147 /* Setup data buffers for rxq */ 3148 err = btintel_pcie_setup_rxq_bufs(data, &data->rxq); 3149 if (err) 3150 goto exit_error_txq; 3151 3152 /* TR Head Index Array */ 3153 data->ia.tr_hia_p_addr = p_addr; 3154 data->ia.tr_hia = v_addr; 3155 p_addr += ci_size; 3156 v_addr += ci_size; 3157 3158 /* TR Tail Index Array */ 3159 data->ia.tr_tia_p_addr = p_addr; 3160 data->ia.tr_tia = v_addr; 3161 p_addr += ci_size; 3162 v_addr += ci_size; 3163 3164 /* CR Head index Array */ 3165 data->ia.cr_hia_p_addr = p_addr; 3166 data->ia.cr_hia = v_addr; 3167 p_addr += ci_size; 3168 v_addr += ci_size; 3169 3170 /* CR Tail Index Array */ 3171 data->ia.cr_tia_p_addr = p_addr; 3172 data->ia.cr_tia = v_addr; 3173 p_addr += ci_size; 3174 v_addr += ci_size; 3175 3176 /* Setup data buffers for dbgc */ 3177 if (btintel_pcie_dbg_to_wifi(data)) { 3178 /* Firmware routes traces to the WiFi DBGC; skip host DBGC 3179 * buffer allocation entirely. 3180 */ 3181 err = 0; 3182 } else if (btintel_pcie_is_mdbgc_supported(data)) { 3183 err = btintel_pcie_setup_mdbgc(data); 3184 } else { 3185 err = btintel_pcie_setup_dbgc(data); 3186 } 3187 if (err) 3188 goto exit_error_txq; 3189 3190 /* Setup Context Information */ 3191 data->ci = v_addr; 3192 data->ci_p_addr = p_addr; 3193 3194 /* Initialize the CI */ 3195 btintel_pcie_init_ci(data, data->ci); 3196 3197 return 0; 3198 3199 exit_error_txq: 3200 btintel_pcie_free_txq_bufs(data, &data->txq); 3201 exit_error_pool: 3202 dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr); 3203 dma_pool_destroy(data->dma_pool); 3204 exit_error: 3205 return err; 3206 } 3207 3208 static int btintel_pcie_open(struct hci_dev *hdev) 3209 { 3210 bt_dev_dbg(hdev, ""); 3211 3212 return 0; 3213 } 3214 3215 static int btintel_pcie_close(struct hci_dev *hdev) 3216 { 3217 bt_dev_dbg(hdev, ""); 3218 3219 return 0; 3220 } 3221 3222 static int btintel_pcie_inject_cmd_complete(struct hci_dev *hdev, __u16 opcode) 3223 { 3224 struct sk_buff *skb; 3225 struct hci_event_hdr *hdr; 3226 struct hci_ev_cmd_complete *evt; 3227 3228 skb = bt_skb_alloc(sizeof(*hdr) + sizeof(*evt) + 1, GFP_KERNEL); 3229 if (!skb) 3230 return -ENOMEM; 3231 3232 hdr = (struct hci_event_hdr *)skb_put(skb, sizeof(*hdr)); 3233 hdr->evt = HCI_EV_CMD_COMPLETE; 3234 hdr->plen = sizeof(*evt) + 1; 3235 3236 evt = (struct hci_ev_cmd_complete *)skb_put(skb, sizeof(*evt)); 3237 evt->ncmd = 0x01; 3238 evt->opcode = cpu_to_le16(opcode); 3239 3240 *(u8 *)skb_put(skb, 1) = 0x00; 3241 3242 hci_skb_pkt_type(skb) = HCI_EVENT_PKT; 3243 3244 return hci_recv_frame(hdev, skb); 3245 } 3246 3247 static int btintel_pcie_send_frame(struct hci_dev *hdev, 3248 struct sk_buff *skb) 3249 { 3250 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 3251 struct hci_command_hdr *cmd; 3252 __u16 opcode = ~0; 3253 int ret; 3254 u32 type; 3255 3256 if (test_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags)) 3257 return -ENODEV; 3258 3259 if (test_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags)) 3260 return -ENODEV; 3261 3262 /* Account for the 4-byte PCIe type header prepended before the 3263 * DMA copy. Written as a subtraction to avoid wrap-around on 3264 * attacker-controlled skb->len. 3265 */ 3266 if (skb->len > BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN) { 3267 bt_dev_err(hdev, "Packet too large: %u > %u", skb->len, 3268 BTINTEL_PCIE_BUFFER_SIZE - BTINTEL_PCIE_HCI_TYPE_LEN); 3269 return -EMSGSIZE; 3270 } 3271 3272 /* Due to the fw limitation, the type header of the packet should be 3273 * 4 bytes unlike 1 byte for UART. In UART, the firmware can read 3274 * the first byte to get the packet type and redirect the rest of data 3275 * packet to the right handler. 3276 * 3277 * But for PCIe, THF(Transfer Flow Handler) fetches the 4 bytes of data 3278 * from DMA memory and by the time it reads the first 4 bytes, it has 3279 * already consumed some part of packet. Thus the packet type indicator 3280 * for iBT PCIe is 4 bytes. 3281 * 3282 * Luckily, when HCI core creates the skb, it allocates 8 bytes of 3283 * head room for profile and driver use, and before sending the data 3284 * to the device, append the iBT PCIe packet type in the front. 3285 */ 3286 switch (hci_skb_pkt_type(skb)) { 3287 case HCI_COMMAND_PKT: 3288 type = BTINTEL_PCIE_HCI_CMD_PKT; 3289 cmd = (void *)skb->data; 3290 opcode = le16_to_cpu(cmd->opcode); 3291 if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) { 3292 struct hci_command_hdr *cmd = (void *)skb->data; 3293 __u16 opcode = le16_to_cpu(cmd->opcode); 3294 3295 /* When the BTINTEL_HCI_OP_RESET command is issued to 3296 * boot into the operational firmware, it will actually 3297 * not send a command complete event. To keep the flow 3298 * control working inject that event here. 3299 */ 3300 if (opcode == BTINTEL_HCI_OP_RESET) 3301 btintel_pcie_inject_cmd_complete(hdev, opcode); 3302 } 3303 3304 hdev->stat.cmd_tx++; 3305 break; 3306 case HCI_ACLDATA_PKT: 3307 type = BTINTEL_PCIE_HCI_ACL_PKT; 3308 hdev->stat.acl_tx++; 3309 break; 3310 case HCI_SCODATA_PKT: 3311 type = BTINTEL_PCIE_HCI_SCO_PKT; 3312 hdev->stat.sco_tx++; 3313 break; 3314 case HCI_ISODATA_PKT: 3315 type = BTINTEL_PCIE_HCI_ISO_PKT; 3316 break; 3317 default: 3318 bt_dev_err(hdev, "Unknown HCI packet type"); 3319 return -EILSEQ; 3320 } 3321 3322 ret = btintel_pcie_send_sync(data, skb, type, opcode); 3323 if (ret) { 3324 hdev->stat.err_tx++; 3325 bt_dev_err(hdev, "Failed to send frame (%d)", ret); 3326 goto exit_error; 3327 } 3328 3329 hdev->stat.byte_tx += skb->len; 3330 kfree_skb(skb); 3331 3332 exit_error: 3333 return ret; 3334 } 3335 3336 static void btintel_pcie_release_hdev(struct btintel_pcie_data *data) 3337 { 3338 struct hci_dev *hdev = data->hdev; 3339 3340 if (!hdev) 3341 return; 3342 3343 hci_unregister_dev(hdev); 3344 hci_free_dev(hdev); 3345 data->hdev = NULL; 3346 } 3347 3348 static void btintel_pcie_disable_interrupts(struct btintel_pcie_data *data) 3349 { 3350 spin_lock(&data->irq_lock); 3351 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, data->fh_init_mask); 3352 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, data->hw_init_mask); 3353 spin_unlock(&data->irq_lock); 3354 } 3355 3356 static void btintel_pcie_enable_interrupts(struct btintel_pcie_data *data) 3357 { 3358 spin_lock(&data->irq_lock); 3359 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, ~data->fh_init_mask); 3360 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, ~data->hw_init_mask); 3361 spin_unlock(&data->irq_lock); 3362 } 3363 3364 static void btintel_pcie_synchronize_irqs(struct btintel_pcie_data *data) 3365 { 3366 for (int i = 0; i < data->alloc_vecs; i++) 3367 synchronize_irq(data->msix_entries[i].vector); 3368 } 3369 3370 static int btintel_pcie_get_debug_info_addr(struct hci_dev *hdev) 3371 { 3372 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 3373 struct btintel_pcie_trigger_evt *evt; 3374 u8 param[1] = {0x10}; 3375 struct sk_buff *skb; 3376 int err = 0; 3377 3378 skb = __hci_cmd_sync(hdev, BTINTEL_HCI_OP_DEBUG, 1, param, 3379 HCI_CMD_TIMEOUT); 3380 if (IS_ERR(skb)) { 3381 bt_dev_err(hdev, "Reading Intel read debug info address command failed (%ld)", 3382 PTR_ERR(skb)); 3383 /* Not all Intel products supports this command */ 3384 if (PTR_ERR(skb) == -EOPNOTSUPP) 3385 return 0; 3386 return PTR_ERR(skb); 3387 } 3388 3389 if (skb->len < (1 + sizeof(*evt))) { 3390 bt_dev_err(hdev, "Debug info response too short (%u bytes)", skb->len); 3391 err = -EIO; 3392 goto exit_error; 3393 } 3394 3395 /* Check the status */ 3396 if (skb->data[0]) { 3397 bt_dev_err(hdev, "Reading Intel read debug info command failed (0x%2.2x)", 3398 skb->data[0]); 3399 err = -EIO; 3400 goto exit_error; 3401 } 3402 3403 /* Consume Command Complete Status field */ 3404 skb_pull(skb, 1); 3405 3406 evt = (void *)skb->data; 3407 3408 data->debug_evt_addr = le32_to_cpu(evt->addr); 3409 data->debug_evt_size = le32_to_cpu(evt->size); 3410 3411 bt_dev_dbg(hdev, "config type: %u config len: %u debug event addr: 0x%8.8x size: 0x%8.8x", 3412 evt->type, evt->len, data->debug_evt_addr, 3413 data->debug_evt_size); 3414 exit_error: 3415 kfree_skb(skb); 3416 return err; 3417 } 3418 3419 static int btintel_pcie_setup_internal(struct hci_dev *hdev) 3420 { 3421 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 3422 const u8 param[1] = { 0xFF }; 3423 struct intel_version_tlv ver_tlv; 3424 struct sk_buff *skb; 3425 int err; 3426 3427 BT_DBG("%s", hdev->name); 3428 3429 skb = __hci_cmd_sync(hdev, 0xfc05, 1, param, HCI_CMD_TIMEOUT); 3430 if (IS_ERR(skb)) { 3431 bt_dev_err(hdev, "Reading Intel version command failed (%ld)", 3432 PTR_ERR(skb)); 3433 return PTR_ERR(skb); 3434 } 3435 3436 /* Check the status */ 3437 if (skb->data[0]) { 3438 bt_dev_err(hdev, "Intel Read Version command failed (%02x)", 3439 skb->data[0]); 3440 err = -EIO; 3441 goto exit_error; 3442 } 3443 3444 /* Apply the common HCI quirks for Intel device */ 3445 hci_set_quirk(hdev, HCI_QUIRK_STRICT_DUPLICATE_FILTER); 3446 hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY); 3447 hci_set_quirk(hdev, HCI_QUIRK_NON_PERSISTENT_DIAG); 3448 3449 /* Set up the quality report callback for Intel devices */ 3450 hdev->set_quality_report = btintel_set_quality_report; 3451 3452 memset(&ver_tlv, 0, sizeof(ver_tlv)); 3453 /* For TLV type device, parse the tlv data */ 3454 err = btintel_parse_version_tlv(hdev, &ver_tlv, skb); 3455 if (err) { 3456 bt_dev_err(hdev, "Failed to parse TLV version information"); 3457 goto exit_error; 3458 } 3459 3460 data->dmp_hdr.cnvi_bt = ver_tlv.cnvi_bt; 3461 switch (INTEL_HW_PLATFORM(ver_tlv.cnvi_bt)) { 3462 case 0x37: 3463 break; 3464 default: 3465 bt_dev_err(hdev, "Unsupported Intel hardware platform (0x%2x)", 3466 INTEL_HW_PLATFORM(ver_tlv.cnvi_bt)); 3467 err = -EINVAL; 3468 goto exit_error; 3469 } 3470 3471 /* Check for supported iBT hardware variants of this firmware 3472 * loading method. 3473 * 3474 * This check has been put in place to ensure correct forward 3475 * compatibility options when newer hardware variants come 3476 * along. 3477 */ 3478 switch (INTEL_HW_VARIANT(ver_tlv.cnvi_bt)) { 3479 case 0x1e: /* BzrI */ 3480 case 0x1f: /* ScP */ 3481 case 0x20: /* ScP2 */ 3482 case 0x21: /* ScP2 F */ 3483 case 0x22: /* BzrIW */ 3484 case 0x23: /* DrC */ 3485 /* Display version information of TLV type */ 3486 btintel_version_info_tlv(hdev, &ver_tlv); 3487 3488 /* Apply the device specific HCI quirks for TLV based devices 3489 * 3490 * All TLV based devices support WBS 3491 */ 3492 hci_set_quirk(hdev, HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED); 3493 3494 /* Setup MSFT Extension support */ 3495 btintel_set_msft_opcode(hdev, 3496 INTEL_HW_VARIANT(ver_tlv.cnvi_bt)); 3497 3498 err = btintel_bootloader_setup_tlv(hdev, &ver_tlv); 3499 if (err) 3500 goto exit_error; 3501 break; 3502 default: 3503 bt_dev_err(hdev, "Unsupported Intel hw variant (%u)", 3504 INTEL_HW_VARIANT(ver_tlv.cnvi_bt)); 3505 err = -EINVAL; 3506 goto exit_error; 3507 } 3508 3509 data->dmp_hdr.cnvi_top = ver_tlv.cnvi_top; 3510 data->dmp_hdr.cnvr_top = ver_tlv.cnvr_top; 3511 data->dmp_hdr.fw_timestamp = ver_tlv.timestamp; 3512 data->dmp_hdr.fw_build_type = ver_tlv.build_type; 3513 data->dmp_hdr.fw_build_num = ver_tlv.build_num; 3514 3515 if (ver_tlv.img_type == 0x02 || ver_tlv.img_type == 0x03) 3516 data->dmp_hdr.fw_sha = ver_tlv.git_sha1; 3517 3518 err = btintel_pcie_get_debug_info_addr(hdev); 3519 if (err) 3520 goto exit_error; 3521 3522 btintel_print_fseq_info(hdev); 3523 exit_error: 3524 kfree_skb(skb); 3525 3526 return err; 3527 } 3528 3529 static int btintel_pcie_setup(struct hci_dev *hdev) 3530 { 3531 int err, fw_dl_retry = 0; 3532 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 3533 3534 while ((err = btintel_pcie_setup_internal(hdev)) && fw_dl_retry++ < 1) { 3535 bt_dev_err(hdev, "Firmware download retry count: %d", 3536 fw_dl_retry); 3537 btintel_pcie_dump_debug_registers(hdev); 3538 btintel_pcie_disable_interrupts(data); 3539 btintel_pcie_synchronize_irqs(data); 3540 err = btintel_pcie_reset_bt(data); 3541 if (err) { 3542 bt_dev_err(hdev, "Failed to do shr reset: %d", err); 3543 break; 3544 } 3545 usleep_range(10000, 12000); 3546 btintel_pcie_reset_ia(data); 3547 btintel_pcie_enable_interrupts(data); 3548 btintel_pcie_config_msix(data); 3549 err = btintel_pcie_enable_bt(data); 3550 if (err) { 3551 bt_dev_err(hdev, "Failed to enable hardware: %d", err); 3552 break; 3553 } 3554 btintel_pcie_start_rx(data); 3555 } 3556 3557 if (!err) 3558 set_bit(BTINTEL_PCIE_SETUP_DONE, &data->flags); 3559 return err; 3560 } 3561 3562 static struct btintel_pcie_dev_recovery * 3563 btintel_pcie_get_recovery(struct pci_dev *pdev, struct device *dev) 3564 { 3565 struct btintel_pcie_dev_recovery *tmp, *data = NULL; 3566 const char *name = pci_name(pdev); 3567 const size_t name_len = strlen(name) + 1; 3568 struct hci_dev *hdev = to_hci_dev(dev); 3569 3570 spin_lock(&btintel_pcie_recovery_lock); 3571 list_for_each_entry(tmp, &btintel_pcie_recovery_list, list) { 3572 if (strcmp(tmp->name, name)) 3573 continue; 3574 data = tmp; 3575 break; 3576 } 3577 spin_unlock(&btintel_pcie_recovery_lock); 3578 3579 if (data) { 3580 bt_dev_dbg(hdev, "Found restart data for BDF: %s", data->name); 3581 return data; 3582 } 3583 3584 data = kzalloc_flex(*data, name, name_len, GFP_ATOMIC); 3585 if (!data) 3586 return NULL; 3587 3588 strscpy(data->name, name, name_len); 3589 spin_lock(&btintel_pcie_recovery_lock); 3590 list_add_tail(&data->list, &btintel_pcie_recovery_list); 3591 spin_unlock(&btintel_pcie_recovery_lock); 3592 3593 return data; 3594 } 3595 3596 static void btintel_pcie_free_restart_list(void) 3597 { 3598 struct btintel_pcie_dev_recovery *tmp; 3599 3600 while ((tmp = list_first_entry_or_null(&btintel_pcie_recovery_list, 3601 typeof(*tmp), list))) { 3602 list_del(&tmp->list); 3603 kfree(tmp); 3604 } 3605 } 3606 3607 static void btintel_pcie_inc_recovery_count(struct pci_dev *pdev, 3608 struct device *dev) 3609 { 3610 struct btintel_pcie_dev_recovery *data; 3611 time64_t retry_window; 3612 3613 data = btintel_pcie_get_recovery(pdev, dev); 3614 if (!data) 3615 return; 3616 3617 retry_window = ktime_get_boottime_seconds() - data->last_error; 3618 if (data->count == 0) { 3619 data->last_error = ktime_get_boottime_seconds(); 3620 data->count++; 3621 } else if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS && 3622 data->count <= BTINTEL_PCIE_FLR_MAX_RETRY) { 3623 data->count++; 3624 } else if (retry_window > BTINTEL_PCIE_RESET_WINDOW_SECS) { 3625 data->last_error = 0; 3626 data->count = 0; 3627 } 3628 } 3629 3630 static int btintel_pcie_acpi_reset_method(struct btintel_pcie_data *data) 3631 { 3632 union acpi_object *obj, argv4; 3633 acpi_handle handle; 3634 int ret; 3635 struct pldr_mode { 3636 __le16 cmd_type; 3637 __le16 cmd_payload; 3638 } __packed; 3639 3640 /* set 1 for _PRR mode 3641 * Product Reset (PLDR Abort flow) 3642 */ 3643 static const struct pldr_mode mode = { 3644 .cmd_type = cpu_to_le16(1), 3645 .cmd_payload = cpu_to_le16(BTINTEL_PCIE_DSM_PLDR_MODE_EN_PROD_RESET | 3646 BTINTEL_PCIE_DSM_PLDR_MODE_EN_WIFI_FLR), 3647 }; 3648 struct hci_dev *hdev = data->hdev; 3649 3650 handle = ACPI_HANDLE(GET_HCIDEV_DEV(data->hdev)); 3651 if (!handle) { 3652 bt_dev_err(data->hdev, "No support for bluetooth device in ACPI firmware"); 3653 return -EACCES; 3654 } 3655 3656 if (!acpi_has_method(handle, "_PRR")) { 3657 bt_dev_err(data->hdev, "No support for _PRR ACPI method, cold boot"); 3658 return -ENODEV; 3659 } 3660 3661 argv4.buffer.type = ACPI_TYPE_BUFFER; 3662 argv4.buffer.length = sizeof(mode); 3663 argv4.buffer.pointer = (void *)&mode; 3664 3665 obj = acpi_evaluate_dsm(handle, &btintel_guid_dsm, 0, 3666 BTINTEL_PCIE_DSM_DYNAMIC_PLDR, &argv4); 3667 if (!obj) { 3668 bt_dev_err(data->hdev, "Failed to call dsm to set reset method"); 3669 return -EIO; 3670 } 3671 ACPI_FREE(obj); 3672 3673 pci_dev_lock(data->pdev); 3674 pci_save_state(data->pdev); 3675 ret = btintel_acpi_reset_method(hdev); 3676 if (ret) 3677 bt_dev_err(data->hdev, "ACPI _PRR reset failed (%d), PLDR incomplete", 3678 ret); 3679 pci_restore_state(data->pdev); 3680 pci_dev_unlock(data->pdev); 3681 return ret; 3682 } 3683 3684 static void btintel_pcie_perform_pldr(struct btintel_pcie_data *data) 3685 { 3686 struct pci_dev *pdev = data->pdev; 3687 struct pci_dev *wifi = NULL; 3688 struct pci_bus *bus; 3689 int ret; 3690 /* on integrated we have to look up by ID (same bus) */ 3691 static const struct pci_device_id wifi_device_ids[] = { 3692 #define WIFI_DEV(_id) { PCI_DEVICE(PCI_VENDOR_ID_INTEL, _id) } 3693 WIFI_DEV(0xA840), /* LNL */ 3694 WIFI_DEV(0xE440), /* PTL-P */ 3695 WIFI_DEV(0xE340), /* PTL-H */ 3696 WIFI_DEV(0xD340), /* NVL-H */ 3697 WIFI_DEV(0x6E70), /* NVL-S */ 3698 WIFI_DEV(0x4D40), /* WCL */ 3699 {} 3700 }; 3701 struct pci_dev *tmp = NULL; 3702 3703 bus = pdev->bus; 3704 if (!bus) 3705 return; 3706 3707 list_for_each_entry(tmp, &bus->devices, bus_list) { 3708 if (pci_match_id(wifi_device_ids, tmp)) { 3709 wifi = pci_dev_get(tmp); 3710 break; 3711 } 3712 } 3713 3714 if (wifi) 3715 device_release_driver(&wifi->dev); 3716 3717 /* Wi-Fi is fully unbound before the reset and fully reprobed after 3718 * the normal PCI probe path handles all state setup from scratch. 3719 * BT needs pci_save_state()/pci_restore_state() because the BT driver 3720 * is still partially attached when the _PRR runs (it hasn't been unbound yet). 3721 * The PCI device needs to remain minimally functional so that 3722 * device_reprobe(&pdev->dev) can work afterward 3723 */ 3724 ret = btintel_pcie_acpi_reset_method(data); 3725 3726 if (wifi) { 3727 if (device_reprobe(&wifi->dev)) 3728 BT_ERR("WiFi reprobe failed for BDF:%s", pci_name(wifi)); 3729 pci_dev_put(wifi); 3730 } 3731 3732 if (!ret) { 3733 if (device_reprobe(&pdev->dev)) 3734 BT_ERR("BT reprobe failed for BDF:%s", pci_name(pdev)); 3735 } 3736 } 3737 3738 /* 3739 * Issue a Function Level Reset and hand teardown/re-init off to the PCI 3740 * core via device_reprobe(), mirroring the PLDR path's contract. 3741 * 3742 * Caller must hold pci_lock_rescan_remove() and must have already 3743 * disabled interrupts and drained both rx_work and coredump_work. 3744 */ 3745 static int btintel_pcie_perform_flr(struct btintel_pcie_data *data) 3746 { 3747 struct pci_dev *pdev = data->pdev; 3748 int err; 3749 3750 /* pci_try_reset_function() avoids the device_lock ABBA against 3751 * btintel_pcie_remove(): .remove() runs with device_lock held and 3752 * then waits for this work via disable_work_sync(); the blocking 3753 * pci_reset_function() would deadlock by trying to re-acquire 3754 * device_lock here. 3755 */ 3756 err = pci_try_reset_function(pdev); 3757 if (err) { 3758 BT_ERR("Failed resetting the pcie device (%d)", err); 3759 return err; 3760 } 3761 3762 /* device_reprobe() always detaches the driver first (running 3763 * .remove(), which frees 'data'); any re-probe failure leaves the 3764 * device unbound but 'data' is already gone, so just log it. 3765 */ 3766 if (device_reprobe(&pdev->dev)) 3767 BT_ERR("BT reprobe failed for BDF:%s", pci_name(pdev)); 3768 3769 return 0; 3770 } 3771 3772 static void btintel_pcie_reset_work(struct work_struct *wk) 3773 { 3774 struct btintel_pcie_data *data = 3775 container_of(wk, struct btintel_pcie_data, reset_work); 3776 struct pci_dev *pdev = data->pdev; 3777 3778 pci_lock_rescan_remove(); 3779 3780 if (!pdev->bus) 3781 goto out; 3782 3783 if (!data) 3784 goto out; 3785 3786 btintel_pcie_disable_interrupts(data); 3787 btintel_pcie_synchronize_irqs(data); 3788 3789 flush_work(&data->rx_work); 3790 /* Drain any in-flight dump workers and block new ones across reset. 3791 * Safe from self-deadlock: they all run on a separate wq. 3792 */ 3793 disable_work_sync(&data->coredump_work); 3794 disable_work_sync(&data->hwexp_work); 3795 disable_work_sync(&data->fwtrigger_work); 3796 disable_work_sync(&data->mbox_work); 3797 3798 bt_dev_dbg(data->hdev, "Release bluetooth interface"); 3799 3800 /* Both reset paths follow the same contract: on success they 3801 * destroy 'data' via device_reprobe() (a fresh probe re-INIT_WORKs 3802 * the dump workers with disable count 0), so enable_work() must 3803 * NOT be called on the success path. Only the FLR path can fail 3804 * with 'data' still alive, in which case we balance the 3805 * disable_work_sync() calls above so a later successful reset is 3806 * not permanently blocked. 3807 * 3808 * pci_lock_rescan_remove() (held above) serializes against PCI 3809 * device addition/removal (hotplug), so no device can be added to 3810 * or removed from the bus list while this code runs. 3811 */ 3812 if (data->reset_type == BTINTEL_PCIE_IOSF_PRR_PLDR) { 3813 btintel_pcie_perform_pldr(data); 3814 goto out; 3815 } 3816 3817 if (btintel_pcie_perform_flr(data)) { 3818 enable_work(&data->coredump_work); 3819 enable_work(&data->hwexp_work); 3820 enable_work(&data->fwtrigger_work); 3821 enable_work(&data->mbox_work); 3822 } 3823 3824 out: 3825 pci_dev_put(pdev); 3826 pci_unlock_rescan_remove(); 3827 } 3828 3829 /* Schedule a device reset of the requested type. 3830 * 3831 * BTINTEL_PCIE_RECOVERY_IN_PROGRESS serializes all reset requesters 3832 * (sysfs reset attribute, hci_cmd_timeout(), hw_error, resume error 3833 * path, etc.) so that: 3834 * 3835 * - dev_data->reset_type is written by exactly one caller (the 3836 * thread that wins test_and_set_bit), eliminating the race where 3837 * a second hw_error could clobber an already-scheduled reset's 3838 * type; 3839 * - the write happens AFTER the bit is set, so reset_work observes 3840 * it through schedule_work()'s memory ordering; 3841 * - losers return without touching reset_type or scheduling the 3842 * work, so concurrent triggers are silently coalesced into the 3843 * in-flight one (whose recovery will reinitialize the device 3844 * regardless of the dropped trigger's variant). 3845 * 3846 * The bit is cleared only by .remove() / re-probe via fresh devm 3847 * allocation, which is the intended one-shot semantics: a reset 3848 * tears down and re-probes 'data', so there is no "in-flight" 3849 * reset to follow up after device_reprobe() succeeds. 3850 */ 3851 static void btintel_pcie_request_reset(struct btintel_pcie_data *data, 3852 enum btintel_pcie_reset_type type) 3853 { 3854 if (!test_bit(BTINTEL_PCIE_SETUP_DONE, &data->flags)) 3855 return; 3856 3857 if (test_and_set_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags)) 3858 return; 3859 3860 data->reset_type = type; 3861 3862 pci_dev_get(data->pdev); 3863 schedule_work(&data->reset_work); 3864 } 3865 3866 static void btintel_pcie_hci_reset(struct hci_dev *hdev) 3867 { 3868 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 3869 3870 btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_FLR); 3871 } 3872 3873 static ssize_t vendor_reset_store(struct device *dev, 3874 struct device_attribute *attr, 3875 const char *buf, size_t count) 3876 { 3877 unsigned int val; 3878 struct pci_dev *pdev = to_pci_dev(dev); 3879 struct btintel_pcie_data *data = pci_get_drvdata(pdev); 3880 3881 if (!data || !data->hdev) 3882 return -ENODEV; 3883 3884 if (kstrtouint(buf, 10, &val) || val != 0) { 3885 bt_dev_warn(data->hdev, "PLDR rejected: invalid input"); 3886 return -EINVAL; 3887 } 3888 3889 bt_dev_info(data->hdev, "PLDR triggered via sysfs"); 3890 btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_PLDR); 3891 3892 return count; 3893 } 3894 3895 static ssize_t vendor_reset_show(struct device *dev, 3896 struct device_attribute *attr, char *buf) 3897 { 3898 return sysfs_emit(buf, "0 - PLDR\n"); 3899 } 3900 3901 static DEVICE_ATTR_RW(vendor_reset); 3902 3903 static struct attribute *btintel_pcie_attrs[] = { 3904 &dev_attr_vendor_reset.attr, 3905 NULL, 3906 }; 3907 3908 ATTRIBUTE_GROUPS(btintel_pcie); 3909 3910 static void btintel_pcie_hw_error(struct hci_dev *hdev, u8 code) 3911 { 3912 struct btintel_pcie_dev_recovery *rec; 3913 struct btintel_pcie_data *dev_data = hci_get_drvdata(hdev); 3914 struct pci_dev *pdev = dev_data->pdev; 3915 enum btintel_pcie_reset_type type; 3916 time64_t retry_window; 3917 3918 if (test_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &dev_data->flags)) 3919 return; 3920 3921 btintel_pcie_dump_debug_registers(hdev); 3922 3923 rec = btintel_pcie_get_recovery(pdev, &hdev->dev); 3924 if (!rec) 3925 return; 3926 3927 type = (code == 0x13) ? BTINTEL_PCIE_IOSF_PRR_PLDR 3928 : BTINTEL_PCIE_IOSF_PRR_FLR; 3929 3930 bt_dev_err(hdev, "Encountered exception err:0x%x triggering: %s", code, 3931 type == BTINTEL_PCIE_IOSF_PRR_PLDR ? "PLDR" : "FLR"); 3932 retry_window = ktime_get_boottime_seconds() - rec->last_error; 3933 3934 if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS && 3935 rec->count >= BTINTEL_PCIE_FLR_MAX_RETRY) { 3936 bt_dev_err(hdev, "Exhausted maximum: %d recovery attempts: %d", 3937 BTINTEL_PCIE_FLR_MAX_RETRY, rec->count); 3938 bt_dev_dbg(hdev, "Boot time: %lld seconds", 3939 ktime_get_boottime_seconds()); 3940 bt_dev_dbg(hdev, "last error at: %lld seconds", 3941 rec->last_error); 3942 return; 3943 } 3944 btintel_pcie_inc_recovery_count(pdev, &hdev->dev); 3945 btintel_pcie_request_reset(dev_data, type); 3946 } 3947 3948 static bool btintel_pcie_wakeup(struct hci_dev *hdev) 3949 { 3950 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 3951 3952 return device_may_wakeup(&data->pdev->dev); 3953 } 3954 3955 static const struct { 3956 u16 opcode; 3957 const char *desc; 3958 } btintel_pcie_hci_drv_supported_commands[] = { 3959 /* Common commands */ 3960 { HCI_DRV_OP_READ_INFO, "Read Info" }, 3961 }; 3962 3963 static int btintel_pcie_hci_drv_read_info(struct hci_dev *hdev, void *data, 3964 u16 data_len) 3965 { 3966 struct hci_drv_rp_read_info *rp; 3967 size_t rp_size; 3968 int err, i; 3969 u16 opcode, num_supported_commands = 3970 ARRAY_SIZE(btintel_pcie_hci_drv_supported_commands); 3971 3972 rp_size = struct_size(rp, supported_commands, num_supported_commands); 3973 3974 rp = kmalloc(rp_size, GFP_KERNEL); 3975 if (!rp) 3976 return -ENOMEM; 3977 3978 strscpy_pad(rp->driver_name, KBUILD_MODNAME); 3979 3980 rp->num_supported_commands = cpu_to_le16(num_supported_commands); 3981 for (i = 0; i < num_supported_commands; i++) { 3982 opcode = btintel_pcie_hci_drv_supported_commands[i].opcode; 3983 bt_dev_dbg(hdev, 3984 "Supported HCI Drv command (0x%02x|0x%04x): %s", 3985 hci_opcode_ogf(opcode), 3986 hci_opcode_ocf(opcode), 3987 btintel_pcie_hci_drv_supported_commands[i].desc); 3988 rp->supported_commands[i] = cpu_to_le16(opcode); 3989 } 3990 3991 err = hci_drv_cmd_complete(hdev, HCI_DRV_OP_READ_INFO, 3992 HCI_DRV_STATUS_SUCCESS, 3993 rp, rp_size); 3994 3995 kfree(rp); 3996 return err; 3997 } 3998 3999 static const struct hci_drv_handler btintel_pcie_hci_drv_common_handlers[] = { 4000 { btintel_pcie_hci_drv_read_info, HCI_DRV_READ_INFO_SIZE }, 4001 }; 4002 4003 static const struct hci_drv_handler btintel_pcie_hci_drv_specific_handlers[] = {}; 4004 4005 static struct hci_drv btintel_pcie_hci_drv = { 4006 .common_handler_count = ARRAY_SIZE(btintel_pcie_hci_drv_common_handlers), 4007 .common_handlers = btintel_pcie_hci_drv_common_handlers, 4008 .specific_handler_count = ARRAY_SIZE(btintel_pcie_hci_drv_specific_handlers), 4009 .specific_handlers = btintel_pcie_hci_drv_specific_handlers, 4010 }; 4011 4012 static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data) 4013 { 4014 int err; 4015 struct hci_dev *hdev; 4016 4017 hdev = hci_alloc_dev_priv(sizeof(struct btintel_data)); 4018 if (!hdev) 4019 return -ENOMEM; 4020 4021 hdev->bus = HCI_PCI; 4022 hci_set_drvdata(hdev, data); 4023 4024 SET_HCIDEV_DEV(hdev, &data->pdev->dev); 4025 4026 hdev->manufacturer = 2; 4027 hdev->open = btintel_pcie_open; 4028 hdev->close = btintel_pcie_close; 4029 hdev->send = btintel_pcie_send_frame; 4030 hdev->setup = btintel_pcie_setup; 4031 hdev->shutdown = btintel_shutdown_combined; 4032 hdev->hw_error = btintel_pcie_hw_error; 4033 hdev->set_diag = btintel_set_diag; 4034 hdev->set_bdaddr = btintel_set_bdaddr; 4035 hdev->reset = btintel_pcie_hci_reset; 4036 hdev->wakeup = btintel_pcie_wakeup; 4037 hdev->hci_drv = &btintel_pcie_hci_drv; 4038 4039 err = hci_register_dev(hdev); 4040 if (err < 0) { 4041 BT_ERR("Failed to register to hdev (%d)", err); 4042 hci_free_dev(hdev); 4043 return err; 4044 } 4045 4046 /* Publish hdev only after successful registration; the coredump 4047 * worker bails on !data->hdev, so it never observes a half-set-up 4048 * device. 4049 */ 4050 data->hdev = hdev; 4051 data->dmp_hdr.driver_name = KBUILD_MODNAME; 4052 return 0; 4053 } 4054 4055 static int btintel_pcie_probe(struct pci_dev *pdev, 4056 const struct pci_device_id *ent) 4057 { 4058 int err; 4059 struct btintel_pcie_data *data; 4060 4061 if (!pdev) 4062 return -ENODEV; 4063 4064 data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); 4065 if (!data) 4066 return -ENOMEM; 4067 4068 data->pdev = pdev; 4069 4070 spin_lock_init(&data->irq_lock); 4071 spin_lock_init(&data->hci_rx_lock); 4072 4073 init_waitqueue_head(&data->gp0_wait_q); 4074 data->gp0_received = false; 4075 4076 init_waitqueue_head(&data->tx_wait_q); 4077 data->tx_wait_done = false; 4078 4079 init_waitqueue_head(&data->mbox_parse_wait_q); 4080 4081 data->workqueue = alloc_ordered_workqueue(KBUILD_MODNAME, WQ_HIGHPRI); 4082 if (!data->workqueue) 4083 return -ENOMEM; 4084 4085 data->dump_workqueue = alloc_ordered_workqueue(KBUILD_MODNAME "_cd", 0); 4086 if (!data->dump_workqueue) { 4087 destroy_workqueue(data->workqueue); 4088 return -ENOMEM; 4089 } 4090 4091 skb_queue_head_init(&data->rx_skb_q); 4092 INIT_WORK(&data->rx_work, btintel_pcie_rx_work); 4093 INIT_WORK(&data->reset_work, btintel_pcie_reset_work); 4094 INIT_WORK(&data->coredump_work, btintel_pcie_coredump_worker); 4095 INIT_WORK(&data->hwexp_work, btintel_pcie_hwexp_worker); 4096 INIT_WORK(&data->fwtrigger_work, btintel_pcie_fwtrigger_worker); 4097 INIT_WORK(&data->mbox_work, btintel_pcie_mbox_worker); 4098 4099 data->boot_stage_cache = 0x00; 4100 data->img_resp_cache = 0x00; 4101 data->dbg_path_cache = BTINTEL_PCIE_WIFI_DBGC; 4102 /* FLR can be invoked by echoing to debugfs path, so explicitly 4103 * initialized 4104 */ 4105 data->reset_type = BTINTEL_PCIE_IOSF_PRR_FLR; 4106 err = btintel_pcie_config_pcie(pdev, data); 4107 if (err) 4108 goto exit_error; 4109 4110 pci_set_drvdata(pdev, data); 4111 4112 err = btintel_pcie_alloc(data); 4113 if (err) 4114 goto exit_error; 4115 4116 err = btintel_pcie_enable_bt(data); 4117 if (err) 4118 goto exit_error; 4119 4120 /* CNV information (CNVi and CNVr) is in CSR */ 4121 data->cnvi = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_HW_REV_REG); 4122 4123 data->cnvr = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_RF_ID_REG); 4124 4125 err = btintel_pcie_start_rx(data); 4126 if (err) 4127 goto exit_error; 4128 4129 err = btintel_pcie_setup_hdev(data); 4130 if (err) 4131 goto exit_error; 4132 4133 bt_dev_dbg(data->hdev, "cnvi: 0x%8.8x cnvr: 0x%8.8x", data->cnvi, 4134 data->cnvr); 4135 return 0; 4136 4137 exit_error: 4138 /* reset device before exit */ 4139 btintel_pcie_reset_bt(data); 4140 4141 destroy_workqueue(data->dump_workqueue); 4142 4143 pci_clear_master(pdev); 4144 4145 pci_set_drvdata(pdev, NULL); 4146 4147 return err; 4148 } 4149 4150 static void btintel_pcie_remove(struct pci_dev *pdev) 4151 { 4152 struct btintel_pcie_data *data; 4153 4154 data = pci_get_drvdata(pdev); 4155 if (!data) { 4156 BT_WARN("PCI driver data is NULL, aborting remove"); 4157 return; 4158 } 4159 4160 /* Permanently block all dump triggers and drain the workers before 4161 * tearing down. Must run before disable_work_sync(&reset_work) so 4162 * the disable counters stay >= 1 even after reset_work()'s 4163 * balanced enable_work() (counter 2 -> 1, never reaching 0). 4164 */ 4165 disable_work_sync(&data->coredump_work); 4166 disable_work_sync(&data->hwexp_work); 4167 disable_work_sync(&data->fwtrigger_work); 4168 disable_work_sync(&data->mbox_work); 4169 4170 /* Cancel pending reset work. Skip only when remove() is called from 4171 * within the reset work itself (PLDR device_reprobe path) to avoid 4172 * deadlock. current_work() returns the work_struct of the caller if 4173 * we are in a workqueue context. 4174 */ 4175 if (current_work() != &data->reset_work) 4176 disable_work_sync(&data->reset_work); 4177 4178 btintel_pcie_disable_interrupts(data); 4179 4180 btintel_pcie_synchronize_irqs(data); 4181 4182 flush_work(&data->rx_work); 4183 4184 btintel_pcie_reset_bt(data); 4185 for (int i = 0; i < data->alloc_vecs; i++) { 4186 struct msix_entry *msix_entry; 4187 4188 msix_entry = &data->msix_entries[i]; 4189 devm_free_irq(&pdev->dev, msix_entry->vector, msix_entry); 4190 } 4191 4192 pci_free_irq_vectors(pdev); 4193 4194 btintel_pcie_release_hdev(data); 4195 4196 destroy_workqueue(data->dump_workqueue); 4197 destroy_workqueue(data->workqueue); 4198 4199 btintel_pcie_free(data); 4200 4201 pci_clear_master(pdev); 4202 4203 pci_set_drvdata(pdev, NULL); 4204 } 4205 4206 #ifdef CONFIG_DEV_COREDUMP 4207 static void btintel_pcie_coredump(struct device *dev) 4208 { 4209 struct pci_dev *pdev = to_pci_dev(dev); 4210 struct btintel_pcie_data *data = pci_get_drvdata(pdev); 4211 4212 if (!data) 4213 return; 4214 4215 btintel_pcie_queue_coredump(data, 4216 BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER); 4217 } 4218 #endif 4219 4220 static int btintel_pcie_set_dxstate(struct btintel_pcie_data *data, u32 dxstate) 4221 { 4222 int retry = 0, err; 4223 long status; 4224 u32 dx_intr_timeout_ms = 200; 4225 4226 do { 4227 data->gp0_received = false; 4228 4229 btintel_pcie_wr_sleep_cntrl(data, dxstate); 4230 4231 status = wait_event_timeout(data->gp0_wait_q, data->gp0_received, 4232 msecs_to_jiffies(dx_intr_timeout_ms)); 4233 4234 if (!status) { 4235 bt_dev_warn(data->hdev, 4236 "Timeout (%u ms) on alive interrupt for D%d entry, retry count %d", 4237 dx_intr_timeout_ms, dxstate, retry); 4238 4239 /* MSIX_HW_INT_CAUSES is W1C. Write only GP0 so other 4240 * pending causes are not acknowledged here. 4241 */ 4242 btintel_pcie_wr_reg32(data, 4243 BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES, 4244 BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0); 4245 } 4246 4247 /* gp0_received is set at the top of the handler, before the switch on 4248 * alive_intr_ctxt. Error and lockdown are filtered out above it, but a 4249 * matched case can still complete without doing anything - D3 breaks 4250 * unchanged while the controller has not reached D0 - and a hardware 4251 * bug may drop the interrupt outright. Either way the flag says a gp0 4252 * was handled, not that the transition completed, and only the register 4253 * knows. Refresh the cache here and retry only if the state check still 4254 * fails. 4255 */ 4256 data->boot_stage_cache = btintel_pcie_rd_reg32(data, 4257 BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 4258 if (dxstate == BTINTEL_PCIE_STATE_D0) { 4259 if (btintel_pcie_in_d0(data)) { 4260 bool armed; 4261 4262 /* Claim the transition under irq_lock so a 4263 * concurrent gp0 handler run cannot also arm RX. 4264 */ 4265 spin_lock(&data->irq_lock); 4266 armed = data->alive_intr_ctxt == BTINTEL_PCIE_D0; 4267 if (!armed) 4268 data->alive_intr_ctxt = BTINTEL_PCIE_D0; 4269 spin_unlock(&data->irq_lock); 4270 4271 if (armed) 4272 return 0; 4273 4274 /* Do what the handler's D3 -> D0 branch 4275 * would have done, unless it already has. 4276 */ 4277 btintel_pcie_reset_ia(data); 4278 err = btintel_pcie_start_rx(data); 4279 if (err) 4280 return err; 4281 4282 /* Complete the mbox<->alive handshake */ 4283 if (test_and_clear_bit(BTINTEL_PCIE_MBOX_PARSE_PENDING, 4284 &data->flags)) { 4285 set_bit(BTINTEL_PCIE_MBOX_PARSE_READY, &data->flags); 4286 wake_up(&data->mbox_parse_wait_q); 4287 } 4288 4289 return 0; 4290 } 4291 } else { 4292 if (btintel_pcie_in_d3(data)) { 4293 /* Do what the handler's D0 -> D3 branch 4294 * would have done, unless it already has. 4295 */ 4296 spin_lock(&data->irq_lock); 4297 if (data->alive_intr_ctxt != BTINTEL_PCIE_D3) 4298 data->alive_intr_ctxt = BTINTEL_PCIE_D3; 4299 spin_unlock(&data->irq_lock); 4300 4301 return 0; 4302 } 4303 } 4304 4305 } while (++retry < BTINTEL_PCIE_DX_TRANSITION_MAX_RETRIES); 4306 4307 return -EBUSY; 4308 } 4309 4310 static int btintel_pcie_suspend_late(struct device *dev, pm_message_t mesg) 4311 { 4312 struct pci_dev *pdev = to_pci_dev(dev); 4313 struct btintel_pcie_data *data; 4314 ktime_t start; 4315 u32 dxstate; 4316 int err; 4317 4318 data = pci_get_drvdata(pdev); 4319 4320 /* S0ix (s2idle) uses D3_HOT; S3, freeze and hibernate use D3_COLD. */ 4321 if (mesg.event == PM_EVENT_SUSPEND && 4322 pm_suspend_target_state == PM_SUSPEND_TO_IDLE) 4323 dxstate = BTINTEL_PCIE_STATE_D3_HOT; 4324 else 4325 dxstate = BTINTEL_PCIE_STATE_D3_COLD; 4326 4327 start = ktime_get(); 4328 4329 /* Refer: 6.4.11.7 -> Platform power management */ 4330 err = btintel_pcie_set_dxstate(data, dxstate); 4331 4332 if (err) { 4333 bt_dev_err(data->hdev, "Failed to set dxstate:%u (%d)", 4334 dxstate, err); 4335 return err; 4336 } 4337 4338 bt_dev_dbg(data->hdev, 4339 "device entered into d3 state from d0 in %lld us", 4340 ktime_to_us(ktime_get() - start)); 4341 return err; 4342 } 4343 4344 static int btintel_pcie_suspend(struct device *dev) 4345 { 4346 return btintel_pcie_suspend_late(dev, PMSG_SUSPEND); 4347 } 4348 4349 static int btintel_pcie_hibernate(struct device *dev) 4350 { 4351 return btintel_pcie_suspend_late(dev, PMSG_HIBERNATE); 4352 } 4353 4354 static int btintel_pcie_freeze(struct device *dev) 4355 { 4356 return btintel_pcie_suspend_late(dev, PMSG_FREEZE); 4357 } 4358 4359 static int btintel_pcie_resume_event(struct device *dev, pm_message_t mesg) 4360 { 4361 struct pci_dev *pdev = to_pci_dev(dev); 4362 struct btintel_pcie_data *data; 4363 ktime_t start; 4364 int err; 4365 4366 data = pci_get_drvdata(pdev); 4367 4368 start = ktime_get(); 4369 4370 /* S3 and S4 may cut power, erasing the firmware. Force FLR to recover 4371 * instead of a normal D0 transition. 4372 */ 4373 if (mesg.event == PM_EVENT_RESTORE || 4374 (mesg.event == PM_EVENT_RESUME && 4375 pm_suspend_target_state == PM_SUSPEND_MEM)) { 4376 set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags); 4377 btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_FLR); 4378 return 0; 4379 } 4380 4381 /* Refer: 6.4.11.7 -> Platform power management */ 4382 err = btintel_pcie_set_dxstate(data, BTINTEL_PCIE_STATE_D0); 4383 4384 if (err) { 4385 bt_dev_err(data->hdev, "Failed to set D0 state (%d)", err); 4386 } else { 4387 bt_dev_dbg(data->hdev, 4388 "device entered into d0 state from d3 in %lld us", 4389 ktime_to_us(ktime_get() - start)); 4390 return err; 4391 } 4392 4393 /* Trigger function level reset if the controller is in error 4394 * state during resume() to bring back the controller to 4395 * operational mode 4396 */ 4397 4398 data->boot_stage_cache = btintel_pcie_rd_reg32(data, 4399 BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 4400 if (btintel_pcie_in_error(data) || 4401 btintel_pcie_in_device_halt(data)) { 4402 bt_dev_err(data->hdev, "Controller in error state for D0 entry"); 4403 btintel_pcie_queue_coredump(data, 4404 BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT); 4405 set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags); 4406 btintel_pcie_request_reset(data, BTINTEL_PCIE_IOSF_PRR_FLR); 4407 } 4408 return err; 4409 } 4410 4411 static int btintel_pcie_resume(struct device *dev) 4412 { 4413 return btintel_pcie_resume_event(dev, PMSG_RESUME); 4414 } 4415 4416 static int btintel_pcie_restore(struct device *dev) 4417 { 4418 return btintel_pcie_resume_event(dev, PMSG_RESTORE); 4419 } 4420 4421 static int btintel_pcie_thaw(struct device *dev) 4422 { 4423 return btintel_pcie_resume_event(dev, PMSG_THAW); 4424 } 4425 4426 static const struct dev_pm_ops btintel_pcie_pm_ops = { 4427 .suspend = btintel_pcie_suspend, 4428 .resume = btintel_pcie_resume, 4429 .freeze = btintel_pcie_freeze, 4430 .thaw = btintel_pcie_thaw, 4431 .poweroff = btintel_pcie_hibernate, 4432 .restore = btintel_pcie_restore, 4433 }; 4434 4435 static struct pci_driver btintel_pcie_driver = { 4436 .name = KBUILD_MODNAME, 4437 .id_table = btintel_pcie_table, 4438 .probe = btintel_pcie_probe, 4439 .remove = btintel_pcie_remove, 4440 .driver.pm = pm_sleep_ptr(&btintel_pcie_pm_ops), 4441 .dev_groups = btintel_pcie_groups, 4442 #ifdef CONFIG_DEV_COREDUMP 4443 .driver.coredump = btintel_pcie_coredump 4444 #endif 4445 }; 4446 4447 static int __init btintel_pcie_init(void) 4448 { 4449 return pci_register_driver(&btintel_pcie_driver); 4450 } 4451 4452 static void __exit btintel_pcie_exit(void) 4453 { 4454 pci_unregister_driver(&btintel_pcie_driver); 4455 btintel_pcie_free_restart_list(); 4456 } 4457 4458 module_init(btintel_pcie_init); 4459 module_exit(btintel_pcie_exit); 4460 4461 MODULE_AUTHOR("Tedd Ho-Jeong An <tedd.an@intel.com>"); 4462 MODULE_DESCRIPTION("Intel Bluetooth PCIe transport driver ver " VERSION); 4463 MODULE_VERSION(VERSION); 4464 MODULE_LICENSE("GPL"); 4465