1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * 4 * Bluetooth support for Intel PCIe devices 5 * 6 * Copyright (C) 2024 Intel Corporation 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/firmware.h> 12 #include <linux/overflow.h> 13 #include <linux/pci.h> 14 #include <linux/string.h> 15 #include <linux/wait.h> 16 #include <linux/delay.h> 17 #include <linux/interrupt.h> 18 19 #include <linux/unaligned.h> 20 #include <linux/devcoredump.h> 21 22 #include <net/bluetooth/bluetooth.h> 23 #include <net/bluetooth/hci_core.h> 24 #include <net/bluetooth/hci_drv.h> 25 26 #include "btintel.h" 27 #include "btintel_pcie.h" 28 29 #define VERSION "0.1" 30 31 #define BTINTEL_PCI_DEVICE(dev, subdev) \ 32 .vendor = PCI_VENDOR_ID_INTEL, \ 33 .device = (dev), \ 34 .subvendor = PCI_ANY_ID, \ 35 .subdevice = (subdev), \ 36 .driver_data = 0 37 38 #define POLL_INTERVAL_US 10 39 40 #define BTINTEL_PCIE_DMA_ALIGN_128B 128 /* 128 byte aligned */ 41 42 /* Intel Bluetooth PCIe device id table */ 43 static const struct pci_device_id btintel_pcie_table[] = { 44 /* BlazarI, Wildcat Lake */ 45 { BTINTEL_PCI_DEVICE(0x4D76, PCI_ANY_ID) }, 46 /* BlazarI, Lunar Lake */ 47 { BTINTEL_PCI_DEVICE(0xA876, PCI_ANY_ID) }, 48 /* Scorpious, Panther Lake-H484 */ 49 { BTINTEL_PCI_DEVICE(0xE376, PCI_ANY_ID) }, 50 /* Scorpious, Panther Lake-H404 */ 51 { BTINTEL_PCI_DEVICE(0xE476, PCI_ANY_ID) }, 52 /* Scorpious2, Nova Lake-PCD-H */ 53 { BTINTEL_PCI_DEVICE(0xD346, PCI_ANY_ID) }, 54 /* Scorpious2, Nova Lake-PCD-S */ 55 { BTINTEL_PCI_DEVICE(0x6E74, PCI_ANY_ID) }, 56 { 0 } 57 }; 58 MODULE_DEVICE_TABLE(pci, btintel_pcie_table); 59 60 struct btintel_pcie_dev_recovery { 61 struct list_head list; 62 u8 count; 63 time64_t last_error; 64 char name[]; 65 }; 66 67 /* Intel PCIe uses 4 bytes of HCI type instead of 1 byte BT SIG HCI type */ 68 #define BTINTEL_PCIE_HCI_TYPE_LEN 4 69 #define BTINTEL_PCIE_HCI_CMD_PKT 0x00000001 70 #define BTINTEL_PCIE_HCI_ACL_PKT 0x00000002 71 #define BTINTEL_PCIE_HCI_SCO_PKT 0x00000003 72 #define BTINTEL_PCIE_HCI_EVT_PKT 0x00000004 73 #define BTINTEL_PCIE_HCI_ISO_PKT 0x00000005 74 75 #define BTINTEL_PCIE_MAGIC_NUM 0xA5A5A5A5 76 77 #define BTINTEL_PCIE_BLZR_HWEXP_SIZE 1024 78 #define BTINTEL_PCIE_BLZR_HWEXP_DMP_ADDR 0xB00A7C00 79 80 #define BTINTEL_PCIE_SCP_HWEXP_SIZE 4096 81 #define BTINTEL_PCIE_SCP_HWEXP_DMP_ADDR 0xB030F800 82 83 #define BTINTEL_PCIE_SCP2_HWEXP_SIZE 4096 84 #define BTINTEL_PCIE_SCP2_HWEXP_DMP_ADDR 0xB031D000 85 86 #define BTINTEL_PCIE_MAGIC_NUM 0xA5A5A5A5 87 88 #define BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER 0x17A2 89 #define BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT 0x1E61 90 91 #define BTINTEL_PCIE_RESET_WINDOW_SECS 5 92 #define BTINTEL_PCIE_FLR_MAX_RETRY 1 93 94 /* Alive interrupt context */ 95 enum { 96 BTINTEL_PCIE_ROM, 97 BTINTEL_PCIE_FW_DL, 98 BTINTEL_PCIE_HCI_RESET, 99 BTINTEL_PCIE_INTEL_HCI_RESET1, 100 BTINTEL_PCIE_INTEL_HCI_RESET2, 101 BTINTEL_PCIE_D0, 102 BTINTEL_PCIE_D3 103 }; 104 105 /* Structure for dbgc fragment buffer 106 * @buf_addr_lsb: LSB of the buffer's physical address 107 * @buf_addr_msb: MSB of the buffer's physical address 108 * @buf_size: Total size of the buffer 109 */ 110 struct btintel_pcie_dbgc_ctxt_buf { 111 u32 buf_addr_lsb; 112 u32 buf_addr_msb; 113 u32 buf_size; 114 }; 115 116 /* Structure for dbgc fragment 117 * @magic_num: 0XA5A5A5A5 118 * @ver: For Driver-FW compatibility 119 * @total_size: Total size of the payload debug info 120 * @num_buf: Num of allocated debug bufs 121 * @bufs: All buffer's addresses and sizes 122 */ 123 struct btintel_pcie_dbgc_ctxt { 124 u32 magic_num; 125 u32 ver; 126 u32 total_size; 127 u32 num_buf; 128 struct btintel_pcie_dbgc_ctxt_buf bufs[BTINTEL_PCIE_DBGC_BUFFER_COUNT]; 129 }; 130 131 struct btintel_pcie_removal { 132 struct pci_dev *pdev; 133 struct work_struct work; 134 }; 135 136 static LIST_HEAD(btintel_pcie_recovery_list); 137 static DEFINE_SPINLOCK(btintel_pcie_recovery_lock); 138 139 static inline char *btintel_pcie_alivectxt_state2str(u32 alive_intr_ctxt) 140 { 141 switch (alive_intr_ctxt) { 142 case BTINTEL_PCIE_ROM: 143 return "rom"; 144 case BTINTEL_PCIE_FW_DL: 145 return "fw_dl"; 146 case BTINTEL_PCIE_D0: 147 return "d0"; 148 case BTINTEL_PCIE_D3: 149 return "d3"; 150 case BTINTEL_PCIE_HCI_RESET: 151 return "hci_reset"; 152 case BTINTEL_PCIE_INTEL_HCI_RESET1: 153 return "intel_reset1"; 154 case BTINTEL_PCIE_INTEL_HCI_RESET2: 155 return "intel_reset2"; 156 default: 157 return "unknown"; 158 } 159 } 160 161 /* This function initializes the memory for DBGC buffers and formats the 162 * DBGC fragment which consists header info and DBGC buffer's LSB, MSB and 163 * size as the payload 164 */ 165 static int btintel_pcie_setup_dbgc(struct btintel_pcie_data *data) 166 { 167 struct btintel_pcie_dbgc_ctxt db_frag; 168 struct data_buf *buf; 169 int i; 170 171 data->dbgc.count = BTINTEL_PCIE_DBGC_BUFFER_COUNT; 172 data->dbgc.bufs = devm_kcalloc(&data->pdev->dev, data->dbgc.count, 173 sizeof(*buf), GFP_KERNEL); 174 if (!data->dbgc.bufs) 175 return -ENOMEM; 176 177 data->dbgc.buf_v_addr = dmam_alloc_coherent(&data->pdev->dev, 178 data->dbgc.count * 179 BTINTEL_PCIE_DBGC_BUFFER_SIZE, 180 &data->dbgc.buf_p_addr, 181 GFP_KERNEL | __GFP_NOWARN); 182 if (!data->dbgc.buf_v_addr) 183 return -ENOMEM; 184 185 data->dbgc.frag_v_addr = dmam_alloc_coherent(&data->pdev->dev, 186 sizeof(struct btintel_pcie_dbgc_ctxt), 187 &data->dbgc.frag_p_addr, 188 GFP_KERNEL | __GFP_NOWARN); 189 if (!data->dbgc.frag_v_addr) 190 return -ENOMEM; 191 192 data->dbgc.frag_size = sizeof(struct btintel_pcie_dbgc_ctxt); 193 194 db_frag.magic_num = BTINTEL_PCIE_MAGIC_NUM; 195 db_frag.ver = BTINTEL_PCIE_DBGC_FRAG_VERSION; 196 db_frag.total_size = BTINTEL_PCIE_DBGC_FRAG_PAYLOAD_SIZE; 197 db_frag.num_buf = BTINTEL_PCIE_DBGC_FRAG_BUFFER_COUNT; 198 199 for (i = 0; i < data->dbgc.count; i++) { 200 buf = &data->dbgc.bufs[i]; 201 buf->data_p_addr = data->dbgc.buf_p_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE; 202 buf->data = data->dbgc.buf_v_addr + i * BTINTEL_PCIE_DBGC_BUFFER_SIZE; 203 db_frag.bufs[i].buf_addr_lsb = lower_32_bits(buf->data_p_addr); 204 db_frag.bufs[i].buf_addr_msb = upper_32_bits(buf->data_p_addr); 205 db_frag.bufs[i].buf_size = BTINTEL_PCIE_DBGC_BUFFER_SIZE; 206 } 207 208 memcpy(data->dbgc.frag_v_addr, &db_frag, sizeof(db_frag)); 209 return 0; 210 } 211 212 static inline void ipc_print_ia_ring(struct hci_dev *hdev, struct ia *ia, 213 u16 queue_num) 214 { 215 bt_dev_dbg(hdev, "IA: %s: tr-h:%02u tr-t:%02u cr-h:%02u cr-t:%02u", 216 queue_num == BTINTEL_PCIE_TXQ_NUM ? "TXQ" : "RXQ", 217 ia->tr_hia[queue_num], ia->tr_tia[queue_num], 218 ia->cr_hia[queue_num], ia->cr_tia[queue_num]); 219 } 220 221 static inline void ipc_print_urbd1(struct hci_dev *hdev, struct urbd1 *urbd1, 222 u16 index) 223 { 224 bt_dev_dbg(hdev, "RXQ:urbd1(%u) frbd_tag:%u status: 0x%x fixed:0x%x", 225 index, urbd1->frbd_tag, urbd1->status, urbd1->fixed); 226 } 227 228 static struct btintel_pcie_data *btintel_pcie_get_data(struct msix_entry *entry) 229 { 230 u8 queue = entry->entry; 231 struct msix_entry *entries = entry - queue; 232 233 return container_of(entries, struct btintel_pcie_data, msix_entries[0]); 234 } 235 236 /* Set the doorbell for TXQ to notify the device that @index (actually index-1) 237 * of the TFD is updated and ready to transmit. 238 */ 239 static void btintel_pcie_set_tx_db(struct btintel_pcie_data *data, u16 index) 240 { 241 u32 val; 242 243 val = index; 244 val |= (BTINTEL_PCIE_TX_DB_VEC << 16); 245 246 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val); 247 } 248 249 /* Copy the data to next(@tfd_index) data buffer and update the TFD(transfer 250 * descriptor) with the data length and the DMA address of the data buffer. 251 */ 252 static void btintel_pcie_prepare_tx(struct txq *txq, u16 tfd_index, 253 struct sk_buff *skb) 254 { 255 struct data_buf *buf; 256 struct tfd *tfd; 257 258 tfd = &txq->tfds[tfd_index]; 259 memset(tfd, 0, sizeof(*tfd)); 260 261 buf = &txq->bufs[tfd_index]; 262 263 tfd->size = skb->len; 264 tfd->addr = buf->data_p_addr; 265 266 /* Copy the outgoing data to DMA buffer */ 267 memcpy(buf->data, skb->data, tfd->size); 268 } 269 270 static inline void btintel_pcie_dump_debug_registers(struct hci_dev *hdev) 271 { 272 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 273 u16 cr_hia, cr_tia; 274 u32 reg, mbox_reg; 275 struct sk_buff *skb; 276 u8 buf[80]; 277 278 skb = alloc_skb(1024, GFP_ATOMIC); 279 if (!skb) 280 return; 281 282 strscpy(buf, "---- Dump of debug registers ---"); 283 bt_dev_dbg(hdev, "%s", buf); 284 skb_put_data(skb, buf, strlen(buf)); 285 286 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 287 snprintf(buf, sizeof(buf), "boot stage: 0x%8.8x", reg); 288 bt_dev_dbg(hdev, "%s", buf); 289 skb_put_data(skb, buf, strlen(buf)); 290 data->boot_stage_cache = reg; 291 292 if (reg & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_WARNING) 293 bt_dev_warn(hdev, "Controller device warning (boot_stage: 0x%8.8x)", reg); 294 295 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_STATUS_REG); 296 snprintf(buf, sizeof(buf), "ipc status: 0x%8.8x", reg); 297 skb_put_data(skb, buf, strlen(buf)); 298 bt_dev_dbg(hdev, "%s", buf); 299 300 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_CONTROL_REG); 301 snprintf(buf, sizeof(buf), "ipc control: 0x%8.8x", reg); 302 skb_put_data(skb, buf, strlen(buf)); 303 bt_dev_dbg(hdev, "%s", buf); 304 305 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IPC_SLEEP_CTL_REG); 306 snprintf(buf, sizeof(buf), "ipc sleep control: 0x%8.8x", reg); 307 skb_put_data(skb, buf, strlen(buf)); 308 bt_dev_dbg(hdev, "%s", buf); 309 310 /*Read the Mail box status and registers*/ 311 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MBOX_STATUS_REG); 312 snprintf(buf, sizeof(buf), "mbox status: 0x%8.8x", reg); 313 skb_put_data(skb, buf, strlen(buf)); 314 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX1) { 315 mbox_reg = btintel_pcie_rd_reg32(data, 316 BTINTEL_PCIE_CSR_MBOX_1_REG); 317 snprintf(buf, sizeof(buf), "mbox_1: 0x%8.8x", mbox_reg); 318 skb_put_data(skb, buf, strlen(buf)); 319 bt_dev_dbg(hdev, "%s", buf); 320 } 321 322 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX2) { 323 mbox_reg = btintel_pcie_rd_reg32(data, 324 BTINTEL_PCIE_CSR_MBOX_2_REG); 325 snprintf(buf, sizeof(buf), "mbox_2: 0x%8.8x", mbox_reg); 326 skb_put_data(skb, buf, strlen(buf)); 327 bt_dev_dbg(hdev, "%s", buf); 328 } 329 330 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX3) { 331 mbox_reg = btintel_pcie_rd_reg32(data, 332 BTINTEL_PCIE_CSR_MBOX_3_REG); 333 snprintf(buf, sizeof(buf), "mbox_3: 0x%8.8x", mbox_reg); 334 skb_put_data(skb, buf, strlen(buf)); 335 bt_dev_dbg(hdev, "%s", buf); 336 } 337 338 if (reg & BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX4) { 339 mbox_reg = btintel_pcie_rd_reg32(data, 340 BTINTEL_PCIE_CSR_MBOX_4_REG); 341 snprintf(buf, sizeof(buf), "mbox_4: 0x%8.8x", mbox_reg); 342 skb_put_data(skb, buf, strlen(buf)); 343 bt_dev_dbg(hdev, "%s", buf); 344 } 345 346 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM]; 347 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; 348 snprintf(buf, sizeof(buf), "rxq: cr_tia: %u cr_hia: %u", cr_tia, cr_hia); 349 skb_put_data(skb, buf, strlen(buf)); 350 bt_dev_dbg(hdev, "%s", buf); 351 352 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM]; 353 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM]; 354 snprintf(buf, sizeof(buf), "txq: cr_tia: %u cr_hia: %u", cr_tia, cr_hia); 355 skb_put_data(skb, buf, strlen(buf)); 356 bt_dev_dbg(hdev, "%s", buf); 357 strscpy(buf, "--------------------------------"); 358 bt_dev_dbg(hdev, "%s", buf); 359 360 hci_recv_diag(hdev, skb); 361 } 362 363 static int btintel_pcie_send_sync(struct btintel_pcie_data *data, 364 struct sk_buff *skb, u32 pkt_type, u16 opcode) 365 { 366 int ret; 367 u16 tfd_index; 368 u32 old_ctxt; 369 bool wait_on_alive = false; 370 struct hci_dev *hdev = data->hdev; 371 372 struct txq *txq = &data->txq; 373 374 tfd_index = data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM]; 375 376 if (tfd_index > txq->count) 377 return -ERANGE; 378 379 /* Firmware raises alive interrupt on HCI_OP_RESET or 380 * BTINTEL_HCI_OP_RESET 381 */ 382 wait_on_alive = (pkt_type == BTINTEL_PCIE_HCI_CMD_PKT && 383 (opcode == BTINTEL_HCI_OP_RESET || opcode == HCI_OP_RESET)); 384 385 if (wait_on_alive) { 386 data->gp0_received = false; 387 old_ctxt = data->alive_intr_ctxt; 388 data->alive_intr_ctxt = 389 (opcode == BTINTEL_HCI_OP_RESET ? BTINTEL_PCIE_INTEL_HCI_RESET1 : 390 BTINTEL_PCIE_HCI_RESET); 391 bt_dev_dbg(data->hdev, "sending cmd: 0x%4.4x alive context changed: %s -> %s", 392 opcode, btintel_pcie_alivectxt_state2str(old_ctxt), 393 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt)); 394 } 395 396 memcpy(skb_push(skb, BTINTEL_PCIE_HCI_TYPE_LEN), &pkt_type, 397 BTINTEL_PCIE_HCI_TYPE_LEN); 398 399 /* Prepare for TX. It updates the TFD with the length of data and 400 * address of the DMA buffer, and copy the data to the DMA buffer 401 */ 402 btintel_pcie_prepare_tx(txq, tfd_index, skb); 403 404 tfd_index = (tfd_index + 1) % txq->count; 405 data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM] = tfd_index; 406 407 /* Arm wait event condition */ 408 data->tx_wait_done = false; 409 410 /* Set the doorbell to notify the device */ 411 btintel_pcie_set_tx_db(data, tfd_index); 412 413 /* Wait for the complete interrupt - URBD0 */ 414 ret = wait_event_timeout(data->tx_wait_q, data->tx_wait_done, 415 msecs_to_jiffies(BTINTEL_PCIE_TX_WAIT_TIMEOUT_MS)); 416 if (!ret) { 417 bt_dev_err(data->hdev, "Timeout (%u ms) on tx completion", 418 BTINTEL_PCIE_TX_WAIT_TIMEOUT_MS); 419 btintel_pcie_dump_debug_registers(data->hdev); 420 return -ETIME; 421 } 422 423 if (wait_on_alive) { 424 ret = wait_event_timeout(data->gp0_wait_q, 425 data->gp0_received, 426 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT_MS)); 427 if (!ret) { 428 hdev->stat.err_tx++; 429 bt_dev_err(hdev, "Timeout (%u ms) on alive interrupt, alive context: %s", 430 BTINTEL_DEFAULT_INTR_TIMEOUT_MS, 431 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt)); 432 return -ETIME; 433 } 434 } 435 return 0; 436 } 437 438 /* Set the doorbell for RXQ to notify the device that @index (actually index-1) 439 * is available to receive the data 440 */ 441 static void btintel_pcie_set_rx_db(struct btintel_pcie_data *data, u16 index) 442 { 443 u32 val; 444 445 val = index; 446 val |= (BTINTEL_PCIE_RX_DB_VEC << 16); 447 448 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val); 449 } 450 451 /* Update the FRBD (free buffer descriptor) with the @frbd_index and the 452 * DMA address of the free buffer. 453 */ 454 static void btintel_pcie_prepare_rx(struct rxq *rxq, u16 frbd_index) 455 { 456 struct data_buf *buf; 457 struct frbd *frbd; 458 459 /* Get the buffer of the FRBD for DMA */ 460 buf = &rxq->bufs[frbd_index]; 461 462 frbd = &rxq->frbds[frbd_index]; 463 memset(frbd, 0, sizeof(*frbd)); 464 465 /* Update FRBD */ 466 frbd->tag = frbd_index; 467 frbd->addr = buf->data_p_addr; 468 } 469 470 static int btintel_pcie_submit_rx(struct btintel_pcie_data *data) 471 { 472 u16 frbd_index; 473 struct rxq *rxq = &data->rxq; 474 475 frbd_index = data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM]; 476 477 if (frbd_index > rxq->count) 478 return -ERANGE; 479 480 /* Prepare for RX submit. It updates the FRBD with the address of DMA 481 * buffer 482 */ 483 btintel_pcie_prepare_rx(rxq, frbd_index); 484 485 frbd_index = (frbd_index + 1) % rxq->count; 486 data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM] = frbd_index; 487 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM); 488 489 /* Set the doorbell to notify the device */ 490 btintel_pcie_set_rx_db(data, frbd_index); 491 492 return 0; 493 } 494 495 static int btintel_pcie_start_rx(struct btintel_pcie_data *data) 496 { 497 int i, ret; 498 struct rxq *rxq = &data->rxq; 499 500 /* Post (BTINTEL_PCIE_RX_DESCS_COUNT - 3) buffers to overcome the 501 * hardware issues leading to race condition at the firmware. 502 */ 503 504 for (i = 0; i < rxq->count - 3; i++) { 505 ret = btintel_pcie_submit_rx(data); 506 if (ret) 507 return ret; 508 } 509 510 return 0; 511 } 512 513 static void btintel_pcie_reset_ia(struct btintel_pcie_data *data) 514 { 515 memset(data->ia.tr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 516 memset(data->ia.tr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 517 memset(data->ia.cr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 518 memset(data->ia.cr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 519 } 520 521 static int btintel_pcie_reset_bt(struct btintel_pcie_data *data) 522 { 523 u32 reg; 524 int retry = 3; 525 526 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 527 528 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 529 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT | 530 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT); 531 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON; 532 533 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 534 535 do { 536 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 537 if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_STS) 538 break; 539 usleep_range(10000, 12000); 540 541 } while (--retry > 0); 542 usleep_range(10000, 12000); 543 544 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 545 546 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 547 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT | 548 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT); 549 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET; 550 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 551 usleep_range(10000, 12000); 552 553 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 554 bt_dev_dbg(data->hdev, "csr register after reset: 0x%8.8x", reg); 555 556 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 557 558 /* If shared hardware reset is success then boot stage register shall be 559 * set to 0 560 */ 561 return reg == 0 ? 0 : -ENODEV; 562 } 563 564 static void btintel_pcie_mac_init(struct btintel_pcie_data *data) 565 { 566 u32 reg; 567 568 /* Set MAC_INIT bit to start primary bootloader */ 569 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 570 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT | 571 BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON | 572 BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET); 573 reg |= (BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 574 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT); 575 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 576 } 577 578 static int btintel_pcie_get_mac_access(struct btintel_pcie_data *data) 579 { 580 u32 reg; 581 int retry = 15; 582 583 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 584 585 if (!(reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ)) { 586 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ; 587 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 588 } 589 590 do { 591 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 592 if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS) 593 return 0; 594 /* Need delay here for Target Access harwdware to settle down*/ 595 usleep_range(1000, 1200); 596 597 } while (--retry > 0); 598 599 return -ETIME; 600 } 601 602 static void btintel_pcie_release_mac_access(struct btintel_pcie_data *data) 603 { 604 u32 reg; 605 606 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 607 608 if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ) { 609 reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ; 610 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 611 } 612 } 613 614 static void *btintel_pcie_copy_tlv(void *dest, enum btintel_pcie_tlv_type type, 615 void *data, size_t size) 616 { 617 struct intel_tlv *tlv; 618 619 tlv = dest; 620 tlv->type = type; 621 tlv->len = size; 622 memcpy(tlv->val, data, tlv->len); 623 return dest + sizeof(*tlv) + size; 624 } 625 626 static int btintel_pcie_read_dram_buffers(struct btintel_pcie_data *data) 627 { 628 u32 offset, prev_size, wr_ptr_status, dump_size, data_len; 629 struct btintel_pcie_dbgc *dbgc = &data->dbgc; 630 struct hci_dev *hdev = data->hdev; 631 u8 *pdata, *p, buf_idx; 632 struct intel_tlv *tlv; 633 struct timespec64 now; 634 struct tm tm_now; 635 char fw_build[128]; 636 char ts[128]; 637 char vendor[64]; 638 char driver[64]; 639 640 if (!IS_ENABLED(CONFIG_DEV_COREDUMP)) 641 return -EOPNOTSUPP; 642 643 644 wr_ptr_status = btintel_pcie_rd_dev_mem(data, BTINTEL_PCIE_DBGC_CUR_DBGBUFF_STATUS); 645 offset = wr_ptr_status & BTINTEL_PCIE_DBG_OFFSET_BIT_MASK; 646 647 buf_idx = BTINTEL_PCIE_DBGC_DBG_BUF_IDX(wr_ptr_status); 648 if (buf_idx > dbgc->count) { 649 bt_dev_warn(hdev, "Buffer index is invalid"); 650 return -EINVAL; 651 } 652 653 prev_size = buf_idx * BTINTEL_PCIE_DBGC_BUFFER_SIZE; 654 if (prev_size + offset >= prev_size) 655 data->dmp_hdr.write_ptr = prev_size + offset; 656 else 657 return -EINVAL; 658 659 strscpy(vendor, "Vendor: Intel\n"); 660 snprintf(driver, sizeof(driver), "Driver: %s\n", 661 data->dmp_hdr.driver_name); 662 663 ktime_get_real_ts64(&now); 664 time64_to_tm(now.tv_sec, 0, &tm_now); 665 snprintf(ts, sizeof(ts), "Dump Time: %02d-%02d-%04ld %02d:%02d:%02d", 666 tm_now.tm_mday, tm_now.tm_mon + 1, tm_now.tm_year + 1900, 667 tm_now.tm_hour, tm_now.tm_min, tm_now.tm_sec); 668 669 snprintf(fw_build, sizeof(fw_build), 670 "Firmware Timestamp: Year %u WW %02u buildtype %u build %u", 671 2000 + (data->dmp_hdr.fw_timestamp >> 8), 672 data->dmp_hdr.fw_timestamp & 0xff, data->dmp_hdr.fw_build_type, 673 data->dmp_hdr.fw_build_num); 674 675 data_len = sizeof(*tlv) + sizeof(data->dmp_hdr.cnvi_bt) + 676 sizeof(*tlv) + sizeof(data->dmp_hdr.write_ptr) + 677 sizeof(*tlv) + sizeof(data->dmp_hdr.wrap_ctr) + 678 sizeof(*tlv) + sizeof(data->dmp_hdr.trigger_reason) + 679 sizeof(*tlv) + sizeof(data->dmp_hdr.fw_git_sha1) + 680 sizeof(*tlv) + sizeof(data->dmp_hdr.cnvr_top) + 681 sizeof(*tlv) + sizeof(data->dmp_hdr.cnvi_top) + 682 sizeof(*tlv) + strlen(ts) + 683 sizeof(*tlv) + strlen(fw_build) + 684 sizeof(*tlv) + strlen(vendor) + 685 sizeof(*tlv) + strlen(driver); 686 687 /* 688 * sizeof(u32) - signature 689 * sizeof(data_len) - to store tlv data size 690 * data_len - TLV data 691 */ 692 dump_size = sizeof(u32) + sizeof(data_len) + data_len; 693 694 695 /* Add debug buffers data length to dump size */ 696 dump_size += BTINTEL_PCIE_DBGC_BUFFER_SIZE * dbgc->count; 697 698 pdata = vmalloc(dump_size); 699 if (!pdata) 700 return -ENOMEM; 701 p = pdata; 702 703 *(u32 *)p = BTINTEL_PCIE_MAGIC_NUM; 704 p += sizeof(u32); 705 706 *(u32 *)p = data_len; 707 p += sizeof(u32); 708 709 710 p = btintel_pcie_copy_tlv(p, BTINTEL_VENDOR, vendor, strlen(vendor)); 711 p = btintel_pcie_copy_tlv(p, BTINTEL_DRIVER, driver, strlen(driver)); 712 p = btintel_pcie_copy_tlv(p, BTINTEL_DUMP_TIME, ts, strlen(ts)); 713 p = btintel_pcie_copy_tlv(p, BTINTEL_FW_BUILD, fw_build, 714 strlen(fw_build)); 715 p = btintel_pcie_copy_tlv(p, BTINTEL_CNVI_BT, &data->dmp_hdr.cnvi_bt, 716 sizeof(data->dmp_hdr.cnvi_bt)); 717 p = btintel_pcie_copy_tlv(p, BTINTEL_WRITE_PTR, &data->dmp_hdr.write_ptr, 718 sizeof(data->dmp_hdr.write_ptr)); 719 p = btintel_pcie_copy_tlv(p, BTINTEL_WRAP_CTR, &data->dmp_hdr.wrap_ctr, 720 sizeof(data->dmp_hdr.wrap_ctr)); 721 722 data->dmp_hdr.wrap_ctr = btintel_pcie_rd_dev_mem(data, 723 BTINTEL_PCIE_DBGC_DBGBUFF_WRAP_ARND); 724 725 p = btintel_pcie_copy_tlv(p, BTINTEL_TRIGGER_REASON, &data->dmp_hdr.trigger_reason, 726 sizeof(data->dmp_hdr.trigger_reason)); 727 p = btintel_pcie_copy_tlv(p, BTINTEL_FW_SHA, &data->dmp_hdr.fw_git_sha1, 728 sizeof(data->dmp_hdr.fw_git_sha1)); 729 p = btintel_pcie_copy_tlv(p, BTINTEL_CNVR_TOP, &data->dmp_hdr.cnvr_top, 730 sizeof(data->dmp_hdr.cnvr_top)); 731 p = btintel_pcie_copy_tlv(p, BTINTEL_CNVI_TOP, &data->dmp_hdr.cnvi_top, 732 sizeof(data->dmp_hdr.cnvi_top)); 733 734 memcpy(p, dbgc->bufs[0].data, dbgc->count * BTINTEL_PCIE_DBGC_BUFFER_SIZE); 735 dev_coredumpv(&hdev->dev, pdata, dump_size, GFP_KERNEL); 736 return 0; 737 } 738 739 static void btintel_pcie_dump_traces(struct hci_dev *hdev) 740 { 741 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 742 int ret = 0; 743 744 ret = btintel_pcie_get_mac_access(data); 745 if (ret) { 746 bt_dev_err(hdev, "Failed to get mac access: (%d)", ret); 747 return; 748 } 749 750 ret = btintel_pcie_read_dram_buffers(data); 751 752 btintel_pcie_release_mac_access(data); 753 754 if (ret) 755 bt_dev_err(hdev, "Failed to dump traces: (%d)", ret); 756 } 757 758 /* This function enables BT function by setting BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT bit in 759 * BTINTEL_PCIE_CSR_FUNC_CTRL_REG register and wait for MSI-X with 760 * BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0. 761 * Then the host reads firmware version from BTINTEL_CSR_F2D_MBX and the boot stage 762 * from BTINTEL_PCIE_CSR_BOOT_STAGE_REG. 763 */ 764 static int btintel_pcie_enable_bt(struct btintel_pcie_data *data) 765 { 766 int err; 767 u32 reg; 768 769 data->gp0_received = false; 770 771 /* Update the DMA address of CI struct to CSR */ 772 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_LSB_REG, 773 data->ci_p_addr & 0xffffffff); 774 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_MSB_REG, 775 (u64)data->ci_p_addr >> 32); 776 777 /* Reset the cached value of boot stage. it is updated by the MSI-X 778 * gp0 interrupt handler. 779 */ 780 data->boot_stage_cache = 0x0; 781 782 /* Set MAC_INIT bit to start primary bootloader */ 783 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 784 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT | 785 BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON | 786 BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET); 787 reg |= (BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 788 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT); 789 790 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 791 792 /* MAC is ready. Enable BT FUNC */ 793 btintel_pcie_set_reg_bits(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, 794 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT); 795 796 btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 797 798 /* wait for interrupt from the device after booting up to primary 799 * bootloader. 800 */ 801 data->alive_intr_ctxt = BTINTEL_PCIE_ROM; 802 err = wait_event_timeout(data->gp0_wait_q, data->gp0_received, 803 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT_MS)); 804 if (!err) 805 return -ETIME; 806 807 /* Check cached boot stage is BTINTEL_PCIE_CSR_BOOT_STAGE_ROM(BIT(0)) */ 808 if (~data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ROM) 809 return -ENODEV; 810 811 return 0; 812 } 813 814 static inline bool btintel_pcie_in_op(struct btintel_pcie_data *data) 815 { 816 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW; 817 } 818 819 static inline bool btintel_pcie_in_iml(struct btintel_pcie_data *data) 820 { 821 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_IML && 822 !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW); 823 } 824 825 static inline bool btintel_pcie_in_d3(struct btintel_pcie_data *data) 826 { 827 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY; 828 } 829 830 static inline bool btintel_pcie_in_d0(struct btintel_pcie_data *data) 831 { 832 return !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY); 833 } 834 835 static inline bool btintel_pcie_in_device_halt(struct btintel_pcie_data *data) 836 { 837 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_HALTED; 838 } 839 840 static void btintel_pcie_wr_sleep_cntrl(struct btintel_pcie_data *data, 841 u32 dxstate) 842 { 843 bt_dev_dbg(data->hdev, "writing sleep_ctl_reg: 0x%8.8x", dxstate); 844 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_IPC_SLEEP_CTL_REG, dxstate); 845 } 846 847 static int btintel_pcie_read_device_mem(struct btintel_pcie_data *data, 848 void *buf, u32 dev_addr, int len) 849 { 850 int err; 851 u32 *val = buf; 852 853 /* Get device mac access */ 854 err = btintel_pcie_get_mac_access(data); 855 if (err) { 856 bt_dev_err(data->hdev, "Failed to get mac access %d", err); 857 return err; 858 } 859 860 for (; len > 0; len -= 4, dev_addr += 4, val++) 861 *val = btintel_pcie_rd_dev_mem(data, dev_addr); 862 863 btintel_pcie_release_mac_access(data); 864 865 return 0; 866 } 867 868 static inline bool btintel_pcie_in_lockdown(struct btintel_pcie_data *data) 869 { 870 return (data->boot_stage_cache & 871 BTINTEL_PCIE_CSR_BOOT_STAGE_ROM_LOCKDOWN) || 872 (data->boot_stage_cache & 873 BTINTEL_PCIE_CSR_BOOT_STAGE_IML_LOCKDOWN); 874 } 875 876 static inline bool btintel_pcie_in_error(struct btintel_pcie_data *data) 877 { 878 if (data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_DEVICE_WARNING) 879 bt_dev_warn(data->hdev, "Controller device warning (boot_stage: 0x%8.8x)", 880 data->boot_stage_cache); 881 882 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ABORT_HANDLER; 883 } 884 885 static void btintel_pcie_msix_gp1_handler(struct btintel_pcie_data *data) 886 { 887 bt_dev_err(data->hdev, "Received gp1 mailbox interrupt"); 888 btintel_pcie_dump_debug_registers(data->hdev); 889 } 890 891 /* This function handles the MSI-X interrupt for gp0 cause (bit 0 in 892 * BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES) which is sent for boot stage and image response. 893 */ 894 static void btintel_pcie_msix_gp0_handler(struct btintel_pcie_data *data) 895 { 896 bool submit_rx, signal_waitq; 897 u32 reg, old_ctxt; 898 899 /* This interrupt is for three different causes and it is not easy to 900 * know what causes the interrupt. So, it compares each register value 901 * with cached value and update it before it wake up the queue. 902 */ 903 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 904 if (reg != data->boot_stage_cache) 905 data->boot_stage_cache = reg; 906 907 bt_dev_dbg(data->hdev, "Alive context: %s old_boot_stage: 0x%8.8x new_boot_stage: 0x%8.8x", 908 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt), 909 data->boot_stage_cache, reg); 910 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IMG_RESPONSE_REG); 911 if (reg != data->img_resp_cache) 912 data->img_resp_cache = reg; 913 914 if (btintel_pcie_in_error(data)) { 915 bt_dev_err(data->hdev, "Controller in error state (boot_stage: 0x%8.8x)", 916 data->boot_stage_cache); 917 btintel_pcie_dump_debug_registers(data->hdev); 918 return; 919 } 920 921 if (btintel_pcie_in_lockdown(data)) { 922 bt_dev_err(data->hdev, "Controller in lockdown state"); 923 btintel_pcie_dump_debug_registers(data->hdev); 924 return; 925 } 926 927 data->gp0_received = true; 928 929 old_ctxt = data->alive_intr_ctxt; 930 submit_rx = false; 931 signal_waitq = false; 932 933 switch (data->alive_intr_ctxt) { 934 case BTINTEL_PCIE_ROM: 935 data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL; 936 signal_waitq = true; 937 break; 938 case BTINTEL_PCIE_FW_DL: 939 /* Error case is already handled. Ideally control shall not 940 * reach here 941 */ 942 break; 943 case BTINTEL_PCIE_INTEL_HCI_RESET1: 944 if (btintel_pcie_in_op(data)) { 945 submit_rx = true; 946 signal_waitq = true; 947 break; 948 } 949 950 if (btintel_pcie_in_iml(data)) { 951 submit_rx = true; 952 signal_waitq = true; 953 data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL; 954 break; 955 } 956 break; 957 case BTINTEL_PCIE_INTEL_HCI_RESET2: 958 if (btintel_test_and_clear_flag(data->hdev, INTEL_WAIT_FOR_D0)) { 959 btintel_wake_up_flag(data->hdev, INTEL_WAIT_FOR_D0); 960 data->alive_intr_ctxt = BTINTEL_PCIE_D0; 961 } 962 break; 963 case BTINTEL_PCIE_D0: 964 if (btintel_pcie_in_d3(data)) { 965 data->alive_intr_ctxt = BTINTEL_PCIE_D3; 966 signal_waitq = true; 967 break; 968 } 969 break; 970 case BTINTEL_PCIE_D3: 971 if (btintel_pcie_in_d0(data)) { 972 data->alive_intr_ctxt = BTINTEL_PCIE_D0; 973 submit_rx = true; 974 signal_waitq = true; 975 break; 976 } 977 break; 978 case BTINTEL_PCIE_HCI_RESET: 979 data->alive_intr_ctxt = BTINTEL_PCIE_D0; 980 submit_rx = true; 981 signal_waitq = true; 982 break; 983 default: 984 bt_dev_err(data->hdev, "Unknown state: 0x%2.2x", 985 data->alive_intr_ctxt); 986 break; 987 } 988 989 if (submit_rx) { 990 btintel_pcie_reset_ia(data); 991 btintel_pcie_start_rx(data); 992 } 993 994 if (signal_waitq) { 995 bt_dev_dbg(data->hdev, "wake up gp0 wait_q"); 996 wake_up(&data->gp0_wait_q); 997 } 998 999 if (old_ctxt != data->alive_intr_ctxt) 1000 bt_dev_dbg(data->hdev, "alive context changed: %s -> %s", 1001 btintel_pcie_alivectxt_state2str(old_ctxt), 1002 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt)); 1003 } 1004 1005 /* This function handles the MSX-X interrupt for rx queue 0 which is for TX 1006 */ 1007 static void btintel_pcie_msix_tx_handle(struct btintel_pcie_data *data) 1008 { 1009 u16 cr_tia, cr_hia; 1010 struct txq *txq; 1011 struct urbd0 *urbd0; 1012 1013 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM]; 1014 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM]; 1015 1016 if (cr_tia == cr_hia) 1017 return; 1018 1019 txq = &data->txq; 1020 1021 while (cr_tia != cr_hia) { 1022 data->tx_wait_done = true; 1023 wake_up(&data->tx_wait_q); 1024 1025 urbd0 = &txq->urbd0s[cr_tia]; 1026 1027 if (urbd0->tfd_index > txq->count) 1028 return; 1029 1030 cr_tia = (cr_tia + 1) % txq->count; 1031 data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM] = cr_tia; 1032 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_TXQ_NUM); 1033 } 1034 } 1035 1036 static int btintel_pcie_recv_event(struct hci_dev *hdev, struct sk_buff *skb) 1037 { 1038 struct hci_event_hdr *hdr = (void *)skb->data; 1039 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 1040 1041 if (skb->len > HCI_EVENT_HDR_SIZE && hdr->evt == 0xff && 1042 hdr->plen > 0) { 1043 const void *ptr = skb->data + HCI_EVENT_HDR_SIZE + 1; 1044 unsigned int len = skb->len - HCI_EVENT_HDR_SIZE - 1; 1045 1046 if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) { 1047 switch (skb->data[2]) { 1048 case 0x02: 1049 /* When switching to the operational firmware 1050 * the device sends a vendor specific event 1051 * indicating that the bootup completed. 1052 */ 1053 btintel_bootup(hdev, ptr, len); 1054 1055 /* If bootup event is from operational image, 1056 * driver needs to write sleep control register to 1057 * move into D0 state 1058 */ 1059 if (btintel_pcie_in_op(data)) { 1060 btintel_pcie_wr_sleep_cntrl(data, BTINTEL_PCIE_STATE_D0); 1061 data->alive_intr_ctxt = BTINTEL_PCIE_INTEL_HCI_RESET2; 1062 kfree_skb(skb); 1063 return 0; 1064 } 1065 1066 if (btintel_pcie_in_iml(data)) { 1067 /* In case of IML, there is no concept 1068 * of D0 transition. Just mimic as if 1069 * IML moved to D0 by clearing INTEL_WAIT_FOR_D0 1070 * bit and waking up the task waiting on 1071 * INTEL_WAIT_FOR_D0. This is required 1072 * as intel_boot() is common function for 1073 * both IML and OP image loading. 1074 */ 1075 if (btintel_test_and_clear_flag(data->hdev, 1076 INTEL_WAIT_FOR_D0)) 1077 btintel_wake_up_flag(data->hdev, 1078 INTEL_WAIT_FOR_D0); 1079 } 1080 kfree_skb(skb); 1081 return 0; 1082 case 0x06: 1083 /* When the firmware loading completes the 1084 * device sends out a vendor specific event 1085 * indicating the result of the firmware 1086 * loading. 1087 */ 1088 btintel_secure_send_result(hdev, ptr, len); 1089 kfree_skb(skb); 1090 return 0; 1091 } 1092 } 1093 1094 /* This is a debug event that comes from IML and OP image when it 1095 * starts execution. There is no need pass this event to stack. 1096 */ 1097 if (skb->data[2] == 0x97) { 1098 hci_recv_diag(hdev, skb); 1099 return 0; 1100 } 1101 } 1102 1103 return hci_recv_frame(hdev, skb); 1104 } 1105 /* Process the received rx data 1106 * It check the frame header to identify the data type and create skb 1107 * and calling HCI API 1108 */ 1109 static int btintel_pcie_recv_frame(struct btintel_pcie_data *data, 1110 struct sk_buff *skb) 1111 { 1112 int ret; 1113 u8 pkt_type; 1114 u16 plen; 1115 u32 pcie_pkt_type; 1116 void *pdata; 1117 struct hci_dev *hdev = data->hdev; 1118 1119 spin_lock(&data->hci_rx_lock); 1120 1121 /* The first 4 bytes indicates the Intel PCIe specific packet type */ 1122 pdata = skb_pull_data(skb, BTINTEL_PCIE_HCI_TYPE_LEN); 1123 if (!pdata) { 1124 bt_dev_err(hdev, "Corrupted packet received"); 1125 ret = -EILSEQ; 1126 goto exit_error; 1127 } 1128 1129 pcie_pkt_type = get_unaligned_le32(pdata); 1130 1131 switch (pcie_pkt_type) { 1132 case BTINTEL_PCIE_HCI_ACL_PKT: 1133 if (skb->len >= HCI_ACL_HDR_SIZE) { 1134 plen = HCI_ACL_HDR_SIZE + __le16_to_cpu(hci_acl_hdr(skb)->dlen); 1135 pkt_type = HCI_ACLDATA_PKT; 1136 } else { 1137 bt_dev_err(hdev, "ACL packet is too short"); 1138 ret = -EILSEQ; 1139 goto exit_error; 1140 } 1141 break; 1142 1143 case BTINTEL_PCIE_HCI_SCO_PKT: 1144 if (skb->len >= HCI_SCO_HDR_SIZE) { 1145 plen = HCI_SCO_HDR_SIZE + hci_sco_hdr(skb)->dlen; 1146 pkt_type = HCI_SCODATA_PKT; 1147 } else { 1148 bt_dev_err(hdev, "SCO packet is too short"); 1149 ret = -EILSEQ; 1150 goto exit_error; 1151 } 1152 break; 1153 1154 case BTINTEL_PCIE_HCI_EVT_PKT: 1155 if (skb->len >= HCI_EVENT_HDR_SIZE) { 1156 plen = HCI_EVENT_HDR_SIZE + hci_event_hdr(skb)->plen; 1157 pkt_type = HCI_EVENT_PKT; 1158 } else { 1159 bt_dev_err(hdev, "Event packet is too short"); 1160 ret = -EILSEQ; 1161 goto exit_error; 1162 } 1163 break; 1164 1165 case BTINTEL_PCIE_HCI_ISO_PKT: 1166 if (skb->len >= HCI_ISO_HDR_SIZE) { 1167 plen = HCI_ISO_HDR_SIZE + __le16_to_cpu(hci_iso_hdr(skb)->dlen); 1168 pkt_type = HCI_ISODATA_PKT; 1169 } else { 1170 bt_dev_err(hdev, "ISO packet is too short"); 1171 ret = -EILSEQ; 1172 goto exit_error; 1173 } 1174 break; 1175 1176 default: 1177 bt_dev_err(hdev, "Invalid packet type received: 0x%4.4x", 1178 pcie_pkt_type); 1179 ret = -EINVAL; 1180 goto exit_error; 1181 } 1182 1183 if (skb->len < plen) { 1184 bt_dev_err(hdev, "Received corrupted packet. type: 0x%2.2x", 1185 pkt_type); 1186 ret = -EILSEQ; 1187 goto exit_error; 1188 } 1189 1190 bt_dev_dbg(hdev, "pkt_type: 0x%2.2x len: %u", pkt_type, plen); 1191 1192 hci_skb_pkt_type(skb) = pkt_type; 1193 hdev->stat.byte_rx += plen; 1194 skb_trim(skb, plen); 1195 1196 if (pcie_pkt_type == BTINTEL_PCIE_HCI_EVT_PKT) 1197 ret = btintel_pcie_recv_event(hdev, skb); 1198 else 1199 ret = hci_recv_frame(hdev, skb); 1200 skb = NULL; /* skb is freed in the callee */ 1201 1202 exit_error: 1203 kfree_skb(skb); 1204 1205 if (ret) 1206 hdev->stat.err_rx++; 1207 1208 spin_unlock(&data->hci_rx_lock); 1209 1210 return ret; 1211 } 1212 1213 static void btintel_pcie_read_hwexp(struct btintel_pcie_data *data) 1214 { 1215 int len, err, offset, pending; 1216 struct sk_buff *skb; 1217 u8 *buf, prefix[64]; 1218 u32 addr, val; 1219 u16 pkt_len; 1220 1221 struct tlv { 1222 u8 type; 1223 __le16 len; 1224 u8 val[]; 1225 } __packed; 1226 1227 struct tlv *tlv; 1228 1229 switch (data->dmp_hdr.cnvi_top & 0xfff) { 1230 case BTINTEL_CNVI_BLAZARI: 1231 case BTINTEL_CNVI_BLAZARIW: 1232 /* only from step B0 onwards */ 1233 if (INTEL_CNVX_TOP_STEP(data->dmp_hdr.cnvi_top) != 0x01) 1234 return; 1235 len = BTINTEL_PCIE_BLZR_HWEXP_SIZE; /* exception data length */ 1236 addr = BTINTEL_PCIE_BLZR_HWEXP_DMP_ADDR; 1237 break; 1238 case BTINTEL_CNVI_SCP: 1239 len = BTINTEL_PCIE_SCP_HWEXP_SIZE; 1240 addr = BTINTEL_PCIE_SCP_HWEXP_DMP_ADDR; 1241 break; 1242 case BTINTEL_CNVI_SCP2: 1243 case BTINTEL_CNVI_SCP2F: 1244 len = BTINTEL_PCIE_SCP2_HWEXP_SIZE; 1245 addr = BTINTEL_PCIE_SCP2_HWEXP_DMP_ADDR; 1246 break; 1247 default: 1248 bt_dev_err(data->hdev, "Unsupported cnvi 0x%8.8x", data->dmp_hdr.cnvi_top); 1249 return; 1250 } 1251 1252 buf = kzalloc(len, GFP_KERNEL); 1253 if (!buf) 1254 goto exit_on_error; 1255 1256 btintel_pcie_mac_init(data); 1257 1258 err = btintel_pcie_read_device_mem(data, buf, addr, len); 1259 if (err) 1260 goto exit_on_error; 1261 1262 val = get_unaligned_le32(buf); 1263 if (val != BTINTEL_PCIE_MAGIC_NUM) { 1264 bt_dev_err(data->hdev, "Invalid exception dump signature: 0x%8.8x", 1265 val); 1266 goto exit_on_error; 1267 } 1268 1269 snprintf(prefix, sizeof(prefix), "Bluetooth: %s: ", bt_dev_name(data->hdev)); 1270 1271 offset = 4; 1272 do { 1273 pending = len - offset; 1274 if (pending < sizeof(*tlv)) 1275 break; 1276 tlv = (struct tlv *)(buf + offset); 1277 1278 /* If type == 0, then there are no more TLVs to be parsed */ 1279 if (!tlv->type) { 1280 bt_dev_dbg(data->hdev, "Invalid TLV type 0"); 1281 break; 1282 } 1283 pkt_len = le16_to_cpu(tlv->len); 1284 offset += sizeof(*tlv); 1285 pending = len - offset; 1286 if (pkt_len > pending) 1287 break; 1288 1289 offset += pkt_len; 1290 1291 /* Only TLVs of type == 1 are HCI events, no need to process other 1292 * TLVs 1293 */ 1294 if (tlv->type != 1) 1295 continue; 1296 1297 bt_dev_dbg(data->hdev, "TLV packet length: %u", pkt_len); 1298 if (pkt_len > HCI_MAX_EVENT_SIZE) 1299 break; 1300 skb = bt_skb_alloc(pkt_len, GFP_KERNEL); 1301 if (!skb) 1302 goto exit_on_error; 1303 hci_skb_pkt_type(skb) = HCI_EVENT_PKT; 1304 skb_put_data(skb, tlv->val, pkt_len); 1305 1306 /* copy Intel specific pcie packet type */ 1307 val = BTINTEL_PCIE_HCI_EVT_PKT; 1308 memcpy(skb_push(skb, BTINTEL_PCIE_HCI_TYPE_LEN), &val, 1309 BTINTEL_PCIE_HCI_TYPE_LEN); 1310 1311 print_hex_dump(KERN_DEBUG, prefix, DUMP_PREFIX_OFFSET, 16, 1, 1312 tlv->val, pkt_len, false); 1313 1314 btintel_pcie_recv_frame(data, skb); 1315 } while (offset < len); 1316 1317 exit_on_error: 1318 kfree(buf); 1319 } 1320 1321 static void btintel_pcie_msix_hw_exp_handler(struct btintel_pcie_data *data) 1322 { 1323 bt_dev_err(data->hdev, "Received hw exception interrupt"); 1324 1325 if (test_and_set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags)) 1326 return; 1327 1328 if (test_and_set_bit(BTINTEL_PCIE_HWEXP_INPROGRESS, &data->flags)) 1329 return; 1330 1331 /* Trigger device core dump when there is HW exception */ 1332 if (!test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags)) 1333 data->dmp_hdr.trigger_reason = BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT; 1334 1335 queue_work(data->workqueue, &data->rx_work); 1336 } 1337 1338 static void btintel_pcie_rx_work(struct work_struct *work) 1339 { 1340 struct btintel_pcie_data *data = container_of(work, 1341 struct btintel_pcie_data, rx_work); 1342 struct sk_buff *skb; 1343 1344 if (test_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags)) { 1345 btintel_pcie_dump_traces(data->hdev); 1346 clear_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags); 1347 } 1348 1349 if (test_bit(BTINTEL_PCIE_HWEXP_INPROGRESS, &data->flags)) { 1350 /* Unlike usb products, controller will not send hardware 1351 * exception event on exception. Instead controller writes the 1352 * hardware event to device memory along with optional debug 1353 * events, raises MSIX and halts. Driver shall read the 1354 * exception event from device memory and passes it stack for 1355 * further processing. 1356 */ 1357 btintel_pcie_read_hwexp(data); 1358 clear_bit(BTINTEL_PCIE_HWEXP_INPROGRESS, &data->flags); 1359 } 1360 1361 /* Process the sk_buf in queue and send to the HCI layer */ 1362 while ((skb = skb_dequeue(&data->rx_skb_q))) { 1363 btintel_pcie_recv_frame(data, skb); 1364 } 1365 } 1366 1367 /* create sk_buff with data and save it to queue and start RX work */ 1368 static int btintel_pcie_submit_rx_work(struct btintel_pcie_data *data, u8 status, 1369 void *buf) 1370 { 1371 int ret, len; 1372 struct rfh_hdr *rfh_hdr; 1373 struct sk_buff *skb; 1374 1375 rfh_hdr = buf; 1376 1377 len = rfh_hdr->packet_len; 1378 if (len <= 0) { 1379 ret = -EINVAL; 1380 goto resubmit; 1381 } 1382 1383 /* Remove RFH header */ 1384 buf += sizeof(*rfh_hdr); 1385 1386 skb = alloc_skb(len, GFP_ATOMIC); 1387 if (!skb) 1388 goto resubmit; 1389 1390 skb_put_data(skb, buf, len); 1391 skb_queue_tail(&data->rx_skb_q, skb); 1392 queue_work(data->workqueue, &data->rx_work); 1393 1394 resubmit: 1395 ret = btintel_pcie_submit_rx(data); 1396 1397 return ret; 1398 } 1399 1400 /* Handles the MSI-X interrupt for rx queue 1 which is for RX */ 1401 static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) 1402 { 1403 u16 cr_hia, cr_tia; 1404 struct rxq *rxq; 1405 struct urbd1 *urbd1; 1406 struct data_buf *buf; 1407 int ret; 1408 struct hci_dev *hdev = data->hdev; 1409 1410 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM]; 1411 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; 1412 1413 bt_dev_dbg(hdev, "RXQ: cr_hia: %u cr_tia: %u", cr_hia, cr_tia); 1414 1415 /* Check CR_TIA and CR_HIA for change */ 1416 if (cr_tia == cr_hia) 1417 return; 1418 1419 rxq = &data->rxq; 1420 1421 /* The firmware sends multiple CD in a single MSI-X and it needs to 1422 * process all received CDs in this interrupt. 1423 */ 1424 while (cr_tia != cr_hia) { 1425 urbd1 = &rxq->urbd1s[cr_tia]; 1426 ipc_print_urbd1(data->hdev, urbd1, cr_tia); 1427 1428 buf = &rxq->bufs[urbd1->frbd_tag]; 1429 if (!buf) { 1430 bt_dev_err(hdev, "RXQ: failed to get the DMA buffer for %d", 1431 urbd1->frbd_tag); 1432 return; 1433 } 1434 1435 ret = btintel_pcie_submit_rx_work(data, urbd1->status, 1436 buf->data); 1437 if (ret) { 1438 bt_dev_err(hdev, "RXQ: failed to submit rx request"); 1439 return; 1440 } 1441 1442 cr_tia = (cr_tia + 1) % rxq->count; 1443 data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM] = cr_tia; 1444 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM); 1445 } 1446 } 1447 1448 static inline bool btintel_pcie_is_rxq_empty(struct btintel_pcie_data *data) 1449 { 1450 return data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM] == data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; 1451 } 1452 1453 static inline bool btintel_pcie_is_txackq_empty(struct btintel_pcie_data *data) 1454 { 1455 return data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM] == data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM]; 1456 } 1457 1458 static irqreturn_t btintel_pcie_irq_msix_handler(int irq, void *dev_id) 1459 { 1460 struct msix_entry *entry = dev_id; 1461 struct btintel_pcie_data *data = btintel_pcie_get_data(entry); 1462 u32 intr_fh, intr_hw; 1463 1464 spin_lock(&data->irq_lock); 1465 intr_fh = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES); 1466 intr_hw = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES); 1467 1468 /* Clear causes registers to avoid being handling the same cause */ 1469 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES, intr_fh); 1470 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES, intr_hw); 1471 spin_unlock(&data->irq_lock); 1472 1473 if (unlikely(!(intr_fh | intr_hw))) { 1474 /* Ignore interrupt, inta == 0 */ 1475 return IRQ_NONE; 1476 } 1477 1478 /* This interrupt is raised when there is an hardware exception */ 1479 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_HWEXP) 1480 btintel_pcie_msix_hw_exp_handler(data); 1481 1482 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP1) 1483 btintel_pcie_msix_gp1_handler(data); 1484 1485 1486 /* For TX */ 1487 if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0) { 1488 btintel_pcie_msix_tx_handle(data); 1489 if (!btintel_pcie_is_rxq_empty(data)) 1490 btintel_pcie_msix_rx_handle(data); 1491 } 1492 1493 /* For RX */ 1494 if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1) { 1495 btintel_pcie_msix_rx_handle(data); 1496 if (!btintel_pcie_is_txackq_empty(data)) 1497 btintel_pcie_msix_tx_handle(data); 1498 } 1499 1500 /* This interrupt is triggered by the firmware after updating 1501 * boot_stage register and image_response register 1502 */ 1503 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0) 1504 btintel_pcie_msix_gp0_handler(data); 1505 1506 /* 1507 * Before sending the interrupt the HW disables it to prevent a nested 1508 * interrupt. This is done by writing 1 to the corresponding bit in 1509 * the mask register. After handling the interrupt, it should be 1510 * re-enabled by clearing this bit. This register is defined as write 1 1511 * clear (W1C) register, meaning that it's cleared by writing 1 1512 * to the bit. 1513 */ 1514 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_AUTOMASK_ST, 1515 BIT(entry->entry)); 1516 1517 return IRQ_HANDLED; 1518 } 1519 1520 /* This function requests the irq for MSI-X and registers the handlers per irq. 1521 * Currently, it requests only 1 irq for all interrupt causes. 1522 */ 1523 static int btintel_pcie_setup_irq(struct btintel_pcie_data *data) 1524 { 1525 int err; 1526 int num_irqs, i; 1527 1528 for (i = 0; i < BTINTEL_PCIE_MSIX_VEC_MAX; i++) 1529 data->msix_entries[i].entry = i; 1530 1531 num_irqs = pci_alloc_irq_vectors(data->pdev, BTINTEL_PCIE_MSIX_VEC_MIN, 1532 BTINTEL_PCIE_MSIX_VEC_MAX, PCI_IRQ_MSIX); 1533 if (num_irqs < 0) 1534 return num_irqs; 1535 1536 data->alloc_vecs = num_irqs; 1537 data->msix_enabled = 1; 1538 data->def_irq = 0; 1539 1540 /* setup irq handler */ 1541 for (i = 0; i < data->alloc_vecs; i++) { 1542 struct msix_entry *msix_entry; 1543 1544 msix_entry = &data->msix_entries[i]; 1545 msix_entry->vector = pci_irq_vector(data->pdev, i); 1546 1547 err = devm_request_threaded_irq(&data->pdev->dev, 1548 msix_entry->vector, 1549 NULL, 1550 btintel_pcie_irq_msix_handler, 1551 IRQF_ONESHOT | IRQF_SHARED, 1552 KBUILD_MODNAME, 1553 msix_entry); 1554 if (err) { 1555 pci_free_irq_vectors(data->pdev); 1556 data->alloc_vecs = 0; 1557 return err; 1558 } 1559 } 1560 return 0; 1561 } 1562 1563 struct btintel_pcie_causes_list { 1564 u32 cause; 1565 u32 mask_reg; 1566 u8 cause_num; 1567 }; 1568 1569 static struct btintel_pcie_causes_list causes_list[] = { 1570 { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x00 }, 1571 { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x01 }, 1572 { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x20 }, 1573 { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_HWEXP, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x23 }, 1574 }; 1575 1576 /* This function configures the interrupt masks for both HW_INT_CAUSES and 1577 * FH_INT_CAUSES which are meaningful to us. 1578 * 1579 * After resetting BT function via PCIE FLR or FUNC_CTRL reset, the driver 1580 * need to call this function again to configure since the masks 1581 * are reset to 0xFFFFFFFF after reset. 1582 */ 1583 static void btintel_pcie_config_msix(struct btintel_pcie_data *data) 1584 { 1585 int i; 1586 int val = data->def_irq | BTINTEL_PCIE_MSIX_NON_AUTO_CLEAR_CAUSE; 1587 1588 /* Set Non Auto Clear Cause */ 1589 for (i = 0; i < ARRAY_SIZE(causes_list); i++) { 1590 btintel_pcie_wr_reg8(data, 1591 BTINTEL_PCIE_CSR_MSIX_IVAR(causes_list[i].cause_num), 1592 val); 1593 btintel_pcie_clr_reg_bits(data, 1594 causes_list[i].mask_reg, 1595 causes_list[i].cause); 1596 } 1597 1598 /* Save the initial interrupt mask */ 1599 data->fh_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK); 1600 data->hw_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK); 1601 } 1602 1603 static int btintel_pcie_config_pcie(struct pci_dev *pdev, 1604 struct btintel_pcie_data *data) 1605 { 1606 int err; 1607 1608 err = pcim_enable_device(pdev); 1609 if (err) 1610 return err; 1611 1612 pci_set_master(pdev); 1613 1614 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1615 if (err) { 1616 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1617 if (err) 1618 return err; 1619 } 1620 1621 data->base_addr = pcim_iomap_region(pdev, 0, KBUILD_MODNAME); 1622 if (IS_ERR(data->base_addr)) 1623 return PTR_ERR(data->base_addr); 1624 1625 err = btintel_pcie_setup_irq(data); 1626 if (err) 1627 return err; 1628 1629 /* Configure MSI-X with causes list */ 1630 btintel_pcie_config_msix(data); 1631 1632 return 0; 1633 } 1634 1635 static void btintel_pcie_init_ci(struct btintel_pcie_data *data, 1636 struct ctx_info *ci) 1637 { 1638 ci->version = 0x1; 1639 ci->size = sizeof(*ci); 1640 ci->config = 0x0000; 1641 ci->addr_cr_hia = data->ia.cr_hia_p_addr; 1642 ci->addr_tr_tia = data->ia.tr_tia_p_addr; 1643 ci->addr_cr_tia = data->ia.cr_tia_p_addr; 1644 ci->addr_tr_hia = data->ia.tr_hia_p_addr; 1645 ci->num_cr_ia = BTINTEL_PCIE_NUM_QUEUES; 1646 ci->num_tr_ia = BTINTEL_PCIE_NUM_QUEUES; 1647 ci->addr_urbdq0 = data->txq.urbd0s_p_addr; 1648 ci->addr_tfdq = data->txq.tfds_p_addr; 1649 ci->num_tfdq = data->txq.count; 1650 ci->num_urbdq0 = data->txq.count; 1651 ci->tfdq_db_vec = BTINTEL_PCIE_TXQ_NUM; 1652 ci->urbdq0_db_vec = BTINTEL_PCIE_TXQ_NUM; 1653 ci->rbd_size = BTINTEL_PCIE_RBD_SIZE_4K; 1654 ci->addr_frbdq = data->rxq.frbds_p_addr; 1655 ci->num_frbdq = data->rxq.count; 1656 ci->frbdq_db_vec = BTINTEL_PCIE_RXQ_NUM; 1657 ci->addr_urbdq1 = data->rxq.urbd1s_p_addr; 1658 ci->num_urbdq1 = data->rxq.count; 1659 ci->urbdq_db_vec = BTINTEL_PCIE_RXQ_NUM; 1660 1661 ci->dbg_output_mode = 0x01; 1662 ci->dbgc_addr = data->dbgc.frag_p_addr; 1663 ci->dbgc_size = data->dbgc.frag_size; 1664 ci->dbg_preset = 0x00; 1665 } 1666 1667 static void btintel_pcie_free_txq_bufs(struct btintel_pcie_data *data, 1668 struct txq *txq) 1669 { 1670 /* Free data buffers first */ 1671 dma_free_coherent(&data->pdev->dev, txq->count * BTINTEL_PCIE_BUFFER_SIZE, 1672 txq->buf_v_addr, txq->buf_p_addr); 1673 kfree(txq->bufs); 1674 } 1675 1676 static int btintel_pcie_setup_txq_bufs(struct btintel_pcie_data *data, 1677 struct txq *txq) 1678 { 1679 int i; 1680 struct data_buf *buf; 1681 1682 /* Allocate the same number of buffers as the descriptor */ 1683 txq->bufs = kmalloc_objs(*buf, txq->count); 1684 if (!txq->bufs) 1685 return -ENOMEM; 1686 1687 /* Allocate full chunk of data buffer for DMA first and do indexing and 1688 * initialization next, so it can be freed easily 1689 */ 1690 txq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev, 1691 txq->count * BTINTEL_PCIE_BUFFER_SIZE, 1692 &txq->buf_p_addr, 1693 GFP_KERNEL | __GFP_NOWARN); 1694 if (!txq->buf_v_addr) { 1695 kfree(txq->bufs); 1696 return -ENOMEM; 1697 } 1698 1699 /* Setup the allocated DMA buffer to bufs. Each data_buf should 1700 * have virtual address and physical address 1701 */ 1702 for (i = 0; i < txq->count; i++) { 1703 buf = &txq->bufs[i]; 1704 buf->data_p_addr = txq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 1705 buf->data = txq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 1706 } 1707 1708 return 0; 1709 } 1710 1711 static void btintel_pcie_free_rxq_bufs(struct btintel_pcie_data *data, 1712 struct rxq *rxq) 1713 { 1714 /* Free data buffers first */ 1715 dma_free_coherent(&data->pdev->dev, rxq->count * BTINTEL_PCIE_BUFFER_SIZE, 1716 rxq->buf_v_addr, rxq->buf_p_addr); 1717 kfree(rxq->bufs); 1718 } 1719 1720 static int btintel_pcie_setup_rxq_bufs(struct btintel_pcie_data *data, 1721 struct rxq *rxq) 1722 { 1723 int i; 1724 struct data_buf *buf; 1725 1726 /* Allocate the same number of buffers as the descriptor */ 1727 rxq->bufs = kmalloc_objs(*buf, rxq->count); 1728 if (!rxq->bufs) 1729 return -ENOMEM; 1730 1731 /* Allocate full chunk of data buffer for DMA first and do indexing and 1732 * initialization next, so it can be freed easily 1733 */ 1734 rxq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev, 1735 rxq->count * BTINTEL_PCIE_BUFFER_SIZE, 1736 &rxq->buf_p_addr, 1737 GFP_KERNEL | __GFP_NOWARN); 1738 if (!rxq->buf_v_addr) { 1739 kfree(rxq->bufs); 1740 return -ENOMEM; 1741 } 1742 1743 /* Setup the allocated DMA buffer to bufs. Each data_buf should 1744 * have virtual address and physical address 1745 */ 1746 for (i = 0; i < rxq->count; i++) { 1747 buf = &rxq->bufs[i]; 1748 buf->data_p_addr = rxq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 1749 buf->data = rxq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 1750 } 1751 1752 return 0; 1753 } 1754 1755 static void btintel_pcie_free(struct btintel_pcie_data *data) 1756 { 1757 btintel_pcie_free_rxq_bufs(data, &data->rxq); 1758 btintel_pcie_free_txq_bufs(data, &data->txq); 1759 1760 dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr); 1761 dma_pool_destroy(data->dma_pool); 1762 } 1763 1764 /* Allocate tx and rx queues, any related data structures and buffers. 1765 */ 1766 static int btintel_pcie_alloc(struct btintel_pcie_data *data) 1767 { 1768 int err = 0; 1769 size_t total; 1770 dma_addr_t p_addr; 1771 void *v_addr; 1772 size_t tfd_size, frbd_size, ctx_size, ci_size, urbd0_size, urbd1_size; 1773 1774 /* Allocate the chunk of DMA memory for descriptors, index array, and 1775 * context information, instead of allocating individually. 1776 * The DMA memory for data buffer is allocated while setting up the 1777 * each queue. 1778 * 1779 * Total size is sum of the following and each of the individual sizes 1780 * are aligned to 128 bytes before adding up. 1781 * 1782 * + size of TFD * Number of descriptors in queue 1783 * + size of URBD0 * Number of descriptors in queue 1784 * + size of FRBD * Number of descriptors in queue 1785 * + size of URBD1 * Number of descriptors in queue 1786 * + size of index * Number of queues(2) * type of index array(4) 1787 * + size of context information 1788 */ 1789 tfd_size = ALIGN(sizeof(struct tfd) * BTINTEL_PCIE_TX_DESCS_COUNT, 1790 BTINTEL_PCIE_DMA_ALIGN_128B); 1791 urbd0_size = ALIGN(sizeof(struct urbd0) * BTINTEL_PCIE_TX_DESCS_COUNT, 1792 BTINTEL_PCIE_DMA_ALIGN_128B); 1793 1794 frbd_size = ALIGN(sizeof(struct frbd) * BTINTEL_PCIE_RX_DESCS_COUNT, 1795 BTINTEL_PCIE_DMA_ALIGN_128B); 1796 urbd1_size = ALIGN(sizeof(struct urbd1) * BTINTEL_PCIE_RX_DESCS_COUNT, 1797 BTINTEL_PCIE_DMA_ALIGN_128B); 1798 1799 ci_size = ALIGN(sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES, 1800 BTINTEL_PCIE_DMA_ALIGN_128B); 1801 1802 ctx_size = ALIGN(sizeof(struct ctx_info), BTINTEL_PCIE_DMA_ALIGN_128B); 1803 1804 total = tfd_size + urbd0_size + frbd_size + urbd1_size + ctx_size + ci_size * 4; 1805 1806 data->dma_pool = dma_pool_create(KBUILD_MODNAME, &data->pdev->dev, 1807 total, BTINTEL_PCIE_DMA_ALIGN_128B, 0); 1808 if (!data->dma_pool) { 1809 err = -ENOMEM; 1810 goto exit_error; 1811 } 1812 1813 v_addr = dma_pool_zalloc(data->dma_pool, GFP_KERNEL | __GFP_NOWARN, 1814 &p_addr); 1815 if (!v_addr) { 1816 dma_pool_destroy(data->dma_pool); 1817 err = -ENOMEM; 1818 goto exit_error; 1819 } 1820 1821 data->dma_p_addr = p_addr; 1822 data->dma_v_addr = v_addr; 1823 1824 /* Setup descriptor count */ 1825 data->txq.count = BTINTEL_PCIE_TX_DESCS_COUNT; 1826 data->rxq.count = BTINTEL_PCIE_RX_DESCS_COUNT; 1827 1828 /* Setup tfds */ 1829 data->txq.tfds_p_addr = p_addr; 1830 data->txq.tfds = v_addr; 1831 1832 p_addr += tfd_size; 1833 v_addr += tfd_size; 1834 1835 /* Setup urbd0 */ 1836 data->txq.urbd0s_p_addr = p_addr; 1837 data->txq.urbd0s = v_addr; 1838 1839 p_addr += urbd0_size; 1840 v_addr += urbd0_size; 1841 1842 /* Setup FRBD*/ 1843 data->rxq.frbds_p_addr = p_addr; 1844 data->rxq.frbds = v_addr; 1845 1846 p_addr += frbd_size; 1847 v_addr += frbd_size; 1848 1849 /* Setup urbd1 */ 1850 data->rxq.urbd1s_p_addr = p_addr; 1851 data->rxq.urbd1s = v_addr; 1852 1853 p_addr += urbd1_size; 1854 v_addr += urbd1_size; 1855 1856 /* Setup data buffers for txq */ 1857 err = btintel_pcie_setup_txq_bufs(data, &data->txq); 1858 if (err) 1859 goto exit_error_pool; 1860 1861 /* Setup data buffers for rxq */ 1862 err = btintel_pcie_setup_rxq_bufs(data, &data->rxq); 1863 if (err) 1864 goto exit_error_txq; 1865 1866 /* TR Head Index Array */ 1867 data->ia.tr_hia_p_addr = p_addr; 1868 data->ia.tr_hia = v_addr; 1869 p_addr += ci_size; 1870 v_addr += ci_size; 1871 1872 /* TR Tail Index Array */ 1873 data->ia.tr_tia_p_addr = p_addr; 1874 data->ia.tr_tia = v_addr; 1875 p_addr += ci_size; 1876 v_addr += ci_size; 1877 1878 /* CR Head index Array */ 1879 data->ia.cr_hia_p_addr = p_addr; 1880 data->ia.cr_hia = v_addr; 1881 p_addr += ci_size; 1882 v_addr += ci_size; 1883 1884 /* CR Tail Index Array */ 1885 data->ia.cr_tia_p_addr = p_addr; 1886 data->ia.cr_tia = v_addr; 1887 p_addr += ci_size; 1888 v_addr += ci_size; 1889 1890 /* Setup data buffers for dbgc */ 1891 err = btintel_pcie_setup_dbgc(data); 1892 if (err) 1893 goto exit_error_txq; 1894 1895 /* Setup Context Information */ 1896 data->ci = v_addr; 1897 data->ci_p_addr = p_addr; 1898 1899 /* Initialize the CI */ 1900 btintel_pcie_init_ci(data, data->ci); 1901 1902 return 0; 1903 1904 exit_error_txq: 1905 btintel_pcie_free_txq_bufs(data, &data->txq); 1906 exit_error_pool: 1907 dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr); 1908 dma_pool_destroy(data->dma_pool); 1909 exit_error: 1910 return err; 1911 } 1912 1913 static int btintel_pcie_open(struct hci_dev *hdev) 1914 { 1915 bt_dev_dbg(hdev, ""); 1916 1917 return 0; 1918 } 1919 1920 static int btintel_pcie_close(struct hci_dev *hdev) 1921 { 1922 bt_dev_dbg(hdev, ""); 1923 1924 return 0; 1925 } 1926 1927 static int btintel_pcie_inject_cmd_complete(struct hci_dev *hdev, __u16 opcode) 1928 { 1929 struct sk_buff *skb; 1930 struct hci_event_hdr *hdr; 1931 struct hci_ev_cmd_complete *evt; 1932 1933 skb = bt_skb_alloc(sizeof(*hdr) + sizeof(*evt) + 1, GFP_KERNEL); 1934 if (!skb) 1935 return -ENOMEM; 1936 1937 hdr = (struct hci_event_hdr *)skb_put(skb, sizeof(*hdr)); 1938 hdr->evt = HCI_EV_CMD_COMPLETE; 1939 hdr->plen = sizeof(*evt) + 1; 1940 1941 evt = (struct hci_ev_cmd_complete *)skb_put(skb, sizeof(*evt)); 1942 evt->ncmd = 0x01; 1943 evt->opcode = cpu_to_le16(opcode); 1944 1945 *(u8 *)skb_put(skb, 1) = 0x00; 1946 1947 hci_skb_pkt_type(skb) = HCI_EVENT_PKT; 1948 1949 return hci_recv_frame(hdev, skb); 1950 } 1951 1952 static int btintel_pcie_send_frame(struct hci_dev *hdev, 1953 struct sk_buff *skb) 1954 { 1955 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 1956 struct hci_command_hdr *cmd; 1957 __u16 opcode = ~0; 1958 int ret; 1959 u32 type; 1960 1961 if (test_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags)) 1962 return -ENODEV; 1963 1964 /* Due to the fw limitation, the type header of the packet should be 1965 * 4 bytes unlike 1 byte for UART. In UART, the firmware can read 1966 * the first byte to get the packet type and redirect the rest of data 1967 * packet to the right handler. 1968 * 1969 * But for PCIe, THF(Transfer Flow Handler) fetches the 4 bytes of data 1970 * from DMA memory and by the time it reads the first 4 bytes, it has 1971 * already consumed some part of packet. Thus the packet type indicator 1972 * for iBT PCIe is 4 bytes. 1973 * 1974 * Luckily, when HCI core creates the skb, it allocates 8 bytes of 1975 * head room for profile and driver use, and before sending the data 1976 * to the device, append the iBT PCIe packet type in the front. 1977 */ 1978 switch (hci_skb_pkt_type(skb)) { 1979 case HCI_COMMAND_PKT: 1980 type = BTINTEL_PCIE_HCI_CMD_PKT; 1981 cmd = (void *)skb->data; 1982 opcode = le16_to_cpu(cmd->opcode); 1983 if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) { 1984 struct hci_command_hdr *cmd = (void *)skb->data; 1985 __u16 opcode = le16_to_cpu(cmd->opcode); 1986 1987 /* When the BTINTEL_HCI_OP_RESET command is issued to 1988 * boot into the operational firmware, it will actually 1989 * not send a command complete event. To keep the flow 1990 * control working inject that event here. 1991 */ 1992 if (opcode == BTINTEL_HCI_OP_RESET) 1993 btintel_pcie_inject_cmd_complete(hdev, opcode); 1994 } 1995 1996 hdev->stat.cmd_tx++; 1997 break; 1998 case HCI_ACLDATA_PKT: 1999 type = BTINTEL_PCIE_HCI_ACL_PKT; 2000 hdev->stat.acl_tx++; 2001 break; 2002 case HCI_SCODATA_PKT: 2003 type = BTINTEL_PCIE_HCI_SCO_PKT; 2004 hdev->stat.sco_tx++; 2005 break; 2006 case HCI_ISODATA_PKT: 2007 type = BTINTEL_PCIE_HCI_ISO_PKT; 2008 break; 2009 default: 2010 bt_dev_err(hdev, "Unknown HCI packet type"); 2011 return -EILSEQ; 2012 } 2013 2014 ret = btintel_pcie_send_sync(data, skb, type, opcode); 2015 if (ret) { 2016 hdev->stat.err_tx++; 2017 bt_dev_err(hdev, "Failed to send frame (%d)", ret); 2018 goto exit_error; 2019 } 2020 2021 hdev->stat.byte_tx += skb->len; 2022 kfree_skb(skb); 2023 2024 exit_error: 2025 return ret; 2026 } 2027 2028 static void btintel_pcie_release_hdev(struct btintel_pcie_data *data) 2029 { 2030 struct hci_dev *hdev; 2031 2032 hdev = data->hdev; 2033 hci_unregister_dev(hdev); 2034 hci_free_dev(hdev); 2035 data->hdev = NULL; 2036 } 2037 2038 static void btintel_pcie_disable_interrupts(struct btintel_pcie_data *data) 2039 { 2040 spin_lock(&data->irq_lock); 2041 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, data->fh_init_mask); 2042 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, data->hw_init_mask); 2043 spin_unlock(&data->irq_lock); 2044 } 2045 2046 static void btintel_pcie_enable_interrupts(struct btintel_pcie_data *data) 2047 { 2048 spin_lock(&data->irq_lock); 2049 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, ~data->fh_init_mask); 2050 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, ~data->hw_init_mask); 2051 spin_unlock(&data->irq_lock); 2052 } 2053 2054 static void btintel_pcie_synchronize_irqs(struct btintel_pcie_data *data) 2055 { 2056 for (int i = 0; i < data->alloc_vecs; i++) 2057 synchronize_irq(data->msix_entries[i].vector); 2058 } 2059 2060 static int btintel_pcie_setup_internal(struct hci_dev *hdev) 2061 { 2062 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 2063 const u8 param[1] = { 0xFF }; 2064 struct intel_version_tlv ver_tlv; 2065 struct sk_buff *skb; 2066 int err; 2067 2068 BT_DBG("%s", hdev->name); 2069 2070 skb = __hci_cmd_sync(hdev, 0xfc05, 1, param, HCI_CMD_TIMEOUT); 2071 if (IS_ERR(skb)) { 2072 bt_dev_err(hdev, "Reading Intel version command failed (%ld)", 2073 PTR_ERR(skb)); 2074 return PTR_ERR(skb); 2075 } 2076 2077 /* Check the status */ 2078 if (skb->data[0]) { 2079 bt_dev_err(hdev, "Intel Read Version command failed (%02x)", 2080 skb->data[0]); 2081 err = -EIO; 2082 goto exit_error; 2083 } 2084 2085 /* Apply the common HCI quirks for Intel device */ 2086 hci_set_quirk(hdev, HCI_QUIRK_STRICT_DUPLICATE_FILTER); 2087 hci_set_quirk(hdev, HCI_QUIRK_SIMULTANEOUS_DISCOVERY); 2088 hci_set_quirk(hdev, HCI_QUIRK_NON_PERSISTENT_DIAG); 2089 2090 /* Set up the quality report callback for Intel devices */ 2091 hdev->set_quality_report = btintel_set_quality_report; 2092 2093 memset(&ver_tlv, 0, sizeof(ver_tlv)); 2094 /* For TLV type device, parse the tlv data */ 2095 err = btintel_parse_version_tlv(hdev, &ver_tlv, skb); 2096 if (err) { 2097 bt_dev_err(hdev, "Failed to parse TLV version information"); 2098 goto exit_error; 2099 } 2100 2101 switch (INTEL_HW_PLATFORM(ver_tlv.cnvi_bt)) { 2102 case 0x37: 2103 break; 2104 default: 2105 bt_dev_err(hdev, "Unsupported Intel hardware platform (0x%2x)", 2106 INTEL_HW_PLATFORM(ver_tlv.cnvi_bt)); 2107 err = -EINVAL; 2108 goto exit_error; 2109 } 2110 2111 /* Check for supported iBT hardware variants of this firmware 2112 * loading method. 2113 * 2114 * This check has been put in place to ensure correct forward 2115 * compatibility options when newer hardware variants come 2116 * along. 2117 */ 2118 switch (INTEL_HW_VARIANT(ver_tlv.cnvi_bt)) { 2119 case 0x1e: /* BzrI */ 2120 case 0x1f: /* ScP */ 2121 case 0x20: /* ScP2 */ 2122 case 0x21: /* ScP2 F */ 2123 case 0x22: /* BzrIW */ 2124 /* Display version information of TLV type */ 2125 btintel_version_info_tlv(hdev, &ver_tlv); 2126 2127 /* Apply the device specific HCI quirks for TLV based devices 2128 * 2129 * All TLV based devices support WBS 2130 */ 2131 hci_set_quirk(hdev, HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED); 2132 2133 /* Setup MSFT Extension support */ 2134 btintel_set_msft_opcode(hdev, 2135 INTEL_HW_VARIANT(ver_tlv.cnvi_bt)); 2136 2137 err = btintel_bootloader_setup_tlv(hdev, &ver_tlv); 2138 if (err) 2139 goto exit_error; 2140 break; 2141 default: 2142 bt_dev_err(hdev, "Unsupported Intel hw variant (%u)", 2143 INTEL_HW_VARIANT(ver_tlv.cnvi_bt)); 2144 err = -EINVAL; 2145 goto exit_error; 2146 break; 2147 } 2148 2149 data->dmp_hdr.cnvi_top = ver_tlv.cnvi_top; 2150 data->dmp_hdr.cnvr_top = ver_tlv.cnvr_top; 2151 data->dmp_hdr.fw_timestamp = ver_tlv.timestamp; 2152 data->dmp_hdr.fw_build_type = ver_tlv.build_type; 2153 data->dmp_hdr.fw_build_num = ver_tlv.build_num; 2154 data->dmp_hdr.cnvi_bt = ver_tlv.cnvi_bt; 2155 2156 if (ver_tlv.img_type == 0x02 || ver_tlv.img_type == 0x03) 2157 data->dmp_hdr.fw_git_sha1 = ver_tlv.git_sha1; 2158 2159 btintel_print_fseq_info(hdev); 2160 exit_error: 2161 kfree_skb(skb); 2162 2163 return err; 2164 } 2165 2166 static int btintel_pcie_setup(struct hci_dev *hdev) 2167 { 2168 int err, fw_dl_retry = 0; 2169 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 2170 2171 while ((err = btintel_pcie_setup_internal(hdev)) && fw_dl_retry++ < 1) { 2172 bt_dev_err(hdev, "Firmware download retry count: %d", 2173 fw_dl_retry); 2174 btintel_pcie_dump_debug_registers(hdev); 2175 btintel_pcie_disable_interrupts(data); 2176 btintel_pcie_synchronize_irqs(data); 2177 err = btintel_pcie_reset_bt(data); 2178 if (err) { 2179 bt_dev_err(hdev, "Failed to do shr reset: %d", err); 2180 break; 2181 } 2182 usleep_range(10000, 12000); 2183 btintel_pcie_reset_ia(data); 2184 btintel_pcie_enable_interrupts(data); 2185 btintel_pcie_config_msix(data); 2186 err = btintel_pcie_enable_bt(data); 2187 if (err) { 2188 bt_dev_err(hdev, "Failed to enable hardware: %d", err); 2189 break; 2190 } 2191 btintel_pcie_start_rx(data); 2192 } 2193 2194 if (!err) 2195 set_bit(BTINTEL_PCIE_SETUP_DONE, &data->flags); 2196 return err; 2197 } 2198 2199 static struct btintel_pcie_dev_recovery * 2200 btintel_pcie_get_recovery(struct pci_dev *pdev, struct device *dev) 2201 { 2202 struct btintel_pcie_dev_recovery *tmp, *data = NULL; 2203 const char *name = pci_name(pdev); 2204 const size_t name_len = strlen(name) + 1; 2205 struct hci_dev *hdev = to_hci_dev(dev); 2206 2207 spin_lock(&btintel_pcie_recovery_lock); 2208 list_for_each_entry(tmp, &btintel_pcie_recovery_list, list) { 2209 if (strcmp(tmp->name, name)) 2210 continue; 2211 data = tmp; 2212 break; 2213 } 2214 spin_unlock(&btintel_pcie_recovery_lock); 2215 2216 if (data) { 2217 bt_dev_dbg(hdev, "Found restart data for BDF: %s", data->name); 2218 return data; 2219 } 2220 2221 data = kzalloc_flex(*data, name, name_len, GFP_ATOMIC); 2222 if (!data) 2223 return NULL; 2224 2225 strscpy(data->name, name, name_len); 2226 spin_lock(&btintel_pcie_recovery_lock); 2227 list_add_tail(&data->list, &btintel_pcie_recovery_list); 2228 spin_unlock(&btintel_pcie_recovery_lock); 2229 2230 return data; 2231 } 2232 2233 static void btintel_pcie_free_restart_list(void) 2234 { 2235 struct btintel_pcie_dev_recovery *tmp; 2236 2237 while ((tmp = list_first_entry_or_null(&btintel_pcie_recovery_list, 2238 typeof(*tmp), list))) { 2239 list_del(&tmp->list); 2240 kfree(tmp); 2241 } 2242 } 2243 2244 static void btintel_pcie_inc_recovery_count(struct pci_dev *pdev, 2245 struct device *dev) 2246 { 2247 struct btintel_pcie_dev_recovery *data; 2248 time64_t retry_window; 2249 2250 data = btintel_pcie_get_recovery(pdev, dev); 2251 if (!data) 2252 return; 2253 2254 retry_window = ktime_get_boottime_seconds() - data->last_error; 2255 if (data->count == 0) { 2256 data->last_error = ktime_get_boottime_seconds(); 2257 data->count++; 2258 } else if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS && 2259 data->count <= BTINTEL_PCIE_FLR_MAX_RETRY) { 2260 data->count++; 2261 } else if (retry_window > BTINTEL_PCIE_RESET_WINDOW_SECS) { 2262 data->last_error = 0; 2263 data->count = 0; 2264 } 2265 } 2266 2267 static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data); 2268 2269 static void btintel_pcie_removal_work(struct work_struct *wk) 2270 { 2271 struct btintel_pcie_removal *removal = 2272 container_of(wk, struct btintel_pcie_removal, work); 2273 struct pci_dev *pdev = removal->pdev; 2274 struct btintel_pcie_data *data; 2275 int err; 2276 2277 pci_lock_rescan_remove(); 2278 2279 if (!pdev->bus) 2280 goto error; 2281 2282 data = pci_get_drvdata(pdev); 2283 2284 btintel_pcie_disable_interrupts(data); 2285 btintel_pcie_synchronize_irqs(data); 2286 2287 flush_work(&data->rx_work); 2288 2289 bt_dev_dbg(data->hdev, "Release bluetooth interface"); 2290 btintel_pcie_release_hdev(data); 2291 2292 err = pci_reset_function(pdev); 2293 if (err) { 2294 BT_ERR("Failed resetting the pcie device (%d)", err); 2295 goto error; 2296 } 2297 2298 btintel_pcie_enable_interrupts(data); 2299 btintel_pcie_config_msix(data); 2300 2301 err = btintel_pcie_enable_bt(data); 2302 if (err) { 2303 BT_ERR("Failed to enable bluetooth hardware after reset (%d)", 2304 err); 2305 goto error; 2306 } 2307 2308 btintel_pcie_reset_ia(data); 2309 btintel_pcie_start_rx(data); 2310 data->flags = 0; 2311 2312 err = btintel_pcie_setup_hdev(data); 2313 if (err) { 2314 BT_ERR("Failed registering hdev (%d)", err); 2315 goto error; 2316 } 2317 error: 2318 pci_dev_put(pdev); 2319 pci_unlock_rescan_remove(); 2320 kfree(removal); 2321 } 2322 2323 static void btintel_pcie_reset(struct hci_dev *hdev) 2324 { 2325 struct btintel_pcie_removal *removal; 2326 struct btintel_pcie_data *data; 2327 2328 data = hci_get_drvdata(hdev); 2329 2330 if (!test_bit(BTINTEL_PCIE_SETUP_DONE, &data->flags)) 2331 return; 2332 2333 if (test_and_set_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags)) 2334 return; 2335 2336 removal = kzalloc_obj(*removal, GFP_ATOMIC); 2337 if (!removal) 2338 return; 2339 2340 removal->pdev = data->pdev; 2341 INIT_WORK(&removal->work, btintel_pcie_removal_work); 2342 pci_dev_get(removal->pdev); 2343 schedule_work(&removal->work); 2344 } 2345 2346 static void btintel_pcie_hw_error(struct hci_dev *hdev, u8 code) 2347 { 2348 struct btintel_pcie_dev_recovery *data; 2349 struct btintel_pcie_data *dev_data = hci_get_drvdata(hdev); 2350 struct pci_dev *pdev = dev_data->pdev; 2351 time64_t retry_window; 2352 2353 if (code == 0x13) { 2354 bt_dev_err(hdev, "Encountered top exception"); 2355 return; 2356 } 2357 2358 data = btintel_pcie_get_recovery(pdev, &hdev->dev); 2359 if (!data) 2360 return; 2361 2362 retry_window = ktime_get_boottime_seconds() - data->last_error; 2363 2364 if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS && 2365 data->count >= BTINTEL_PCIE_FLR_MAX_RETRY) { 2366 bt_dev_err(hdev, "Exhausted maximum: %d recovery attempts: %d", 2367 BTINTEL_PCIE_FLR_MAX_RETRY, data->count); 2368 bt_dev_dbg(hdev, "Boot time: %lld seconds", 2369 ktime_get_boottime_seconds()); 2370 bt_dev_dbg(hdev, "last error at: %lld seconds", 2371 data->last_error); 2372 return; 2373 } 2374 btintel_pcie_inc_recovery_count(pdev, &hdev->dev); 2375 btintel_pcie_reset(hdev); 2376 } 2377 2378 static bool btintel_pcie_wakeup(struct hci_dev *hdev) 2379 { 2380 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 2381 2382 return device_may_wakeup(&data->pdev->dev); 2383 } 2384 2385 static const struct { 2386 u16 opcode; 2387 const char *desc; 2388 } btintel_pcie_hci_drv_supported_commands[] = { 2389 /* Common commands */ 2390 { HCI_DRV_OP_READ_INFO, "Read Info" }, 2391 }; 2392 2393 static int btintel_pcie_hci_drv_read_info(struct hci_dev *hdev, void *data, 2394 u16 data_len) 2395 { 2396 struct hci_drv_rp_read_info *rp; 2397 size_t rp_size; 2398 int err, i; 2399 u16 opcode, num_supported_commands = 2400 ARRAY_SIZE(btintel_pcie_hci_drv_supported_commands); 2401 2402 rp_size = struct_size(rp, supported_commands, num_supported_commands); 2403 2404 rp = kmalloc(rp_size, GFP_KERNEL); 2405 if (!rp) 2406 return -ENOMEM; 2407 2408 strscpy_pad(rp->driver_name, KBUILD_MODNAME); 2409 2410 rp->num_supported_commands = cpu_to_le16(num_supported_commands); 2411 for (i = 0; i < num_supported_commands; i++) { 2412 opcode = btintel_pcie_hci_drv_supported_commands[i].opcode; 2413 bt_dev_dbg(hdev, 2414 "Supported HCI Drv command (0x%02x|0x%04x): %s", 2415 hci_opcode_ogf(opcode), 2416 hci_opcode_ocf(opcode), 2417 btintel_pcie_hci_drv_supported_commands[i].desc); 2418 rp->supported_commands[i] = cpu_to_le16(opcode); 2419 } 2420 2421 err = hci_drv_cmd_complete(hdev, HCI_DRV_OP_READ_INFO, 2422 HCI_DRV_STATUS_SUCCESS, 2423 rp, rp_size); 2424 2425 kfree(rp); 2426 return err; 2427 } 2428 2429 static const struct hci_drv_handler btintel_pcie_hci_drv_common_handlers[] = { 2430 { btintel_pcie_hci_drv_read_info, HCI_DRV_READ_INFO_SIZE }, 2431 }; 2432 2433 static const struct hci_drv_handler btintel_pcie_hci_drv_specific_handlers[] = {}; 2434 2435 static struct hci_drv btintel_pcie_hci_drv = { 2436 .common_handler_count = ARRAY_SIZE(btintel_pcie_hci_drv_common_handlers), 2437 .common_handlers = btintel_pcie_hci_drv_common_handlers, 2438 .specific_handler_count = ARRAY_SIZE(btintel_pcie_hci_drv_specific_handlers), 2439 .specific_handlers = btintel_pcie_hci_drv_specific_handlers, 2440 }; 2441 2442 static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data) 2443 { 2444 int err; 2445 struct hci_dev *hdev; 2446 2447 hdev = hci_alloc_dev_priv(sizeof(struct btintel_data)); 2448 if (!hdev) 2449 return -ENOMEM; 2450 2451 hdev->bus = HCI_PCI; 2452 hci_set_drvdata(hdev, data); 2453 2454 data->hdev = hdev; 2455 SET_HCIDEV_DEV(hdev, &data->pdev->dev); 2456 2457 hdev->manufacturer = 2; 2458 hdev->open = btintel_pcie_open; 2459 hdev->close = btintel_pcie_close; 2460 hdev->send = btintel_pcie_send_frame; 2461 hdev->setup = btintel_pcie_setup; 2462 hdev->shutdown = btintel_shutdown_combined; 2463 hdev->hw_error = btintel_pcie_hw_error; 2464 hdev->set_diag = btintel_set_diag; 2465 hdev->set_bdaddr = btintel_set_bdaddr; 2466 hdev->reset = btintel_pcie_reset; 2467 hdev->wakeup = btintel_pcie_wakeup; 2468 hdev->hci_drv = &btintel_pcie_hci_drv; 2469 2470 err = hci_register_dev(hdev); 2471 if (err < 0) { 2472 BT_ERR("Failed to register to hdev (%d)", err); 2473 goto exit_error; 2474 } 2475 2476 data->dmp_hdr.driver_name = KBUILD_MODNAME; 2477 return 0; 2478 2479 exit_error: 2480 hci_free_dev(hdev); 2481 return err; 2482 } 2483 2484 static int btintel_pcie_probe(struct pci_dev *pdev, 2485 const struct pci_device_id *ent) 2486 { 2487 int err; 2488 struct btintel_pcie_data *data; 2489 2490 if (!pdev) 2491 return -ENODEV; 2492 2493 data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); 2494 if (!data) 2495 return -ENOMEM; 2496 2497 data->pdev = pdev; 2498 2499 spin_lock_init(&data->irq_lock); 2500 spin_lock_init(&data->hci_rx_lock); 2501 2502 init_waitqueue_head(&data->gp0_wait_q); 2503 data->gp0_received = false; 2504 2505 init_waitqueue_head(&data->tx_wait_q); 2506 data->tx_wait_done = false; 2507 2508 data->workqueue = alloc_ordered_workqueue(KBUILD_MODNAME, WQ_HIGHPRI); 2509 if (!data->workqueue) 2510 return -ENOMEM; 2511 2512 skb_queue_head_init(&data->rx_skb_q); 2513 INIT_WORK(&data->rx_work, btintel_pcie_rx_work); 2514 2515 data->boot_stage_cache = 0x00; 2516 data->img_resp_cache = 0x00; 2517 2518 err = btintel_pcie_config_pcie(pdev, data); 2519 if (err) 2520 goto exit_error; 2521 2522 pci_set_drvdata(pdev, data); 2523 2524 err = btintel_pcie_alloc(data); 2525 if (err) 2526 goto exit_error; 2527 2528 err = btintel_pcie_enable_bt(data); 2529 if (err) 2530 goto exit_error; 2531 2532 /* CNV information (CNVi and CNVr) is in CSR */ 2533 data->cnvi = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_HW_REV_REG); 2534 2535 data->cnvr = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_RF_ID_REG); 2536 2537 err = btintel_pcie_start_rx(data); 2538 if (err) 2539 goto exit_error; 2540 2541 err = btintel_pcie_setup_hdev(data); 2542 if (err) 2543 goto exit_error; 2544 2545 bt_dev_dbg(data->hdev, "cnvi: 0x%8.8x cnvr: 0x%8.8x", data->cnvi, 2546 data->cnvr); 2547 return 0; 2548 2549 exit_error: 2550 /* reset device before exit */ 2551 btintel_pcie_reset_bt(data); 2552 2553 pci_clear_master(pdev); 2554 2555 pci_set_drvdata(pdev, NULL); 2556 2557 return err; 2558 } 2559 2560 static void btintel_pcie_remove(struct pci_dev *pdev) 2561 { 2562 struct btintel_pcie_data *data; 2563 2564 data = pci_get_drvdata(pdev); 2565 2566 btintel_pcie_disable_interrupts(data); 2567 2568 btintel_pcie_synchronize_irqs(data); 2569 2570 flush_work(&data->rx_work); 2571 2572 btintel_pcie_reset_bt(data); 2573 for (int i = 0; i < data->alloc_vecs; i++) { 2574 struct msix_entry *msix_entry; 2575 2576 msix_entry = &data->msix_entries[i]; 2577 free_irq(msix_entry->vector, msix_entry); 2578 } 2579 2580 pci_free_irq_vectors(pdev); 2581 2582 btintel_pcie_release_hdev(data); 2583 2584 destroy_workqueue(data->workqueue); 2585 2586 btintel_pcie_free(data); 2587 2588 pci_clear_master(pdev); 2589 2590 pci_set_drvdata(pdev, NULL); 2591 } 2592 2593 #ifdef CONFIG_DEV_COREDUMP 2594 static void btintel_pcie_coredump(struct device *dev) 2595 { 2596 struct pci_dev *pdev = to_pci_dev(dev); 2597 struct btintel_pcie_data *data = pci_get_drvdata(pdev); 2598 2599 if (test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, &data->flags)) 2600 return; 2601 2602 data->dmp_hdr.trigger_reason = BTINTEL_PCIE_TRIGGER_REASON_USER_TRIGGER; 2603 queue_work(data->workqueue, &data->rx_work); 2604 } 2605 #endif 2606 2607 static int btintel_pcie_set_dxstate(struct btintel_pcie_data *data, u32 dxstate) 2608 { 2609 int retry = 0, status; 2610 u32 dx_intr_timeout_ms = 200; 2611 2612 do { 2613 data->gp0_received = false; 2614 2615 btintel_pcie_wr_sleep_cntrl(data, dxstate); 2616 2617 status = wait_event_timeout(data->gp0_wait_q, data->gp0_received, 2618 msecs_to_jiffies(dx_intr_timeout_ms)); 2619 2620 if (status) 2621 return 0; 2622 2623 bt_dev_warn(data->hdev, 2624 "Timeout (%u ms) on alive interrupt for D%d entry, retry count %d", 2625 dx_intr_timeout_ms, dxstate, retry); 2626 2627 /* clear gp0 cause */ 2628 btintel_pcie_clr_reg_bits(data, 2629 BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES, 2630 BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0); 2631 2632 /* A hardware bug may cause the alive interrupt to be missed. 2633 * Check if the controller reached the expected state and retry 2634 * the operation only if it hasn't. 2635 */ 2636 if (dxstate == BTINTEL_PCIE_STATE_D0) { 2637 if (btintel_pcie_in_d0(data)) 2638 return 0; 2639 } else { 2640 if (btintel_pcie_in_d3(data)) 2641 return 0; 2642 } 2643 2644 } while (++retry < BTINTEL_PCIE_DX_TRANSITION_MAX_RETRIES); 2645 2646 return -EBUSY; 2647 } 2648 2649 static int btintel_pcie_suspend_late(struct device *dev, pm_message_t mesg) 2650 { 2651 struct pci_dev *pdev = to_pci_dev(dev); 2652 struct btintel_pcie_data *data; 2653 ktime_t start; 2654 u32 dxstate; 2655 int err; 2656 2657 data = pci_get_drvdata(pdev); 2658 2659 dxstate = (mesg.event == PM_EVENT_SUSPEND ? 2660 BTINTEL_PCIE_STATE_D3_HOT : BTINTEL_PCIE_STATE_D3_COLD); 2661 2662 data->pm_sx_event = mesg.event; 2663 2664 start = ktime_get(); 2665 2666 /* Refer: 6.4.11.7 -> Platform power management */ 2667 err = btintel_pcie_set_dxstate(data, dxstate); 2668 2669 if (err) 2670 return err; 2671 2672 bt_dev_dbg(data->hdev, 2673 "device entered into d3 state from d0 in %lld us", 2674 ktime_to_us(ktime_get() - start)); 2675 return err; 2676 } 2677 2678 static int btintel_pcie_suspend(struct device *dev) 2679 { 2680 return btintel_pcie_suspend_late(dev, PMSG_SUSPEND); 2681 } 2682 2683 static int btintel_pcie_hibernate(struct device *dev) 2684 { 2685 return btintel_pcie_suspend_late(dev, PMSG_HIBERNATE); 2686 } 2687 2688 static int btintel_pcie_freeze(struct device *dev) 2689 { 2690 return btintel_pcie_suspend_late(dev, PMSG_FREEZE); 2691 } 2692 2693 static int btintel_pcie_resume(struct device *dev) 2694 { 2695 struct pci_dev *pdev = to_pci_dev(dev); 2696 struct btintel_pcie_data *data; 2697 ktime_t start; 2698 int err; 2699 2700 data = pci_get_drvdata(pdev); 2701 data->gp0_received = false; 2702 2703 start = ktime_get(); 2704 2705 /* When the system enters S4 (hibernate) mode, bluetooth device loses 2706 * power, which results in the erasure of its loaded firmware. 2707 * Consequently, function level reset (flr) is required on system 2708 * resume to bring the controller back into an operational state by 2709 * initiating a new firmware download. 2710 */ 2711 2712 if (data->pm_sx_event == PM_EVENT_FREEZE || 2713 data->pm_sx_event == PM_EVENT_HIBERNATE) { 2714 set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags); 2715 btintel_pcie_reset(data->hdev); 2716 return 0; 2717 } 2718 2719 /* Refer: 6.4.11.7 -> Platform power management */ 2720 err = btintel_pcie_set_dxstate(data, BTINTEL_PCIE_STATE_D0); 2721 2722 if (err == 0) { 2723 bt_dev_dbg(data->hdev, 2724 "device entered into d0 state from d3 in %lld us", 2725 ktime_to_us(ktime_get() - start)); 2726 return err; 2727 } 2728 2729 /* Trigger function level reset if the controller is in error 2730 * state during resume() to bring back the controller to 2731 * operational mode 2732 */ 2733 2734 data->boot_stage_cache = btintel_pcie_rd_reg32(data, 2735 BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 2736 if (btintel_pcie_in_error(data) || 2737 btintel_pcie_in_device_halt(data)) { 2738 bt_dev_err(data->hdev, "Controller in error state for D0 entry"); 2739 if (!test_and_set_bit(BTINTEL_PCIE_COREDUMP_INPROGRESS, 2740 &data->flags)) { 2741 data->dmp_hdr.trigger_reason = 2742 BTINTEL_PCIE_TRIGGER_REASON_FW_ASSERT; 2743 queue_work(data->workqueue, &data->rx_work); 2744 } 2745 set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags); 2746 btintel_pcie_reset(data->hdev); 2747 } 2748 return err; 2749 } 2750 2751 static const struct dev_pm_ops btintel_pcie_pm_ops = { 2752 .suspend = btintel_pcie_suspend, 2753 .resume = btintel_pcie_resume, 2754 .freeze = btintel_pcie_freeze, 2755 .thaw = btintel_pcie_resume, 2756 .poweroff = btintel_pcie_hibernate, 2757 .restore = btintel_pcie_resume, 2758 }; 2759 2760 static struct pci_driver btintel_pcie_driver = { 2761 .name = KBUILD_MODNAME, 2762 .id_table = btintel_pcie_table, 2763 .probe = btintel_pcie_probe, 2764 .remove = btintel_pcie_remove, 2765 .driver.pm = pm_sleep_ptr(&btintel_pcie_pm_ops), 2766 #ifdef CONFIG_DEV_COREDUMP 2767 .driver.coredump = btintel_pcie_coredump 2768 #endif 2769 }; 2770 2771 static int __init btintel_pcie_init(void) 2772 { 2773 return pci_register_driver(&btintel_pcie_driver); 2774 } 2775 2776 static void __exit btintel_pcie_exit(void) 2777 { 2778 pci_unregister_driver(&btintel_pcie_driver); 2779 btintel_pcie_free_restart_list(); 2780 } 2781 2782 module_init(btintel_pcie_init); 2783 module_exit(btintel_pcie_exit); 2784 2785 MODULE_AUTHOR("Tedd Ho-Jeong An <tedd.an@intel.com>"); 2786 MODULE_DESCRIPTION("Intel Bluetooth PCIe transport driver ver " VERSION); 2787 MODULE_VERSION(VERSION); 2788 MODULE_LICENSE("GPL"); 2789