1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * 4 * Bluetooth support for Intel PCIe devices 5 * 6 * Copyright (C) 2024 Intel Corporation 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/module.h> 11 #include <linux/firmware.h> 12 #include <linux/pci.h> 13 #include <linux/wait.h> 14 #include <linux/delay.h> 15 #include <linux/interrupt.h> 16 17 #include <linux/unaligned.h> 18 19 #include <net/bluetooth/bluetooth.h> 20 #include <net/bluetooth/hci_core.h> 21 22 #include "btintel.h" 23 #include "btintel_pcie.h" 24 25 #define VERSION "0.1" 26 27 #define BTINTEL_PCI_DEVICE(dev, subdev) \ 28 .vendor = PCI_VENDOR_ID_INTEL, \ 29 .device = (dev), \ 30 .subvendor = PCI_ANY_ID, \ 31 .subdevice = (subdev), \ 32 .driver_data = 0 33 34 #define POLL_INTERVAL_US 10 35 36 /* Intel Bluetooth PCIe device id table */ 37 static const struct pci_device_id btintel_pcie_table[] = { 38 { BTINTEL_PCI_DEVICE(0xA876, PCI_ANY_ID) }, 39 { 0 } 40 }; 41 MODULE_DEVICE_TABLE(pci, btintel_pcie_table); 42 43 /* Intel PCIe uses 4 bytes of HCI type instead of 1 byte BT SIG HCI type */ 44 #define BTINTEL_PCIE_HCI_TYPE_LEN 4 45 #define BTINTEL_PCIE_HCI_CMD_PKT 0x00000001 46 #define BTINTEL_PCIE_HCI_ACL_PKT 0x00000002 47 #define BTINTEL_PCIE_HCI_SCO_PKT 0x00000003 48 #define BTINTEL_PCIE_HCI_EVT_PKT 0x00000004 49 #define BTINTEL_PCIE_HCI_ISO_PKT 0x00000005 50 51 /* Alive interrupt context */ 52 enum { 53 BTINTEL_PCIE_ROM, 54 BTINTEL_PCIE_FW_DL, 55 BTINTEL_PCIE_HCI_RESET, 56 BTINTEL_PCIE_INTEL_HCI_RESET1, 57 BTINTEL_PCIE_INTEL_HCI_RESET2, 58 BTINTEL_PCIE_D0, 59 BTINTEL_PCIE_D3 60 }; 61 62 static inline void ipc_print_ia_ring(struct hci_dev *hdev, struct ia *ia, 63 u16 queue_num) 64 { 65 bt_dev_dbg(hdev, "IA: %s: tr-h:%02u tr-t:%02u cr-h:%02u cr-t:%02u", 66 queue_num == BTINTEL_PCIE_TXQ_NUM ? "TXQ" : "RXQ", 67 ia->tr_hia[queue_num], ia->tr_tia[queue_num], 68 ia->cr_hia[queue_num], ia->cr_tia[queue_num]); 69 } 70 71 static inline void ipc_print_urbd1(struct hci_dev *hdev, struct urbd1 *urbd1, 72 u16 index) 73 { 74 bt_dev_dbg(hdev, "RXQ:urbd1(%u) frbd_tag:%u status: 0x%x fixed:0x%x", 75 index, urbd1->frbd_tag, urbd1->status, urbd1->fixed); 76 } 77 78 static struct btintel_pcie_data *btintel_pcie_get_data(struct msix_entry *entry) 79 { 80 u8 queue = entry->entry; 81 struct msix_entry *entries = entry - queue; 82 83 return container_of(entries, struct btintel_pcie_data, msix_entries[0]); 84 } 85 86 /* Set the doorbell for TXQ to notify the device that @index (actually index-1) 87 * of the TFD is updated and ready to transmit. 88 */ 89 static void btintel_pcie_set_tx_db(struct btintel_pcie_data *data, u16 index) 90 { 91 u32 val; 92 93 val = index; 94 val |= (BTINTEL_PCIE_TX_DB_VEC << 16); 95 96 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val); 97 } 98 99 /* Copy the data to next(@tfd_index) data buffer and update the TFD(transfer 100 * descriptor) with the data length and the DMA address of the data buffer. 101 */ 102 static void btintel_pcie_prepare_tx(struct txq *txq, u16 tfd_index, 103 struct sk_buff *skb) 104 { 105 struct data_buf *buf; 106 struct tfd *tfd; 107 108 tfd = &txq->tfds[tfd_index]; 109 memset(tfd, 0, sizeof(*tfd)); 110 111 buf = &txq->bufs[tfd_index]; 112 113 tfd->size = skb->len; 114 tfd->addr = buf->data_p_addr; 115 116 /* Copy the outgoing data to DMA buffer */ 117 memcpy(buf->data, skb->data, tfd->size); 118 } 119 120 static int btintel_pcie_send_sync(struct btintel_pcie_data *data, 121 struct sk_buff *skb) 122 { 123 int ret; 124 u16 tfd_index; 125 struct txq *txq = &data->txq; 126 127 tfd_index = data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM]; 128 129 if (tfd_index > txq->count) 130 return -ERANGE; 131 132 /* Prepare for TX. It updates the TFD with the length of data and 133 * address of the DMA buffer, and copy the data to the DMA buffer 134 */ 135 btintel_pcie_prepare_tx(txq, tfd_index, skb); 136 137 tfd_index = (tfd_index + 1) % txq->count; 138 data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM] = tfd_index; 139 140 /* Arm wait event condition */ 141 data->tx_wait_done = false; 142 143 /* Set the doorbell to notify the device */ 144 btintel_pcie_set_tx_db(data, tfd_index); 145 146 /* Wait for the complete interrupt - URBD0 */ 147 ret = wait_event_timeout(data->tx_wait_q, data->tx_wait_done, 148 msecs_to_jiffies(BTINTEL_PCIE_TX_WAIT_TIMEOUT_MS)); 149 if (!ret) 150 return -ETIME; 151 152 return 0; 153 } 154 155 /* Set the doorbell for RXQ to notify the device that @index (actually index-1) 156 * is available to receive the data 157 */ 158 static void btintel_pcie_set_rx_db(struct btintel_pcie_data *data, u16 index) 159 { 160 u32 val; 161 162 val = index; 163 val |= (BTINTEL_PCIE_RX_DB_VEC << 16); 164 165 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val); 166 } 167 168 /* Update the FRBD (free buffer descriptor) with the @frbd_index and the 169 * DMA address of the free buffer. 170 */ 171 static void btintel_pcie_prepare_rx(struct rxq *rxq, u16 frbd_index) 172 { 173 struct data_buf *buf; 174 struct frbd *frbd; 175 176 /* Get the buffer of the FRBD for DMA */ 177 buf = &rxq->bufs[frbd_index]; 178 179 frbd = &rxq->frbds[frbd_index]; 180 memset(frbd, 0, sizeof(*frbd)); 181 182 /* Update FRBD */ 183 frbd->tag = frbd_index; 184 frbd->addr = buf->data_p_addr; 185 } 186 187 static int btintel_pcie_submit_rx(struct btintel_pcie_data *data) 188 { 189 u16 frbd_index; 190 struct rxq *rxq = &data->rxq; 191 192 frbd_index = data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM]; 193 194 if (frbd_index > rxq->count) 195 return -ERANGE; 196 197 /* Prepare for RX submit. It updates the FRBD with the address of DMA 198 * buffer 199 */ 200 btintel_pcie_prepare_rx(rxq, frbd_index); 201 202 frbd_index = (frbd_index + 1) % rxq->count; 203 data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM] = frbd_index; 204 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM); 205 206 /* Set the doorbell to notify the device */ 207 btintel_pcie_set_rx_db(data, frbd_index); 208 209 return 0; 210 } 211 212 static int btintel_pcie_start_rx(struct btintel_pcie_data *data) 213 { 214 int i, ret; 215 216 for (i = 0; i < BTINTEL_PCIE_RX_MAX_QUEUE; i++) { 217 ret = btintel_pcie_submit_rx(data); 218 if (ret) 219 return ret; 220 } 221 222 return 0; 223 } 224 225 static void btintel_pcie_reset_ia(struct btintel_pcie_data *data) 226 { 227 memset(data->ia.tr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 228 memset(data->ia.tr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 229 memset(data->ia.cr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 230 memset(data->ia.cr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES); 231 } 232 233 static int btintel_pcie_reset_bt(struct btintel_pcie_data *data) 234 { 235 u32 reg; 236 int retry = 3; 237 238 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 239 240 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 241 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT | 242 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT); 243 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON; 244 245 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 246 247 do { 248 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 249 if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_STS) 250 break; 251 usleep_range(10000, 12000); 252 253 } while (--retry > 0); 254 usleep_range(10000, 12000); 255 256 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 257 258 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 259 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT | 260 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT); 261 reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET; 262 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 263 usleep_range(10000, 12000); 264 265 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 266 bt_dev_dbg(data->hdev, "csr register after reset: 0x%8.8x", reg); 267 268 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 269 270 /* If shared hardware reset is success then boot stage register shall be 271 * set to 0 272 */ 273 return reg == 0 ? 0 : -ENODEV; 274 } 275 276 /* This function enables BT function by setting BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT bit in 277 * BTINTEL_PCIE_CSR_FUNC_CTRL_REG register and wait for MSI-X with 278 * BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0. 279 * Then the host reads firmware version from BTINTEL_CSR_F2D_MBX and the boot stage 280 * from BTINTEL_PCIE_CSR_BOOT_STAGE_REG. 281 */ 282 static int btintel_pcie_enable_bt(struct btintel_pcie_data *data) 283 { 284 int err; 285 u32 reg; 286 287 data->gp0_received = false; 288 289 /* Update the DMA address of CI struct to CSR */ 290 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_LSB_REG, 291 data->ci_p_addr & 0xffffffff); 292 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_MSB_REG, 293 (u64)data->ci_p_addr >> 32); 294 295 /* Reset the cached value of boot stage. it is updated by the MSI-X 296 * gp0 interrupt handler. 297 */ 298 data->boot_stage_cache = 0x0; 299 300 /* Set MAC_INIT bit to start primary bootloader */ 301 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 302 reg &= ~(BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT | 303 BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON | 304 BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET); 305 reg |= (BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA | 306 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT); 307 308 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg); 309 310 /* MAC is ready. Enable BT FUNC */ 311 btintel_pcie_set_reg_bits(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, 312 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT); 313 314 btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG); 315 316 /* wait for interrupt from the device after booting up to primary 317 * bootloader. 318 */ 319 data->alive_intr_ctxt = BTINTEL_PCIE_ROM; 320 err = wait_event_timeout(data->gp0_wait_q, data->gp0_received, 321 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT_MS)); 322 if (!err) 323 return -ETIME; 324 325 /* Check cached boot stage is BTINTEL_PCIE_CSR_BOOT_STAGE_ROM(BIT(0)) */ 326 if (~data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ROM) 327 return -ENODEV; 328 329 return 0; 330 } 331 332 /* BIT(0) - ROM, BIT(1) - IML and BIT(3) - OP 333 * Sometimes during firmware image switching from ROM to IML or IML to OP image, 334 * the previous image bit is not cleared by firmware when alive interrupt is 335 * received. Driver needs to take care of these sticky bits when deciding the 336 * current image running on controller. 337 * Ex: 0x10 and 0x11 - both represents that controller is running IML 338 */ 339 static inline bool btintel_pcie_in_rom(struct btintel_pcie_data *data) 340 { 341 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ROM && 342 !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_IML) && 343 !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW); 344 } 345 346 static inline bool btintel_pcie_in_op(struct btintel_pcie_data *data) 347 { 348 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW; 349 } 350 351 static inline bool btintel_pcie_in_iml(struct btintel_pcie_data *data) 352 { 353 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_IML && 354 !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW); 355 } 356 357 static inline bool btintel_pcie_in_d3(struct btintel_pcie_data *data) 358 { 359 return data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY; 360 } 361 362 static inline bool btintel_pcie_in_d0(struct btintel_pcie_data *data) 363 { 364 return !(data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_D3_STATE_READY); 365 } 366 367 static void btintel_pcie_wr_sleep_cntrl(struct btintel_pcie_data *data, 368 u32 dxstate) 369 { 370 bt_dev_dbg(data->hdev, "writing sleep_ctl_reg: 0x%8.8x", dxstate); 371 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_IPC_SLEEP_CTL_REG, dxstate); 372 } 373 374 static inline char *btintel_pcie_alivectxt_state2str(u32 alive_intr_ctxt) 375 { 376 switch (alive_intr_ctxt) { 377 case BTINTEL_PCIE_ROM: 378 return "rom"; 379 case BTINTEL_PCIE_FW_DL: 380 return "fw_dl"; 381 case BTINTEL_PCIE_D0: 382 return "d0"; 383 case BTINTEL_PCIE_D3: 384 return "d3"; 385 case BTINTEL_PCIE_HCI_RESET: 386 return "hci_reset"; 387 case BTINTEL_PCIE_INTEL_HCI_RESET1: 388 return "intel_reset1"; 389 case BTINTEL_PCIE_INTEL_HCI_RESET2: 390 return "intel_reset2"; 391 default: 392 return "unknown"; 393 } 394 } 395 396 /* This function handles the MSI-X interrupt for gp0 cause (bit 0 in 397 * BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES) which is sent for boot stage and image response. 398 */ 399 static void btintel_pcie_msix_gp0_handler(struct btintel_pcie_data *data) 400 { 401 bool submit_rx, signal_waitq; 402 u32 reg, old_ctxt; 403 404 /* This interrupt is for three different causes and it is not easy to 405 * know what causes the interrupt. So, it compares each register value 406 * with cached value and update it before it wake up the queue. 407 */ 408 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG); 409 if (reg != data->boot_stage_cache) 410 data->boot_stage_cache = reg; 411 412 bt_dev_dbg(data->hdev, "Alive context: %s old_boot_stage: 0x%8.8x new_boot_stage: 0x%8.8x", 413 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt), 414 data->boot_stage_cache, reg); 415 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IMG_RESPONSE_REG); 416 if (reg != data->img_resp_cache) 417 data->img_resp_cache = reg; 418 419 data->gp0_received = true; 420 421 old_ctxt = data->alive_intr_ctxt; 422 submit_rx = false; 423 signal_waitq = false; 424 425 switch (data->alive_intr_ctxt) { 426 case BTINTEL_PCIE_ROM: 427 data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL; 428 signal_waitq = true; 429 break; 430 case BTINTEL_PCIE_FW_DL: 431 /* Error case is already handled. Ideally control shall not 432 * reach here 433 */ 434 break; 435 case BTINTEL_PCIE_INTEL_HCI_RESET1: 436 if (btintel_pcie_in_op(data)) { 437 submit_rx = true; 438 break; 439 } 440 441 if (btintel_pcie_in_iml(data)) { 442 submit_rx = true; 443 data->alive_intr_ctxt = BTINTEL_PCIE_FW_DL; 444 break; 445 } 446 break; 447 case BTINTEL_PCIE_INTEL_HCI_RESET2: 448 if (btintel_test_and_clear_flag(data->hdev, INTEL_WAIT_FOR_D0)) { 449 btintel_wake_up_flag(data->hdev, INTEL_WAIT_FOR_D0); 450 data->alive_intr_ctxt = BTINTEL_PCIE_D0; 451 } 452 break; 453 case BTINTEL_PCIE_D0: 454 if (btintel_pcie_in_d3(data)) { 455 data->alive_intr_ctxt = BTINTEL_PCIE_D3; 456 signal_waitq = true; 457 break; 458 } 459 break; 460 case BTINTEL_PCIE_D3: 461 if (btintel_pcie_in_d0(data)) { 462 data->alive_intr_ctxt = BTINTEL_PCIE_D0; 463 submit_rx = true; 464 signal_waitq = true; 465 break; 466 } 467 break; 468 case BTINTEL_PCIE_HCI_RESET: 469 data->alive_intr_ctxt = BTINTEL_PCIE_D0; 470 submit_rx = true; 471 signal_waitq = true; 472 break; 473 default: 474 bt_dev_err(data->hdev, "Unknown state: 0x%2.2x", 475 data->alive_intr_ctxt); 476 break; 477 } 478 479 if (submit_rx) { 480 btintel_pcie_reset_ia(data); 481 btintel_pcie_start_rx(data); 482 } 483 484 if (signal_waitq) { 485 bt_dev_dbg(data->hdev, "wake up gp0 wait_q"); 486 wake_up(&data->gp0_wait_q); 487 } 488 489 if (old_ctxt != data->alive_intr_ctxt) 490 bt_dev_dbg(data->hdev, "alive context changed: %s -> %s", 491 btintel_pcie_alivectxt_state2str(old_ctxt), 492 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt)); 493 } 494 495 /* This function handles the MSX-X interrupt for rx queue 0 which is for TX 496 */ 497 static void btintel_pcie_msix_tx_handle(struct btintel_pcie_data *data) 498 { 499 u16 cr_tia, cr_hia; 500 struct txq *txq; 501 struct urbd0 *urbd0; 502 503 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM]; 504 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM]; 505 506 if (cr_tia == cr_hia) 507 return; 508 509 txq = &data->txq; 510 511 while (cr_tia != cr_hia) { 512 data->tx_wait_done = true; 513 wake_up(&data->tx_wait_q); 514 515 urbd0 = &txq->urbd0s[cr_tia]; 516 517 if (urbd0->tfd_index > txq->count) 518 return; 519 520 cr_tia = (cr_tia + 1) % txq->count; 521 data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM] = cr_tia; 522 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_TXQ_NUM); 523 } 524 } 525 526 static int btintel_pcie_recv_event(struct hci_dev *hdev, struct sk_buff *skb) 527 { 528 struct hci_event_hdr *hdr = (void *)skb->data; 529 const char diagnostics_hdr[] = { 0x87, 0x80, 0x03 }; 530 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 531 532 if (skb->len > HCI_EVENT_HDR_SIZE && hdr->evt == 0xff && 533 hdr->plen > 0) { 534 const void *ptr = skb->data + HCI_EVENT_HDR_SIZE + 1; 535 unsigned int len = skb->len - HCI_EVENT_HDR_SIZE - 1; 536 537 if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) { 538 switch (skb->data[2]) { 539 case 0x02: 540 /* When switching to the operational firmware 541 * the device sends a vendor specific event 542 * indicating that the bootup completed. 543 */ 544 btintel_bootup(hdev, ptr, len); 545 546 /* If bootup event is from operational image, 547 * driver needs to write sleep control register to 548 * move into D0 state 549 */ 550 if (btintel_pcie_in_op(data)) { 551 btintel_pcie_wr_sleep_cntrl(data, BTINTEL_PCIE_STATE_D0); 552 data->alive_intr_ctxt = BTINTEL_PCIE_INTEL_HCI_RESET2; 553 kfree_skb(skb); 554 return 0; 555 } 556 557 if (btintel_pcie_in_iml(data)) { 558 /* In case of IML, there is no concept 559 * of D0 transition. Just mimic as if 560 * IML moved to D0 by clearing INTEL_WAIT_FOR_D0 561 * bit and waking up the task waiting on 562 * INTEL_WAIT_FOR_D0. This is required 563 * as intel_boot() is common function for 564 * both IML and OP image loading. 565 */ 566 if (btintel_test_and_clear_flag(data->hdev, 567 INTEL_WAIT_FOR_D0)) 568 btintel_wake_up_flag(data->hdev, 569 INTEL_WAIT_FOR_D0); 570 } 571 kfree_skb(skb); 572 return 0; 573 case 0x06: 574 /* When the firmware loading completes the 575 * device sends out a vendor specific event 576 * indicating the result of the firmware 577 * loading. 578 */ 579 btintel_secure_send_result(hdev, ptr, len); 580 kfree_skb(skb); 581 return 0; 582 } 583 } 584 585 /* Handle all diagnostics events separately. May still call 586 * hci_recv_frame. 587 */ 588 if (len >= sizeof(diagnostics_hdr) && 589 memcmp(&skb->data[2], diagnostics_hdr, 590 sizeof(diagnostics_hdr)) == 0) { 591 return btintel_diagnostics(hdev, skb); 592 } 593 594 /* This is a debug event that comes from IML and OP image when it 595 * starts execution. There is no need pass this event to stack. 596 */ 597 if (skb->data[2] == 0x97) 598 return 0; 599 } 600 601 return hci_recv_frame(hdev, skb); 602 } 603 /* Process the received rx data 604 * It check the frame header to identify the data type and create skb 605 * and calling HCI API 606 */ 607 static int btintel_pcie_recv_frame(struct btintel_pcie_data *data, 608 struct sk_buff *skb) 609 { 610 int ret; 611 u8 pkt_type; 612 u16 plen; 613 u32 pcie_pkt_type; 614 struct sk_buff *new_skb; 615 void *pdata; 616 struct hci_dev *hdev = data->hdev; 617 618 spin_lock(&data->hci_rx_lock); 619 620 /* The first 4 bytes indicates the Intel PCIe specific packet type */ 621 pdata = skb_pull_data(skb, BTINTEL_PCIE_HCI_TYPE_LEN); 622 if (!pdata) { 623 bt_dev_err(hdev, "Corrupted packet received"); 624 ret = -EILSEQ; 625 goto exit_error; 626 } 627 628 pcie_pkt_type = get_unaligned_le32(pdata); 629 630 switch (pcie_pkt_type) { 631 case BTINTEL_PCIE_HCI_ACL_PKT: 632 if (skb->len >= HCI_ACL_HDR_SIZE) { 633 plen = HCI_ACL_HDR_SIZE + __le16_to_cpu(hci_acl_hdr(skb)->dlen); 634 pkt_type = HCI_ACLDATA_PKT; 635 } else { 636 bt_dev_err(hdev, "ACL packet is too short"); 637 ret = -EILSEQ; 638 goto exit_error; 639 } 640 break; 641 642 case BTINTEL_PCIE_HCI_SCO_PKT: 643 if (skb->len >= HCI_SCO_HDR_SIZE) { 644 plen = HCI_SCO_HDR_SIZE + hci_sco_hdr(skb)->dlen; 645 pkt_type = HCI_SCODATA_PKT; 646 } else { 647 bt_dev_err(hdev, "SCO packet is too short"); 648 ret = -EILSEQ; 649 goto exit_error; 650 } 651 break; 652 653 case BTINTEL_PCIE_HCI_EVT_PKT: 654 if (skb->len >= HCI_EVENT_HDR_SIZE) { 655 plen = HCI_EVENT_HDR_SIZE + hci_event_hdr(skb)->plen; 656 pkt_type = HCI_EVENT_PKT; 657 } else { 658 bt_dev_err(hdev, "Event packet is too short"); 659 ret = -EILSEQ; 660 goto exit_error; 661 } 662 break; 663 664 case BTINTEL_PCIE_HCI_ISO_PKT: 665 if (skb->len >= HCI_ISO_HDR_SIZE) { 666 plen = HCI_ISO_HDR_SIZE + __le16_to_cpu(hci_iso_hdr(skb)->dlen); 667 pkt_type = HCI_ISODATA_PKT; 668 } else { 669 bt_dev_err(hdev, "ISO packet is too short"); 670 ret = -EILSEQ; 671 goto exit_error; 672 } 673 break; 674 675 default: 676 bt_dev_err(hdev, "Invalid packet type received: 0x%4.4x", 677 pcie_pkt_type); 678 ret = -EINVAL; 679 goto exit_error; 680 } 681 682 if (skb->len < plen) { 683 bt_dev_err(hdev, "Received corrupted packet. type: 0x%2.2x", 684 pkt_type); 685 ret = -EILSEQ; 686 goto exit_error; 687 } 688 689 bt_dev_dbg(hdev, "pkt_type: 0x%2.2x len: %u", pkt_type, plen); 690 691 new_skb = bt_skb_alloc(plen, GFP_ATOMIC); 692 if (!new_skb) { 693 bt_dev_err(hdev, "Failed to allocate memory for skb of len: %u", 694 skb->len); 695 ret = -ENOMEM; 696 goto exit_error; 697 } 698 699 hci_skb_pkt_type(new_skb) = pkt_type; 700 skb_put_data(new_skb, skb->data, plen); 701 hdev->stat.byte_rx += plen; 702 703 if (pcie_pkt_type == BTINTEL_PCIE_HCI_EVT_PKT) 704 ret = btintel_pcie_recv_event(hdev, new_skb); 705 else 706 ret = hci_recv_frame(hdev, new_skb); 707 708 exit_error: 709 if (ret) 710 hdev->stat.err_rx++; 711 712 spin_unlock(&data->hci_rx_lock); 713 714 return ret; 715 } 716 717 static void btintel_pcie_rx_work(struct work_struct *work) 718 { 719 struct btintel_pcie_data *data = container_of(work, 720 struct btintel_pcie_data, rx_work); 721 struct sk_buff *skb; 722 int err; 723 struct hci_dev *hdev = data->hdev; 724 725 /* Process the sk_buf in queue and send to the HCI layer */ 726 while ((skb = skb_dequeue(&data->rx_skb_q))) { 727 err = btintel_pcie_recv_frame(data, skb); 728 if (err) 729 bt_dev_err(hdev, "Failed to send received frame: %d", 730 err); 731 kfree_skb(skb); 732 } 733 } 734 735 /* create sk_buff with data and save it to queue and start RX work */ 736 static int btintel_pcie_submit_rx_work(struct btintel_pcie_data *data, u8 status, 737 void *buf) 738 { 739 int ret, len; 740 struct rfh_hdr *rfh_hdr; 741 struct sk_buff *skb; 742 743 rfh_hdr = buf; 744 745 len = rfh_hdr->packet_len; 746 if (len <= 0) { 747 ret = -EINVAL; 748 goto resubmit; 749 } 750 751 /* Remove RFH header */ 752 buf += sizeof(*rfh_hdr); 753 754 skb = alloc_skb(len, GFP_ATOMIC); 755 if (!skb) 756 goto resubmit; 757 758 skb_put_data(skb, buf, len); 759 skb_queue_tail(&data->rx_skb_q, skb); 760 queue_work(data->workqueue, &data->rx_work); 761 762 resubmit: 763 ret = btintel_pcie_submit_rx(data); 764 765 return ret; 766 } 767 768 /* Handles the MSI-X interrupt for rx queue 1 which is for RX */ 769 static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data) 770 { 771 u16 cr_hia, cr_tia; 772 struct rxq *rxq; 773 struct urbd1 *urbd1; 774 struct data_buf *buf; 775 int ret; 776 struct hci_dev *hdev = data->hdev; 777 778 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM]; 779 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM]; 780 781 bt_dev_dbg(hdev, "RXQ: cr_hia: %u cr_tia: %u", cr_hia, cr_tia); 782 783 /* Check CR_TIA and CR_HIA for change */ 784 if (cr_tia == cr_hia) { 785 bt_dev_warn(hdev, "RXQ: no new CD found"); 786 return; 787 } 788 789 rxq = &data->rxq; 790 791 /* The firmware sends multiple CD in a single MSI-X and it needs to 792 * process all received CDs in this interrupt. 793 */ 794 while (cr_tia != cr_hia) { 795 urbd1 = &rxq->urbd1s[cr_tia]; 796 ipc_print_urbd1(data->hdev, urbd1, cr_tia); 797 798 buf = &rxq->bufs[urbd1->frbd_tag]; 799 if (!buf) { 800 bt_dev_err(hdev, "RXQ: failed to get the DMA buffer for %d", 801 urbd1->frbd_tag); 802 return; 803 } 804 805 ret = btintel_pcie_submit_rx_work(data, urbd1->status, 806 buf->data); 807 if (ret) { 808 bt_dev_err(hdev, "RXQ: failed to submit rx request"); 809 return; 810 } 811 812 cr_tia = (cr_tia + 1) % rxq->count; 813 data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM] = cr_tia; 814 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM); 815 } 816 } 817 818 static irqreturn_t btintel_pcie_msix_isr(int irq, void *data) 819 { 820 return IRQ_WAKE_THREAD; 821 } 822 823 static irqreturn_t btintel_pcie_irq_msix_handler(int irq, void *dev_id) 824 { 825 struct msix_entry *entry = dev_id; 826 struct btintel_pcie_data *data = btintel_pcie_get_data(entry); 827 u32 intr_fh, intr_hw; 828 829 spin_lock(&data->irq_lock); 830 intr_fh = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES); 831 intr_hw = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES); 832 833 /* Clear causes registers to avoid being handling the same cause */ 834 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES, intr_fh); 835 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES, intr_hw); 836 spin_unlock(&data->irq_lock); 837 838 if (unlikely(!(intr_fh | intr_hw))) { 839 /* Ignore interrupt, inta == 0 */ 840 return IRQ_NONE; 841 } 842 843 /* This interrupt is triggered by the firmware after updating 844 * boot_stage register and image_response register 845 */ 846 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0) 847 btintel_pcie_msix_gp0_handler(data); 848 849 /* For TX */ 850 if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0) 851 btintel_pcie_msix_tx_handle(data); 852 853 /* For RX */ 854 if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1) 855 btintel_pcie_msix_rx_handle(data); 856 857 /* 858 * Before sending the interrupt the HW disables it to prevent a nested 859 * interrupt. This is done by writing 1 to the corresponding bit in 860 * the mask register. After handling the interrupt, it should be 861 * re-enabled by clearing this bit. This register is defined as write 1 862 * clear (W1C) register, meaning that it's cleared by writing 1 863 * to the bit. 864 */ 865 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_AUTOMASK_ST, 866 BIT(entry->entry)); 867 868 return IRQ_HANDLED; 869 } 870 871 /* This function requests the irq for MSI-X and registers the handlers per irq. 872 * Currently, it requests only 1 irq for all interrupt causes. 873 */ 874 static int btintel_pcie_setup_irq(struct btintel_pcie_data *data) 875 { 876 int err; 877 int num_irqs, i; 878 879 for (i = 0; i < BTINTEL_PCIE_MSIX_VEC_MAX; i++) 880 data->msix_entries[i].entry = i; 881 882 num_irqs = pci_alloc_irq_vectors(data->pdev, BTINTEL_PCIE_MSIX_VEC_MIN, 883 BTINTEL_PCIE_MSIX_VEC_MAX, PCI_IRQ_MSIX); 884 if (num_irqs < 0) 885 return num_irqs; 886 887 data->alloc_vecs = num_irqs; 888 data->msix_enabled = 1; 889 data->def_irq = 0; 890 891 /* setup irq handler */ 892 for (i = 0; i < data->alloc_vecs; i++) { 893 struct msix_entry *msix_entry; 894 895 msix_entry = &data->msix_entries[i]; 896 msix_entry->vector = pci_irq_vector(data->pdev, i); 897 898 err = devm_request_threaded_irq(&data->pdev->dev, 899 msix_entry->vector, 900 btintel_pcie_msix_isr, 901 btintel_pcie_irq_msix_handler, 902 IRQF_SHARED, 903 KBUILD_MODNAME, 904 msix_entry); 905 if (err) { 906 pci_free_irq_vectors(data->pdev); 907 data->alloc_vecs = 0; 908 return err; 909 } 910 } 911 return 0; 912 } 913 914 struct btintel_pcie_causes_list { 915 u32 cause; 916 u32 mask_reg; 917 u8 cause_num; 918 }; 919 920 static struct btintel_pcie_causes_list causes_list[] = { 921 { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x00 }, 922 { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x01 }, 923 { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x20 }, 924 }; 925 926 /* This function configures the interrupt masks for both HW_INT_CAUSES and 927 * FH_INT_CAUSES which are meaningful to us. 928 * 929 * After resetting BT function via PCIE FLR or FUNC_CTRL reset, the driver 930 * need to call this function again to configure since the masks 931 * are reset to 0xFFFFFFFF after reset. 932 */ 933 static void btintel_pcie_config_msix(struct btintel_pcie_data *data) 934 { 935 int i; 936 int val = data->def_irq | BTINTEL_PCIE_MSIX_NON_AUTO_CLEAR_CAUSE; 937 938 /* Set Non Auto Clear Cause */ 939 for (i = 0; i < ARRAY_SIZE(causes_list); i++) { 940 btintel_pcie_wr_reg8(data, 941 BTINTEL_PCIE_CSR_MSIX_IVAR(causes_list[i].cause_num), 942 val); 943 btintel_pcie_clr_reg_bits(data, 944 causes_list[i].mask_reg, 945 causes_list[i].cause); 946 } 947 948 /* Save the initial interrupt mask */ 949 data->fh_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK); 950 data->hw_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK); 951 } 952 953 static int btintel_pcie_config_pcie(struct pci_dev *pdev, 954 struct btintel_pcie_data *data) 955 { 956 int err; 957 958 err = pcim_enable_device(pdev); 959 if (err) 960 return err; 961 962 pci_set_master(pdev); 963 964 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 965 if (err) { 966 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 967 if (err) 968 return err; 969 } 970 971 data->base_addr = pcim_iomap_region(pdev, 0, KBUILD_MODNAME); 972 if (IS_ERR(data->base_addr)) 973 return PTR_ERR(data->base_addr); 974 975 err = btintel_pcie_setup_irq(data); 976 if (err) 977 return err; 978 979 /* Configure MSI-X with causes list */ 980 btintel_pcie_config_msix(data); 981 982 return 0; 983 } 984 985 static void btintel_pcie_init_ci(struct btintel_pcie_data *data, 986 struct ctx_info *ci) 987 { 988 ci->version = 0x1; 989 ci->size = sizeof(*ci); 990 ci->config = 0x0000; 991 ci->addr_cr_hia = data->ia.cr_hia_p_addr; 992 ci->addr_tr_tia = data->ia.tr_tia_p_addr; 993 ci->addr_cr_tia = data->ia.cr_tia_p_addr; 994 ci->addr_tr_hia = data->ia.tr_hia_p_addr; 995 ci->num_cr_ia = BTINTEL_PCIE_NUM_QUEUES; 996 ci->num_tr_ia = BTINTEL_PCIE_NUM_QUEUES; 997 ci->addr_urbdq0 = data->txq.urbd0s_p_addr; 998 ci->addr_tfdq = data->txq.tfds_p_addr; 999 ci->num_tfdq = data->txq.count; 1000 ci->num_urbdq0 = data->txq.count; 1001 ci->tfdq_db_vec = BTINTEL_PCIE_TXQ_NUM; 1002 ci->urbdq0_db_vec = BTINTEL_PCIE_TXQ_NUM; 1003 ci->rbd_size = BTINTEL_PCIE_RBD_SIZE_4K; 1004 ci->addr_frbdq = data->rxq.frbds_p_addr; 1005 ci->num_frbdq = data->rxq.count; 1006 ci->frbdq_db_vec = BTINTEL_PCIE_RXQ_NUM; 1007 ci->addr_urbdq1 = data->rxq.urbd1s_p_addr; 1008 ci->num_urbdq1 = data->rxq.count; 1009 ci->urbdq_db_vec = BTINTEL_PCIE_RXQ_NUM; 1010 } 1011 1012 static void btintel_pcie_free_txq_bufs(struct btintel_pcie_data *data, 1013 struct txq *txq) 1014 { 1015 /* Free data buffers first */ 1016 dma_free_coherent(&data->pdev->dev, txq->count * BTINTEL_PCIE_BUFFER_SIZE, 1017 txq->buf_v_addr, txq->buf_p_addr); 1018 kfree(txq->bufs); 1019 } 1020 1021 static int btintel_pcie_setup_txq_bufs(struct btintel_pcie_data *data, 1022 struct txq *txq) 1023 { 1024 int i; 1025 struct data_buf *buf; 1026 1027 /* Allocate the same number of buffers as the descriptor */ 1028 txq->bufs = kmalloc_array(txq->count, sizeof(*buf), GFP_KERNEL); 1029 if (!txq->bufs) 1030 return -ENOMEM; 1031 1032 /* Allocate full chunk of data buffer for DMA first and do indexing and 1033 * initialization next, so it can be freed easily 1034 */ 1035 txq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev, 1036 txq->count * BTINTEL_PCIE_BUFFER_SIZE, 1037 &txq->buf_p_addr, 1038 GFP_KERNEL | __GFP_NOWARN); 1039 if (!txq->buf_v_addr) { 1040 kfree(txq->bufs); 1041 return -ENOMEM; 1042 } 1043 1044 /* Setup the allocated DMA buffer to bufs. Each data_buf should 1045 * have virtual address and physical address 1046 */ 1047 for (i = 0; i < txq->count; i++) { 1048 buf = &txq->bufs[i]; 1049 buf->data_p_addr = txq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 1050 buf->data = txq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 1051 } 1052 1053 return 0; 1054 } 1055 1056 static void btintel_pcie_free_rxq_bufs(struct btintel_pcie_data *data, 1057 struct rxq *rxq) 1058 { 1059 /* Free data buffers first */ 1060 dma_free_coherent(&data->pdev->dev, rxq->count * BTINTEL_PCIE_BUFFER_SIZE, 1061 rxq->buf_v_addr, rxq->buf_p_addr); 1062 kfree(rxq->bufs); 1063 } 1064 1065 static int btintel_pcie_setup_rxq_bufs(struct btintel_pcie_data *data, 1066 struct rxq *rxq) 1067 { 1068 int i; 1069 struct data_buf *buf; 1070 1071 /* Allocate the same number of buffers as the descriptor */ 1072 rxq->bufs = kmalloc_array(rxq->count, sizeof(*buf), GFP_KERNEL); 1073 if (!rxq->bufs) 1074 return -ENOMEM; 1075 1076 /* Allocate full chunk of data buffer for DMA first and do indexing and 1077 * initialization next, so it can be freed easily 1078 */ 1079 rxq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev, 1080 rxq->count * BTINTEL_PCIE_BUFFER_SIZE, 1081 &rxq->buf_p_addr, 1082 GFP_KERNEL | __GFP_NOWARN); 1083 if (!rxq->buf_v_addr) { 1084 kfree(rxq->bufs); 1085 return -ENOMEM; 1086 } 1087 1088 /* Setup the allocated DMA buffer to bufs. Each data_buf should 1089 * have virtual address and physical address 1090 */ 1091 for (i = 0; i < rxq->count; i++) { 1092 buf = &rxq->bufs[i]; 1093 buf->data_p_addr = rxq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 1094 buf->data = rxq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE); 1095 } 1096 1097 return 0; 1098 } 1099 1100 static void btintel_pcie_setup_ia(struct btintel_pcie_data *data, 1101 dma_addr_t p_addr, void *v_addr, 1102 struct ia *ia) 1103 { 1104 /* TR Head Index Array */ 1105 ia->tr_hia_p_addr = p_addr; 1106 ia->tr_hia = v_addr; 1107 1108 /* TR Tail Index Array */ 1109 ia->tr_tia_p_addr = p_addr + sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES; 1110 ia->tr_tia = v_addr + sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES; 1111 1112 /* CR Head index Array */ 1113 ia->cr_hia_p_addr = p_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 2); 1114 ia->cr_hia = v_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 2); 1115 1116 /* CR Tail Index Array */ 1117 ia->cr_tia_p_addr = p_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 3); 1118 ia->cr_tia = v_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 3); 1119 } 1120 1121 static void btintel_pcie_free(struct btintel_pcie_data *data) 1122 { 1123 btintel_pcie_free_rxq_bufs(data, &data->rxq); 1124 btintel_pcie_free_txq_bufs(data, &data->txq); 1125 1126 dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr); 1127 dma_pool_destroy(data->dma_pool); 1128 } 1129 1130 /* Allocate tx and rx queues, any related data structures and buffers. 1131 */ 1132 static int btintel_pcie_alloc(struct btintel_pcie_data *data) 1133 { 1134 int err = 0; 1135 size_t total; 1136 dma_addr_t p_addr; 1137 void *v_addr; 1138 1139 /* Allocate the chunk of DMA memory for descriptors, index array, and 1140 * context information, instead of allocating individually. 1141 * The DMA memory for data buffer is allocated while setting up the 1142 * each queue. 1143 * 1144 * Total size is sum of the following 1145 * + size of TFD * Number of descriptors in queue 1146 * + size of URBD0 * Number of descriptors in queue 1147 * + size of FRBD * Number of descriptors in queue 1148 * + size of URBD1 * Number of descriptors in queue 1149 * + size of index * Number of queues(2) * type of index array(4) 1150 * + size of context information 1151 */ 1152 total = (sizeof(struct tfd) + sizeof(struct urbd0) + sizeof(struct frbd) 1153 + sizeof(struct urbd1)) * BTINTEL_DESCS_COUNT; 1154 1155 /* Add the sum of size of index array and size of ci struct */ 1156 total += (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 4) + sizeof(struct ctx_info); 1157 1158 /* Allocate DMA Pool */ 1159 data->dma_pool = dma_pool_create(KBUILD_MODNAME, &data->pdev->dev, 1160 total, BTINTEL_PCIE_DMA_POOL_ALIGNMENT, 0); 1161 if (!data->dma_pool) { 1162 err = -ENOMEM; 1163 goto exit_error; 1164 } 1165 1166 v_addr = dma_pool_zalloc(data->dma_pool, GFP_KERNEL | __GFP_NOWARN, 1167 &p_addr); 1168 if (!v_addr) { 1169 dma_pool_destroy(data->dma_pool); 1170 err = -ENOMEM; 1171 goto exit_error; 1172 } 1173 1174 data->dma_p_addr = p_addr; 1175 data->dma_v_addr = v_addr; 1176 1177 /* Setup descriptor count */ 1178 data->txq.count = BTINTEL_DESCS_COUNT; 1179 data->rxq.count = BTINTEL_DESCS_COUNT; 1180 1181 /* Setup tfds */ 1182 data->txq.tfds_p_addr = p_addr; 1183 data->txq.tfds = v_addr; 1184 1185 p_addr += (sizeof(struct tfd) * BTINTEL_DESCS_COUNT); 1186 v_addr += (sizeof(struct tfd) * BTINTEL_DESCS_COUNT); 1187 1188 /* Setup urbd0 */ 1189 data->txq.urbd0s_p_addr = p_addr; 1190 data->txq.urbd0s = v_addr; 1191 1192 p_addr += (sizeof(struct urbd0) * BTINTEL_DESCS_COUNT); 1193 v_addr += (sizeof(struct urbd0) * BTINTEL_DESCS_COUNT); 1194 1195 /* Setup FRBD*/ 1196 data->rxq.frbds_p_addr = p_addr; 1197 data->rxq.frbds = v_addr; 1198 1199 p_addr += (sizeof(struct frbd) * BTINTEL_DESCS_COUNT); 1200 v_addr += (sizeof(struct frbd) * BTINTEL_DESCS_COUNT); 1201 1202 /* Setup urbd1 */ 1203 data->rxq.urbd1s_p_addr = p_addr; 1204 data->rxq.urbd1s = v_addr; 1205 1206 p_addr += (sizeof(struct urbd1) * BTINTEL_DESCS_COUNT); 1207 v_addr += (sizeof(struct urbd1) * BTINTEL_DESCS_COUNT); 1208 1209 /* Setup data buffers for txq */ 1210 err = btintel_pcie_setup_txq_bufs(data, &data->txq); 1211 if (err) 1212 goto exit_error_pool; 1213 1214 /* Setup data buffers for rxq */ 1215 err = btintel_pcie_setup_rxq_bufs(data, &data->rxq); 1216 if (err) 1217 goto exit_error_txq; 1218 1219 /* Setup Index Array */ 1220 btintel_pcie_setup_ia(data, p_addr, v_addr, &data->ia); 1221 1222 /* Setup Context Information */ 1223 p_addr += sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 4; 1224 v_addr += sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 4; 1225 1226 data->ci = v_addr; 1227 data->ci_p_addr = p_addr; 1228 1229 /* Initialize the CI */ 1230 btintel_pcie_init_ci(data, data->ci); 1231 1232 return 0; 1233 1234 exit_error_txq: 1235 btintel_pcie_free_txq_bufs(data, &data->txq); 1236 exit_error_pool: 1237 dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr); 1238 dma_pool_destroy(data->dma_pool); 1239 exit_error: 1240 return err; 1241 } 1242 1243 static int btintel_pcie_open(struct hci_dev *hdev) 1244 { 1245 bt_dev_dbg(hdev, ""); 1246 1247 return 0; 1248 } 1249 1250 static int btintel_pcie_close(struct hci_dev *hdev) 1251 { 1252 bt_dev_dbg(hdev, ""); 1253 1254 return 0; 1255 } 1256 1257 static int btintel_pcie_inject_cmd_complete(struct hci_dev *hdev, __u16 opcode) 1258 { 1259 struct sk_buff *skb; 1260 struct hci_event_hdr *hdr; 1261 struct hci_ev_cmd_complete *evt; 1262 1263 skb = bt_skb_alloc(sizeof(*hdr) + sizeof(*evt) + 1, GFP_KERNEL); 1264 if (!skb) 1265 return -ENOMEM; 1266 1267 hdr = (struct hci_event_hdr *)skb_put(skb, sizeof(*hdr)); 1268 hdr->evt = HCI_EV_CMD_COMPLETE; 1269 hdr->plen = sizeof(*evt) + 1; 1270 1271 evt = (struct hci_ev_cmd_complete *)skb_put(skb, sizeof(*evt)); 1272 evt->ncmd = 0x01; 1273 evt->opcode = cpu_to_le16(opcode); 1274 1275 *(u8 *)skb_put(skb, 1) = 0x00; 1276 1277 hci_skb_pkt_type(skb) = HCI_EVENT_PKT; 1278 1279 return hci_recv_frame(hdev, skb); 1280 } 1281 1282 static int btintel_pcie_send_frame(struct hci_dev *hdev, 1283 struct sk_buff *skb) 1284 { 1285 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 1286 struct hci_command_hdr *cmd; 1287 __u16 opcode = ~0; 1288 int ret; 1289 u32 type; 1290 u32 old_ctxt; 1291 1292 /* Due to the fw limitation, the type header of the packet should be 1293 * 4 bytes unlike 1 byte for UART. In UART, the firmware can read 1294 * the first byte to get the packet type and redirect the rest of data 1295 * packet to the right handler. 1296 * 1297 * But for PCIe, THF(Transfer Flow Handler) fetches the 4 bytes of data 1298 * from DMA memory and by the time it reads the first 4 bytes, it has 1299 * already consumed some part of packet. Thus the packet type indicator 1300 * for iBT PCIe is 4 bytes. 1301 * 1302 * Luckily, when HCI core creates the skb, it allocates 8 bytes of 1303 * head room for profile and driver use, and before sending the data 1304 * to the device, append the iBT PCIe packet type in the front. 1305 */ 1306 switch (hci_skb_pkt_type(skb)) { 1307 case HCI_COMMAND_PKT: 1308 type = BTINTEL_PCIE_HCI_CMD_PKT; 1309 cmd = (void *)skb->data; 1310 opcode = le16_to_cpu(cmd->opcode); 1311 if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) { 1312 struct hci_command_hdr *cmd = (void *)skb->data; 1313 __u16 opcode = le16_to_cpu(cmd->opcode); 1314 1315 /* When the 0xfc01 command is issued to boot into 1316 * the operational firmware, it will actually not 1317 * send a command complete event. To keep the flow 1318 * control working inject that event here. 1319 */ 1320 if (opcode == 0xfc01) 1321 btintel_pcie_inject_cmd_complete(hdev, opcode); 1322 } 1323 hdev->stat.cmd_tx++; 1324 break; 1325 case HCI_ACLDATA_PKT: 1326 type = BTINTEL_PCIE_HCI_ACL_PKT; 1327 hdev->stat.acl_tx++; 1328 break; 1329 case HCI_SCODATA_PKT: 1330 type = BTINTEL_PCIE_HCI_SCO_PKT; 1331 hdev->stat.sco_tx++; 1332 break; 1333 case HCI_ISODATA_PKT: 1334 type = BTINTEL_PCIE_HCI_ISO_PKT; 1335 break; 1336 default: 1337 bt_dev_err(hdev, "Unknown HCI packet type"); 1338 return -EILSEQ; 1339 } 1340 memcpy(skb_push(skb, BTINTEL_PCIE_HCI_TYPE_LEN), &type, 1341 BTINTEL_PCIE_HCI_TYPE_LEN); 1342 1343 ret = btintel_pcie_send_sync(data, skb); 1344 if (ret) { 1345 hdev->stat.err_tx++; 1346 bt_dev_err(hdev, "Failed to send frame (%d)", ret); 1347 goto exit_error; 1348 } 1349 1350 if (type == BTINTEL_PCIE_HCI_CMD_PKT && 1351 (opcode == HCI_OP_RESET || opcode == 0xfc01)) { 1352 old_ctxt = data->alive_intr_ctxt; 1353 data->alive_intr_ctxt = 1354 (opcode == 0xfc01 ? BTINTEL_PCIE_INTEL_HCI_RESET1 : 1355 BTINTEL_PCIE_HCI_RESET); 1356 bt_dev_dbg(data->hdev, "sent cmd: 0x%4.4x alive context changed: %s -> %s", 1357 opcode, btintel_pcie_alivectxt_state2str(old_ctxt), 1358 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt)); 1359 if (opcode == HCI_OP_RESET) { 1360 data->gp0_received = false; 1361 ret = wait_event_timeout(data->gp0_wait_q, 1362 data->gp0_received, 1363 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT_MS)); 1364 if (!ret) { 1365 hdev->stat.err_tx++; 1366 bt_dev_err(hdev, "No alive interrupt received for %s", 1367 btintel_pcie_alivectxt_state2str(data->alive_intr_ctxt)); 1368 ret = -ETIME; 1369 goto exit_error; 1370 } 1371 } 1372 } 1373 hdev->stat.byte_tx += skb->len; 1374 kfree_skb(skb); 1375 1376 exit_error: 1377 return ret; 1378 } 1379 1380 static void btintel_pcie_release_hdev(struct btintel_pcie_data *data) 1381 { 1382 struct hci_dev *hdev; 1383 1384 hdev = data->hdev; 1385 hci_unregister_dev(hdev); 1386 hci_free_dev(hdev); 1387 data->hdev = NULL; 1388 } 1389 1390 static int btintel_pcie_setup_internal(struct hci_dev *hdev) 1391 { 1392 const u8 param[1] = { 0xFF }; 1393 struct intel_version_tlv ver_tlv; 1394 struct sk_buff *skb; 1395 int err; 1396 1397 BT_DBG("%s", hdev->name); 1398 1399 skb = __hci_cmd_sync(hdev, 0xfc05, 1, param, HCI_CMD_TIMEOUT); 1400 if (IS_ERR(skb)) { 1401 bt_dev_err(hdev, "Reading Intel version command failed (%ld)", 1402 PTR_ERR(skb)); 1403 return PTR_ERR(skb); 1404 } 1405 1406 /* Check the status */ 1407 if (skb->data[0]) { 1408 bt_dev_err(hdev, "Intel Read Version command failed (%02x)", 1409 skb->data[0]); 1410 err = -EIO; 1411 goto exit_error; 1412 } 1413 1414 /* Apply the common HCI quirks for Intel device */ 1415 set_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks); 1416 set_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks); 1417 set_bit(HCI_QUIRK_NON_PERSISTENT_DIAG, &hdev->quirks); 1418 1419 /* Set up the quality report callback for Intel devices */ 1420 hdev->set_quality_report = btintel_set_quality_report; 1421 1422 memset(&ver_tlv, 0, sizeof(ver_tlv)); 1423 /* For TLV type device, parse the tlv data */ 1424 err = btintel_parse_version_tlv(hdev, &ver_tlv, skb); 1425 if (err) { 1426 bt_dev_err(hdev, "Failed to parse TLV version information"); 1427 goto exit_error; 1428 } 1429 1430 switch (INTEL_HW_PLATFORM(ver_tlv.cnvi_bt)) { 1431 case 0x37: 1432 break; 1433 default: 1434 bt_dev_err(hdev, "Unsupported Intel hardware platform (0x%2x)", 1435 INTEL_HW_PLATFORM(ver_tlv.cnvi_bt)); 1436 err = -EINVAL; 1437 goto exit_error; 1438 } 1439 1440 /* Check for supported iBT hardware variants of this firmware 1441 * loading method. 1442 * 1443 * This check has been put in place to ensure correct forward 1444 * compatibility options when newer hardware variants come 1445 * along. 1446 */ 1447 switch (INTEL_HW_VARIANT(ver_tlv.cnvi_bt)) { 1448 case 0x1e: /* BzrI */ 1449 /* Display version information of TLV type */ 1450 btintel_version_info_tlv(hdev, &ver_tlv); 1451 1452 /* Apply the device specific HCI quirks for TLV based devices 1453 * 1454 * All TLV based devices support WBS 1455 */ 1456 set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED, &hdev->quirks); 1457 1458 /* Setup MSFT Extension support */ 1459 btintel_set_msft_opcode(hdev, 1460 INTEL_HW_VARIANT(ver_tlv.cnvi_bt)); 1461 1462 err = btintel_bootloader_setup_tlv(hdev, &ver_tlv); 1463 if (err) 1464 goto exit_error; 1465 break; 1466 default: 1467 bt_dev_err(hdev, "Unsupported Intel hw variant (%u)", 1468 INTEL_HW_VARIANT(ver_tlv.cnvi_bt)); 1469 err = -EINVAL; 1470 goto exit_error; 1471 break; 1472 } 1473 1474 btintel_print_fseq_info(hdev); 1475 exit_error: 1476 kfree_skb(skb); 1477 1478 return err; 1479 } 1480 1481 static int btintel_pcie_setup(struct hci_dev *hdev) 1482 { 1483 int err, fw_dl_retry = 0; 1484 struct btintel_pcie_data *data = hci_get_drvdata(hdev); 1485 1486 while ((err = btintel_pcie_setup_internal(hdev)) && fw_dl_retry++ < 1) { 1487 bt_dev_err(hdev, "Firmware download retry count: %d", 1488 fw_dl_retry); 1489 err = btintel_pcie_reset_bt(data); 1490 if (err) { 1491 bt_dev_err(hdev, "Failed to do shr reset: %d", err); 1492 break; 1493 } 1494 usleep_range(10000, 12000); 1495 btintel_pcie_reset_ia(data); 1496 btintel_pcie_config_msix(data); 1497 err = btintel_pcie_enable_bt(data); 1498 if (err) { 1499 bt_dev_err(hdev, "Failed to enable hardware: %d", err); 1500 break; 1501 } 1502 btintel_pcie_start_rx(data); 1503 } 1504 return err; 1505 } 1506 1507 static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data) 1508 { 1509 int err; 1510 struct hci_dev *hdev; 1511 1512 hdev = hci_alloc_dev_priv(sizeof(struct btintel_data)); 1513 if (!hdev) 1514 return -ENOMEM; 1515 1516 hdev->bus = HCI_PCI; 1517 hci_set_drvdata(hdev, data); 1518 1519 data->hdev = hdev; 1520 SET_HCIDEV_DEV(hdev, &data->pdev->dev); 1521 1522 hdev->manufacturer = 2; 1523 hdev->open = btintel_pcie_open; 1524 hdev->close = btintel_pcie_close; 1525 hdev->send = btintel_pcie_send_frame; 1526 hdev->setup = btintel_pcie_setup; 1527 hdev->shutdown = btintel_shutdown_combined; 1528 hdev->hw_error = btintel_hw_error; 1529 hdev->set_diag = btintel_set_diag; 1530 hdev->set_bdaddr = btintel_set_bdaddr; 1531 1532 err = hci_register_dev(hdev); 1533 if (err < 0) { 1534 BT_ERR("Failed to register to hdev (%d)", err); 1535 goto exit_error; 1536 } 1537 1538 return 0; 1539 1540 exit_error: 1541 hci_free_dev(hdev); 1542 return err; 1543 } 1544 1545 static int btintel_pcie_probe(struct pci_dev *pdev, 1546 const struct pci_device_id *ent) 1547 { 1548 int err; 1549 struct btintel_pcie_data *data; 1550 1551 if (!pdev) 1552 return -ENODEV; 1553 1554 data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL); 1555 if (!data) 1556 return -ENOMEM; 1557 1558 data->pdev = pdev; 1559 1560 spin_lock_init(&data->irq_lock); 1561 spin_lock_init(&data->hci_rx_lock); 1562 1563 init_waitqueue_head(&data->gp0_wait_q); 1564 data->gp0_received = false; 1565 1566 init_waitqueue_head(&data->tx_wait_q); 1567 data->tx_wait_done = false; 1568 1569 data->workqueue = alloc_ordered_workqueue(KBUILD_MODNAME, WQ_HIGHPRI); 1570 if (!data->workqueue) 1571 return -ENOMEM; 1572 1573 skb_queue_head_init(&data->rx_skb_q); 1574 INIT_WORK(&data->rx_work, btintel_pcie_rx_work); 1575 1576 data->boot_stage_cache = 0x00; 1577 data->img_resp_cache = 0x00; 1578 1579 err = btintel_pcie_config_pcie(pdev, data); 1580 if (err) 1581 goto exit_error; 1582 1583 pci_set_drvdata(pdev, data); 1584 1585 err = btintel_pcie_alloc(data); 1586 if (err) 1587 goto exit_error; 1588 1589 err = btintel_pcie_enable_bt(data); 1590 if (err) 1591 goto exit_error; 1592 1593 /* CNV information (CNVi and CNVr) is in CSR */ 1594 data->cnvi = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_HW_REV_REG); 1595 1596 data->cnvr = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_RF_ID_REG); 1597 1598 err = btintel_pcie_start_rx(data); 1599 if (err) 1600 goto exit_error; 1601 1602 err = btintel_pcie_setup_hdev(data); 1603 if (err) 1604 goto exit_error; 1605 1606 bt_dev_dbg(data->hdev, "cnvi: 0x%8.8x cnvr: 0x%8.8x", data->cnvi, 1607 data->cnvr); 1608 return 0; 1609 1610 exit_error: 1611 /* reset device before exit */ 1612 btintel_pcie_reset_bt(data); 1613 1614 pci_clear_master(pdev); 1615 1616 pci_set_drvdata(pdev, NULL); 1617 1618 return err; 1619 } 1620 1621 static void btintel_pcie_remove(struct pci_dev *pdev) 1622 { 1623 struct btintel_pcie_data *data; 1624 1625 data = pci_get_drvdata(pdev); 1626 1627 btintel_pcie_reset_bt(data); 1628 for (int i = 0; i < data->alloc_vecs; i++) { 1629 struct msix_entry *msix_entry; 1630 1631 msix_entry = &data->msix_entries[i]; 1632 free_irq(msix_entry->vector, msix_entry); 1633 } 1634 1635 pci_free_irq_vectors(pdev); 1636 1637 btintel_pcie_release_hdev(data); 1638 1639 flush_work(&data->rx_work); 1640 1641 destroy_workqueue(data->workqueue); 1642 1643 btintel_pcie_free(data); 1644 1645 pci_clear_master(pdev); 1646 1647 pci_set_drvdata(pdev, NULL); 1648 } 1649 1650 static struct pci_driver btintel_pcie_driver = { 1651 .name = KBUILD_MODNAME, 1652 .id_table = btintel_pcie_table, 1653 .probe = btintel_pcie_probe, 1654 .remove = btintel_pcie_remove, 1655 }; 1656 module_pci_driver(btintel_pcie_driver); 1657 1658 MODULE_AUTHOR("Tedd Ho-Jeong An <tedd.an@intel.com>"); 1659 MODULE_DESCRIPTION("Intel Bluetooth PCIe transport driver ver " VERSION); 1660 MODULE_VERSION(VERSION); 1661 MODULE_LICENSE("GPL"); 1662