1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright (c) 2020, MIPI Alliance, Inc. 4 * 5 * Author: Nicolas Pitre <npitre@baylibre.com> 6 * 7 * Note: The I3C HCI v2.0 spec is still in flux. The IBI support is based on 8 * v1.x of the spec and v2.0 will likely be split out. 9 */ 10 11 #include <linux/bitfield.h> 12 #include <linux/delay.h> 13 #include <linux/device.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/errno.h> 16 #include <linux/i3c/master.h> 17 #include <linux/io.h> 18 #include <linux/pci.h> 19 20 #include "hci.h" 21 #include "cmd.h" 22 #include "ibi.h" 23 24 /* 25 * Software Parameter Values (somewhat arb itrary for now). 26 * Some of them could be determined at run time eventually. 27 */ 28 29 #define XFER_RINGS 1 /* max: 8 */ 30 #define XFER_RING_ENTRIES 255 /* max: 255 */ 31 32 #define IBI_RINGS 1 /* max: 8 */ 33 #define IBI_STATUS_RING_ENTRIES 32 /* max: 255 */ 34 #define IBI_CHUNK_CACHELINES 1 /* max: 256 bytes equivalent */ 35 #define IBI_CHUNK_POOL_SIZE 128 /* max: 1023 */ 36 37 /* 38 * Ring Header Preamble 39 */ 40 41 #define rhs_reg_read(r) readl(hci->RHS_regs + (RHS_##r)) 42 #define rhs_reg_write(r, v) writel(v, hci->RHS_regs + (RHS_##r)) 43 44 #define RHS_CONTROL 0x00 45 #define PREAMBLE_SIZE GENMASK(31, 24) /* Preamble Section Size */ 46 #define HEADER_SIZE GENMASK(23, 16) /* Ring Header Size */ 47 #define MAX_HEADER_COUNT_CAP GENMASK(7, 4) /* HC Max Header Count */ 48 #define MAX_HEADER_COUNT GENMASK(3, 0) /* Driver Max Header Count */ 49 50 #define RHS_RHn_OFFSET(n) (0x04 + (n)*4) 51 52 /* 53 * Ring Header (Per-Ring Bundle) 54 */ 55 56 #define rh_reg_read(r) readl(rh->regs + (RH_##r)) 57 #define rh_reg_write(r, v) writel(v, rh->regs + (RH_##r)) 58 59 #define RH_CR_SETUP 0x00 /* Command/Response Ring */ 60 #define CR_XFER_STRUCT_SIZE GENMASK(31, 24) 61 #define CR_RESP_STRUCT_SIZE GENMASK(23, 16) 62 #define CR_RING_SIZE GENMASK(8, 0) 63 64 #define RH_IBI_SETUP 0x04 65 #define IBI_STATUS_STRUCT_SIZE GENMASK(31, 24) 66 #define IBI_STATUS_RING_SIZE GENMASK(23, 16) 67 #define IBI_DATA_CHUNK_SIZE GENMASK(12, 10) 68 #define IBI_DATA_CHUNK_COUNT GENMASK(9, 0) 69 70 #define RH_CHUNK_CONTROL 0x08 71 72 #define RH_INTR_STATUS 0x10 73 #define RH_INTR_STATUS_ENABLE 0x14 74 #define RH_INTR_SIGNAL_ENABLE 0x18 75 #define RH_INTR_FORCE 0x1c 76 #define INTR_IBI_READY BIT(12) 77 #define INTR_TRANSFER_COMPLETION BIT(11) 78 #define INTR_RING_OP BIT(10) 79 #define INTR_TRANSFER_ERR BIT(9) 80 #define INTR_IBI_RING_FULL BIT(6) 81 #define INTR_TRANSFER_ABORT BIT(5) 82 83 #define RH_RING_STATUS 0x20 84 #define RING_STATUS_LOCKED BIT(3) 85 #define RING_STATUS_ABORTED BIT(2) 86 #define RING_STATUS_RUNNING BIT(1) 87 #define RING_STATUS_ENABLED BIT(0) 88 89 #define RH_RING_CONTROL 0x24 90 #define RING_CTRL_ABORT BIT(2) 91 #define RING_CTRL_RUN_STOP BIT(1) 92 #define RING_CTRL_ENABLE BIT(0) 93 94 #define RH_RING_OPERATION1 0x28 95 #define RING_OP1_IBI_DEQ_PTR GENMASK(23, 16) 96 #define RING_OP1_CR_SW_DEQ_PTR GENMASK(15, 8) 97 #define RING_OP1_CR_ENQ_PTR GENMASK(7, 0) 98 99 #define RH_RING_OPERATION2 0x2c 100 #define RING_OP2_IBI_ENQ_PTR GENMASK(23, 16) 101 #define RING_OP2_CR_DEQ_PTR GENMASK(7, 0) 102 103 #define RH_CMD_RING_BASE_LO 0x30 104 #define RH_CMD_RING_BASE_HI 0x34 105 #define RH_RESP_RING_BASE_LO 0x38 106 #define RH_RESP_RING_BASE_HI 0x3c 107 #define RH_IBI_STATUS_RING_BASE_LO 0x40 108 #define RH_IBI_STATUS_RING_BASE_HI 0x44 109 #define RH_IBI_DATA_RING_BASE_LO 0x48 110 #define RH_IBI_DATA_RING_BASE_HI 0x4c 111 112 #define RH_CMD_RING_SG 0x50 /* Ring Scatter Gather Support */ 113 #define RH_RESP_RING_SG 0x54 114 #define RH_IBI_STATUS_RING_SG 0x58 115 #define RH_IBI_DATA_RING_SG 0x5c 116 #define RING_SG_BLP BIT(31) /* Buffer Vs. List Pointer */ 117 #define RING_SG_LIST_SIZE GENMASK(15, 0) 118 119 /* 120 * Data Buffer Descriptor (in memory) 121 */ 122 123 #define DATA_BUF_BLP BIT(31) /* Buffer Vs. List Pointer */ 124 #define DATA_BUF_IOC BIT(30) /* Interrupt on Completion */ 125 #define DATA_BUF_BLOCK_SIZE GENMASK(15, 0) 126 127 struct hci_rh_data { 128 void __iomem *regs; 129 void *xfer, *resp, *ibi_status, *ibi_data; 130 dma_addr_t xfer_dma, resp_dma, ibi_status_dma, ibi_data_dma; 131 unsigned int xfer_entries, ibi_status_entries, ibi_chunks_total; 132 unsigned int xfer_struct_sz, resp_struct_sz, ibi_status_sz, ibi_chunk_sz; 133 unsigned int xfer_alloc_sz, done_ptr, ibi_chunk_ptr, xfer_space; 134 struct hci_xfer **src_xfers; 135 struct completion op_done; 136 }; 137 138 struct hci_rings_data { 139 struct device *sysdev; 140 unsigned int total; 141 struct hci_rh_data headers[] __counted_by(total); 142 }; 143 144 struct hci_dma_dev_ibi_data { 145 struct i3c_generic_ibi_pool *pool; 146 unsigned int max_len; 147 }; 148 149 static void hci_dma_cleanup(struct i3c_hci *hci) 150 { 151 struct hci_rings_data *rings = hci->io_data; 152 struct hci_rh_data *rh; 153 unsigned int i; 154 155 if (!rings) 156 return; 157 158 for (i = 0; i < rings->total; i++) { 159 rh = &rings->headers[i]; 160 161 rh_reg_write(INTR_SIGNAL_ENABLE, 0); 162 rh_reg_write(RING_CONTROL, 0); 163 } 164 165 i3c_hci_sync_irq_inactive(hci); 166 167 for (i = 0; i < rings->total; i++) { 168 rh = &rings->headers[i]; 169 170 rh_reg_write(CR_SETUP, 0); 171 rh_reg_write(IBI_SETUP, 0); 172 } 173 174 rhs_reg_write(CONTROL, 0); 175 } 176 177 static void hci_dma_free(void *data) 178 { 179 struct i3c_hci *hci = data; 180 struct hci_rings_data *rings = hci->io_data; 181 struct hci_rh_data *rh; 182 183 if (!rings) 184 return; 185 186 for (int i = 0; i < rings->total; i++) { 187 rh = &rings->headers[i]; 188 189 if (rh->xfer) 190 dma_free_coherent(rings->sysdev, rh->xfer_alloc_sz, rh->xfer, rh->xfer_dma); 191 kfree(rh->src_xfers); 192 if (rh->ibi_status) 193 dma_free_coherent(rings->sysdev, 194 rh->ibi_status_sz * rh->ibi_status_entries, 195 rh->ibi_status, rh->ibi_status_dma); 196 if (rh->ibi_data_dma) 197 dma_unmap_single(rings->sysdev, rh->ibi_data_dma, 198 rh->ibi_chunk_sz * rh->ibi_chunks_total, 199 DMA_FROM_DEVICE); 200 kfree(rh->ibi_data); 201 } 202 203 kfree(rings); 204 hci->io_data = NULL; 205 } 206 207 static void hci_dma_init_rh(struct i3c_hci *hci, struct hci_rh_data *rh, int i) 208 { 209 u32 regval; 210 211 rh_reg_write(CMD_RING_BASE_LO, lower_32_bits(rh->xfer_dma)); 212 rh_reg_write(CMD_RING_BASE_HI, upper_32_bits(rh->xfer_dma)); 213 rh_reg_write(RESP_RING_BASE_LO, lower_32_bits(rh->resp_dma)); 214 rh_reg_write(RESP_RING_BASE_HI, upper_32_bits(rh->resp_dma)); 215 216 regval = FIELD_PREP(CR_RING_SIZE, rh->xfer_entries); 217 rh_reg_write(CR_SETUP, regval); 218 219 rh_reg_write(INTR_STATUS_ENABLE, 0xffffffff); 220 rh_reg_write(INTR_SIGNAL_ENABLE, INTR_IBI_READY | 221 INTR_TRANSFER_COMPLETION | 222 INTR_RING_OP | 223 INTR_TRANSFER_ERR | 224 INTR_IBI_RING_FULL | 225 INTR_TRANSFER_ABORT); 226 227 if (i >= IBI_RINGS) 228 goto ring_ready; 229 230 rh_reg_write(IBI_STATUS_RING_BASE_LO, lower_32_bits(rh->ibi_status_dma)); 231 rh_reg_write(IBI_STATUS_RING_BASE_HI, upper_32_bits(rh->ibi_status_dma)); 232 rh_reg_write(IBI_DATA_RING_BASE_LO, lower_32_bits(rh->ibi_data_dma)); 233 rh_reg_write(IBI_DATA_RING_BASE_HI, upper_32_bits(rh->ibi_data_dma)); 234 235 regval = FIELD_PREP(IBI_STATUS_RING_SIZE, rh->ibi_status_entries) | 236 FIELD_PREP(IBI_DATA_CHUNK_SIZE, ilog2(rh->ibi_chunk_sz) - 2) | 237 FIELD_PREP(IBI_DATA_CHUNK_COUNT, rh->ibi_chunks_total); 238 rh_reg_write(IBI_SETUP, regval); 239 240 regval = rh_reg_read(INTR_SIGNAL_ENABLE); 241 regval |= INTR_IBI_READY; 242 rh_reg_write(INTR_SIGNAL_ENABLE, regval); 243 244 ring_ready: 245 /* 246 * The MIPI I3C HCI specification does not document reset values for 247 * RING_OPERATION1 fields and some controllers (e.g. Intel controllers) 248 * do not reset the values, so ensure the ring pointers are set to zero 249 * here. 250 */ 251 rh_reg_write(RING_OPERATION1, 0); 252 253 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE); 254 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_RUN_STOP); 255 256 /* 257 * Do not clear the entries of rh->src_xfers because the recovery uses 258 * them. In other cases they should be NULL anyway. 259 */ 260 rh->done_ptr = 0; 261 rh->ibi_chunk_ptr = 0; 262 rh->xfer_space = rh->xfer_entries; 263 } 264 265 static void hci_dma_init_rings(struct i3c_hci *hci) 266 { 267 struct hci_rings_data *rings = hci->io_data; 268 u32 regval; 269 270 regval = FIELD_PREP(MAX_HEADER_COUNT, rings->total); 271 rhs_reg_write(CONTROL, regval); 272 273 for (int i = 0; i < rings->total; i++) 274 hci_dma_init_rh(hci, &rings->headers[i], i); 275 } 276 277 static void hci_dma_suspend(struct i3c_hci *hci) 278 { 279 struct hci_rings_data *rings = hci->io_data; 280 int n = rings ? rings->total : 0; 281 282 for (int i = 0; i < n; i++) { 283 struct hci_rh_data *rh = &rings->headers[i]; 284 285 rh_reg_write(INTR_SIGNAL_ENABLE, 0); 286 rh_reg_write(RING_CONTROL, 0); 287 } 288 289 i3c_hci_sync_irq_inactive(hci); 290 } 291 292 static void hci_dma_resume(struct i3c_hci *hci) 293 { 294 struct hci_rings_data *rings = hci->io_data; 295 296 if (rings) 297 hci_dma_init_rings(hci); 298 } 299 300 static int hci_dma_init(struct i3c_hci *hci) 301 { 302 struct hci_rings_data *rings; 303 struct hci_rh_data *rh; 304 struct device *sysdev; 305 u32 regval; 306 unsigned int i, nr_rings, xfers_sz, resps_sz; 307 unsigned int ibi_status_ring_sz, ibi_data_ring_sz; 308 int ret; 309 310 /* 311 * Set pointer to a physical device that does DMA and has IOMMU setup 312 * done for it in case of enabled IOMMU and use it with the DMA API. 313 * Here such device is either 314 * "mipi-i3c-hci" platform device (OF/ACPI enumeration) parent or 315 * grandparent (PCI enumeration). 316 */ 317 sysdev = hci->master.dev.parent; 318 if (sysdev->parent && dev_is_pci(sysdev->parent)) 319 sysdev = sysdev->parent; 320 321 regval = rhs_reg_read(CONTROL); 322 nr_rings = FIELD_GET(MAX_HEADER_COUNT_CAP, regval); 323 dev_dbg(&hci->master.dev, "%d DMA rings available\n", nr_rings); 324 if (unlikely(nr_rings > 8)) { 325 dev_err(&hci->master.dev, "number of rings should be <= 8\n"); 326 nr_rings = 8; 327 } 328 if (nr_rings > XFER_RINGS) 329 nr_rings = XFER_RINGS; 330 rings = kzalloc_flex(*rings, headers, nr_rings); 331 if (!rings) 332 return -ENOMEM; 333 hci->io_data = rings; 334 rings->total = nr_rings; 335 rings->sysdev = sysdev; 336 337 for (i = 0; i < rings->total; i++) { 338 u32 offset = rhs_reg_read(RHn_OFFSET(i)); 339 340 dev_dbg(&hci->master.dev, "Ring %d at offset %#x\n", i, offset); 341 ret = -EINVAL; 342 if (!offset) 343 goto err_out; 344 rh = &rings->headers[i]; 345 rh->regs = hci->base_regs + offset; 346 init_completion(&rh->op_done); 347 348 rh->xfer_entries = XFER_RING_ENTRIES; 349 350 regval = rh_reg_read(CR_SETUP); 351 rh->xfer_struct_sz = FIELD_GET(CR_XFER_STRUCT_SIZE, regval); 352 rh->resp_struct_sz = FIELD_GET(CR_RESP_STRUCT_SIZE, regval); 353 dev_dbg(&hci->master.dev, 354 "xfer_struct_sz = %d, resp_struct_sz = %d", 355 rh->xfer_struct_sz, rh->resp_struct_sz); 356 xfers_sz = round_up(rh->xfer_struct_sz * rh->xfer_entries, 4); 357 resps_sz = rh->resp_struct_sz * rh->xfer_entries; 358 rh->xfer_alloc_sz = xfers_sz + resps_sz; 359 360 rh->xfer = dma_alloc_coherent(rings->sysdev, rh->xfer_alloc_sz, 361 &rh->xfer_dma, GFP_KERNEL); 362 rh->src_xfers = 363 kzalloc_objs(*rh->src_xfers, rh->xfer_entries); 364 ret = -ENOMEM; 365 if (!rh->xfer || !rh->src_xfers) 366 goto err_out; 367 rh->resp = rh->xfer + xfers_sz; 368 rh->resp_dma = rh->xfer_dma + xfers_sz; 369 370 /* IBIs */ 371 372 if (i >= IBI_RINGS) 373 continue; 374 375 regval = rh_reg_read(IBI_SETUP); 376 rh->ibi_status_sz = FIELD_GET(IBI_STATUS_STRUCT_SIZE, regval); 377 rh->ibi_status_entries = IBI_STATUS_RING_ENTRIES; 378 rh->ibi_chunks_total = IBI_CHUNK_POOL_SIZE; 379 380 rh->ibi_chunk_sz = dma_get_cache_alignment(); 381 rh->ibi_chunk_sz *= IBI_CHUNK_CACHELINES; 382 /* 383 * Round IBI data chunk size to number of bytes supported by 384 * the HW. Chunk size can be 2^n number of DWORDs which is the 385 * same as 2^(n+2) bytes, where n is 0..6. 386 */ 387 rh->ibi_chunk_sz = umax(4, rh->ibi_chunk_sz); 388 rh->ibi_chunk_sz = roundup_pow_of_two(rh->ibi_chunk_sz); 389 if (rh->ibi_chunk_sz > 256) { 390 ret = -EINVAL; 391 goto err_out; 392 } 393 394 ibi_status_ring_sz = rh->ibi_status_sz * rh->ibi_status_entries; 395 ibi_data_ring_sz = rh->ibi_chunk_sz * rh->ibi_chunks_total; 396 397 rh->ibi_status = 398 dma_alloc_coherent(rings->sysdev, ibi_status_ring_sz, 399 &rh->ibi_status_dma, GFP_KERNEL); 400 rh->ibi_data = kmalloc(ibi_data_ring_sz, GFP_KERNEL); 401 ret = -ENOMEM; 402 if (!rh->ibi_status || !rh->ibi_data) 403 goto err_out; 404 rh->ibi_data_dma = 405 dma_map_single(rings->sysdev, rh->ibi_data, 406 ibi_data_ring_sz, DMA_FROM_DEVICE); 407 if (dma_mapping_error(rings->sysdev, rh->ibi_data_dma)) { 408 rh->ibi_data_dma = 0; 409 ret = -ENOMEM; 410 goto err_out; 411 } 412 } 413 414 ret = devm_add_action(hci->master.dev.parent, hci_dma_free, hci); 415 if (ret) 416 goto err_out; 417 418 hci_dma_init_rings(hci); 419 420 return 0; 421 422 err_out: 423 hci_dma_free(hci); 424 return ret; 425 } 426 427 static void hci_dma_unmap_xfer(struct i3c_hci *hci, 428 struct hci_xfer *xfer_list, unsigned int n) 429 { 430 struct hci_xfer *xfer; 431 unsigned int i; 432 433 for (i = 0; i < n; i++) { 434 xfer = xfer_list + i; 435 if (!xfer->data) 436 continue; 437 i3c_master_dma_unmap_single(xfer->dma); 438 } 439 } 440 441 static struct i3c_dma *hci_dma_map_xfer(struct device *dev, struct hci_xfer *xfer) 442 { 443 enum dma_data_direction dir = xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 444 bool need_bounce = device_iommu_mapped(dev) && xfer->rnw && (xfer->data_len & 3); 445 446 return i3c_master_dma_map_single(dev, xfer->data, xfer->data_len, need_bounce, dir); 447 } 448 449 static int hci_dma_map_xfer_list(struct i3c_hci *hci, struct device *dev, 450 struct hci_xfer *xfer_list, int n) 451 { 452 for (int i = 0; i < n; i++) { 453 struct hci_xfer *xfer = xfer_list + i; 454 455 if (!xfer->data) 456 continue; 457 458 xfer->dma = hci_dma_map_xfer(dev, xfer); 459 if (!xfer->dma) { 460 hci_dma_unmap_xfer(hci, xfer_list, i); 461 return -ENOMEM; 462 } 463 } 464 465 return 0; 466 } 467 468 static int hci_dma_queue_xfer(struct i3c_hci *hci, 469 struct hci_xfer *xfer_list, int n) 470 { 471 struct hci_rings_data *rings = hci->io_data; 472 struct hci_rh_data *rh; 473 unsigned int i, ring, enqueue_ptr; 474 u32 op1_val; 475 int ret; 476 477 ret = hci_dma_map_xfer_list(hci, rings->sysdev, xfer_list, n); 478 if (ret) 479 return ret; 480 481 /* For now we only use ring 0 */ 482 ring = 0; 483 rh = &rings->headers[ring]; 484 485 spin_lock_irq(&hci->lock); 486 487 while (unlikely(hci->enqueue_blocked)) { 488 spin_unlock_irq(&hci->lock); 489 wait_event(hci->enqueue_wait_queue, !READ_ONCE(hci->enqueue_blocked)); 490 spin_lock_irq(&hci->lock); 491 } 492 493 if (n > rh->xfer_space) { 494 spin_unlock_irq(&hci->lock); 495 hci_dma_unmap_xfer(hci, xfer_list, n); 496 return -EBUSY; 497 } 498 499 op1_val = rh_reg_read(RING_OPERATION1); 500 enqueue_ptr = FIELD_GET(RING_OP1_CR_ENQ_PTR, op1_val); 501 for (i = 0; i < n; i++) { 502 struct hci_xfer *xfer = xfer_list + i; 503 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr; 504 505 xfer->final_xfer = xfer_list + n - 1; 506 xfer->xfer_list_pos = i; 507 508 /* store cmd descriptor */ 509 *ring_data++ = xfer->cmd_desc[0]; 510 *ring_data++ = xfer->cmd_desc[1]; 511 if (hci->cmd == &mipi_i3c_hci_cmd_v2) { 512 *ring_data++ = xfer->cmd_desc[2]; 513 *ring_data++ = xfer->cmd_desc[3]; 514 } 515 516 /* first word of Data Buffer Descriptor Structure */ 517 if (!xfer->data) 518 xfer->data_len = 0; 519 *ring_data++ = 520 FIELD_PREP(DATA_BUF_BLOCK_SIZE, xfer->data_len) | 521 ((i == n - 1) ? DATA_BUF_IOC : 0); 522 523 /* 2nd and 3rd words of Data Buffer Descriptor Structure */ 524 if (xfer->data) { 525 *ring_data++ = lower_32_bits(xfer->dma->addr); 526 *ring_data++ = upper_32_bits(xfer->dma->addr); 527 } else { 528 *ring_data++ = 0; 529 *ring_data++ = 0; 530 } 531 532 /* remember corresponding xfer struct */ 533 rh->src_xfers[enqueue_ptr] = xfer; 534 /* remember corresponding ring/entry for this xfer structure */ 535 xfer->ring_number = ring; 536 xfer->ring_entry = enqueue_ptr; 537 538 enqueue_ptr = (enqueue_ptr + 1) % rh->xfer_entries; 539 } 540 541 if (rh->xfer_space == rh->xfer_entries) 542 hci_start_xfer(xfer_list); 543 544 rh->xfer_space -= n; 545 546 op1_val &= ~RING_OP1_CR_ENQ_PTR; 547 op1_val |= FIELD_PREP(RING_OP1_CR_ENQ_PTR, enqueue_ptr); 548 rh_reg_write(RING_OPERATION1, op1_val); 549 spin_unlock_irq(&hci->lock); 550 551 return 0; 552 } 553 554 static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh) 555 { 556 u32 op1_val, op2_val, resp, *ring_resp; 557 unsigned int tid, done_ptr = rh->done_ptr; 558 unsigned int done_cnt = 0; 559 bool start_next = false; 560 struct hci_xfer *xfer; 561 562 for (;;) { 563 op2_val = rh_reg_read(RING_OPERATION2); 564 if (done_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) 565 break; 566 567 ring_resp = rh->resp + rh->resp_struct_sz * done_ptr; 568 resp = *ring_resp; 569 tid = RESP_TID(resp); 570 dev_dbg(&hci->master.dev, "resp = 0x%08x", resp); 571 572 xfer = rh->src_xfers[done_ptr]; 573 if (!xfer) { 574 dev_dbg(&hci->master.dev, "orphaned ring entry"); 575 } else { 576 hci_dma_unmap_xfer(hci, xfer, 1); 577 rh->src_xfers[done_ptr] = NULL; 578 xfer->ring_entry = -1; 579 if (tid != xfer->cmd_tid) { 580 dev_err(&hci->master.dev, 581 "response tid=%d when expecting %d\n", 582 tid, xfer->cmd_tid); 583 hci->recovery_needed = true; 584 if (!RESP_STATUS(resp)) 585 hci_cmd_set_resp_err(&resp, RESP_ERR_HC_TERMINATED); 586 } 587 xfer->response = resp; 588 if (xfer == xfer->final_xfer || RESP_STATUS(resp)) 589 complete(xfer->final_xfer->completion); 590 else 591 hci_start_xfer(xfer); 592 if (RESP_STATUS(resp)) { 593 hci->enqueue_blocked = true; 594 start_next = false; 595 } else { 596 start_next = true; 597 } 598 } 599 600 done_ptr = (done_ptr + 1) % rh->xfer_entries; 601 rh->done_ptr = done_ptr; 602 done_cnt += 1; 603 } 604 605 rh->xfer_space += done_cnt; 606 if (start_next && rh->xfer_space < rh->xfer_entries) { 607 xfer = rh->src_xfers[done_ptr]; 608 hci_start_xfer(xfer); 609 } 610 op1_val = rh_reg_read(RING_OPERATION1); 611 op1_val &= ~RING_OP1_CR_SW_DEQ_PTR; 612 op1_val |= FIELD_PREP(RING_OP1_CR_SW_DEQ_PTR, done_ptr); 613 rh_reg_write(RING_OPERATION1, op1_val); 614 } 615 616 static void hci_dma_requires_hc_abort_quirk(struct i3c_hci *hci, struct hci_rh_data *rh) 617 { 618 reinit_completion(&rh->op_done); 619 mipi_i3c_hci_abort(hci); 620 wait_for_completion_timeout(&rh->op_done, HZ); 621 rh_reg_write(RING_CONTROL, rh_reg_read(RING_CONTROL) | RING_CTRL_ABORT); 622 } 623 624 static void hci_dma_abort(struct i3c_hci *hci, struct hci_rh_data *rh) 625 { 626 if (hci->quirks & HCI_QUIRK_DMA_REQUIRES_HC_ABORT) { 627 hci_dma_requires_hc_abort_quirk(hci, rh); 628 return; 629 } 630 631 reinit_completion(&rh->op_done); 632 rh_reg_write(RING_CONTROL, rh_reg_read(RING_CONTROL) | RING_CTRL_ABORT); 633 wait_for_completion_timeout(&rh->op_done, HZ); 634 } 635 636 static void hci_dma_unblock_enqueue(struct i3c_hci *hci) 637 { 638 if (hci->enqueue_blocked) { 639 hci->enqueue_blocked = false; 640 wake_up_all(&hci->enqueue_wait_queue); 641 } 642 } 643 644 static void hci_dma_error_out_rh(struct i3c_hci *hci, struct hci_rh_data *rh) 645 { 646 /* 647 * The entries of rh->src_xfers are not cleared by 648 * i3c_hci_reset_and_restore(), so can be used here. Do 2 passes so 649 * that the final_xfer of an xfer list is always processed last. 650 */ 651 for (int pass = 0; pass < 2; pass++) 652 for (int i = 0; i < rh->xfer_entries; i++) { 653 struct hci_xfer *xfer = rh->src_xfers[i]; 654 655 if (!xfer || (!pass && xfer == xfer->final_xfer)) 656 continue; 657 hci_dma_unmap_xfer(hci, xfer, 1); 658 rh->src_xfers[i] = NULL; 659 xfer->ring_entry = -1; 660 hci_cmd_set_resp_err(&xfer->response, RESP_ERR_HC_TERMINATED); 661 if (xfer == xfer->final_xfer) 662 complete(xfer->final_xfer->completion); 663 } 664 } 665 666 static void hci_dma_error_out_all(struct i3c_hci *hci) 667 { 668 struct hci_rings_data *rings = hci->io_data; 669 670 for (int i = 0; i < rings->total; i++) 671 hci_dma_error_out_rh(hci, &rings->headers[i]); 672 } 673 674 static void hci_dma_recovery(struct i3c_hci *hci) 675 { 676 int ret; 677 678 dev_err(&hci->master.dev, "Attempting to recover from internal errors\n"); 679 680 for (int i = 0; i < 3; i++) { 681 ret = i3c_hci_reset_and_restore(hci); 682 if (!ret) 683 break; 684 dev_err(&hci->master.dev, "Reset and restore failed, error %d\n", ret); 685 /* Just in case the controller is busy, give it some time */ 686 msleep(1000); 687 } 688 689 spin_lock_irq(&hci->lock); 690 hci_dma_error_out_all(hci); 691 hci_dma_unblock_enqueue(hci); 692 hci->recovery_needed = false; 693 spin_unlock_irq(&hci->lock); 694 695 dev_err(&hci->master.dev, "Recovery %s\n", ret ? "failed!" : "done"); 696 } 697 698 static bool hci_dma_wait_for_noop(struct i3c_hci *hci, struct hci_xfer *xfer_list, int n, 699 int noop_pos) 700 { 701 struct completion *done = xfer_list->final_xfer->completion; 702 bool timeout = !wait_for_completion_timeout(done, HZ); 703 u32 error = timeout; 704 705 for (int i = noop_pos; i < n && !error; i++) 706 error = RESP_STATUS(xfer_list[i].response); 707 708 if (!error) 709 return true; 710 711 if (timeout) 712 dev_err(&hci->master.dev, "NoOp timeout error\n"); 713 else 714 dev_err(&hci->master.dev, "NoOp error %u\n", error); 715 716 return false; 717 } 718 719 static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, 720 struct hci_xfer *xfer_list, int n) 721 { 722 struct hci_rings_data *rings = hci->io_data; 723 struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number]; 724 int noop_pos = -1; 725 unsigned int i; 726 bool did_unqueue = false; 727 u32 ring_status; 728 729 guard(mutex)(&hci->control_mutex); 730 731 spin_lock_irq(&hci->lock); 732 restart: 733 ring_status = rh_reg_read(RING_STATUS); 734 if (ring_status & RING_STATUS_RUNNING) { 735 /* 736 * The transfer may have already completed, especially 737 * if recovery has just run. Do nothing in that case. 738 */ 739 hci_dma_xfer_done(hci, rh); 740 if (xfer_list->final_xfer->ring_entry < 0 && 741 !hci->recovery_needed && !hci->enqueue_blocked && 742 ring_status == (RING_STATUS_ENABLED | RING_STATUS_RUNNING)) { 743 spin_unlock_irq(&hci->lock); 744 return false; 745 } 746 hci->enqueue_blocked = true; 747 spin_unlock_irq(&hci->lock); 748 /* stop the ring */ 749 hci_dma_abort(hci, rh); 750 spin_lock_irq(&hci->lock); 751 ring_status = rh_reg_read(RING_STATUS); 752 if (ring_status & RING_STATUS_RUNNING) { 753 dev_err(&hci->master.dev, "Unable to abort the DMA ring\n"); 754 hci->recovery_needed = true; 755 } 756 } 757 758 if ((hci->quirks & HCI_QUIRK_DMA_ABORT_REQUIRES_PIO_RESET) && 759 (rh_reg_read(RING_STATUS) & RING_STATUS_ABORTED)) 760 mipi_i3c_hci_pio_reset_all_queues(hci); 761 762 hci_dma_xfer_done(hci, rh); 763 764 if (hci->recovery_needed) { 765 hci->enqueue_blocked = true; 766 spin_unlock_irq(&hci->lock); 767 hci_dma_recovery(hci); 768 return true; 769 } 770 771 for (i = 0; i < n; i++) { 772 struct hci_xfer *xfer = xfer_list + i; 773 int idx = xfer->ring_entry; 774 775 /* 776 * At the time the abort happened, the xfer might have 777 * completed already. If not then replace corresponding 778 * descriptor entries with a no-op. 779 */ 780 if (idx >= 0) { 781 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx; 782 783 /* store no-op cmd descriptor */ 784 *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7) | FIELD_PREP(CMD_0_TID, xfer->cmd_tid); 785 *ring_data++ = 0; 786 if (hci->cmd == &mipi_i3c_hci_cmd_v2) { 787 *ring_data++ = 0; 788 *ring_data++ = 0; 789 } 790 791 if (noop_pos < 0) { 792 reinit_completion(xfer->final_xfer->completion); 793 noop_pos = i; 794 } 795 796 did_unqueue = true; 797 } 798 } 799 800 /* 801 * A software ABORT may race with transfer completion and abort the next 802 * transfer list instead. Detect that case, and do not restart the ring. 803 * It will be handled by a subsequent dequeue. 804 */ 805 if (!did_unqueue) { 806 struct hci_xfer *xfer = rh->src_xfers[rh->done_ptr]; 807 808 if (xfer && xfer->xfer_list_pos && xfer->final_xfer != xfer_list->final_xfer) { 809 spin_unlock_irq(&hci->lock); 810 return false; 811 } 812 } 813 814 /* restart the ring */ 815 reinit_completion(&rh->op_done); 816 mipi_i3c_hci_resume(hci); 817 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE); 818 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_RUN_STOP); 819 820 hci_dma_unblock_enqueue(hci); 821 822 if (rh->xfer_space < rh->xfer_entries) 823 hci_start_xfer(rh->src_xfers[rh->done_ptr]); 824 825 spin_unlock_irq(&hci->lock); 826 827 wait_for_completion_timeout(&rh->op_done, HZ); 828 829 if (did_unqueue && !hci_dma_wait_for_noop(hci, xfer_list, n, noop_pos)) { 830 spin_lock_irq(&hci->lock); 831 hci->recovery_needed = true; 832 goto restart; 833 } 834 835 return did_unqueue; 836 } 837 838 static int hci_dma_handle_error(struct i3c_hci *hci, struct hci_xfer *xfer_list, int n) 839 { 840 return hci_dma_dequeue_xfer(hci, xfer_list, n) ? -EIO : 0; 841 } 842 843 static int hci_dma_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev, 844 const struct i3c_ibi_setup *req) 845 { 846 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 847 struct i3c_generic_ibi_pool *pool; 848 struct hci_dma_dev_ibi_data *dev_ibi; 849 850 dev_ibi = kmalloc_obj(*dev_ibi); 851 if (!dev_ibi) 852 return -ENOMEM; 853 pool = i3c_generic_ibi_alloc_pool(dev, req); 854 if (IS_ERR(pool)) { 855 kfree(dev_ibi); 856 return PTR_ERR(pool); 857 } 858 dev_ibi->pool = pool; 859 dev_ibi->max_len = req->max_payload_len; 860 dev_data->ibi_data = dev_ibi; 861 return 0; 862 } 863 864 static void hci_dma_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev) 865 { 866 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 867 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data; 868 869 dev_data->ibi_data = NULL; 870 i3c_generic_ibi_free_pool(dev_ibi->pool); 871 kfree(dev_ibi); 872 } 873 874 static void hci_dma_recycle_ibi_slot(struct i3c_hci *hci, 875 struct i3c_dev_desc *dev, 876 struct i3c_ibi_slot *slot) 877 { 878 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 879 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data; 880 881 i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot); 882 } 883 884 static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh) 885 { 886 struct hci_rings_data *rings = hci->io_data; 887 struct i3c_dev_desc *dev; 888 struct i3c_hci_dev_data *dev_data; 889 struct hci_dma_dev_ibi_data *dev_ibi; 890 struct i3c_ibi_slot *slot; 891 u32 op1_val, op2_val, ibi_status_error; 892 unsigned int ptr, enq_ptr, deq_ptr; 893 unsigned int ibi_size, ibi_chunks, ibi_data_offset, first_part; 894 int ibi_addr, last_ptr; 895 void *ring_ibi_data; 896 dma_addr_t ring_ibi_data_dma; 897 898 op1_val = rh_reg_read(RING_OPERATION1); 899 deq_ptr = FIELD_GET(RING_OP1_IBI_DEQ_PTR, op1_val); 900 901 op2_val = rh_reg_read(RING_OPERATION2); 902 enq_ptr = FIELD_GET(RING_OP2_IBI_ENQ_PTR, op2_val); 903 904 ibi_status_error = 0; 905 ibi_addr = -1; 906 ibi_chunks = 0; 907 ibi_size = 0; 908 last_ptr = -1; 909 910 /* let's find all we can about this IBI */ 911 for (ptr = deq_ptr; ptr != enq_ptr; 912 ptr = (ptr + 1) % rh->ibi_status_entries) { 913 u32 ibi_status, *ring_ibi_status; 914 unsigned int chunks; 915 916 ring_ibi_status = rh->ibi_status + rh->ibi_status_sz * ptr; 917 ibi_status = *ring_ibi_status; 918 dev_dbg(&hci->master.dev, "status = %#x", ibi_status); 919 920 if (ibi_status_error) { 921 /* we no longer care */ 922 } else if (ibi_status & IBI_ERROR) { 923 ibi_status_error = ibi_status; 924 } else if (ibi_addr == -1) { 925 ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status); 926 } else if (ibi_addr != FIELD_GET(IBI_TARGET_ADDR, ibi_status)) { 927 /* the address changed unexpectedly */ 928 ibi_status_error = ibi_status; 929 } 930 931 chunks = FIELD_GET(IBI_CHUNKS, ibi_status); 932 ibi_chunks += chunks; 933 if (!(ibi_status & IBI_LAST_STATUS)) { 934 ibi_size += chunks * rh->ibi_chunk_sz; 935 } else { 936 if (chunks) { 937 ibi_size += (chunks - 1) * rh->ibi_chunk_sz; 938 ibi_size += FIELD_GET(IBI_DATA_LENGTH, ibi_status); 939 } 940 last_ptr = ptr; 941 break; 942 } 943 } 944 945 /* validate what we've got */ 946 947 if (last_ptr == -1) { 948 /* this IBI sequence is not yet complete */ 949 dev_dbg(&hci->master.dev, 950 "no LAST_STATUS available (e=%d d=%d)", 951 enq_ptr, deq_ptr); 952 return; 953 } 954 deq_ptr = last_ptr + 1; 955 deq_ptr %= rh->ibi_status_entries; 956 957 if (ibi_status_error) { 958 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi_addr); 959 goto done; 960 } 961 962 /* determine who this is for */ 963 if (ibi_addr == I3C_HOT_JOIN_ADDR) { 964 i3c_master_queue_hotjoin(&hci->master); 965 goto done; 966 } 967 968 dev = i3c_hci_addr_to_dev(hci, ibi_addr); 969 if (!dev) { 970 /* 971 * Either an IBI received just before IBI's were disabled, or 972 * the controller is broken. Assume the former. 973 */ 974 dev_dbg(&hci->master.dev, "IBI when not enabled at address %#x\n", ibi_addr); 975 goto done; 976 } 977 978 dev_data = i3c_dev_get_master_data(dev); 979 dev_ibi = dev_data->ibi_data; 980 if (ibi_size > dev_ibi->max_len) { 981 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n", 982 ibi_size, dev_ibi->max_len); 983 goto done; 984 } 985 986 /* 987 * This ring model is not suitable for zero-copy processing of IBIs. 988 * We have the data chunk ring wrap-around to deal with, meaning 989 * that the payload might span multiple chunks beginning at the 990 * end of the ring and wrap to the start of the ring. Furthermore 991 * there is no guarantee that those chunks will be released in order 992 * and in a timely manner by the upper driver. So let's just copy 993 * them to a discrete buffer. In practice they're supposed to be 994 * small anyway. 995 */ 996 slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool); 997 if (!slot) { 998 dev_err(&hci->master.dev, "no free slot for IBI\n"); 999 goto done; 1000 } 1001 1002 /* copy first part of the payload */ 1003 ibi_data_offset = rh->ibi_chunk_sz * rh->ibi_chunk_ptr; 1004 ring_ibi_data = rh->ibi_data + ibi_data_offset; 1005 ring_ibi_data_dma = rh->ibi_data_dma + ibi_data_offset; 1006 first_part = (rh->ibi_chunks_total - rh->ibi_chunk_ptr) 1007 * rh->ibi_chunk_sz; 1008 if (first_part > ibi_size) 1009 first_part = ibi_size; 1010 dma_sync_single_for_cpu(rings->sysdev, ring_ibi_data_dma, 1011 first_part, DMA_FROM_DEVICE); 1012 memcpy(slot->data, ring_ibi_data, first_part); 1013 1014 /* copy second part if any */ 1015 if (ibi_size > first_part) { 1016 /* we wrap back to the start and copy remaining data */ 1017 ring_ibi_data = rh->ibi_data; 1018 ring_ibi_data_dma = rh->ibi_data_dma; 1019 dma_sync_single_for_cpu(rings->sysdev, ring_ibi_data_dma, 1020 ibi_size - first_part, DMA_FROM_DEVICE); 1021 memcpy(slot->data + first_part, ring_ibi_data, 1022 ibi_size - first_part); 1023 } 1024 1025 /* submit it */ 1026 slot->dev = dev; 1027 slot->len = ibi_size; 1028 i3c_master_queue_ibi(dev, slot); 1029 1030 done: 1031 op1_val = rh_reg_read(RING_OPERATION1); 1032 op1_val &= ~RING_OP1_IBI_DEQ_PTR; 1033 op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr); 1034 rh_reg_write(RING_OPERATION1, op1_val); 1035 1036 /* update the chunk pointer */ 1037 rh->ibi_chunk_ptr += ibi_chunks; 1038 rh->ibi_chunk_ptr %= rh->ibi_chunks_total; 1039 1040 /* and tell the hardware about freed chunks */ 1041 rh_reg_write(CHUNK_CONTROL, rh_reg_read(CHUNK_CONTROL) + ibi_chunks); 1042 } 1043 1044 static bool hci_dma_irq_handler(struct i3c_hci *hci) 1045 { 1046 struct hci_rings_data *rings = hci->io_data; 1047 unsigned int i; 1048 bool handled = false; 1049 1050 for (i = 0; i < rings->total; i++) { 1051 struct hci_rh_data *rh; 1052 u32 status; 1053 1054 rh = &rings->headers[i]; 1055 status = rh_reg_read(INTR_STATUS); 1056 dev_dbg(&hci->master.dev, "Ring %d: RH_INTR_STATUS %#x", 1057 i, status); 1058 if (!status) 1059 continue; 1060 rh_reg_write(INTR_STATUS, status); 1061 1062 if (status & INTR_IBI_READY) 1063 hci_dma_process_ibi(hci, rh); 1064 if (status & (INTR_TRANSFER_COMPLETION | INTR_TRANSFER_ERR)) 1065 hci_dma_xfer_done(hci, rh); 1066 if (status & INTR_RING_OP) 1067 complete(&rh->op_done); 1068 if (status & INTR_TRANSFER_ABORT) 1069 dev_dbg(&hci->master.dev, "Ring %d: Transfer Aborted\n", i); 1070 if (status & INTR_IBI_RING_FULL) 1071 dev_err_ratelimited(&hci->master.dev, 1072 "Ring %d: IBI Ring Full Condition\n", i); 1073 1074 handled = true; 1075 } 1076 1077 return handled; 1078 } 1079 1080 const struct hci_io_ops mipi_i3c_hci_dma = { 1081 .init = hci_dma_init, 1082 .cleanup = hci_dma_cleanup, 1083 .queue_xfer = hci_dma_queue_xfer, 1084 .dequeue_xfer = hci_dma_dequeue_xfer, 1085 .handle_error = hci_dma_handle_error, 1086 .irq_handler = hci_dma_irq_handler, 1087 .request_ibi = hci_dma_request_ibi, 1088 .free_ibi = hci_dma_free_ibi, 1089 .recycle_ibi_slot = hci_dma_recycle_ibi_slot, 1090 .suspend = hci_dma_suspend, 1091 .resume = hci_dma_resume, 1092 }; 1093