1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright (c) 2020, MIPI Alliance, Inc. 4 * 5 * Author: Nicolas Pitre <npitre@baylibre.com> 6 * 7 * Note: The I3C HCI v2.0 spec is still in flux. The IBI support is based on 8 * v1.x of the spec and v2.0 will likely be split out. 9 */ 10 11 #include <linux/bitfield.h> 12 #include <linux/device.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/errno.h> 15 #include <linux/i3c/master.h> 16 #include <linux/io.h> 17 #include <linux/pci.h> 18 19 #include "hci.h" 20 #include "cmd.h" 21 #include "ibi.h" 22 23 24 /* 25 * Software Parameter Values (somewhat arb itrary for now). 26 * Some of them could be determined at run time eventually. 27 */ 28 29 #define XFER_RINGS 1 /* max: 8 */ 30 #define XFER_RING_ENTRIES 16 /* max: 255 */ 31 32 #define IBI_RINGS 1 /* max: 8 */ 33 #define IBI_STATUS_RING_ENTRIES 32 /* max: 255 */ 34 #define IBI_CHUNK_CACHELINES 1 /* max: 256 bytes equivalent */ 35 #define IBI_CHUNK_POOL_SIZE 128 /* max: 1023 */ 36 37 /* 38 * Ring Header Preamble 39 */ 40 41 #define rhs_reg_read(r) readl(hci->RHS_regs + (RHS_##r)) 42 #define rhs_reg_write(r, v) writel(v, hci->RHS_regs + (RHS_##r)) 43 44 #define RHS_CONTROL 0x00 45 #define PREAMBLE_SIZE GENMASK(31, 24) /* Preamble Section Size */ 46 #define HEADER_SIZE GENMASK(23, 16) /* Ring Header Size */ 47 #define MAX_HEADER_COUNT_CAP GENMASK(7, 4) /* HC Max Header Count */ 48 #define MAX_HEADER_COUNT GENMASK(3, 0) /* Driver Max Header Count */ 49 50 #define RHS_RHn_OFFSET(n) (0x04 + (n)*4) 51 52 /* 53 * Ring Header (Per-Ring Bundle) 54 */ 55 56 #define rh_reg_read(r) readl(rh->regs + (RH_##r)) 57 #define rh_reg_write(r, v) writel(v, rh->regs + (RH_##r)) 58 59 #define RH_CR_SETUP 0x00 /* Command/Response Ring */ 60 #define CR_XFER_STRUCT_SIZE GENMASK(31, 24) 61 #define CR_RESP_STRUCT_SIZE GENMASK(23, 16) 62 #define CR_RING_SIZE GENMASK(8, 0) 63 64 #define RH_IBI_SETUP 0x04 65 #define IBI_STATUS_STRUCT_SIZE GENMASK(31, 24) 66 #define IBI_STATUS_RING_SIZE GENMASK(23, 16) 67 #define IBI_DATA_CHUNK_SIZE GENMASK(12, 10) 68 #define IBI_DATA_CHUNK_COUNT GENMASK(9, 0) 69 70 #define RH_CHUNK_CONTROL 0x08 71 72 #define RH_INTR_STATUS 0x10 73 #define RH_INTR_STATUS_ENABLE 0x14 74 #define RH_INTR_SIGNAL_ENABLE 0x18 75 #define RH_INTR_FORCE 0x1c 76 #define INTR_IBI_READY BIT(12) 77 #define INTR_TRANSFER_COMPLETION BIT(11) 78 #define INTR_RING_OP BIT(10) 79 #define INTR_TRANSFER_ERR BIT(9) 80 #define INTR_IBI_RING_FULL BIT(6) 81 #define INTR_TRANSFER_ABORT BIT(5) 82 83 #define RH_RING_STATUS 0x20 84 #define RING_STATUS_LOCKED BIT(3) 85 #define RING_STATUS_ABORTED BIT(2) 86 #define RING_STATUS_RUNNING BIT(1) 87 #define RING_STATUS_ENABLED BIT(0) 88 89 #define RH_RING_CONTROL 0x24 90 #define RING_CTRL_ABORT BIT(2) 91 #define RING_CTRL_RUN_STOP BIT(1) 92 #define RING_CTRL_ENABLE BIT(0) 93 94 #define RH_RING_OPERATION1 0x28 95 #define RING_OP1_IBI_DEQ_PTR GENMASK(23, 16) 96 #define RING_OP1_CR_SW_DEQ_PTR GENMASK(15, 8) 97 #define RING_OP1_CR_ENQ_PTR GENMASK(7, 0) 98 99 #define RH_RING_OPERATION2 0x2c 100 #define RING_OP2_IBI_ENQ_PTR GENMASK(23, 16) 101 #define RING_OP2_CR_DEQ_PTR GENMASK(7, 0) 102 103 #define RH_CMD_RING_BASE_LO 0x30 104 #define RH_CMD_RING_BASE_HI 0x34 105 #define RH_RESP_RING_BASE_LO 0x38 106 #define RH_RESP_RING_BASE_HI 0x3c 107 #define RH_IBI_STATUS_RING_BASE_LO 0x40 108 #define RH_IBI_STATUS_RING_BASE_HI 0x44 109 #define RH_IBI_DATA_RING_BASE_LO 0x48 110 #define RH_IBI_DATA_RING_BASE_HI 0x4c 111 112 #define RH_CMD_RING_SG 0x50 /* Ring Scatter Gather Support */ 113 #define RH_RESP_RING_SG 0x54 114 #define RH_IBI_STATUS_RING_SG 0x58 115 #define RH_IBI_DATA_RING_SG 0x5c 116 #define RING_SG_BLP BIT(31) /* Buffer Vs. List Pointer */ 117 #define RING_SG_LIST_SIZE GENMASK(15, 0) 118 119 /* 120 * Data Buffer Descriptor (in memory) 121 */ 122 123 #define DATA_BUF_BLP BIT(31) /* Buffer Vs. List Pointer */ 124 #define DATA_BUF_IOC BIT(30) /* Interrupt on Completion */ 125 #define DATA_BUF_BLOCK_SIZE GENMASK(15, 0) 126 127 128 struct hci_rh_data { 129 void __iomem *regs; 130 void *xfer, *resp, *ibi_status, *ibi_data; 131 dma_addr_t xfer_dma, resp_dma, ibi_status_dma, ibi_data_dma; 132 unsigned int xfer_entries, ibi_status_entries, ibi_chunks_total; 133 unsigned int xfer_struct_sz, resp_struct_sz, ibi_status_sz, ibi_chunk_sz; 134 unsigned int done_ptr, ibi_chunk_ptr; 135 struct hci_xfer **src_xfers; 136 spinlock_t lock; 137 struct completion op_done; 138 }; 139 140 struct hci_rings_data { 141 struct device *sysdev; 142 unsigned int total; 143 struct hci_rh_data headers[] __counted_by(total); 144 }; 145 146 struct hci_dma_dev_ibi_data { 147 struct i3c_generic_ibi_pool *pool; 148 unsigned int max_len; 149 }; 150 151 static void hci_dma_cleanup(struct i3c_hci *hci) 152 { 153 struct hci_rings_data *rings = hci->io_data; 154 struct hci_rh_data *rh; 155 unsigned int i; 156 157 if (!rings) 158 return; 159 160 for (i = 0; i < rings->total; i++) { 161 rh = &rings->headers[i]; 162 163 rh_reg_write(INTR_SIGNAL_ENABLE, 0); 164 rh_reg_write(RING_CONTROL, 0); 165 rh_reg_write(CR_SETUP, 0); 166 rh_reg_write(IBI_SETUP, 0); 167 168 if (rh->xfer) 169 dma_free_coherent(rings->sysdev, 170 rh->xfer_struct_sz * rh->xfer_entries, 171 rh->xfer, rh->xfer_dma); 172 if (rh->resp) 173 dma_free_coherent(rings->sysdev, 174 rh->resp_struct_sz * rh->xfer_entries, 175 rh->resp, rh->resp_dma); 176 kfree(rh->src_xfers); 177 if (rh->ibi_status) 178 dma_free_coherent(rings->sysdev, 179 rh->ibi_status_sz * rh->ibi_status_entries, 180 rh->ibi_status, rh->ibi_status_dma); 181 if (rh->ibi_data_dma) 182 dma_unmap_single(rings->sysdev, rh->ibi_data_dma, 183 rh->ibi_chunk_sz * rh->ibi_chunks_total, 184 DMA_FROM_DEVICE); 185 kfree(rh->ibi_data); 186 } 187 188 rhs_reg_write(CONTROL, 0); 189 190 kfree(rings); 191 hci->io_data = NULL; 192 } 193 194 static int hci_dma_init(struct i3c_hci *hci) 195 { 196 struct hci_rings_data *rings; 197 struct hci_rh_data *rh; 198 struct device *sysdev; 199 u32 regval; 200 unsigned int i, nr_rings, xfers_sz, resps_sz; 201 unsigned int ibi_status_ring_sz, ibi_data_ring_sz; 202 int ret; 203 204 /* 205 * Set pointer to a physical device that does DMA and has IOMMU setup 206 * done for it in case of enabled IOMMU and use it with the DMA API. 207 * Here such device is either 208 * "mipi-i3c-hci" platform device (OF/ACPI enumeration) parent or 209 * grandparent (PCI enumeration). 210 */ 211 sysdev = hci->master.dev.parent; 212 if (sysdev->parent && dev_is_pci(sysdev->parent)) 213 sysdev = sysdev->parent; 214 215 regval = rhs_reg_read(CONTROL); 216 nr_rings = FIELD_GET(MAX_HEADER_COUNT_CAP, regval); 217 dev_info(&hci->master.dev, "%d DMA rings available\n", nr_rings); 218 if (unlikely(nr_rings > 8)) { 219 dev_err(&hci->master.dev, "number of rings should be <= 8\n"); 220 nr_rings = 8; 221 } 222 if (nr_rings > XFER_RINGS) 223 nr_rings = XFER_RINGS; 224 rings = kzalloc(struct_size(rings, headers, nr_rings), GFP_KERNEL); 225 if (!rings) 226 return -ENOMEM; 227 hci->io_data = rings; 228 rings->total = nr_rings; 229 rings->sysdev = sysdev; 230 231 regval = FIELD_PREP(MAX_HEADER_COUNT, rings->total); 232 rhs_reg_write(CONTROL, regval); 233 234 for (i = 0; i < rings->total; i++) { 235 u32 offset = rhs_reg_read(RHn_OFFSET(i)); 236 237 dev_info(&hci->master.dev, "Ring %d at offset %#x\n", i, offset); 238 ret = -EINVAL; 239 if (!offset) 240 goto err_out; 241 rh = &rings->headers[i]; 242 rh->regs = hci->base_regs + offset; 243 spin_lock_init(&rh->lock); 244 init_completion(&rh->op_done); 245 246 rh->xfer_entries = XFER_RING_ENTRIES; 247 248 regval = rh_reg_read(CR_SETUP); 249 rh->xfer_struct_sz = FIELD_GET(CR_XFER_STRUCT_SIZE, regval); 250 rh->resp_struct_sz = FIELD_GET(CR_RESP_STRUCT_SIZE, regval); 251 dev_dbg(&hci->master.dev, 252 "xfer_struct_sz = %d, resp_struct_sz = %d", 253 rh->xfer_struct_sz, rh->resp_struct_sz); 254 xfers_sz = rh->xfer_struct_sz * rh->xfer_entries; 255 resps_sz = rh->resp_struct_sz * rh->xfer_entries; 256 257 rh->xfer = dma_alloc_coherent(rings->sysdev, xfers_sz, 258 &rh->xfer_dma, GFP_KERNEL); 259 rh->resp = dma_alloc_coherent(rings->sysdev, resps_sz, 260 &rh->resp_dma, GFP_KERNEL); 261 rh->src_xfers = 262 kmalloc_array(rh->xfer_entries, sizeof(*rh->src_xfers), 263 GFP_KERNEL); 264 ret = -ENOMEM; 265 if (!rh->xfer || !rh->resp || !rh->src_xfers) 266 goto err_out; 267 268 rh_reg_write(CMD_RING_BASE_LO, lower_32_bits(rh->xfer_dma)); 269 rh_reg_write(CMD_RING_BASE_HI, upper_32_bits(rh->xfer_dma)); 270 rh_reg_write(RESP_RING_BASE_LO, lower_32_bits(rh->resp_dma)); 271 rh_reg_write(RESP_RING_BASE_HI, upper_32_bits(rh->resp_dma)); 272 273 regval = FIELD_PREP(CR_RING_SIZE, rh->xfer_entries); 274 rh_reg_write(CR_SETUP, regval); 275 276 rh_reg_write(INTR_STATUS_ENABLE, 0xffffffff); 277 rh_reg_write(INTR_SIGNAL_ENABLE, INTR_IBI_READY | 278 INTR_TRANSFER_COMPLETION | 279 INTR_RING_OP | 280 INTR_TRANSFER_ERR | 281 INTR_IBI_RING_FULL | 282 INTR_TRANSFER_ABORT); 283 284 /* IBIs */ 285 286 if (i >= IBI_RINGS) 287 goto ring_ready; 288 289 regval = rh_reg_read(IBI_SETUP); 290 rh->ibi_status_sz = FIELD_GET(IBI_STATUS_STRUCT_SIZE, regval); 291 rh->ibi_status_entries = IBI_STATUS_RING_ENTRIES; 292 rh->ibi_chunks_total = IBI_CHUNK_POOL_SIZE; 293 294 rh->ibi_chunk_sz = dma_get_cache_alignment(); 295 rh->ibi_chunk_sz *= IBI_CHUNK_CACHELINES; 296 /* 297 * Round IBI data chunk size to number of bytes supported by 298 * the HW. Chunk size can be 2^n number of DWORDs which is the 299 * same as 2^(n+2) bytes, where n is 0..6. 300 */ 301 rh->ibi_chunk_sz = umax(4, rh->ibi_chunk_sz); 302 rh->ibi_chunk_sz = roundup_pow_of_two(rh->ibi_chunk_sz); 303 if (rh->ibi_chunk_sz > 256) { 304 ret = -EINVAL; 305 goto err_out; 306 } 307 308 ibi_status_ring_sz = rh->ibi_status_sz * rh->ibi_status_entries; 309 ibi_data_ring_sz = rh->ibi_chunk_sz * rh->ibi_chunks_total; 310 311 rh->ibi_status = 312 dma_alloc_coherent(rings->sysdev, ibi_status_ring_sz, 313 &rh->ibi_status_dma, GFP_KERNEL); 314 rh->ibi_data = kmalloc(ibi_data_ring_sz, GFP_KERNEL); 315 ret = -ENOMEM; 316 if (!rh->ibi_status || !rh->ibi_data) 317 goto err_out; 318 rh->ibi_data_dma = 319 dma_map_single(rings->sysdev, rh->ibi_data, 320 ibi_data_ring_sz, DMA_FROM_DEVICE); 321 if (dma_mapping_error(rings->sysdev, rh->ibi_data_dma)) { 322 rh->ibi_data_dma = 0; 323 ret = -ENOMEM; 324 goto err_out; 325 } 326 327 rh_reg_write(IBI_STATUS_RING_BASE_LO, lower_32_bits(rh->ibi_status_dma)); 328 rh_reg_write(IBI_STATUS_RING_BASE_HI, upper_32_bits(rh->ibi_status_dma)); 329 rh_reg_write(IBI_DATA_RING_BASE_LO, lower_32_bits(rh->ibi_data_dma)); 330 rh_reg_write(IBI_DATA_RING_BASE_HI, upper_32_bits(rh->ibi_data_dma)); 331 332 regval = FIELD_PREP(IBI_STATUS_RING_SIZE, 333 rh->ibi_status_entries) | 334 FIELD_PREP(IBI_DATA_CHUNK_SIZE, 335 ilog2(rh->ibi_chunk_sz) - 2) | 336 FIELD_PREP(IBI_DATA_CHUNK_COUNT, 337 rh->ibi_chunks_total); 338 rh_reg_write(IBI_SETUP, regval); 339 340 regval = rh_reg_read(INTR_SIGNAL_ENABLE); 341 regval |= INTR_IBI_READY; 342 rh_reg_write(INTR_SIGNAL_ENABLE, regval); 343 344 ring_ready: 345 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | 346 RING_CTRL_RUN_STOP); 347 } 348 349 return 0; 350 351 err_out: 352 hci_dma_cleanup(hci); 353 return ret; 354 } 355 356 static void hci_dma_unmap_xfer(struct i3c_hci *hci, 357 struct hci_xfer *xfer_list, unsigned int n) 358 { 359 struct hci_xfer *xfer; 360 unsigned int i; 361 362 for (i = 0; i < n; i++) { 363 xfer = xfer_list + i; 364 if (!xfer->data) 365 continue; 366 i3c_master_dma_unmap_single(xfer->dma); 367 } 368 } 369 370 static int hci_dma_queue_xfer(struct i3c_hci *hci, 371 struct hci_xfer *xfer_list, int n) 372 { 373 struct hci_rings_data *rings = hci->io_data; 374 struct hci_rh_data *rh; 375 unsigned int i, ring, enqueue_ptr; 376 u32 op1_val, op2_val; 377 378 /* For now we only use ring 0 */ 379 ring = 0; 380 rh = &rings->headers[ring]; 381 382 op1_val = rh_reg_read(RING_OPERATION1); 383 enqueue_ptr = FIELD_GET(RING_OP1_CR_ENQ_PTR, op1_val); 384 for (i = 0; i < n; i++) { 385 struct hci_xfer *xfer = xfer_list + i; 386 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr; 387 enum dma_data_direction dir = xfer->rnw ? DMA_FROM_DEVICE : 388 DMA_TO_DEVICE; 389 bool need_bounce; 390 391 /* store cmd descriptor */ 392 *ring_data++ = xfer->cmd_desc[0]; 393 *ring_data++ = xfer->cmd_desc[1]; 394 if (hci->cmd == &mipi_i3c_hci_cmd_v2) { 395 *ring_data++ = xfer->cmd_desc[2]; 396 *ring_data++ = xfer->cmd_desc[3]; 397 } 398 399 /* first word of Data Buffer Descriptor Structure */ 400 if (!xfer->data) 401 xfer->data_len = 0; 402 *ring_data++ = 403 FIELD_PREP(DATA_BUF_BLOCK_SIZE, xfer->data_len) | 404 ((i == n - 1) ? DATA_BUF_IOC : 0); 405 406 /* 2nd and 3rd words of Data Buffer Descriptor Structure */ 407 if (xfer->data) { 408 need_bounce = device_iommu_mapped(rings->sysdev) && 409 xfer->rnw && 410 xfer->data_len != ALIGN(xfer->data_len, 4); 411 xfer->dma = i3c_master_dma_map_single(rings->sysdev, 412 xfer->data, 413 xfer->data_len, 414 need_bounce, 415 dir); 416 if (!xfer->dma) { 417 hci_dma_unmap_xfer(hci, xfer_list, i); 418 return -ENOMEM; 419 } 420 *ring_data++ = lower_32_bits(xfer->dma->addr); 421 *ring_data++ = upper_32_bits(xfer->dma->addr); 422 } else { 423 *ring_data++ = 0; 424 *ring_data++ = 0; 425 } 426 427 /* remember corresponding xfer struct */ 428 rh->src_xfers[enqueue_ptr] = xfer; 429 /* remember corresponding ring/entry for this xfer structure */ 430 xfer->ring_number = ring; 431 xfer->ring_entry = enqueue_ptr; 432 433 enqueue_ptr = (enqueue_ptr + 1) % rh->xfer_entries; 434 435 /* 436 * We may update the hardware view of the enqueue pointer 437 * only if we didn't reach its dequeue pointer. 438 */ 439 op2_val = rh_reg_read(RING_OPERATION2); 440 if (enqueue_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) { 441 /* the ring is full */ 442 hci_dma_unmap_xfer(hci, xfer_list, i + 1); 443 return -EBUSY; 444 } 445 } 446 447 /* take care to update the hardware enqueue pointer atomically */ 448 spin_lock_irq(&rh->lock); 449 op1_val = rh_reg_read(RING_OPERATION1); 450 op1_val &= ~RING_OP1_CR_ENQ_PTR; 451 op1_val |= FIELD_PREP(RING_OP1_CR_ENQ_PTR, enqueue_ptr); 452 rh_reg_write(RING_OPERATION1, op1_val); 453 spin_unlock_irq(&rh->lock); 454 455 return 0; 456 } 457 458 static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, 459 struct hci_xfer *xfer_list, int n) 460 { 461 struct hci_rings_data *rings = hci->io_data; 462 struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number]; 463 unsigned int i; 464 bool did_unqueue = false; 465 466 /* stop the ring */ 467 rh_reg_write(RING_CONTROL, RING_CTRL_ABORT); 468 if (wait_for_completion_timeout(&rh->op_done, HZ) == 0) { 469 /* 470 * We're deep in it if ever this condition is ever met. 471 * Hardware might still be writing to memory, etc. 472 */ 473 dev_crit(&hci->master.dev, "unable to abort the ring\n"); 474 WARN_ON(1); 475 } 476 477 for (i = 0; i < n; i++) { 478 struct hci_xfer *xfer = xfer_list + i; 479 int idx = xfer->ring_entry; 480 481 /* 482 * At the time the abort happened, the xfer might have 483 * completed already. If not then replace corresponding 484 * descriptor entries with a no-op. 485 */ 486 if (idx >= 0) { 487 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx; 488 489 /* store no-op cmd descriptor */ 490 *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7); 491 *ring_data++ = 0; 492 if (hci->cmd == &mipi_i3c_hci_cmd_v2) { 493 *ring_data++ = 0; 494 *ring_data++ = 0; 495 } 496 497 /* disassociate this xfer struct */ 498 rh->src_xfers[idx] = NULL; 499 500 /* and unmap it */ 501 hci_dma_unmap_xfer(hci, xfer, 1); 502 503 did_unqueue = true; 504 } 505 } 506 507 /* restart the ring */ 508 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE); 509 510 return did_unqueue; 511 } 512 513 static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh) 514 { 515 u32 op1_val, op2_val, resp, *ring_resp; 516 unsigned int tid, done_ptr = rh->done_ptr; 517 struct hci_xfer *xfer; 518 519 for (;;) { 520 op2_val = rh_reg_read(RING_OPERATION2); 521 if (done_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) 522 break; 523 524 ring_resp = rh->resp + rh->resp_struct_sz * done_ptr; 525 resp = *ring_resp; 526 tid = RESP_TID(resp); 527 dev_dbg(&hci->master.dev, "resp = 0x%08x", resp); 528 529 xfer = rh->src_xfers[done_ptr]; 530 if (!xfer) { 531 dev_dbg(&hci->master.dev, "orphaned ring entry"); 532 } else { 533 hci_dma_unmap_xfer(hci, xfer, 1); 534 xfer->ring_entry = -1; 535 xfer->response = resp; 536 if (tid != xfer->cmd_tid) { 537 dev_err(&hci->master.dev, 538 "response tid=%d when expecting %d\n", 539 tid, xfer->cmd_tid); 540 /* TODO: do something about it? */ 541 } 542 if (xfer->completion) 543 complete(xfer->completion); 544 } 545 546 done_ptr = (done_ptr + 1) % rh->xfer_entries; 547 rh->done_ptr = done_ptr; 548 } 549 550 /* take care to update the software dequeue pointer atomically */ 551 spin_lock(&rh->lock); 552 op1_val = rh_reg_read(RING_OPERATION1); 553 op1_val &= ~RING_OP1_CR_SW_DEQ_PTR; 554 op1_val |= FIELD_PREP(RING_OP1_CR_SW_DEQ_PTR, done_ptr); 555 rh_reg_write(RING_OPERATION1, op1_val); 556 spin_unlock(&rh->lock); 557 } 558 559 static int hci_dma_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev, 560 const struct i3c_ibi_setup *req) 561 { 562 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 563 struct i3c_generic_ibi_pool *pool; 564 struct hci_dma_dev_ibi_data *dev_ibi; 565 566 dev_ibi = kmalloc(sizeof(*dev_ibi), GFP_KERNEL); 567 if (!dev_ibi) 568 return -ENOMEM; 569 pool = i3c_generic_ibi_alloc_pool(dev, req); 570 if (IS_ERR(pool)) { 571 kfree(dev_ibi); 572 return PTR_ERR(pool); 573 } 574 dev_ibi->pool = pool; 575 dev_ibi->max_len = req->max_payload_len; 576 dev_data->ibi_data = dev_ibi; 577 return 0; 578 } 579 580 static void hci_dma_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev) 581 { 582 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 583 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data; 584 585 dev_data->ibi_data = NULL; 586 i3c_generic_ibi_free_pool(dev_ibi->pool); 587 kfree(dev_ibi); 588 } 589 590 static void hci_dma_recycle_ibi_slot(struct i3c_hci *hci, 591 struct i3c_dev_desc *dev, 592 struct i3c_ibi_slot *slot) 593 { 594 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 595 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data; 596 597 i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot); 598 } 599 600 static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh) 601 { 602 struct hci_rings_data *rings = hci->io_data; 603 struct i3c_dev_desc *dev; 604 struct i3c_hci_dev_data *dev_data; 605 struct hci_dma_dev_ibi_data *dev_ibi; 606 struct i3c_ibi_slot *slot; 607 u32 op1_val, op2_val, ibi_status_error; 608 unsigned int ptr, enq_ptr, deq_ptr; 609 unsigned int ibi_size, ibi_chunks, ibi_data_offset, first_part; 610 int ibi_addr, last_ptr; 611 void *ring_ibi_data; 612 dma_addr_t ring_ibi_data_dma; 613 614 op1_val = rh_reg_read(RING_OPERATION1); 615 deq_ptr = FIELD_GET(RING_OP1_IBI_DEQ_PTR, op1_val); 616 617 op2_val = rh_reg_read(RING_OPERATION2); 618 enq_ptr = FIELD_GET(RING_OP2_IBI_ENQ_PTR, op2_val); 619 620 ibi_status_error = 0; 621 ibi_addr = -1; 622 ibi_chunks = 0; 623 ibi_size = 0; 624 last_ptr = -1; 625 626 /* let's find all we can about this IBI */ 627 for (ptr = deq_ptr; ptr != enq_ptr; 628 ptr = (ptr + 1) % rh->ibi_status_entries) { 629 u32 ibi_status, *ring_ibi_status; 630 unsigned int chunks; 631 632 ring_ibi_status = rh->ibi_status + rh->ibi_status_sz * ptr; 633 ibi_status = *ring_ibi_status; 634 dev_dbg(&hci->master.dev, "status = %#x", ibi_status); 635 636 if (ibi_status_error) { 637 /* we no longer care */ 638 } else if (ibi_status & IBI_ERROR) { 639 ibi_status_error = ibi_status; 640 } else if (ibi_addr == -1) { 641 ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status); 642 } else if (ibi_addr != FIELD_GET(IBI_TARGET_ADDR, ibi_status)) { 643 /* the address changed unexpectedly */ 644 ibi_status_error = ibi_status; 645 } 646 647 chunks = FIELD_GET(IBI_CHUNKS, ibi_status); 648 ibi_chunks += chunks; 649 if (!(ibi_status & IBI_LAST_STATUS)) { 650 ibi_size += chunks * rh->ibi_chunk_sz; 651 } else { 652 ibi_size += FIELD_GET(IBI_DATA_LENGTH, ibi_status); 653 last_ptr = ptr; 654 break; 655 } 656 } 657 658 /* validate what we've got */ 659 660 if (last_ptr == -1) { 661 /* this IBI sequence is not yet complete */ 662 dev_dbg(&hci->master.dev, 663 "no LAST_STATUS available (e=%d d=%d)", 664 enq_ptr, deq_ptr); 665 return; 666 } 667 deq_ptr = last_ptr + 1; 668 deq_ptr %= rh->ibi_status_entries; 669 670 if (ibi_status_error) { 671 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi_addr); 672 goto done; 673 } 674 675 /* determine who this is for */ 676 dev = i3c_hci_addr_to_dev(hci, ibi_addr); 677 if (!dev) { 678 dev_err(&hci->master.dev, 679 "IBI for unknown device %#x\n", ibi_addr); 680 goto done; 681 } 682 683 dev_data = i3c_dev_get_master_data(dev); 684 dev_ibi = dev_data->ibi_data; 685 if (ibi_size > dev_ibi->max_len) { 686 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n", 687 ibi_size, dev_ibi->max_len); 688 goto done; 689 } 690 691 /* 692 * This ring model is not suitable for zero-copy processing of IBIs. 693 * We have the data chunk ring wrap-around to deal with, meaning 694 * that the payload might span multiple chunks beginning at the 695 * end of the ring and wrap to the start of the ring. Furthermore 696 * there is no guarantee that those chunks will be released in order 697 * and in a timely manner by the upper driver. So let's just copy 698 * them to a discrete buffer. In practice they're supposed to be 699 * small anyway. 700 */ 701 slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool); 702 if (!slot) { 703 dev_err(&hci->master.dev, "no free slot for IBI\n"); 704 goto done; 705 } 706 707 /* copy first part of the payload */ 708 ibi_data_offset = rh->ibi_chunk_sz * rh->ibi_chunk_ptr; 709 ring_ibi_data = rh->ibi_data + ibi_data_offset; 710 ring_ibi_data_dma = rh->ibi_data_dma + ibi_data_offset; 711 first_part = (rh->ibi_chunks_total - rh->ibi_chunk_ptr) 712 * rh->ibi_chunk_sz; 713 if (first_part > ibi_size) 714 first_part = ibi_size; 715 dma_sync_single_for_cpu(rings->sysdev, ring_ibi_data_dma, 716 first_part, DMA_FROM_DEVICE); 717 memcpy(slot->data, ring_ibi_data, first_part); 718 719 /* copy second part if any */ 720 if (ibi_size > first_part) { 721 /* we wrap back to the start and copy remaining data */ 722 ring_ibi_data = rh->ibi_data; 723 ring_ibi_data_dma = rh->ibi_data_dma; 724 dma_sync_single_for_cpu(rings->sysdev, ring_ibi_data_dma, 725 ibi_size - first_part, DMA_FROM_DEVICE); 726 memcpy(slot->data + first_part, ring_ibi_data, 727 ibi_size - first_part); 728 } 729 730 /* submit it */ 731 slot->dev = dev; 732 slot->len = ibi_size; 733 i3c_master_queue_ibi(dev, slot); 734 735 done: 736 /* take care to update the ibi dequeue pointer atomically */ 737 spin_lock(&rh->lock); 738 op1_val = rh_reg_read(RING_OPERATION1); 739 op1_val &= ~RING_OP1_IBI_DEQ_PTR; 740 op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr); 741 rh_reg_write(RING_OPERATION1, op1_val); 742 spin_unlock(&rh->lock); 743 744 /* update the chunk pointer */ 745 rh->ibi_chunk_ptr += ibi_chunks; 746 rh->ibi_chunk_ptr %= rh->ibi_chunks_total; 747 748 /* and tell the hardware about freed chunks */ 749 rh_reg_write(CHUNK_CONTROL, rh_reg_read(CHUNK_CONTROL) + ibi_chunks); 750 } 751 752 static bool hci_dma_irq_handler(struct i3c_hci *hci) 753 { 754 struct hci_rings_data *rings = hci->io_data; 755 unsigned int i; 756 bool handled = false; 757 758 for (i = 0; i < rings->total; i++) { 759 struct hci_rh_data *rh; 760 u32 status; 761 762 rh = &rings->headers[i]; 763 status = rh_reg_read(INTR_STATUS); 764 dev_dbg(&hci->master.dev, "Ring %d: RH_INTR_STATUS %#x", 765 i, status); 766 if (!status) 767 continue; 768 rh_reg_write(INTR_STATUS, status); 769 770 if (status & INTR_IBI_READY) 771 hci_dma_process_ibi(hci, rh); 772 if (status & (INTR_TRANSFER_COMPLETION | INTR_TRANSFER_ERR)) 773 hci_dma_xfer_done(hci, rh); 774 if (status & INTR_RING_OP) 775 complete(&rh->op_done); 776 777 if (status & INTR_TRANSFER_ABORT) { 778 u32 ring_status; 779 780 dev_notice_ratelimited(&hci->master.dev, 781 "Ring %d: Transfer Aborted\n", i); 782 mipi_i3c_hci_resume(hci); 783 ring_status = rh_reg_read(RING_STATUS); 784 if (!(ring_status & RING_STATUS_RUNNING) && 785 status & INTR_TRANSFER_COMPLETION && 786 status & INTR_TRANSFER_ERR) { 787 /* 788 * Ring stop followed by run is an Intel 789 * specific required quirk after resuming the 790 * halted controller. Do it only when the ring 791 * is not in running state after a transfer 792 * error. 793 */ 794 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE); 795 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | 796 RING_CTRL_RUN_STOP); 797 } 798 } 799 if (status & INTR_IBI_RING_FULL) 800 dev_err_ratelimited(&hci->master.dev, 801 "Ring %d: IBI Ring Full Condition\n", i); 802 803 handled = true; 804 } 805 806 return handled; 807 } 808 809 const struct hci_io_ops mipi_i3c_hci_dma = { 810 .init = hci_dma_init, 811 .cleanup = hci_dma_cleanup, 812 .queue_xfer = hci_dma_queue_xfer, 813 .dequeue_xfer = hci_dma_dequeue_xfer, 814 .irq_handler = hci_dma_irq_handler, 815 .request_ibi = hci_dma_request_ibi, 816 .free_ibi = hci_dma_free_ibi, 817 .recycle_ibi_slot = hci_dma_recycle_ibi_slot, 818 }; 819