1 // SPDX-License-Identifier: BSD-3-Clause 2 /* 3 * Copyright (c) 2020, MIPI Alliance, Inc. 4 * 5 * Author: Nicolas Pitre <npitre@baylibre.com> 6 * 7 * Note: The I3C HCI v2.0 spec is still in flux. The IBI support is based on 8 * v1.x of the spec and v2.0 will likely be split out. 9 */ 10 11 #include <linux/bitfield.h> 12 #include <linux/device.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/errno.h> 15 #include <linux/i3c/master.h> 16 #include <linux/io.h> 17 18 #include "hci.h" 19 #include "cmd.h" 20 #include "ibi.h" 21 22 23 /* 24 * Software Parameter Values (somewhat arb itrary for now). 25 * Some of them could be determined at run time eventually. 26 */ 27 28 #define XFER_RINGS 1 /* max: 8 */ 29 #define XFER_RING_ENTRIES 16 /* max: 255 */ 30 31 #define IBI_RINGS 1 /* max: 8 */ 32 #define IBI_STATUS_RING_ENTRIES 32 /* max: 255 */ 33 #define IBI_CHUNK_CACHELINES 1 /* max: 256 bytes equivalent */ 34 #define IBI_CHUNK_POOL_SIZE 128 /* max: 1023 */ 35 36 /* 37 * Ring Header Preamble 38 */ 39 40 #define rhs_reg_read(r) readl(hci->RHS_regs + (RHS_##r)) 41 #define rhs_reg_write(r, v) writel(v, hci->RHS_regs + (RHS_##r)) 42 43 #define RHS_CONTROL 0x00 44 #define PREAMBLE_SIZE GENMASK(31, 24) /* Preamble Section Size */ 45 #define HEADER_SIZE GENMASK(23, 16) /* Ring Header Size */ 46 #define MAX_HEADER_COUNT_CAP GENMASK(7, 4) /* HC Max Header Count */ 47 #define MAX_HEADER_COUNT GENMASK(3, 0) /* Driver Max Header Count */ 48 49 #define RHS_RHn_OFFSET(n) (0x04 + (n)*4) 50 51 /* 52 * Ring Header (Per-Ring Bundle) 53 */ 54 55 #define rh_reg_read(r) readl(rh->regs + (RH_##r)) 56 #define rh_reg_write(r, v) writel(v, rh->regs + (RH_##r)) 57 58 #define RH_CR_SETUP 0x00 /* Command/Response Ring */ 59 #define CR_XFER_STRUCT_SIZE GENMASK(31, 24) 60 #define CR_RESP_STRUCT_SIZE GENMASK(23, 16) 61 #define CR_RING_SIZE GENMASK(8, 0) 62 63 #define RH_IBI_SETUP 0x04 64 #define IBI_STATUS_STRUCT_SIZE GENMASK(31, 24) 65 #define IBI_STATUS_RING_SIZE GENMASK(23, 16) 66 #define IBI_DATA_CHUNK_SIZE GENMASK(12, 10) 67 #define IBI_DATA_CHUNK_COUNT GENMASK(9, 0) 68 69 #define RH_CHUNK_CONTROL 0x08 70 71 #define RH_INTR_STATUS 0x10 72 #define RH_INTR_STATUS_ENABLE 0x14 73 #define RH_INTR_SIGNAL_ENABLE 0x18 74 #define RH_INTR_FORCE 0x1c 75 #define INTR_IBI_READY BIT(12) 76 #define INTR_TRANSFER_COMPLETION BIT(11) 77 #define INTR_RING_OP BIT(10) 78 #define INTR_TRANSFER_ERR BIT(9) 79 #define INTR_WARN_INS_STOP_MODE BIT(7) 80 #define INTR_IBI_RING_FULL BIT(6) 81 #define INTR_TRANSFER_ABORT BIT(5) 82 83 #define RH_RING_STATUS 0x20 84 #define RING_STATUS_LOCKED BIT(3) 85 #define RING_STATUS_ABORTED BIT(2) 86 #define RING_STATUS_RUNNING BIT(1) 87 #define RING_STATUS_ENABLED BIT(0) 88 89 #define RH_RING_CONTROL 0x24 90 #define RING_CTRL_ABORT BIT(2) 91 #define RING_CTRL_RUN_STOP BIT(1) 92 #define RING_CTRL_ENABLE BIT(0) 93 94 #define RH_RING_OPERATION1 0x28 95 #define RING_OP1_IBI_DEQ_PTR GENMASK(23, 16) 96 #define RING_OP1_CR_SW_DEQ_PTR GENMASK(15, 8) 97 #define RING_OP1_CR_ENQ_PTR GENMASK(7, 0) 98 99 #define RH_RING_OPERATION2 0x2c 100 #define RING_OP2_IBI_ENQ_PTR GENMASK(23, 16) 101 #define RING_OP2_CR_DEQ_PTR GENMASK(7, 0) 102 103 #define RH_CMD_RING_BASE_LO 0x30 104 #define RH_CMD_RING_BASE_HI 0x34 105 #define RH_RESP_RING_BASE_LO 0x38 106 #define RH_RESP_RING_BASE_HI 0x3c 107 #define RH_IBI_STATUS_RING_BASE_LO 0x40 108 #define RH_IBI_STATUS_RING_BASE_HI 0x44 109 #define RH_IBI_DATA_RING_BASE_LO 0x48 110 #define RH_IBI_DATA_RING_BASE_HI 0x4c 111 112 #define RH_CMD_RING_SG 0x50 /* Ring Scatter Gather Support */ 113 #define RH_RESP_RING_SG 0x54 114 #define RH_IBI_STATUS_RING_SG 0x58 115 #define RH_IBI_DATA_RING_SG 0x5c 116 #define RING_SG_BLP BIT(31) /* Buffer Vs. List Pointer */ 117 #define RING_SG_LIST_SIZE GENMASK(15, 0) 118 119 /* 120 * Data Buffer Descriptor (in memory) 121 */ 122 123 #define DATA_BUF_BLP BIT(31) /* Buffer Vs. List Pointer */ 124 #define DATA_BUF_IOC BIT(30) /* Interrupt on Completion */ 125 #define DATA_BUF_BLOCK_SIZE GENMASK(15, 0) 126 127 128 struct hci_rh_data { 129 void __iomem *regs; 130 void *xfer, *resp, *ibi_status, *ibi_data; 131 dma_addr_t xfer_dma, resp_dma, ibi_status_dma, ibi_data_dma; 132 unsigned int xfer_entries, ibi_status_entries, ibi_chunks_total; 133 unsigned int xfer_struct_sz, resp_struct_sz, ibi_status_sz, ibi_chunk_sz; 134 unsigned int done_ptr, ibi_chunk_ptr; 135 struct hci_xfer **src_xfers; 136 spinlock_t lock; 137 struct completion op_done; 138 }; 139 140 struct hci_rings_data { 141 unsigned int total; 142 struct hci_rh_data headers[] __counted_by(total); 143 }; 144 145 struct hci_dma_dev_ibi_data { 146 struct i3c_generic_ibi_pool *pool; 147 unsigned int max_len; 148 }; 149 150 static void hci_dma_cleanup(struct i3c_hci *hci) 151 { 152 struct hci_rings_data *rings = hci->io_data; 153 struct hci_rh_data *rh; 154 unsigned int i; 155 156 if (!rings) 157 return; 158 159 for (i = 0; i < rings->total; i++) { 160 rh = &rings->headers[i]; 161 162 rh_reg_write(INTR_SIGNAL_ENABLE, 0); 163 rh_reg_write(RING_CONTROL, 0); 164 rh_reg_write(CR_SETUP, 0); 165 rh_reg_write(IBI_SETUP, 0); 166 167 if (rh->xfer) 168 dma_free_coherent(&hci->master.dev, 169 rh->xfer_struct_sz * rh->xfer_entries, 170 rh->xfer, rh->xfer_dma); 171 if (rh->resp) 172 dma_free_coherent(&hci->master.dev, 173 rh->resp_struct_sz * rh->xfer_entries, 174 rh->resp, rh->resp_dma); 175 kfree(rh->src_xfers); 176 if (rh->ibi_status) 177 dma_free_coherent(&hci->master.dev, 178 rh->ibi_status_sz * rh->ibi_status_entries, 179 rh->ibi_status, rh->ibi_status_dma); 180 if (rh->ibi_data_dma) 181 dma_unmap_single(&hci->master.dev, rh->ibi_data_dma, 182 rh->ibi_chunk_sz * rh->ibi_chunks_total, 183 DMA_FROM_DEVICE); 184 kfree(rh->ibi_data); 185 } 186 187 rhs_reg_write(CONTROL, 0); 188 189 kfree(rings); 190 hci->io_data = NULL; 191 } 192 193 static int hci_dma_init(struct i3c_hci *hci) 194 { 195 struct hci_rings_data *rings; 196 struct hci_rh_data *rh; 197 u32 regval; 198 unsigned int i, nr_rings, xfers_sz, resps_sz; 199 unsigned int ibi_status_ring_sz, ibi_data_ring_sz; 200 int ret; 201 202 regval = rhs_reg_read(CONTROL); 203 nr_rings = FIELD_GET(MAX_HEADER_COUNT_CAP, regval); 204 dev_info(&hci->master.dev, "%d DMA rings available\n", nr_rings); 205 if (unlikely(nr_rings > 8)) { 206 dev_err(&hci->master.dev, "number of rings should be <= 8\n"); 207 nr_rings = 8; 208 } 209 if (nr_rings > XFER_RINGS) 210 nr_rings = XFER_RINGS; 211 rings = kzalloc(struct_size(rings, headers, nr_rings), GFP_KERNEL); 212 if (!rings) 213 return -ENOMEM; 214 hci->io_data = rings; 215 rings->total = nr_rings; 216 217 regval = FIELD_PREP(MAX_HEADER_COUNT, rings->total); 218 rhs_reg_write(CONTROL, regval); 219 220 for (i = 0; i < rings->total; i++) { 221 u32 offset = rhs_reg_read(RHn_OFFSET(i)); 222 223 dev_info(&hci->master.dev, "Ring %d at offset %#x\n", i, offset); 224 ret = -EINVAL; 225 if (!offset) 226 goto err_out; 227 rh = &rings->headers[i]; 228 rh->regs = hci->base_regs + offset; 229 spin_lock_init(&rh->lock); 230 init_completion(&rh->op_done); 231 232 rh->xfer_entries = XFER_RING_ENTRIES; 233 234 regval = rh_reg_read(CR_SETUP); 235 rh->xfer_struct_sz = FIELD_GET(CR_XFER_STRUCT_SIZE, regval); 236 rh->resp_struct_sz = FIELD_GET(CR_RESP_STRUCT_SIZE, regval); 237 DBG("xfer_struct_sz = %d, resp_struct_sz = %d", 238 rh->xfer_struct_sz, rh->resp_struct_sz); 239 xfers_sz = rh->xfer_struct_sz * rh->xfer_entries; 240 resps_sz = rh->resp_struct_sz * rh->xfer_entries; 241 242 rh->xfer = dma_alloc_coherent(&hci->master.dev, xfers_sz, 243 &rh->xfer_dma, GFP_KERNEL); 244 rh->resp = dma_alloc_coherent(&hci->master.dev, resps_sz, 245 &rh->resp_dma, GFP_KERNEL); 246 rh->src_xfers = 247 kmalloc_array(rh->xfer_entries, sizeof(*rh->src_xfers), 248 GFP_KERNEL); 249 ret = -ENOMEM; 250 if (!rh->xfer || !rh->resp || !rh->src_xfers) 251 goto err_out; 252 253 rh_reg_write(CMD_RING_BASE_LO, lower_32_bits(rh->xfer_dma)); 254 rh_reg_write(CMD_RING_BASE_HI, upper_32_bits(rh->xfer_dma)); 255 rh_reg_write(RESP_RING_BASE_LO, lower_32_bits(rh->resp_dma)); 256 rh_reg_write(RESP_RING_BASE_HI, upper_32_bits(rh->resp_dma)); 257 258 regval = FIELD_PREP(CR_RING_SIZE, rh->xfer_entries); 259 rh_reg_write(CR_SETUP, regval); 260 261 rh_reg_write(INTR_STATUS_ENABLE, 0xffffffff); 262 rh_reg_write(INTR_SIGNAL_ENABLE, INTR_IBI_READY | 263 INTR_TRANSFER_COMPLETION | 264 INTR_RING_OP | 265 INTR_TRANSFER_ERR | 266 INTR_WARN_INS_STOP_MODE | 267 INTR_IBI_RING_FULL | 268 INTR_TRANSFER_ABORT); 269 270 /* IBIs */ 271 272 if (i >= IBI_RINGS) 273 goto ring_ready; 274 275 regval = rh_reg_read(IBI_SETUP); 276 rh->ibi_status_sz = FIELD_GET(IBI_STATUS_STRUCT_SIZE, regval); 277 rh->ibi_status_entries = IBI_STATUS_RING_ENTRIES; 278 rh->ibi_chunks_total = IBI_CHUNK_POOL_SIZE; 279 280 rh->ibi_chunk_sz = dma_get_cache_alignment(); 281 rh->ibi_chunk_sz *= IBI_CHUNK_CACHELINES; 282 /* 283 * Round IBI data chunk size to number of bytes supported by 284 * the HW. Chunk size can be 2^n number of DWORDs which is the 285 * same as 2^(n+2) bytes, where n is 0..6. 286 */ 287 rh->ibi_chunk_sz = umax(4, rh->ibi_chunk_sz); 288 rh->ibi_chunk_sz = roundup_pow_of_two(rh->ibi_chunk_sz); 289 if (rh->ibi_chunk_sz > 256) { 290 ret = -EINVAL; 291 goto err_out; 292 } 293 294 ibi_status_ring_sz = rh->ibi_status_sz * rh->ibi_status_entries; 295 ibi_data_ring_sz = rh->ibi_chunk_sz * rh->ibi_chunks_total; 296 297 rh->ibi_status = 298 dma_alloc_coherent(&hci->master.dev, ibi_status_ring_sz, 299 &rh->ibi_status_dma, GFP_KERNEL); 300 rh->ibi_data = kmalloc(ibi_data_ring_sz, GFP_KERNEL); 301 ret = -ENOMEM; 302 if (!rh->ibi_status || !rh->ibi_data) 303 goto err_out; 304 rh->ibi_data_dma = 305 dma_map_single(&hci->master.dev, rh->ibi_data, 306 ibi_data_ring_sz, DMA_FROM_DEVICE); 307 if (dma_mapping_error(&hci->master.dev, rh->ibi_data_dma)) { 308 rh->ibi_data_dma = 0; 309 ret = -ENOMEM; 310 goto err_out; 311 } 312 313 rh_reg_write(IBI_STATUS_RING_BASE_LO, lower_32_bits(rh->ibi_status_dma)); 314 rh_reg_write(IBI_STATUS_RING_BASE_HI, upper_32_bits(rh->ibi_status_dma)); 315 rh_reg_write(IBI_DATA_RING_BASE_LO, lower_32_bits(rh->ibi_data_dma)); 316 rh_reg_write(IBI_DATA_RING_BASE_HI, upper_32_bits(rh->ibi_data_dma)); 317 318 regval = FIELD_PREP(IBI_STATUS_RING_SIZE, 319 rh->ibi_status_entries) | 320 FIELD_PREP(IBI_DATA_CHUNK_SIZE, 321 ilog2(rh->ibi_chunk_sz) - 2) | 322 FIELD_PREP(IBI_DATA_CHUNK_COUNT, 323 rh->ibi_chunks_total); 324 rh_reg_write(IBI_SETUP, regval); 325 326 regval = rh_reg_read(INTR_SIGNAL_ENABLE); 327 regval |= INTR_IBI_READY; 328 rh_reg_write(INTR_SIGNAL_ENABLE, regval); 329 330 ring_ready: 331 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | 332 RING_CTRL_RUN_STOP); 333 } 334 335 return 0; 336 337 err_out: 338 hci_dma_cleanup(hci); 339 return ret; 340 } 341 342 static void hci_dma_unmap_xfer(struct i3c_hci *hci, 343 struct hci_xfer *xfer_list, unsigned int n) 344 { 345 struct hci_xfer *xfer; 346 unsigned int i; 347 348 for (i = 0; i < n; i++) { 349 xfer = xfer_list + i; 350 if (!xfer->data) 351 continue; 352 dma_unmap_single(&hci->master.dev, 353 xfer->data_dma, xfer->data_len, 354 xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE); 355 } 356 } 357 358 static int hci_dma_queue_xfer(struct i3c_hci *hci, 359 struct hci_xfer *xfer_list, int n) 360 { 361 struct hci_rings_data *rings = hci->io_data; 362 struct hci_rh_data *rh; 363 unsigned int i, ring, enqueue_ptr; 364 u32 op1_val, op2_val; 365 void *buf; 366 367 /* For now we only use ring 0 */ 368 ring = 0; 369 rh = &rings->headers[ring]; 370 371 op1_val = rh_reg_read(RING_OPERATION1); 372 enqueue_ptr = FIELD_GET(RING_OP1_CR_ENQ_PTR, op1_val); 373 for (i = 0; i < n; i++) { 374 struct hci_xfer *xfer = xfer_list + i; 375 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr; 376 377 /* store cmd descriptor */ 378 *ring_data++ = xfer->cmd_desc[0]; 379 *ring_data++ = xfer->cmd_desc[1]; 380 if (hci->cmd == &mipi_i3c_hci_cmd_v2) { 381 *ring_data++ = xfer->cmd_desc[2]; 382 *ring_data++ = xfer->cmd_desc[3]; 383 } 384 385 /* first word of Data Buffer Descriptor Structure */ 386 if (!xfer->data) 387 xfer->data_len = 0; 388 *ring_data++ = 389 FIELD_PREP(DATA_BUF_BLOCK_SIZE, xfer->data_len) | 390 ((i == n - 1) ? DATA_BUF_IOC : 0); 391 392 /* 2nd and 3rd words of Data Buffer Descriptor Structure */ 393 if (xfer->data) { 394 buf = xfer->bounce_buf ? xfer->bounce_buf : xfer->data; 395 xfer->data_dma = 396 dma_map_single(&hci->master.dev, 397 buf, 398 xfer->data_len, 399 xfer->rnw ? 400 DMA_FROM_DEVICE : 401 DMA_TO_DEVICE); 402 if (dma_mapping_error(&hci->master.dev, 403 xfer->data_dma)) { 404 hci_dma_unmap_xfer(hci, xfer_list, i); 405 return -ENOMEM; 406 } 407 *ring_data++ = lower_32_bits(xfer->data_dma); 408 *ring_data++ = upper_32_bits(xfer->data_dma); 409 } else { 410 *ring_data++ = 0; 411 *ring_data++ = 0; 412 } 413 414 /* remember corresponding xfer struct */ 415 rh->src_xfers[enqueue_ptr] = xfer; 416 /* remember corresponding ring/entry for this xfer structure */ 417 xfer->ring_number = ring; 418 xfer->ring_entry = enqueue_ptr; 419 420 enqueue_ptr = (enqueue_ptr + 1) % rh->xfer_entries; 421 422 /* 423 * We may update the hardware view of the enqueue pointer 424 * only if we didn't reach its dequeue pointer. 425 */ 426 op2_val = rh_reg_read(RING_OPERATION2); 427 if (enqueue_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) { 428 /* the ring is full */ 429 hci_dma_unmap_xfer(hci, xfer_list, i + 1); 430 return -EBUSY; 431 } 432 } 433 434 /* take care to update the hardware enqueue pointer atomically */ 435 spin_lock_irq(&rh->lock); 436 op1_val = rh_reg_read(RING_OPERATION1); 437 op1_val &= ~RING_OP1_CR_ENQ_PTR; 438 op1_val |= FIELD_PREP(RING_OP1_CR_ENQ_PTR, enqueue_ptr); 439 rh_reg_write(RING_OPERATION1, op1_val); 440 spin_unlock_irq(&rh->lock); 441 442 return 0; 443 } 444 445 static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, 446 struct hci_xfer *xfer_list, int n) 447 { 448 struct hci_rings_data *rings = hci->io_data; 449 struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number]; 450 unsigned int i; 451 bool did_unqueue = false; 452 453 /* stop the ring */ 454 rh_reg_write(RING_CONTROL, RING_CTRL_ABORT); 455 if (wait_for_completion_timeout(&rh->op_done, HZ) == 0) { 456 /* 457 * We're deep in it if ever this condition is ever met. 458 * Hardware might still be writing to memory, etc. 459 */ 460 dev_crit(&hci->master.dev, "unable to abort the ring\n"); 461 WARN_ON(1); 462 } 463 464 for (i = 0; i < n; i++) { 465 struct hci_xfer *xfer = xfer_list + i; 466 int idx = xfer->ring_entry; 467 468 /* 469 * At the time the abort happened, the xfer might have 470 * completed already. If not then replace corresponding 471 * descriptor entries with a no-op. 472 */ 473 if (idx >= 0) { 474 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx; 475 476 /* store no-op cmd descriptor */ 477 *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7); 478 *ring_data++ = 0; 479 if (hci->cmd == &mipi_i3c_hci_cmd_v2) { 480 *ring_data++ = 0; 481 *ring_data++ = 0; 482 } 483 484 /* disassociate this xfer struct */ 485 rh->src_xfers[idx] = NULL; 486 487 /* and unmap it */ 488 hci_dma_unmap_xfer(hci, xfer, 1); 489 490 did_unqueue = true; 491 } 492 } 493 494 /* restart the ring */ 495 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE); 496 497 return did_unqueue; 498 } 499 500 static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh) 501 { 502 u32 op1_val, op2_val, resp, *ring_resp; 503 unsigned int tid, done_ptr = rh->done_ptr; 504 struct hci_xfer *xfer; 505 506 for (;;) { 507 op2_val = rh_reg_read(RING_OPERATION2); 508 if (done_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) 509 break; 510 511 ring_resp = rh->resp + rh->resp_struct_sz * done_ptr; 512 resp = *ring_resp; 513 tid = RESP_TID(resp); 514 DBG("resp = 0x%08x", resp); 515 516 xfer = rh->src_xfers[done_ptr]; 517 if (!xfer) { 518 DBG("orphaned ring entry"); 519 } else { 520 hci_dma_unmap_xfer(hci, xfer, 1); 521 xfer->ring_entry = -1; 522 xfer->response = resp; 523 if (tid != xfer->cmd_tid) { 524 dev_err(&hci->master.dev, 525 "response tid=%d when expecting %d\n", 526 tid, xfer->cmd_tid); 527 /* TODO: do something about it? */ 528 } 529 if (xfer->completion) 530 complete(xfer->completion); 531 } 532 533 done_ptr = (done_ptr + 1) % rh->xfer_entries; 534 rh->done_ptr = done_ptr; 535 } 536 537 /* take care to update the software dequeue pointer atomically */ 538 spin_lock(&rh->lock); 539 op1_val = rh_reg_read(RING_OPERATION1); 540 op1_val &= ~RING_OP1_CR_SW_DEQ_PTR; 541 op1_val |= FIELD_PREP(RING_OP1_CR_SW_DEQ_PTR, done_ptr); 542 rh_reg_write(RING_OPERATION1, op1_val); 543 spin_unlock(&rh->lock); 544 } 545 546 static int hci_dma_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev, 547 const struct i3c_ibi_setup *req) 548 { 549 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 550 struct i3c_generic_ibi_pool *pool; 551 struct hci_dma_dev_ibi_data *dev_ibi; 552 553 dev_ibi = kmalloc(sizeof(*dev_ibi), GFP_KERNEL); 554 if (!dev_ibi) 555 return -ENOMEM; 556 pool = i3c_generic_ibi_alloc_pool(dev, req); 557 if (IS_ERR(pool)) { 558 kfree(dev_ibi); 559 return PTR_ERR(pool); 560 } 561 dev_ibi->pool = pool; 562 dev_ibi->max_len = req->max_payload_len; 563 dev_data->ibi_data = dev_ibi; 564 return 0; 565 } 566 567 static void hci_dma_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev) 568 { 569 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 570 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data; 571 572 dev_data->ibi_data = NULL; 573 i3c_generic_ibi_free_pool(dev_ibi->pool); 574 kfree(dev_ibi); 575 } 576 577 static void hci_dma_recycle_ibi_slot(struct i3c_hci *hci, 578 struct i3c_dev_desc *dev, 579 struct i3c_ibi_slot *slot) 580 { 581 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); 582 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data; 583 584 i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot); 585 } 586 587 static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh) 588 { 589 struct i3c_dev_desc *dev; 590 struct i3c_hci_dev_data *dev_data; 591 struct hci_dma_dev_ibi_data *dev_ibi; 592 struct i3c_ibi_slot *slot; 593 u32 op1_val, op2_val, ibi_status_error; 594 unsigned int ptr, enq_ptr, deq_ptr; 595 unsigned int ibi_size, ibi_chunks, ibi_data_offset, first_part; 596 int ibi_addr, last_ptr; 597 void *ring_ibi_data; 598 dma_addr_t ring_ibi_data_dma; 599 600 op1_val = rh_reg_read(RING_OPERATION1); 601 deq_ptr = FIELD_GET(RING_OP1_IBI_DEQ_PTR, op1_val); 602 603 op2_val = rh_reg_read(RING_OPERATION2); 604 enq_ptr = FIELD_GET(RING_OP2_IBI_ENQ_PTR, op2_val); 605 606 ibi_status_error = 0; 607 ibi_addr = -1; 608 ibi_chunks = 0; 609 ibi_size = 0; 610 last_ptr = -1; 611 612 /* let's find all we can about this IBI */ 613 for (ptr = deq_ptr; ptr != enq_ptr; 614 ptr = (ptr + 1) % rh->ibi_status_entries) { 615 u32 ibi_status, *ring_ibi_status; 616 unsigned int chunks; 617 618 ring_ibi_status = rh->ibi_status + rh->ibi_status_sz * ptr; 619 ibi_status = *ring_ibi_status; 620 DBG("status = %#x", ibi_status); 621 622 if (ibi_status_error) { 623 /* we no longer care */ 624 } else if (ibi_status & IBI_ERROR) { 625 ibi_status_error = ibi_status; 626 } else if (ibi_addr == -1) { 627 ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status); 628 } else if (ibi_addr != FIELD_GET(IBI_TARGET_ADDR, ibi_status)) { 629 /* the address changed unexpectedly */ 630 ibi_status_error = ibi_status; 631 } 632 633 chunks = FIELD_GET(IBI_CHUNKS, ibi_status); 634 ibi_chunks += chunks; 635 if (!(ibi_status & IBI_LAST_STATUS)) { 636 ibi_size += chunks * rh->ibi_chunk_sz; 637 } else { 638 ibi_size += FIELD_GET(IBI_DATA_LENGTH, ibi_status); 639 last_ptr = ptr; 640 break; 641 } 642 } 643 644 /* validate what we've got */ 645 646 if (last_ptr == -1) { 647 /* this IBI sequence is not yet complete */ 648 DBG("no LAST_STATUS available (e=%d d=%d)", enq_ptr, deq_ptr); 649 return; 650 } 651 deq_ptr = last_ptr + 1; 652 deq_ptr %= rh->ibi_status_entries; 653 654 if (ibi_status_error) { 655 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi_addr); 656 goto done; 657 } 658 659 /* determine who this is for */ 660 dev = i3c_hci_addr_to_dev(hci, ibi_addr); 661 if (!dev) { 662 dev_err(&hci->master.dev, 663 "IBI for unknown device %#x\n", ibi_addr); 664 goto done; 665 } 666 667 dev_data = i3c_dev_get_master_data(dev); 668 dev_ibi = dev_data->ibi_data; 669 if (ibi_size > dev_ibi->max_len) { 670 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n", 671 ibi_size, dev_ibi->max_len); 672 goto done; 673 } 674 675 /* 676 * This ring model is not suitable for zero-copy processing of IBIs. 677 * We have the data chunk ring wrap-around to deal with, meaning 678 * that the payload might span multiple chunks beginning at the 679 * end of the ring and wrap to the start of the ring. Furthermore 680 * there is no guarantee that those chunks will be released in order 681 * and in a timely manner by the upper driver. So let's just copy 682 * them to a discrete buffer. In practice they're supposed to be 683 * small anyway. 684 */ 685 slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool); 686 if (!slot) { 687 dev_err(&hci->master.dev, "no free slot for IBI\n"); 688 goto done; 689 } 690 691 /* copy first part of the payload */ 692 ibi_data_offset = rh->ibi_chunk_sz * rh->ibi_chunk_ptr; 693 ring_ibi_data = rh->ibi_data + ibi_data_offset; 694 ring_ibi_data_dma = rh->ibi_data_dma + ibi_data_offset; 695 first_part = (rh->ibi_chunks_total - rh->ibi_chunk_ptr) 696 * rh->ibi_chunk_sz; 697 if (first_part > ibi_size) 698 first_part = ibi_size; 699 dma_sync_single_for_cpu(&hci->master.dev, ring_ibi_data_dma, 700 first_part, DMA_FROM_DEVICE); 701 memcpy(slot->data, ring_ibi_data, first_part); 702 703 /* copy second part if any */ 704 if (ibi_size > first_part) { 705 /* we wrap back to the start and copy remaining data */ 706 ring_ibi_data = rh->ibi_data; 707 ring_ibi_data_dma = rh->ibi_data_dma; 708 dma_sync_single_for_cpu(&hci->master.dev, ring_ibi_data_dma, 709 ibi_size - first_part, DMA_FROM_DEVICE); 710 memcpy(slot->data + first_part, ring_ibi_data, 711 ibi_size - first_part); 712 } 713 714 /* submit it */ 715 slot->dev = dev; 716 slot->len = ibi_size; 717 i3c_master_queue_ibi(dev, slot); 718 719 done: 720 /* take care to update the ibi dequeue pointer atomically */ 721 spin_lock(&rh->lock); 722 op1_val = rh_reg_read(RING_OPERATION1); 723 op1_val &= ~RING_OP1_IBI_DEQ_PTR; 724 op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr); 725 rh_reg_write(RING_OPERATION1, op1_val); 726 spin_unlock(&rh->lock); 727 728 /* update the chunk pointer */ 729 rh->ibi_chunk_ptr += ibi_chunks; 730 rh->ibi_chunk_ptr %= rh->ibi_chunks_total; 731 732 /* and tell the hardware about freed chunks */ 733 rh_reg_write(CHUNK_CONTROL, rh_reg_read(CHUNK_CONTROL) + ibi_chunks); 734 } 735 736 static bool hci_dma_irq_handler(struct i3c_hci *hci) 737 { 738 struct hci_rings_data *rings = hci->io_data; 739 unsigned int i; 740 bool handled = false; 741 742 for (i = 0; i < rings->total; i++) { 743 struct hci_rh_data *rh; 744 u32 status; 745 746 rh = &rings->headers[i]; 747 status = rh_reg_read(INTR_STATUS); 748 DBG("rh%d status: %#x", i, status); 749 if (!status) 750 continue; 751 rh_reg_write(INTR_STATUS, status); 752 753 if (status & INTR_IBI_READY) 754 hci_dma_process_ibi(hci, rh); 755 if (status & (INTR_TRANSFER_COMPLETION | INTR_TRANSFER_ERR)) 756 hci_dma_xfer_done(hci, rh); 757 if (status & INTR_RING_OP) 758 complete(&rh->op_done); 759 760 if (status & INTR_TRANSFER_ABORT) { 761 dev_notice_ratelimited(&hci->master.dev, 762 "ring %d: Transfer Aborted\n", i); 763 mipi_i3c_hci_resume(hci); 764 } 765 if (status & INTR_WARN_INS_STOP_MODE) 766 dev_warn_ratelimited(&hci->master.dev, 767 "ring %d: Inserted Stop on Mode Change\n", i); 768 if (status & INTR_IBI_RING_FULL) 769 dev_err_ratelimited(&hci->master.dev, 770 "ring %d: IBI Ring Full Condition\n", i); 771 772 handled = true; 773 } 774 775 return handled; 776 } 777 778 const struct hci_io_ops mipi_i3c_hci_dma = { 779 .init = hci_dma_init, 780 .cleanup = hci_dma_cleanup, 781 .queue_xfer = hci_dma_queue_xfer, 782 .dequeue_xfer = hci_dma_dequeue_xfer, 783 .irq_handler = hci_dma_irq_handler, 784 .request_ibi = hci_dma_request_ibi, 785 .free_ibi = hci_dma_free_ibi, 786 .recycle_ibi_slot = hci_dma_recycle_ibi_slot, 787 }; 788