1 // SPDX-License-Identifier: BSD-3-Clause
2 /*
3 * Copyright (c) 2020, MIPI Alliance, Inc.
4 *
5 * Author: Nicolas Pitre <npitre@baylibre.com>
6 */
7
8 #include <linux/bitfield.h>
9 #include <linux/device.h>
10 #include <linux/errno.h>
11 #include <linux/i3c/master.h>
12 #include <linux/io.h>
13
14 #include "hci.h"
15 #include "cmd.h"
16 #include "ibi.h"
17
18 /*
19 * PIO Access Area
20 */
21
22 #define pio_reg_read(r) readl(hci->PIO_regs + (PIO_##r))
23 #define pio_reg_write(r, v) writel(v, hci->PIO_regs + (PIO_##r))
24
25 #define PIO_COMMAND_QUEUE_PORT 0x00
26 #define PIO_RESPONSE_QUEUE_PORT 0x04
27 #define PIO_XFER_DATA_PORT 0x08
28 #define PIO_IBI_PORT 0x0c
29
30 #define PIO_QUEUE_THLD_CTRL 0x10
31 #define QUEUE_IBI_STATUS_THLD GENMASK(31, 24)
32 #define QUEUE_IBI_DATA_THLD GENMASK(23, 16)
33 #define QUEUE_RESP_BUF_THLD GENMASK(15, 8)
34 #define QUEUE_CMD_EMPTY_BUF_THLD GENMASK(7, 0)
35
36 #define PIO_DATA_BUFFER_THLD_CTRL 0x14
37 #define DATA_RX_START_THLD GENMASK(26, 24)
38 #define DATA_TX_START_THLD GENMASK(18, 16)
39 #define DATA_RX_BUF_THLD GENMASK(10, 8)
40 #define DATA_TX_BUF_THLD GENMASK(2, 0)
41
42 #define PIO_QUEUE_SIZE 0x18
43 #define TX_DATA_BUFFER_SIZE GENMASK(31, 24)
44 #define RX_DATA_BUFFER_SIZE GENMASK(23, 16)
45 #define IBI_STATUS_SIZE GENMASK(15, 8)
46 #define CR_QUEUE_SIZE GENMASK(7, 0)
47
48 #define PIO_INTR_STATUS 0x20
49 #define PIO_INTR_STATUS_ENABLE 0x24
50 #define PIO_INTR_SIGNAL_ENABLE 0x28
51 #define PIO_INTR_FORCE 0x2c
52 #define STAT_TRANSFER_BLOCKED BIT(25)
53 #define STAT_PERR_RESP_UFLOW BIT(24)
54 #define STAT_PERR_CMD_OFLOW BIT(23)
55 #define STAT_PERR_IBI_UFLOW BIT(22)
56 #define STAT_PERR_RX_UFLOW BIT(21)
57 #define STAT_PERR_TX_OFLOW BIT(20)
58 #define STAT_ERR_RESP_QUEUE_FULL BIT(19)
59 #define STAT_WARN_RESP_QUEUE_FULL BIT(18)
60 #define STAT_ERR_IBI_QUEUE_FULL BIT(17)
61 #define STAT_WARN_IBI_QUEUE_FULL BIT(16)
62 #define STAT_ERR_RX_DATA_FULL BIT(15)
63 #define STAT_WARN_RX_DATA_FULL BIT(14)
64 #define STAT_ERR_TX_DATA_EMPTY BIT(13)
65 #define STAT_WARN_TX_DATA_EMPTY BIT(12)
66 #define STAT_TRANSFER_ERR BIT(9)
67 #define STAT_WARN_INS_STOP_MODE BIT(7)
68 #define STAT_TRANSFER_ABORT BIT(5)
69 #define STAT_RESP_READY BIT(4)
70 #define STAT_CMD_QUEUE_READY BIT(3)
71 #define STAT_IBI_STATUS_THLD BIT(2)
72 #define STAT_RX_THLD BIT(1)
73 #define STAT_TX_THLD BIT(0)
74
75 #define PIO_QUEUE_CUR_STATUS 0x38
76 #define CUR_IBI_Q_LEVEL GENMASK(28, 20)
77 #define CUR_RESP_Q_LEVEL GENMASK(18, 10)
78 #define CUR_CMD_Q_EMPTY_LEVEL GENMASK(8, 0)
79
80 #define PIO_DATA_BUFFER_CUR_STATUS 0x3c
81 #define CUR_RX_BUF_LVL GENMASK(26, 16)
82 #define CUR_TX_BUF_LVL GENMASK(10, 0)
83
84 /*
85 * Handy status bit combinations
86 */
87
88 #define STAT_LATENCY_WARNINGS (STAT_WARN_RESP_QUEUE_FULL | \
89 STAT_WARN_IBI_QUEUE_FULL | \
90 STAT_WARN_RX_DATA_FULL | \
91 STAT_WARN_TX_DATA_EMPTY | \
92 STAT_WARN_INS_STOP_MODE)
93
94 #define STAT_LATENCY_ERRORS (STAT_ERR_RESP_QUEUE_FULL | \
95 STAT_ERR_IBI_QUEUE_FULL | \
96 STAT_ERR_RX_DATA_FULL | \
97 STAT_ERR_TX_DATA_EMPTY)
98
99 #define STAT_PROG_ERRORS (STAT_TRANSFER_BLOCKED | \
100 STAT_PERR_RESP_UFLOW | \
101 STAT_PERR_CMD_OFLOW | \
102 STAT_PERR_IBI_UFLOW | \
103 STAT_PERR_RX_UFLOW | \
104 STAT_PERR_TX_OFLOW)
105
106 #define STAT_ALL_ERRORS (STAT_TRANSFER_ABORT | \
107 STAT_TRANSFER_ERR | \
108 STAT_LATENCY_ERRORS | \
109 STAT_PROG_ERRORS)
110
111 struct hci_pio_dev_ibi_data {
112 struct i3c_generic_ibi_pool *pool;
113 unsigned int max_len;
114 };
115
116 struct hci_pio_ibi_data {
117 struct i3c_ibi_slot *slot;
118 void *data_ptr;
119 unsigned int addr;
120 unsigned int seg_len, seg_cnt;
121 unsigned int max_len;
122 bool last_seg;
123 };
124
125 struct hci_pio_data {
126 struct hci_xfer *curr_xfer, *xfer_queue;
127 struct hci_xfer *curr_rx, *rx_queue;
128 struct hci_xfer *curr_tx, *tx_queue;
129 struct hci_xfer *curr_resp, *resp_queue;
130 struct hci_pio_ibi_data ibi;
131 unsigned int rx_thresh_size, tx_thresh_size;
132 unsigned int max_ibi_thresh;
133 u32 reg_queue_thresh;
134 u32 enabled_irqs;
135 };
136
__hci_pio_init(struct i3c_hci * hci,u32 * size_val_ptr)137 static void __hci_pio_init(struct i3c_hci *hci, u32 *size_val_ptr)
138 {
139 u32 val, size_val, rx_thresh, tx_thresh, ibi_val;
140 struct hci_pio_data *pio = hci->io_data;
141
142 size_val = pio_reg_read(QUEUE_SIZE);
143 if (size_val_ptr)
144 *size_val_ptr = size_val;
145
146 /*
147 * Let's initialize data thresholds to half of the actual FIFO size.
148 * The start thresholds aren't used (set to 0) as the FIFO is always
149 * serviced before the corresponding command is queued.
150 */
151 rx_thresh = FIELD_GET(RX_DATA_BUFFER_SIZE, size_val);
152 tx_thresh = FIELD_GET(TX_DATA_BUFFER_SIZE, size_val);
153 if (hci->version_major == 1) {
154 /* those are expressed as 2^[n+1), so just sub 1 if not 0 */
155 if (rx_thresh)
156 rx_thresh -= 1;
157 if (tx_thresh)
158 tx_thresh -= 1;
159 pio->rx_thresh_size = 2 << rx_thresh;
160 pio->tx_thresh_size = 2 << tx_thresh;
161 } else {
162 /* size is 2^(n+1) and threshold is 2^n i.e. already halved */
163 pio->rx_thresh_size = 1 << rx_thresh;
164 pio->tx_thresh_size = 1 << tx_thresh;
165 }
166 val = FIELD_PREP(DATA_RX_BUF_THLD, rx_thresh) |
167 FIELD_PREP(DATA_TX_BUF_THLD, tx_thresh);
168 pio_reg_write(DATA_BUFFER_THLD_CTRL, val);
169
170 /*
171 * Let's raise an interrupt as soon as there is one free cmd slot
172 * or one available response or IBI. For IBI data let's use half the
173 * IBI queue size within allowed bounds.
174 */
175 ibi_val = FIELD_GET(IBI_STATUS_SIZE, size_val);
176 pio->max_ibi_thresh = clamp_val(ibi_val/2, 1, 63);
177 val = FIELD_PREP(QUEUE_IBI_STATUS_THLD, 1) |
178 FIELD_PREP(QUEUE_IBI_DATA_THLD, pio->max_ibi_thresh) |
179 FIELD_PREP(QUEUE_RESP_BUF_THLD, 1) |
180 FIELD_PREP(QUEUE_CMD_EMPTY_BUF_THLD, 1);
181 pio_reg_write(QUEUE_THLD_CTRL, val);
182 pio->reg_queue_thresh = val;
183
184 /* Disable all IRQs but allow all status bits */
185 pio_reg_write(INTR_SIGNAL_ENABLE, 0x0);
186 pio_reg_write(INTR_STATUS_ENABLE, 0xffffffff);
187
188 /* Always accept error interrupts (will be activated on first xfer) */
189 pio->enabled_irqs = STAT_ALL_ERRORS;
190 }
191
hci_pio_suspend(struct i3c_hci * hci)192 static void hci_pio_suspend(struct i3c_hci *hci)
193 {
194 pio_reg_write(INTR_SIGNAL_ENABLE, 0);
195
196 i3c_hci_sync_irq_inactive(hci);
197 }
198
hci_pio_resume(struct i3c_hci * hci)199 static void hci_pio_resume(struct i3c_hci *hci)
200 {
201 __hci_pio_init(hci, NULL);
202 }
203
hci_pio_init(struct i3c_hci * hci)204 static int hci_pio_init(struct i3c_hci *hci)
205 {
206 struct hci_pio_data *pio;
207 u32 size_val;
208
209 pio = devm_kzalloc(hci->master.dev.parent, sizeof(*pio), GFP_KERNEL);
210 if (!pio)
211 return -ENOMEM;
212
213 hci->io_data = pio;
214
215 __hci_pio_init(hci, &size_val);
216
217 dev_dbg(&hci->master.dev, "CMD/RESP FIFO = %ld entries\n",
218 FIELD_GET(CR_QUEUE_SIZE, size_val));
219 dev_dbg(&hci->master.dev, "IBI FIFO = %ld bytes\n",
220 4 * FIELD_GET(IBI_STATUS_SIZE, size_val));
221 dev_dbg(&hci->master.dev, "RX data FIFO = %d bytes\n",
222 4 * (2 << FIELD_GET(RX_DATA_BUFFER_SIZE, size_val)));
223 dev_dbg(&hci->master.dev, "TX data FIFO = %d bytes\n",
224 4 * (2 << FIELD_GET(TX_DATA_BUFFER_SIZE, size_val)));
225
226 return 0;
227 }
228
hci_pio_cleanup(struct i3c_hci * hci)229 static void hci_pio_cleanup(struct i3c_hci *hci)
230 {
231 struct hci_pio_data *pio = hci->io_data;
232
233 pio_reg_write(INTR_SIGNAL_ENABLE, 0x0);
234
235 i3c_hci_sync_irq_inactive(hci);
236
237 if (pio) {
238 dev_dbg(&hci->master.dev, "status = %#x/%#x",
239 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE));
240 BUG_ON(pio->curr_xfer);
241 BUG_ON(pio->curr_rx);
242 BUG_ON(pio->curr_tx);
243 BUG_ON(pio->curr_resp);
244 }
245 }
246
hci_pio_write_cmd(struct i3c_hci * hci,struct hci_xfer * xfer)247 static void hci_pio_write_cmd(struct i3c_hci *hci, struct hci_xfer *xfer)
248 {
249 dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x",
250 0, xfer->cmd_desc[0]);
251 dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x",
252 1, xfer->cmd_desc[1]);
253 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[0]);
254 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[1]);
255 if (hci->cmd == &mipi_i3c_hci_cmd_v2) {
256 dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x",
257 2, xfer->cmd_desc[2]);
258 dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x",
259 3, xfer->cmd_desc[3]);
260 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[2]);
261 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[3]);
262 }
263 }
264
hci_pio_do_rx(struct i3c_hci * hci,struct hci_pio_data * pio)265 static bool hci_pio_do_rx(struct i3c_hci *hci, struct hci_pio_data *pio)
266 {
267 struct hci_xfer *xfer = pio->curr_rx;
268 unsigned int nr_words;
269 u32 *p;
270
271 p = xfer->data;
272 p += (xfer->data_len - xfer->data_left) / 4;
273
274 while (xfer->data_left >= 4) {
275 /* bail out if FIFO hasn't reached the threshold value yet */
276 if (!(pio_reg_read(INTR_STATUS) & STAT_RX_THLD))
277 return false;
278 nr_words = min(xfer->data_left / 4, pio->rx_thresh_size);
279 /* extract data from FIFO */
280 xfer->data_left -= nr_words * 4;
281 dev_dbg(&hci->master.dev, "now %d left %d",
282 nr_words * 4, xfer->data_left);
283 while (nr_words--)
284 *p++ = pio_reg_read(XFER_DATA_PORT);
285 }
286
287 /* trailing data is retrieved upon response reception */
288 return !xfer->data_left;
289 }
290
hci_pio_do_trailing_rx(struct i3c_hci * hci,struct hci_pio_data * pio,unsigned int count)291 static void hci_pio_do_trailing_rx(struct i3c_hci *hci,
292 struct hci_pio_data *pio, unsigned int count)
293 {
294 struct hci_xfer *xfer = pio->curr_rx;
295 u32 *p;
296
297 dev_dbg(&hci->master.dev, "%d remaining", count);
298
299 p = xfer->data;
300 p += (xfer->data_len - xfer->data_left) / 4;
301
302 if (count >= 4) {
303 unsigned int nr_words = count / 4;
304 /* extract data from FIFO */
305 xfer->data_left -= nr_words * 4;
306 dev_dbg(&hci->master.dev, "now %d left %d",
307 nr_words * 4, xfer->data_left);
308 while (nr_words--)
309 *p++ = pio_reg_read(XFER_DATA_PORT);
310 }
311
312 count &= 3;
313 if (count) {
314 /*
315 * There are trailing bytes in the last word.
316 * Fetch it and extract bytes in an endian independent way.
317 * Unlike the TX case, we must not write memory past the
318 * end of the destination buffer.
319 */
320 u8 *p_byte = (u8 *)p;
321 u32 data = pio_reg_read(XFER_DATA_PORT);
322
323 xfer->data_word_before_partial = data;
324 xfer->data_left -= count;
325 data = (__force u32) cpu_to_le32(data);
326 while (count--) {
327 *p_byte++ = data;
328 data >>= 8;
329 }
330 }
331 }
332
hci_pio_do_tx(struct i3c_hci * hci,struct hci_pio_data * pio)333 static bool hci_pio_do_tx(struct i3c_hci *hci, struct hci_pio_data *pio)
334 {
335 struct hci_xfer *xfer = pio->curr_tx;
336 unsigned int nr_words;
337 u32 *p;
338
339 p = xfer->data;
340 p += (xfer->data_len - xfer->data_left) / 4;
341
342 while (xfer->data_left >= 4) {
343 /* bail out if FIFO free space is below set threshold */
344 if (!(pio_reg_read(INTR_STATUS) & STAT_TX_THLD))
345 return false;
346 /* we can fill up to that TX threshold */
347 nr_words = min(xfer->data_left / 4, pio->tx_thresh_size);
348 /* push data into the FIFO */
349 xfer->data_left -= nr_words * 4;
350 dev_dbg(&hci->master.dev, "now %d left %d",
351 nr_words * 4, xfer->data_left);
352 while (nr_words--)
353 pio_reg_write(XFER_DATA_PORT, *p++);
354 }
355
356 if (xfer->data_left) {
357 /*
358 * There are trailing bytes to send. We can simply load
359 * them from memory as a word which will keep those bytes
360 * in their proper place even on a BE system. This will
361 * also get some bytes past the actual buffer but no one
362 * should care as they won't be sent out.
363 */
364 if (!(pio_reg_read(INTR_STATUS) & STAT_TX_THLD))
365 return false;
366 dev_dbg(&hci->master.dev, "trailing %d", xfer->data_left);
367 pio_reg_write(XFER_DATA_PORT, *p);
368 xfer->data_left = 0;
369 }
370
371 return true;
372 }
373
hci_pio_process_rx(struct i3c_hci * hci,struct hci_pio_data * pio)374 static bool hci_pio_process_rx(struct i3c_hci *hci, struct hci_pio_data *pio)
375 {
376 while (pio->curr_rx && hci_pio_do_rx(hci, pio))
377 pio->curr_rx = pio->curr_rx->next_data;
378 return !pio->curr_rx;
379 }
380
hci_pio_process_tx(struct i3c_hci * hci,struct hci_pio_data * pio)381 static bool hci_pio_process_tx(struct i3c_hci *hci, struct hci_pio_data *pio)
382 {
383 while (pio->curr_tx && hci_pio_do_tx(hci, pio))
384 pio->curr_tx = pio->curr_tx->next_data;
385 return !pio->curr_tx;
386 }
387
hci_pio_queue_data(struct i3c_hci * hci,struct hci_pio_data * pio)388 static void hci_pio_queue_data(struct i3c_hci *hci, struct hci_pio_data *pio)
389 {
390 struct hci_xfer *xfer = pio->curr_xfer;
391 struct hci_xfer *prev_queue_tail;
392
393 if (!xfer->data) {
394 xfer->data_len = xfer->data_left = 0;
395 return;
396 }
397
398 if (xfer->rnw) {
399 prev_queue_tail = pio->rx_queue;
400 pio->rx_queue = xfer;
401 if (pio->curr_rx) {
402 prev_queue_tail->next_data = xfer;
403 } else {
404 pio->curr_rx = xfer;
405 if (!hci_pio_process_rx(hci, pio))
406 pio->enabled_irqs |= STAT_RX_THLD;
407 }
408 } else {
409 prev_queue_tail = pio->tx_queue;
410 pio->tx_queue = xfer;
411 if (pio->curr_tx) {
412 prev_queue_tail->next_data = xfer;
413 } else {
414 pio->curr_tx = xfer;
415 if (!hci_pio_process_tx(hci, pio))
416 pio->enabled_irqs |= STAT_TX_THLD;
417 }
418 }
419 }
420
hci_pio_push_to_next_rx(struct i3c_hci * hci,struct hci_xfer * xfer,unsigned int words_to_keep)421 static void hci_pio_push_to_next_rx(struct i3c_hci *hci, struct hci_xfer *xfer,
422 unsigned int words_to_keep)
423 {
424 u32 *from = xfer->data;
425 u32 from_last;
426 unsigned int received, count;
427
428 received = (xfer->data_len - xfer->data_left) / 4;
429 if ((xfer->data_len - xfer->data_left) & 3) {
430 from_last = xfer->data_word_before_partial;
431 received += 1;
432 } else {
433 from_last = from[received];
434 }
435 from += words_to_keep;
436 count = received - words_to_keep;
437
438 while (count) {
439 unsigned int room, left, chunk, bytes_to_move;
440 u32 last_word;
441
442 xfer = xfer->next_data;
443 if (!xfer) {
444 dev_err(&hci->master.dev, "pushing RX data to unexistent xfer\n");
445 return;
446 }
447
448 room = DIV_ROUND_UP(xfer->data_len, 4);
449 left = DIV_ROUND_UP(xfer->data_left, 4);
450 chunk = min(count, room);
451 if (chunk > left) {
452 hci_pio_push_to_next_rx(hci, xfer, chunk - left);
453 left = chunk;
454 xfer->data_left = left * 4;
455 }
456
457 bytes_to_move = xfer->data_len - xfer->data_left;
458 if (bytes_to_move & 3) {
459 /* preserve word to become partial */
460 u32 *p = xfer->data;
461
462 xfer->data_word_before_partial = p[bytes_to_move / 4];
463 }
464 memmove(xfer->data + chunk, xfer->data, bytes_to_move);
465
466 /* treat last word specially because of partial word issues */
467 chunk -= 1;
468
469 memcpy(xfer->data, from, chunk * 4);
470 xfer->data_left -= chunk * 4;
471 from += chunk;
472 count -= chunk;
473
474 last_word = (count == 1) ? from_last : *from++;
475 if (xfer->data_left < 4) {
476 /*
477 * Like in hci_pio_do_trailing_rx(), preserve original
478 * word to be stored partially then store bytes it
479 * in an endian independent way.
480 */
481 u8 *p_byte = xfer->data;
482
483 p_byte += chunk * 4;
484 xfer->data_word_before_partial = last_word;
485 last_word = (__force u32) cpu_to_le32(last_word);
486 while (xfer->data_left--) {
487 *p_byte++ = last_word;
488 last_word >>= 8;
489 }
490 } else {
491 u32 *p = xfer->data;
492
493 p[chunk] = last_word;
494 xfer->data_left -= 4;
495 }
496 count--;
497 }
498 }
499
500 static void hci_pio_err(struct i3c_hci *hci, struct hci_pio_data *pio,
501 u32 status);
502
hci_pio_process_resp(struct i3c_hci * hci,struct hci_pio_data * pio)503 static bool hci_pio_process_resp(struct i3c_hci *hci, struct hci_pio_data *pio)
504 {
505 while (pio->curr_resp &&
506 (pio_reg_read(INTR_STATUS) & STAT_RESP_READY)) {
507 struct hci_xfer *xfer = pio->curr_resp;
508 u32 resp = pio_reg_read(RESPONSE_QUEUE_PORT);
509 unsigned int tid = RESP_TID(resp);
510
511 dev_dbg(&hci->master.dev, "resp = 0x%08x", resp);
512 if (tid != xfer->cmd_tid) {
513 dev_err(&hci->master.dev,
514 "response tid=%d when expecting %d\n",
515 tid, xfer->cmd_tid);
516 /* let's pretend it is a prog error... any of them */
517 hci_pio_err(hci, pio, STAT_PROG_ERRORS);
518 return false;
519 }
520 xfer->response = resp;
521
522 if (pio->curr_rx == xfer) {
523 /*
524 * Response availability implies RX completion.
525 * Retrieve trailing RX data if any.
526 * Note that short reads are possible.
527 */
528 unsigned int received, expected, to_keep;
529
530 received = xfer->data_len - xfer->data_left;
531 expected = RESP_DATA_LENGTH(xfer->response);
532 if (expected > received) {
533 hci_pio_do_trailing_rx(hci, pio,
534 expected - received);
535 } else if (received > expected) {
536 /* we consumed data meant for next xfer */
537 to_keep = DIV_ROUND_UP(expected, 4);
538 hci_pio_push_to_next_rx(hci, xfer, to_keep);
539 }
540
541 /* then process the RX list pointer */
542 if (hci_pio_process_rx(hci, pio))
543 pio->enabled_irqs &= ~STAT_RX_THLD;
544 }
545
546 /*
547 * We're about to give back ownership of the xfer structure
548 * to the waiting instance. Make sure no reference to it
549 * still exists.
550 */
551 if (pio->curr_rx == xfer) {
552 dev_dbg(&hci->master.dev, "short RX ?");
553 pio->curr_rx = pio->curr_rx->next_data;
554 } else if (pio->curr_tx == xfer) {
555 dev_dbg(&hci->master.dev, "short TX ?");
556 pio->curr_tx = pio->curr_tx->next_data;
557 } else if (xfer->data_left) {
558 dev_dbg(&hci->master.dev,
559 "PIO xfer count = %d after response",
560 xfer->data_left);
561 }
562
563 pio->curr_resp = xfer->next_resp;
564 if (xfer->completion)
565 complete(xfer->completion);
566 }
567 return !pio->curr_resp;
568 }
569
hci_pio_queue_resp(struct i3c_hci * hci,struct hci_pio_data * pio)570 static void hci_pio_queue_resp(struct i3c_hci *hci, struct hci_pio_data *pio)
571 {
572 struct hci_xfer *xfer = pio->curr_xfer;
573 struct hci_xfer *prev_queue_tail;
574
575 if (!(xfer->cmd_desc[0] & CMD_0_ROC))
576 return;
577
578 prev_queue_tail = pio->resp_queue;
579 pio->resp_queue = xfer;
580 if (pio->curr_resp) {
581 prev_queue_tail->next_resp = xfer;
582 } else {
583 pio->curr_resp = xfer;
584 if (!hci_pio_process_resp(hci, pio))
585 pio->enabled_irqs |= STAT_RESP_READY;
586 }
587 }
588
hci_pio_process_cmd(struct i3c_hci * hci,struct hci_pio_data * pio)589 static bool hci_pio_process_cmd(struct i3c_hci *hci, struct hci_pio_data *pio)
590 {
591 while (pio->curr_xfer &&
592 (pio_reg_read(INTR_STATUS) & STAT_CMD_QUEUE_READY)) {
593 /*
594 * Always process the data FIFO before sending the command
595 * so needed TX data or RX space is available upfront.
596 */
597 hci_pio_queue_data(hci, pio);
598 /*
599 * Then queue our response request. This will also process
600 * the response FIFO in case it got suddenly filled up
601 * with results from previous commands.
602 */
603 hci_pio_queue_resp(hci, pio);
604 /*
605 * Finally send the command.
606 */
607 hci_pio_write_cmd(hci, pio->curr_xfer);
608 /*
609 * And move on.
610 */
611 pio->curr_xfer = pio->curr_xfer->next_xfer;
612 }
613 return !pio->curr_xfer;
614 }
615
hci_pio_queue_xfer(struct i3c_hci * hci,struct hci_xfer * xfer,int n)616 static int hci_pio_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n)
617 {
618 struct hci_pio_data *pio = hci->io_data;
619 struct hci_xfer *prev_queue_tail;
620 int i;
621
622 dev_dbg(&hci->master.dev, "n = %d", n);
623
624 /* link xfer instances together and initialize data count */
625 for (i = 0; i < n; i++) {
626 xfer[i].next_xfer = (i + 1 < n) ? &xfer[i + 1] : NULL;
627 xfer[i].next_data = NULL;
628 xfer[i].next_resp = NULL;
629 xfer[i].data_left = xfer[i].data_len;
630 }
631
632 spin_lock_irq(&hci->lock);
633 prev_queue_tail = pio->xfer_queue;
634 pio->xfer_queue = &xfer[n - 1];
635 if (pio->curr_xfer) {
636 prev_queue_tail->next_xfer = xfer;
637 } else {
638 pio->curr_xfer = xfer;
639 if (!hci_pio_process_cmd(hci, pio))
640 pio->enabled_irqs |= STAT_CMD_QUEUE_READY;
641 pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs);
642 dev_dbg(&hci->master.dev, "status = %#x/%#x",
643 pio_reg_read(INTR_STATUS),
644 pio_reg_read(INTR_SIGNAL_ENABLE));
645 }
646 spin_unlock_irq(&hci->lock);
647 return 0;
648 }
649
hci_pio_dequeue_xfer_common(struct i3c_hci * hci,struct hci_pio_data * pio,struct hci_xfer * xfer,int n)650 static bool hci_pio_dequeue_xfer_common(struct i3c_hci *hci,
651 struct hci_pio_data *pio,
652 struct hci_xfer *xfer, int n)
653 {
654 struct hci_xfer *p, **p_prev_next;
655 int i;
656
657 /*
658 * To safely dequeue a transfer request, it must be either entirely
659 * processed, or not yet processed at all. If our request tail is
660 * reachable from either the data or resp list that means the command
661 * was submitted and not yet completed.
662 */
663 for (p = pio->curr_resp; p; p = p->next_resp)
664 for (i = 0; i < n; i++)
665 if (p == &xfer[i])
666 goto pio_screwed;
667 for (p = pio->curr_rx; p; p = p->next_data)
668 for (i = 0; i < n; i++)
669 if (p == &xfer[i])
670 goto pio_screwed;
671 for (p = pio->curr_tx; p; p = p->next_data)
672 for (i = 0; i < n; i++)
673 if (p == &xfer[i])
674 goto pio_screwed;
675
676 /*
677 * The command was completed, or wasn't yet submitted.
678 * Unlink it from the que if the later.
679 */
680 p_prev_next = &pio->curr_xfer;
681 for (p = pio->curr_xfer; p; p = p->next_xfer) {
682 if (p == &xfer[0]) {
683 *p_prev_next = xfer[n - 1].next_xfer;
684 break;
685 }
686 p_prev_next = &p->next_xfer;
687 }
688
689 /* return true if we actually unqueued something */
690 return !!p;
691
692 pio_screwed:
693 /*
694 * Life is tough. We must invalidate the hardware state and
695 * discard everything that is still queued.
696 */
697 for (p = pio->curr_resp; p; p = p->next_resp) {
698 p->response = FIELD_PREP(RESP_ERR_FIELD, RESP_ERR_HC_TERMINATED);
699 if (p->completion)
700 complete(p->completion);
701 }
702 for (p = pio->curr_xfer; p; p = p->next_xfer) {
703 p->response = FIELD_PREP(RESP_ERR_FIELD, RESP_ERR_HC_TERMINATED);
704 if (p->completion)
705 complete(p->completion);
706 }
707 pio->curr_xfer = pio->curr_rx = pio->curr_tx = pio->curr_resp = NULL;
708
709 return true;
710 }
711
hci_pio_dequeue_xfer(struct i3c_hci * hci,struct hci_xfer * xfer,int n)712 static bool hci_pio_dequeue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n)
713 {
714 struct hci_pio_data *pio = hci->io_data;
715 int ret;
716
717 spin_lock_irq(&hci->lock);
718 dev_dbg(&hci->master.dev, "n=%d status=%#x/%#x", n,
719 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE));
720 dev_dbg(&hci->master.dev, "main_status = %#x/%#x",
721 readl(hci->base_regs + 0x20), readl(hci->base_regs + 0x28));
722
723 ret = hci_pio_dequeue_xfer_common(hci, pio, xfer, n);
724 spin_unlock_irq(&hci->lock);
725 return ret;
726 }
727
hci_pio_err(struct i3c_hci * hci,struct hci_pio_data * pio,u32 status)728 static void hci_pio_err(struct i3c_hci *hci, struct hci_pio_data *pio,
729 u32 status)
730 {
731 /* TODO: this ought to be more sophisticated eventually */
732
733 if (pio_reg_read(INTR_STATUS) & STAT_RESP_READY) {
734 /* this may happen when an error is signaled with ROC unset */
735 u32 resp = pio_reg_read(RESPONSE_QUEUE_PORT);
736
737 dev_err(&hci->master.dev,
738 "orphan response (%#x) on error\n", resp);
739 }
740
741 /* dump states on programming errors */
742 if (status & STAT_PROG_ERRORS) {
743 u32 queue = pio_reg_read(QUEUE_CUR_STATUS);
744 u32 data = pio_reg_read(DATA_BUFFER_CUR_STATUS);
745
746 dev_err(&hci->master.dev,
747 "prog error %#lx (C/R/I = %ld/%ld/%ld, TX/RX = %ld/%ld)\n",
748 status & STAT_PROG_ERRORS,
749 FIELD_GET(CUR_CMD_Q_EMPTY_LEVEL, queue),
750 FIELD_GET(CUR_RESP_Q_LEVEL, queue),
751 FIELD_GET(CUR_IBI_Q_LEVEL, queue),
752 FIELD_GET(CUR_TX_BUF_LVL, data),
753 FIELD_GET(CUR_RX_BUF_LVL, data));
754 }
755
756 /* just bust out everything with pending responses for now */
757 hci_pio_dequeue_xfer_common(hci, pio, pio->curr_resp, 1);
758 /* ... and half-way TX transfers if any */
759 if (pio->curr_tx && pio->curr_tx->data_left != pio->curr_tx->data_len)
760 hci_pio_dequeue_xfer_common(hci, pio, pio->curr_tx, 1);
761 /* then reset the hardware */
762 mipi_i3c_hci_pio_reset(hci);
763 mipi_i3c_hci_resume(hci);
764
765 dev_dbg(&hci->master.dev, "status=%#x/%#x",
766 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE));
767 }
768
hci_pio_set_ibi_thresh(struct i3c_hci * hci,struct hci_pio_data * pio,unsigned int thresh_val)769 static void hci_pio_set_ibi_thresh(struct i3c_hci *hci,
770 struct hci_pio_data *pio,
771 unsigned int thresh_val)
772 {
773 u32 regval = pio->reg_queue_thresh;
774
775 regval &= ~QUEUE_IBI_STATUS_THLD;
776 regval |= FIELD_PREP(QUEUE_IBI_STATUS_THLD, thresh_val);
777 /* write the threshold reg only if it changes */
778 if (regval != pio->reg_queue_thresh) {
779 pio_reg_write(QUEUE_THLD_CTRL, regval);
780 pio->reg_queue_thresh = regval;
781 dev_dbg(&hci->master.dev, "%d", thresh_val);
782 }
783 }
784
hci_pio_get_ibi_segment(struct i3c_hci * hci,struct hci_pio_data * pio)785 static bool hci_pio_get_ibi_segment(struct i3c_hci *hci,
786 struct hci_pio_data *pio)
787 {
788 struct hci_pio_ibi_data *ibi = &pio->ibi;
789 unsigned int nr_words, thresh_val;
790 u32 *p;
791
792 p = ibi->data_ptr;
793 p += (ibi->seg_len - ibi->seg_cnt) / 4;
794
795 while ((nr_words = ibi->seg_cnt/4)) {
796 /* determine our IBI queue threshold value */
797 thresh_val = min(nr_words, pio->max_ibi_thresh);
798 hci_pio_set_ibi_thresh(hci, pio, thresh_val);
799 /* bail out if we don't have that amount of data ready */
800 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD))
801 return false;
802 /* extract the data from the IBI port */
803 nr_words = thresh_val;
804 ibi->seg_cnt -= nr_words * 4;
805 dev_dbg(&hci->master.dev, "now %d left %d",
806 nr_words * 4, ibi->seg_cnt);
807 while (nr_words--)
808 *p++ = pio_reg_read(IBI_PORT);
809 }
810
811 if (ibi->seg_cnt) {
812 /*
813 * There are trailing bytes in the last word.
814 * Fetch it and extract bytes in an endian independent way.
815 * Unlike the TX case, we must not write past the end of
816 * the destination buffer.
817 */
818 u32 data;
819 u8 *p_byte = (u8 *)p;
820
821 hci_pio_set_ibi_thresh(hci, pio, 1);
822 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD))
823 return false;
824 dev_dbg(&hci->master.dev, "trailing %d", ibi->seg_cnt);
825 data = pio_reg_read(IBI_PORT);
826 data = (__force u32) cpu_to_le32(data);
827 while (ibi->seg_cnt--) {
828 *p_byte++ = data;
829 data >>= 8;
830 }
831 }
832
833 return true;
834 }
835
hci_pio_prep_new_ibi(struct i3c_hci * hci,struct hci_pio_data * pio)836 static bool hci_pio_prep_new_ibi(struct i3c_hci *hci, struct hci_pio_data *pio)
837 {
838 struct hci_pio_ibi_data *ibi = &pio->ibi;
839 struct i3c_dev_desc *dev;
840 struct i3c_hci_dev_data *dev_data;
841 struct hci_pio_dev_ibi_data *dev_ibi;
842 u32 ibi_status;
843
844 /*
845 * We have a new IBI. Try to set up its payload retrieval.
846 * When returning true, the IBI data has to be consumed whether
847 * or not we are set up to capture it. If we return true with
848 * ibi->slot == NULL that means the data payload has to be
849 * drained out of the IBI port and dropped.
850 */
851
852 ibi_status = pio_reg_read(IBI_PORT);
853 dev_dbg(&hci->master.dev, "status = %#x", ibi_status);
854 ibi->addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status);
855 if (ibi_status & IBI_ERROR) {
856 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi->addr);
857 return false;
858 }
859
860 ibi->last_seg = ibi_status & IBI_LAST_STATUS;
861 ibi->seg_len = FIELD_GET(IBI_DATA_LENGTH, ibi_status);
862 ibi->seg_cnt = ibi->seg_len;
863
864 dev = i3c_hci_addr_to_dev(hci, ibi->addr);
865 if (!dev) {
866 dev_err(&hci->master.dev,
867 "IBI for unknown device %#x\n", ibi->addr);
868 return true;
869 }
870
871 dev_data = i3c_dev_get_master_data(dev);
872 dev_ibi = dev_data->ibi_data;
873 ibi->max_len = dev_ibi->max_len;
874
875 if (ibi->seg_len > ibi->max_len) {
876 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n",
877 ibi->seg_len, ibi->max_len);
878 return true;
879 }
880
881 ibi->slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool);
882 if (!ibi->slot) {
883 dev_err(&hci->master.dev, "no free slot for IBI\n");
884 } else {
885 ibi->slot->len = 0;
886 ibi->data_ptr = ibi->slot->data;
887 }
888 return true;
889 }
890
hci_pio_free_ibi_slot(struct i3c_hci * hci,struct hci_pio_data * pio)891 static void hci_pio_free_ibi_slot(struct i3c_hci *hci, struct hci_pio_data *pio)
892 {
893 struct hci_pio_ibi_data *ibi = &pio->ibi;
894 struct hci_pio_dev_ibi_data *dev_ibi;
895
896 if (ibi->slot) {
897 dev_ibi = ibi->slot->dev->common.master_priv;
898 i3c_generic_ibi_recycle_slot(dev_ibi->pool, ibi->slot);
899 ibi->slot = NULL;
900 }
901 }
902
hci_pio_process_ibi(struct i3c_hci * hci,struct hci_pio_data * pio)903 static bool hci_pio_process_ibi(struct i3c_hci *hci, struct hci_pio_data *pio)
904 {
905 struct hci_pio_ibi_data *ibi = &pio->ibi;
906
907 if (!ibi->slot && !ibi->seg_cnt && ibi->last_seg)
908 if (!hci_pio_prep_new_ibi(hci, pio))
909 return false;
910
911 for (;;) {
912 u32 ibi_status;
913 unsigned int ibi_addr;
914
915 if (ibi->slot) {
916 if (!hci_pio_get_ibi_segment(hci, pio))
917 return false;
918 ibi->slot->len += ibi->seg_len;
919 ibi->data_ptr += ibi->seg_len;
920 if (ibi->last_seg) {
921 /* was the last segment: submit it and leave */
922 i3c_master_queue_ibi(ibi->slot->dev, ibi->slot);
923 ibi->slot = NULL;
924 hci_pio_set_ibi_thresh(hci, pio, 1);
925 return true;
926 }
927 } else if (ibi->seg_cnt) {
928 /*
929 * No slot but a non-zero count. This is the result
930 * of some error and the payload must be drained.
931 * This normally does not happen therefore no need
932 * to be extra optimized here.
933 */
934 hci_pio_set_ibi_thresh(hci, pio, 1);
935 do {
936 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD))
937 return false;
938 pio_reg_read(IBI_PORT);
939 } while (--ibi->seg_cnt);
940 if (ibi->last_seg)
941 return true;
942 }
943
944 /* try to move to the next segment right away */
945 hci_pio_set_ibi_thresh(hci, pio, 1);
946 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD))
947 return false;
948 ibi_status = pio_reg_read(IBI_PORT);
949 ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status);
950 if (ibi->addr != ibi_addr) {
951 /* target address changed before last segment */
952 dev_err(&hci->master.dev,
953 "unexp IBI address changed from %d to %d\n",
954 ibi->addr, ibi_addr);
955 hci_pio_free_ibi_slot(hci, pio);
956 }
957 ibi->last_seg = ibi_status & IBI_LAST_STATUS;
958 ibi->seg_len = FIELD_GET(IBI_DATA_LENGTH, ibi_status);
959 ibi->seg_cnt = ibi->seg_len;
960 if (ibi->slot && ibi->slot->len + ibi->seg_len > ibi->max_len) {
961 dev_err(&hci->master.dev,
962 "IBI payload too big (%d > %d)\n",
963 ibi->slot->len + ibi->seg_len, ibi->max_len);
964 hci_pio_free_ibi_slot(hci, pio);
965 }
966 }
967
968 return false;
969 }
970
hci_pio_request_ibi(struct i3c_hci * hci,struct i3c_dev_desc * dev,const struct i3c_ibi_setup * req)971 static int hci_pio_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev,
972 const struct i3c_ibi_setup *req)
973 {
974 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
975 struct i3c_generic_ibi_pool *pool;
976 struct hci_pio_dev_ibi_data *dev_ibi;
977
978 dev_ibi = kmalloc_obj(*dev_ibi);
979 if (!dev_ibi)
980 return -ENOMEM;
981 pool = i3c_generic_ibi_alloc_pool(dev, req);
982 if (IS_ERR(pool)) {
983 kfree(dev_ibi);
984 return PTR_ERR(pool);
985 }
986 dev_ibi->pool = pool;
987 dev_ibi->max_len = req->max_payload_len;
988 dev_data->ibi_data = dev_ibi;
989 return 0;
990 }
991
hci_pio_free_ibi(struct i3c_hci * hci,struct i3c_dev_desc * dev)992 static void hci_pio_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev)
993 {
994 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
995 struct hci_pio_dev_ibi_data *dev_ibi = dev_data->ibi_data;
996
997 dev_data->ibi_data = NULL;
998 i3c_generic_ibi_free_pool(dev_ibi->pool);
999 kfree(dev_ibi);
1000 }
1001
hci_pio_recycle_ibi_slot(struct i3c_hci * hci,struct i3c_dev_desc * dev,struct i3c_ibi_slot * slot)1002 static void hci_pio_recycle_ibi_slot(struct i3c_hci *hci,
1003 struct i3c_dev_desc *dev,
1004 struct i3c_ibi_slot *slot)
1005 {
1006 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
1007 struct hci_pio_dev_ibi_data *dev_ibi = dev_data->ibi_data;
1008
1009 i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot);
1010 }
1011
hci_pio_irq_handler(struct i3c_hci * hci)1012 static bool hci_pio_irq_handler(struct i3c_hci *hci)
1013 {
1014 struct hci_pio_data *pio = hci->io_data;
1015 u32 status;
1016
1017 status = pio_reg_read(INTR_STATUS);
1018 dev_dbg(&hci->master.dev, "PIO_INTR_STATUS %#x/%#x",
1019 status, pio->enabled_irqs);
1020 status &= pio->enabled_irqs | STAT_LATENCY_WARNINGS;
1021 if (!status)
1022 return false;
1023
1024 if (status & STAT_IBI_STATUS_THLD)
1025 hci_pio_process_ibi(hci, pio);
1026
1027 if (status & STAT_RX_THLD)
1028 if (hci_pio_process_rx(hci, pio))
1029 pio->enabled_irqs &= ~STAT_RX_THLD;
1030 if (status & STAT_TX_THLD)
1031 if (hci_pio_process_tx(hci, pio))
1032 pio->enabled_irqs &= ~STAT_TX_THLD;
1033 if (status & STAT_RESP_READY)
1034 if (hci_pio_process_resp(hci, pio))
1035 pio->enabled_irqs &= ~STAT_RESP_READY;
1036
1037 if (unlikely(status & STAT_LATENCY_WARNINGS)) {
1038 pio_reg_write(INTR_STATUS, status & STAT_LATENCY_WARNINGS);
1039 dev_warn_ratelimited(&hci->master.dev,
1040 "encountered warning condition %#lx\n",
1041 status & STAT_LATENCY_WARNINGS);
1042 }
1043
1044 if (unlikely(status & STAT_ALL_ERRORS)) {
1045 pio_reg_write(INTR_STATUS, status & STAT_ALL_ERRORS);
1046 hci_pio_err(hci, pio, status & STAT_ALL_ERRORS);
1047 }
1048
1049 if (status & STAT_CMD_QUEUE_READY)
1050 if (hci_pio_process_cmd(hci, pio))
1051 pio->enabled_irqs &= ~STAT_CMD_QUEUE_READY;
1052
1053 pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs);
1054 dev_dbg(&hci->master.dev, "PIO_INTR_STATUS %#x/%#x",
1055 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE));
1056 return true;
1057 }
1058
1059 const struct hci_io_ops mipi_i3c_hci_pio = {
1060 .init = hci_pio_init,
1061 .cleanup = hci_pio_cleanup,
1062 .queue_xfer = hci_pio_queue_xfer,
1063 .dequeue_xfer = hci_pio_dequeue_xfer,
1064 .irq_handler = hci_pio_irq_handler,
1065 .request_ibi = hci_pio_request_ibi,
1066 .free_ibi = hci_pio_free_ibi,
1067 .recycle_ibi_slot = hci_pio_recycle_ibi_slot,
1068 .suspend = hci_pio_suspend,
1069 .resume = hci_pio_resume,
1070 };
1071