1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
3
4 #include <linux/clk.h>
5 #include <linux/dmapool.h>
6 #include <linux/dma-mapping.h>
7 #include <linux/interconnect.h>
8 #include <linux/interrupt.h>
9 #include <linux/io.h>
10 #include <linux/module.h>
11 #include <linux/of.h>
12 #include <linux/platform_device.h>
13 #include <linux/pinctrl/consumer.h>
14 #include <linux/pm_runtime.h>
15 #include <linux/pm_opp.h>
16 #include <linux/spi/spi.h>
17 #include <linux/spi/spi-mem.h>
18
19
20 #define QSPI_NUM_CS 2
21 #define QSPI_BYTES_PER_WORD 4
22
23 #define MSTR_CONFIG 0x0000
24 #define FULL_CYCLE_MODE BIT(3)
25 #define FB_CLK_EN BIT(4)
26 #define PIN_HOLDN BIT(6)
27 #define PIN_WPN BIT(7)
28 #define DMA_ENABLE BIT(8)
29 #define BIG_ENDIAN_MODE BIT(9)
30 #define SPI_MODE_MSK 0xc00
31 #define SPI_MODE_SHFT 10
32 #define CHIP_SELECT_NUM BIT(12)
33 #define SBL_EN BIT(13)
34 #define LPA_BASE_MSK 0x3c000
35 #define LPA_BASE_SHFT 14
36 #define TX_DATA_DELAY_MSK 0xc0000
37 #define TX_DATA_DELAY_SHFT 18
38 #define TX_CLK_DELAY_MSK 0x300000
39 #define TX_CLK_DELAY_SHFT 20
40 #define TX_CS_N_DELAY_MSK 0xc00000
41 #define TX_CS_N_DELAY_SHFT 22
42 #define TX_DATA_OE_DELAY_MSK 0x3000000
43 #define TX_DATA_OE_DELAY_SHFT 24
44
45 #define AHB_MASTER_CFG 0x0004
46 #define HMEM_TYPE_START_MID_TRANS_MSK 0x7
47 #define HMEM_TYPE_START_MID_TRANS_SHFT 0
48 #define HMEM_TYPE_LAST_TRANS_MSK 0x38
49 #define HMEM_TYPE_LAST_TRANS_SHFT 3
50 #define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_MSK 0xc0
51 #define USE_HMEMTYPE_LAST_ON_DESC_OR_CHAIN_SHFT 6
52 #define HMEMTYPE_READ_TRANS_MSK 0x700
53 #define HMEMTYPE_READ_TRANS_SHFT 8
54 #define HSHARED BIT(11)
55 #define HINNERSHARED BIT(12)
56
57 #define MSTR_INT_EN 0x000C
58 #define MSTR_INT_STATUS 0x0010
59 #define RESP_FIFO_UNDERRUN BIT(0)
60 #define RESP_FIFO_NOT_EMPTY BIT(1)
61 #define RESP_FIFO_RDY BIT(2)
62 #define HRESP_FROM_NOC_ERR BIT(3)
63 #define WR_FIFO_EMPTY BIT(9)
64 #define WR_FIFO_FULL BIT(10)
65 #define WR_FIFO_OVERRUN BIT(11)
66 #define TRANSACTION_DONE BIT(16)
67 #define DMA_CHAIN_DONE BIT(31)
68 #define QSPI_ERR_IRQS (RESP_FIFO_UNDERRUN | HRESP_FROM_NOC_ERR | \
69 WR_FIFO_OVERRUN)
70 #define QSPI_ALL_IRQS (QSPI_ERR_IRQS | RESP_FIFO_RDY | \
71 WR_FIFO_EMPTY | WR_FIFO_FULL | \
72 TRANSACTION_DONE | DMA_CHAIN_DONE)
73
74 #define PIO_XFER_CTRL 0x0014
75 #define REQUEST_COUNT_MSK 0xffff
76
77 #define PIO_XFER_CFG 0x0018
78 #define TRANSFER_DIRECTION BIT(0)
79 #define MULTI_IO_MODE_MSK 0xe
80 #define MULTI_IO_MODE_SHFT 1
81 #define TRANSFER_FRAGMENT BIT(8)
82 #define SDR_1BIT 1
83 #define SDR_2BIT 2
84 #define SDR_4BIT 3
85 #define DDR_1BIT 5
86 #define DDR_2BIT 6
87 #define DDR_4BIT 7
88 #define DMA_DESC_SINGLE_SPI 1
89 #define DMA_DESC_DUAL_SPI 2
90 #define DMA_DESC_QUAD_SPI 3
91
92 #define PIO_XFER_STATUS 0x001c
93 #define WR_FIFO_BYTES_MSK 0xffff0000
94 #define WR_FIFO_BYTES_SHFT 16
95
96 #define PIO_DATAOUT_1B 0x0020
97 #define PIO_DATAOUT_4B 0x0024
98
99 #define RD_FIFO_CFG 0x0028
100 #define CONTINUOUS_MODE BIT(0)
101
102 #define RD_FIFO_STATUS 0x002c
103 #define FIFO_EMPTY BIT(11)
104 #define WR_CNTS_MSK 0x7f0
105 #define WR_CNTS_SHFT 4
106 #define RDY_64BYTE BIT(3)
107 #define RDY_32BYTE BIT(2)
108 #define RDY_16BYTE BIT(1)
109 #define FIFO_RDY BIT(0)
110
111 #define RD_FIFO_RESET 0x0030
112 #define RESET_FIFO BIT(0)
113
114 #define NEXT_DMA_DESC_ADDR 0x0040
115 #define CURRENT_DMA_DESC_ADDR 0x0044
116 #define CURRENT_MEM_ADDR 0x0048
117
118 #define CUR_MEM_ADDR 0x0048
119 #define HW_VERSION 0x004c
120 #define RD_FIFO 0x0050
121 #define SAMPLING_CLK_CFG 0x0090
122 #define SAMPLING_CLK_STATUS 0x0094
123
124 #define QSPI_ALIGN_REQ 32
125
126 enum qspi_dir {
127 QSPI_READ,
128 QSPI_WRITE,
129 };
130
131 struct qspi_cmd_desc {
132 u32 data_address;
133 u32 next_descriptor;
134 u32 direction:1;
135 u32 multi_io_mode:3;
136 u32 reserved1:4;
137 u32 fragment:1;
138 u32 reserved2:7;
139 u32 length:16;
140 };
141
142 struct qspi_xfer {
143 union {
144 const void *tx_buf;
145 void *rx_buf;
146 };
147 unsigned int rem_bytes;
148 unsigned int buswidth;
149 enum qspi_dir dir;
150 bool is_last;
151 };
152
153 enum qspi_clocks {
154 QSPI_CLK_CORE,
155 QSPI_CLK_IFACE,
156 QSPI_NUM_CLKS
157 };
158
159 /*
160 * Number of entries in sgt returned from spi framework that-
161 * will be supported. Can be modified as required.
162 * In practice, given max_dma_len is 64KB, the number of
163 * entries is not expected to exceed 1.
164 */
165 #define QSPI_MAX_SG 5
166
167 struct qcom_qspi {
168 void __iomem *base;
169 struct device *dev;
170 struct clk_bulk_data *clks;
171 struct qspi_xfer xfer;
172 struct dma_pool *dma_cmd_pool;
173 dma_addr_t dma_cmd_desc[QSPI_MAX_SG];
174 void *virt_cmd_desc[QSPI_MAX_SG];
175 unsigned int n_cmd_desc;
176 struct icc_path *icc_path_cpu_to_qspi;
177 struct icc_path *icc_path_mem;
178 unsigned long last_speed;
179 /* Lock to protect data accessed by IRQs */
180 spinlock_t lock;
181 };
182
qspi_buswidth_to_iomode(struct qcom_qspi * ctrl,unsigned int buswidth)183 static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl,
184 unsigned int buswidth)
185 {
186 switch (buswidth) {
187 case 1:
188 return SDR_1BIT;
189 case 2:
190 return SDR_2BIT;
191 case 4:
192 return SDR_4BIT;
193 default:
194 dev_warn_once(ctrl->dev,
195 "Unexpected bus width: %u\n", buswidth);
196 return SDR_1BIT;
197 }
198 }
199
qcom_qspi_pio_xfer_cfg(struct qcom_qspi * ctrl)200 static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl)
201 {
202 u32 pio_xfer_cfg;
203 u32 iomode;
204 const struct qspi_xfer *xfer;
205
206 xfer = &ctrl->xfer;
207 pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG);
208 pio_xfer_cfg &= ~TRANSFER_DIRECTION;
209 pio_xfer_cfg |= xfer->dir;
210 if (xfer->is_last)
211 pio_xfer_cfg &= ~TRANSFER_FRAGMENT;
212 else
213 pio_xfer_cfg |= TRANSFER_FRAGMENT;
214 pio_xfer_cfg &= ~MULTI_IO_MODE_MSK;
215 iomode = qspi_buswidth_to_iomode(ctrl, xfer->buswidth);
216 pio_xfer_cfg |= iomode << MULTI_IO_MODE_SHFT;
217
218 writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG);
219 }
220
qcom_qspi_pio_xfer_ctrl(struct qcom_qspi * ctrl)221 static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl)
222 {
223 u32 pio_xfer_ctrl;
224
225 pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL);
226 pio_xfer_ctrl &= ~REQUEST_COUNT_MSK;
227 pio_xfer_ctrl |= ctrl->xfer.rem_bytes;
228 writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL);
229 }
230
qcom_qspi_pio_xfer(struct qcom_qspi * ctrl)231 static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl)
232 {
233 u32 ints;
234
235 qcom_qspi_pio_xfer_cfg(ctrl);
236
237 /* Ack any previous interrupts that might be hanging around */
238 writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS);
239
240 /* Setup new interrupts */
241 if (ctrl->xfer.dir == QSPI_WRITE)
242 ints = QSPI_ERR_IRQS | WR_FIFO_EMPTY;
243 else
244 ints = QSPI_ERR_IRQS | RESP_FIFO_RDY;
245 writel(ints, ctrl->base + MSTR_INT_EN);
246
247 /* Kick off the transfer */
248 qcom_qspi_pio_xfer_ctrl(ctrl);
249 }
250
qcom_qspi_handle_err(struct spi_controller * host,struct spi_message * msg)251 static void qcom_qspi_handle_err(struct spi_controller *host,
252 struct spi_message *msg)
253 {
254 u32 int_status;
255 struct qcom_qspi *ctrl = spi_controller_get_devdata(host);
256 unsigned long flags;
257 int i;
258
259 spin_lock_irqsave(&ctrl->lock, flags);
260 writel(0, ctrl->base + MSTR_INT_EN);
261 int_status = readl(ctrl->base + MSTR_INT_STATUS);
262 writel(int_status, ctrl->base + MSTR_INT_STATUS);
263 ctrl->xfer.rem_bytes = 0;
264
265 /* free cmd descriptors if they are around (DMA mode) */
266 for (i = 0; i < ctrl->n_cmd_desc; i++)
267 dma_pool_free(ctrl->dma_cmd_pool, ctrl->virt_cmd_desc[i],
268 ctrl->dma_cmd_desc[i]);
269 ctrl->n_cmd_desc = 0;
270 spin_unlock_irqrestore(&ctrl->lock, flags);
271 }
272
qcom_qspi_set_speed(struct qcom_qspi * ctrl,unsigned long speed_hz)273 static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
274 {
275 int ret;
276 unsigned int avg_bw_cpu, avg_bw_mem;
277
278 if (speed_hz == ctrl->last_speed)
279 return 0;
280
281 /* In regular operation (SBL_EN=1) core must be 4x transfer clock */
282 ret = dev_pm_opp_set_rate(ctrl->dev, speed_hz * 4);
283 if (ret) {
284 dev_err(ctrl->dev, "Failed to set core clk %d\n", ret);
285 return ret;
286 }
287
288 /*
289 * Set BW quota for CPU and memory paths.
290 * We don't have explicit peak requirement so keep it equal to avg_bw.
291 */
292 avg_bw_cpu = Bps_to_icc(speed_hz);
293 ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, avg_bw_cpu, avg_bw_cpu);
294 if (ret) {
295 dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
296 __func__, ret);
297 return ret;
298 }
299
300 avg_bw_mem = Bps_to_icc(speed_hz);
301 ret = icc_set_bw(ctrl->icc_path_mem, avg_bw_mem, avg_bw_mem);
302 if (ret) {
303 dev_err(ctrl->dev, "ICC BW voting failed for memory: %d\n", ret);
304 return ret;
305 }
306
307 ctrl->last_speed = speed_hz;
308
309 return 0;
310 }
311
qcom_qspi_alloc_desc(struct qcom_qspi * ctrl,dma_addr_t dma_ptr,uint32_t n_bytes)312 static int qcom_qspi_alloc_desc(struct qcom_qspi *ctrl, dma_addr_t dma_ptr,
313 uint32_t n_bytes)
314 {
315 struct qspi_cmd_desc *virt_cmd_desc, *prev;
316 dma_addr_t dma_cmd_desc;
317
318 /* allocate for dma cmd descriptor */
319 virt_cmd_desc = dma_pool_alloc(ctrl->dma_cmd_pool, GFP_ATOMIC | __GFP_ZERO, &dma_cmd_desc);
320 if (!virt_cmd_desc) {
321 dev_warn_once(ctrl->dev, "Couldn't find memory for descriptor\n");
322 return -EAGAIN;
323 }
324
325 ctrl->virt_cmd_desc[ctrl->n_cmd_desc] = virt_cmd_desc;
326 ctrl->dma_cmd_desc[ctrl->n_cmd_desc] = dma_cmd_desc;
327 ctrl->n_cmd_desc++;
328
329 /* setup cmd descriptor */
330 virt_cmd_desc->data_address = dma_ptr;
331 virt_cmd_desc->direction = ctrl->xfer.dir;
332 virt_cmd_desc->multi_io_mode = qspi_buswidth_to_iomode(ctrl, ctrl->xfer.buswidth);
333 virt_cmd_desc->fragment = !ctrl->xfer.is_last;
334 virt_cmd_desc->length = n_bytes;
335
336 /* update previous descriptor */
337 if (ctrl->n_cmd_desc >= 2) {
338 prev = (ctrl->virt_cmd_desc)[ctrl->n_cmd_desc - 2];
339 prev->next_descriptor = dma_cmd_desc;
340 prev->fragment = 1;
341 }
342
343 return 0;
344 }
345
qcom_qspi_setup_dma_desc(struct qcom_qspi * ctrl,struct spi_transfer * xfer)346 static int qcom_qspi_setup_dma_desc(struct qcom_qspi *ctrl,
347 struct spi_transfer *xfer)
348 {
349 int ret;
350 struct sg_table *sgt;
351 dma_addr_t dma_ptr_sg;
352 unsigned int dma_len_sg;
353 int i;
354
355 if (ctrl->n_cmd_desc) {
356 dev_err(ctrl->dev, "Remnant dma buffers n_cmd_desc-%d\n", ctrl->n_cmd_desc);
357 return -EIO;
358 }
359
360 sgt = (ctrl->xfer.dir == QSPI_READ) ? &xfer->rx_sg : &xfer->tx_sg;
361 if (!sgt->nents || sgt->nents > QSPI_MAX_SG) {
362 dev_warn_once(ctrl->dev, "Cannot handle %d entries in scatter list\n", sgt->nents);
363 return -EAGAIN;
364 }
365
366 for (i = 0; i < sgt->nents; i++) {
367 dma_ptr_sg = sg_dma_address(sgt->sgl + i);
368 dma_len_sg = sg_dma_len(sgt->sgl + i);
369 if (!IS_ALIGNED(dma_ptr_sg, QSPI_ALIGN_REQ)) {
370 dev_warn_once(ctrl->dev, "dma_address not aligned to %d\n", QSPI_ALIGN_REQ);
371 return -EAGAIN;
372 }
373 /*
374 * When reading with DMA the controller writes to memory 1 word
375 * at a time. If the length isn't a multiple of 4 bytes then
376 * the controller can clobber the things later in memory.
377 * Fallback to PIO to be safe.
378 */
379 if (ctrl->xfer.dir == QSPI_READ && (dma_len_sg & 0x03)) {
380 dev_warn_once(ctrl->dev, "fallback to PIO for read of size %#010x\n",
381 dma_len_sg);
382 return -EAGAIN;
383 }
384 }
385
386 for (i = 0; i < sgt->nents; i++) {
387 dma_ptr_sg = sg_dma_address(sgt->sgl + i);
388 dma_len_sg = sg_dma_len(sgt->sgl + i);
389
390 ret = qcom_qspi_alloc_desc(ctrl, dma_ptr_sg, dma_len_sg);
391 if (ret)
392 goto cleanup;
393 }
394 return 0;
395
396 cleanup:
397 for (i = 0; i < ctrl->n_cmd_desc; i++)
398 dma_pool_free(ctrl->dma_cmd_pool, ctrl->virt_cmd_desc[i],
399 ctrl->dma_cmd_desc[i]);
400 ctrl->n_cmd_desc = 0;
401 return ret;
402 }
403
qcom_qspi_dma_xfer(struct qcom_qspi * ctrl)404 static void qcom_qspi_dma_xfer(struct qcom_qspi *ctrl)
405 {
406 /* Setup new interrupts */
407 writel(DMA_CHAIN_DONE, ctrl->base + MSTR_INT_EN);
408
409 /* kick off transfer */
410 writel((u32)((ctrl->dma_cmd_desc)[0]), ctrl->base + NEXT_DMA_DESC_ADDR);
411 }
412
413 /* Switch to DMA if transfer length exceeds this */
414 #define QSPI_MAX_BYTES_FIFO 64
415
qcom_qspi_can_dma(struct spi_controller * ctlr,struct spi_device * slv,struct spi_transfer * xfer)416 static bool qcom_qspi_can_dma(struct spi_controller *ctlr,
417 struct spi_device *slv, struct spi_transfer *xfer)
418 {
419 return xfer->len > QSPI_MAX_BYTES_FIFO;
420 }
421
qcom_qspi_transfer_one(struct spi_controller * host,struct spi_device * slv,struct spi_transfer * xfer)422 static int qcom_qspi_transfer_one(struct spi_controller *host,
423 struct spi_device *slv,
424 struct spi_transfer *xfer)
425 {
426 struct qcom_qspi *ctrl = spi_controller_get_devdata(host);
427 int ret;
428 unsigned long speed_hz;
429 unsigned long flags;
430 u32 mstr_cfg;
431
432 speed_hz = slv->max_speed_hz;
433 if (xfer->speed_hz)
434 speed_hz = xfer->speed_hz;
435
436 ret = qcom_qspi_set_speed(ctrl, speed_hz);
437 if (ret)
438 return ret;
439
440 spin_lock_irqsave(&ctrl->lock, flags);
441 mstr_cfg = readl(ctrl->base + MSTR_CONFIG);
442
443 /* We are half duplex, so either rx or tx will be set */
444 if (xfer->rx_buf) {
445 ctrl->xfer.dir = QSPI_READ;
446 ctrl->xfer.buswidth = xfer->rx_nbits;
447 ctrl->xfer.rx_buf = xfer->rx_buf;
448 } else {
449 ctrl->xfer.dir = QSPI_WRITE;
450 ctrl->xfer.buswidth = xfer->tx_nbits;
451 ctrl->xfer.tx_buf = xfer->tx_buf;
452 }
453 ctrl->xfer.is_last = list_is_last(&xfer->transfer_list,
454 &host->cur_msg->transfers);
455 ctrl->xfer.rem_bytes = xfer->len;
456
457 if (xfer->rx_sg.nents || xfer->tx_sg.nents) {
458 /* do DMA transfer */
459 if (!(mstr_cfg & DMA_ENABLE)) {
460 mstr_cfg |= DMA_ENABLE;
461 writel(mstr_cfg, ctrl->base + MSTR_CONFIG);
462 }
463
464 ret = qcom_qspi_setup_dma_desc(ctrl, xfer);
465 if (ret != -EAGAIN) {
466 if (!ret) {
467 dma_wmb();
468 qcom_qspi_dma_xfer(ctrl);
469 }
470 goto exit;
471 }
472 dev_warn_once(ctrl->dev, "DMA failure, falling back to PIO\n");
473 ret = 0; /* We'll retry w/ PIO */
474 }
475
476 if (mstr_cfg & DMA_ENABLE) {
477 mstr_cfg &= ~DMA_ENABLE;
478 writel(mstr_cfg, ctrl->base + MSTR_CONFIG);
479 }
480 qcom_qspi_pio_xfer(ctrl);
481
482 exit:
483 spin_unlock_irqrestore(&ctrl->lock, flags);
484
485 if (ret)
486 return ret;
487
488 /* We'll call spi_finalize_current_transfer() when done */
489 return 1;
490 }
491
qcom_qspi_prepare_message(struct spi_controller * host,struct spi_message * message)492 static int qcom_qspi_prepare_message(struct spi_controller *host,
493 struct spi_message *message)
494 {
495 u32 mstr_cfg;
496 struct qcom_qspi *ctrl;
497 int tx_data_oe_delay = 1;
498 int tx_data_delay = 1;
499 unsigned long flags;
500
501 ctrl = spi_controller_get_devdata(host);
502 spin_lock_irqsave(&ctrl->lock, flags);
503
504 mstr_cfg = readl(ctrl->base + MSTR_CONFIG);
505 mstr_cfg &= ~CHIP_SELECT_NUM;
506 if (spi_get_chipselect(message->spi, 0))
507 mstr_cfg |= CHIP_SELECT_NUM;
508
509 mstr_cfg |= FB_CLK_EN | PIN_WPN | PIN_HOLDN | SBL_EN | FULL_CYCLE_MODE;
510 mstr_cfg &= ~(SPI_MODE_MSK | TX_DATA_OE_DELAY_MSK | TX_DATA_DELAY_MSK);
511 mstr_cfg |= message->spi->mode << SPI_MODE_SHFT;
512 mstr_cfg |= tx_data_oe_delay << TX_DATA_OE_DELAY_SHFT;
513 mstr_cfg |= tx_data_delay << TX_DATA_DELAY_SHFT;
514 mstr_cfg &= ~DMA_ENABLE;
515
516 writel(mstr_cfg, ctrl->base + MSTR_CONFIG);
517 spin_unlock_irqrestore(&ctrl->lock, flags);
518
519 return 0;
520 }
521
qcom_qspi_alloc_dma(struct qcom_qspi * ctrl)522 static int qcom_qspi_alloc_dma(struct qcom_qspi *ctrl)
523 {
524 ctrl->dma_cmd_pool = dmam_pool_create("qspi cmd desc pool",
525 ctrl->dev, sizeof(struct qspi_cmd_desc), 0, 0);
526 if (!ctrl->dma_cmd_pool)
527 return -ENOMEM;
528
529 return 0;
530 }
531
pio_read(struct qcom_qspi * ctrl)532 static irqreturn_t pio_read(struct qcom_qspi *ctrl)
533 {
534 u32 rd_fifo_status;
535 u32 rd_fifo;
536 unsigned int wr_cnts;
537 unsigned int bytes_to_read;
538 unsigned int words_to_read;
539 u32 *word_buf;
540 u8 *byte_buf;
541 int i;
542
543 rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS);
544
545 if (!(rd_fifo_status & FIFO_RDY)) {
546 dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status);
547 return IRQ_NONE;
548 }
549
550 wr_cnts = (rd_fifo_status & WR_CNTS_MSK) >> WR_CNTS_SHFT;
551 wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes);
552
553 words_to_read = wr_cnts / QSPI_BYTES_PER_WORD;
554 bytes_to_read = wr_cnts % QSPI_BYTES_PER_WORD;
555
556 if (words_to_read) {
557 word_buf = ctrl->xfer.rx_buf;
558 ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD;
559 ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read);
560 ctrl->xfer.rx_buf = word_buf + words_to_read;
561 }
562
563 if (bytes_to_read) {
564 byte_buf = ctrl->xfer.rx_buf;
565 rd_fifo = readl(ctrl->base + RD_FIFO);
566 ctrl->xfer.rem_bytes -= bytes_to_read;
567 for (i = 0; i < bytes_to_read; i++)
568 *byte_buf++ = rd_fifo >> (i * BITS_PER_BYTE);
569 ctrl->xfer.rx_buf = byte_buf;
570 }
571
572 return IRQ_HANDLED;
573 }
574
pio_write(struct qcom_qspi * ctrl)575 static irqreturn_t pio_write(struct qcom_qspi *ctrl)
576 {
577 const void *xfer_buf = ctrl->xfer.tx_buf;
578 const int *word_buf;
579 const char *byte_buf;
580 unsigned int wr_fifo_bytes;
581 unsigned int wr_fifo_words;
582 unsigned int wr_size;
583 unsigned int rem_words;
584
585 wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS);
586 wr_fifo_bytes >>= WR_FIFO_BYTES_SHFT;
587
588 if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) {
589 /* Process the last 1-3 bytes */
590 wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes);
591 ctrl->xfer.rem_bytes -= wr_size;
592
593 byte_buf = xfer_buf;
594 while (wr_size--)
595 writel(*byte_buf++,
596 ctrl->base + PIO_DATAOUT_1B);
597 ctrl->xfer.tx_buf = byte_buf;
598 } else {
599 /*
600 * Process all the whole words; to keep things simple we'll
601 * just wait for the next interrupt to handle the last 1-3
602 * bytes if we don't have an even number of words.
603 */
604 rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD;
605 wr_fifo_words = wr_fifo_bytes / QSPI_BYTES_PER_WORD;
606
607 wr_size = min(rem_words, wr_fifo_words);
608 ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD;
609
610 word_buf = xfer_buf;
611 iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size);
612 ctrl->xfer.tx_buf = word_buf + wr_size;
613
614 }
615
616 return IRQ_HANDLED;
617 }
618
qcom_qspi_irq(int irq,void * dev_id)619 static irqreturn_t qcom_qspi_irq(int irq, void *dev_id)
620 {
621 u32 int_status;
622 struct qcom_qspi *ctrl = dev_id;
623 irqreturn_t ret = IRQ_NONE;
624
625 spin_lock(&ctrl->lock);
626
627 int_status = readl(ctrl->base + MSTR_INT_STATUS);
628 writel(int_status, ctrl->base + MSTR_INT_STATUS);
629
630 /* Ignore disabled interrupts */
631 int_status &= readl(ctrl->base + MSTR_INT_EN);
632
633 /* PIO mode handling */
634 if (ctrl->xfer.dir == QSPI_WRITE) {
635 if (int_status & WR_FIFO_EMPTY)
636 ret = pio_write(ctrl);
637 } else {
638 if (int_status & RESP_FIFO_RDY)
639 ret = pio_read(ctrl);
640 }
641
642 if (int_status & QSPI_ERR_IRQS) {
643 if (int_status & RESP_FIFO_UNDERRUN)
644 dev_err(ctrl->dev, "IRQ error: FIFO underrun\n");
645 if (int_status & WR_FIFO_OVERRUN)
646 dev_err(ctrl->dev, "IRQ error: FIFO overrun\n");
647 if (int_status & HRESP_FROM_NOC_ERR)
648 dev_err(ctrl->dev, "IRQ error: NOC response error\n");
649 ret = IRQ_HANDLED;
650 }
651
652 if (!ctrl->xfer.rem_bytes) {
653 writel(0, ctrl->base + MSTR_INT_EN);
654 spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));
655 }
656
657 /* DMA mode handling */
658 if (int_status & DMA_CHAIN_DONE) {
659 int i;
660
661 writel(0, ctrl->base + MSTR_INT_EN);
662 ctrl->xfer.rem_bytes = 0;
663
664 for (i = 0; i < ctrl->n_cmd_desc; i++)
665 dma_pool_free(ctrl->dma_cmd_pool, ctrl->virt_cmd_desc[i],
666 ctrl->dma_cmd_desc[i]);
667 ctrl->n_cmd_desc = 0;
668
669 ret = IRQ_HANDLED;
670 spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));
671 }
672
673 spin_unlock(&ctrl->lock);
674 return ret;
675 }
676
qcom_qspi_adjust_op_size(struct spi_mem * mem,struct spi_mem_op * op)677 static int qcom_qspi_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
678 {
679 /*
680 * If qcom_qspi_can_dma() is going to return false we don't need to
681 * adjust anything.
682 */
683 if (op->data.nbytes <= QSPI_MAX_BYTES_FIFO)
684 return 0;
685
686 /*
687 * When reading, the transfer needs to be a multiple of 4 bytes so
688 * shrink the transfer if that's not true. The caller will then do a
689 * second transfer to finish things up.
690 */
691 if (op->data.dir == SPI_MEM_DATA_IN && (op->data.nbytes & 0x3))
692 op->data.nbytes &= ~0x3;
693
694 return 0;
695 }
696
697 static const struct spi_controller_mem_ops qcom_qspi_mem_ops = {
698 .adjust_op_size = qcom_qspi_adjust_op_size,
699 };
700
qcom_qspi_probe(struct platform_device * pdev)701 static int qcom_qspi_probe(struct platform_device *pdev)
702 {
703 int ret;
704 struct device *dev;
705 struct spi_controller *host;
706 struct qcom_qspi *ctrl;
707
708 dev = &pdev->dev;
709
710 host = devm_spi_alloc_host(dev, sizeof(*ctrl));
711 if (!host)
712 return -ENOMEM;
713
714 platform_set_drvdata(pdev, host);
715
716 ctrl = spi_controller_get_devdata(host);
717
718 spin_lock_init(&ctrl->lock);
719 ctrl->dev = dev;
720 ctrl->base = devm_platform_ioremap_resource(pdev, 0);
721 if (IS_ERR(ctrl->base))
722 return PTR_ERR(ctrl->base);
723
724 ctrl->clks = devm_kcalloc(dev, QSPI_NUM_CLKS,
725 sizeof(*ctrl->clks), GFP_KERNEL);
726 if (!ctrl->clks)
727 return -ENOMEM;
728
729 ctrl->clks[QSPI_CLK_CORE].id = "core";
730 ctrl->clks[QSPI_CLK_IFACE].id = "iface";
731 ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks);
732 if (ret)
733 return ret;
734
735 ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");
736 if (IS_ERR(ctrl->icc_path_cpu_to_qspi))
737 return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),
738 "Failed to get cpu path\n");
739
740 ctrl->icc_path_mem = devm_of_icc_get(dev, "qspi-memory");
741 if (IS_ERR(ctrl->icc_path_mem)) {
742 if (PTR_ERR(ctrl->icc_path_mem) != -ENODATA)
743 return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_mem),
744 "Failed to get memory path\n");
745 ctrl->icc_path_mem = NULL;
746 }
747
748 /* Set BW vote for register access */
749 ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),
750 Bps_to_icc(1000));
751 if (ret) {
752 dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",
753 __func__, ret);
754 return ret;
755 }
756
757 ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
758 if (ret) {
759 dev_err(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
760 __func__, ret);
761 return ret;
762 }
763
764 ret = platform_get_irq(pdev, 0);
765 if (ret < 0)
766 return ret;
767 ret = devm_request_irq(dev, ret, qcom_qspi_irq, 0, dev_name(dev), ctrl);
768 if (ret) {
769 dev_err(dev, "Failed to request irq %d\n", ret);
770 return ret;
771 }
772
773 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
774 if (ret)
775 return dev_err_probe(dev, ret, "could not set DMA mask\n");
776
777 host->max_speed_hz = 300000000;
778 /* as per HPG, it is 64KB, limit to 60KB to avoid boundary condition failures */
779 host->max_dma_len = 0xf000;
780 host->dma_alignment = QSPI_ALIGN_REQ;
781 host->num_chipselect = QSPI_NUM_CS;
782 host->bus_num = -1;
783 host->mode_bits = SPI_MODE_0 |
784 SPI_TX_DUAL | SPI_RX_DUAL |
785 SPI_TX_QUAD | SPI_RX_QUAD;
786 host->flags = SPI_CONTROLLER_HALF_DUPLEX;
787 host->prepare_message = qcom_qspi_prepare_message;
788 host->transfer_one = qcom_qspi_transfer_one;
789 host->handle_err = qcom_qspi_handle_err;
790 if (of_property_present(pdev->dev.of_node, "iommus"))
791 host->can_dma = qcom_qspi_can_dma;
792 host->auto_runtime_pm = true;
793 host->mem_ops = &qcom_qspi_mem_ops;
794
795 ret = devm_pm_opp_set_clkname(&pdev->dev, "core");
796 if (ret)
797 return ret;
798 /* OPP table is optional */
799 ret = devm_pm_opp_of_add_table(&pdev->dev);
800 if (ret && ret != -ENODEV) {
801 dev_err(&pdev->dev, "invalid OPP table in device tree\n");
802 return ret;
803 }
804
805 ret = qcom_qspi_alloc_dma(ctrl);
806 if (ret)
807 return ret;
808
809 pm_runtime_use_autosuspend(dev);
810 pm_runtime_set_autosuspend_delay(dev, 250);
811 pm_runtime_enable(dev);
812
813 ret = spi_register_controller(host);
814 if (!ret)
815 return 0;
816
817 pm_runtime_disable(dev);
818
819 return ret;
820 }
821
qcom_qspi_remove(struct platform_device * pdev)822 static void qcom_qspi_remove(struct platform_device *pdev)
823 {
824 struct spi_controller *host = platform_get_drvdata(pdev);
825
826 /* Unregister _before_ disabling pm_runtime() so we stop transfers */
827 spi_unregister_controller(host);
828
829 pm_runtime_disable(&pdev->dev);
830 }
831
qcom_qspi_runtime_suspend(struct device * dev)832 static int qcom_qspi_runtime_suspend(struct device *dev)
833 {
834 struct spi_controller *host = dev_get_drvdata(dev);
835 struct qcom_qspi *ctrl = spi_controller_get_devdata(host);
836 int ret;
837
838 clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);
839
840 ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
841 if (ret) {
842 dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
843 __func__, ret);
844 goto err_enable_clk;
845 }
846
847 ret = icc_disable(ctrl->icc_path_mem);
848 if (ret) {
849 dev_err_ratelimited(ctrl->dev, "ICC disable failed for memory: %d\n", ret);
850 goto err_enable_icc_cpu;
851 }
852
853 ret = pinctrl_pm_select_sleep_state(dev);
854 if (ret)
855 goto err_enable_icc_mem;
856
857 /* Drop the performance state vote */
858 ret = dev_pm_opp_set_rate(dev, 0);
859 if (ret)
860 goto err_select_default_state;
861
862 return 0;
863
864 err_select_default_state:
865 pinctrl_pm_select_default_state(dev);
866 err_enable_icc_mem:
867 icc_enable(ctrl->icc_path_mem);
868 err_enable_icc_cpu:
869 icc_enable(ctrl->icc_path_cpu_to_qspi);
870 err_enable_clk:
871 if (clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks))
872 dev_err_ratelimited(ctrl->dev, "Failed to re-enable clocks\n");
873 return ret;
874 }
875
qcom_qspi_runtime_resume(struct device * dev)876 static int qcom_qspi_runtime_resume(struct device *dev)
877 {
878 struct spi_controller *host = dev_get_drvdata(dev);
879 struct qcom_qspi *ctrl = spi_controller_get_devdata(host);
880 int ret;
881
882 ret = dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);
883 if (ret)
884 return ret;
885
886 ret = pinctrl_pm_select_default_state(dev);
887 if (ret)
888 goto err_opp_set_rate_zero;
889
890 ret = icc_enable(ctrl->icc_path_cpu_to_qspi);
891 if (ret) {
892 dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",
893 __func__, ret);
894 goto err_select_sleep_state;
895 }
896
897 ret = icc_enable(ctrl->icc_path_mem);
898 if (ret) {
899 dev_err_ratelimited(ctrl->dev, "ICC enable failed for memory: %d\n", ret);
900 goto err_disable_icc_cpu;
901 }
902
903 ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);
904 if (ret)
905 goto err_disable_icc_mem;
906
907 return 0;
908
909 err_disable_icc_mem:
910 icc_disable(ctrl->icc_path_mem);
911 err_disable_icc_cpu:
912 icc_disable(ctrl->icc_path_cpu_to_qspi);
913 err_select_sleep_state:
914 pinctrl_pm_select_sleep_state(dev);
915 err_opp_set_rate_zero:
916 dev_pm_opp_set_rate(dev, 0);
917 return ret;
918 }
919
qcom_qspi_suspend(struct device * dev)920 static int qcom_qspi_suspend(struct device *dev)
921 {
922 struct spi_controller *host = dev_get_drvdata(dev);
923 int ret;
924
925 ret = spi_controller_suspend(host);
926 if (ret)
927 return ret;
928
929 ret = pm_runtime_force_suspend(dev);
930 if (ret)
931 spi_controller_resume(host);
932
933 return ret;
934 }
935
qcom_qspi_resume(struct device * dev)936 static int qcom_qspi_resume(struct device *dev)
937 {
938 struct spi_controller *host = dev_get_drvdata(dev);
939 int ret;
940
941 ret = pm_runtime_force_resume(dev);
942 if (ret)
943 return ret;
944
945 ret = spi_controller_resume(host);
946 if (ret)
947 pm_runtime_force_suspend(dev);
948
949 return ret;
950 }
951
952 static const struct dev_pm_ops qcom_qspi_dev_pm_ops = {
953 RUNTIME_PM_OPS(qcom_qspi_runtime_suspend,
954 qcom_qspi_runtime_resume, NULL)
955 SYSTEM_SLEEP_PM_OPS(qcom_qspi_suspend, qcom_qspi_resume)
956 };
957
958 static const struct of_device_id qcom_qspi_dt_match[] = {
959 { .compatible = "qcom,qspi-v1", },
960 { }
961 };
962 MODULE_DEVICE_TABLE(of, qcom_qspi_dt_match);
963
964 static struct platform_driver qcom_qspi_driver = {
965 .driver = {
966 .name = "qcom_qspi",
967 .pm = pm_ptr(&qcom_qspi_dev_pm_ops),
968 .of_match_table = qcom_qspi_dt_match,
969 },
970 .probe = qcom_qspi_probe,
971 .remove = qcom_qspi_remove,
972 };
973 module_platform_driver(qcom_qspi_driver);
974
975 MODULE_DESCRIPTION("SPI driver for QSPI cores");
976 MODULE_LICENSE("GPL v2");
977