1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Designware SPI core controller driver (refer pxa2xx_spi.c) 4 * 5 * Copyright (c) 2009, Intel Corporation. 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/bitops.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/interrupt.h> 12 #include <linux/module.h> 13 #include <linux/preempt.h> 14 #include <linux/highmem.h> 15 #include <linux/delay.h> 16 #include <linux/slab.h> 17 #include <linux/spi/spi.h> 18 #include <linux/spi/spi-mem.h> 19 #include <linux/string.h> 20 #include <linux/of.h> 21 22 #include "internals.h" 23 #include "spi-dw.h" 24 25 #ifdef CONFIG_DEBUG_FS 26 #include <linux/debugfs.h> 27 #endif 28 29 /* Slave spi_device related */ 30 struct dw_spi_chip_data { 31 u32 cr0; 32 u32 rx_sample_dly; /* RX sample delay */ 33 }; 34 35 /* Slave spi_transfer/spi_mem_op related */ 36 struct dw_spi_cfg { 37 u8 tmode; 38 u8 dfs; 39 u32 ndf; 40 u32 freq; 41 u8 spi_frf; 42 }; 43 44 struct dw_spi_enh_cfg { 45 u8 wait_c; 46 u8 inst_l; 47 u8 addr_l; 48 u8 trans_t; 49 }; 50 51 #ifdef CONFIG_DEBUG_FS 52 53 #define DW_SPI_DBGFS_REG(_name, _off) \ 54 { \ 55 .name = _name, \ 56 .offset = _off, \ 57 } 58 59 static const struct debugfs_reg32 dw_spi_dbgfs_regs[] = { 60 DW_SPI_DBGFS_REG("CTRLR0", DW_SPI_CTRLR0), 61 DW_SPI_DBGFS_REG("CTRLR1", DW_SPI_CTRLR1), 62 DW_SPI_DBGFS_REG("SSIENR", DW_SPI_SSIENR), 63 DW_SPI_DBGFS_REG("SER", DW_SPI_SER), 64 DW_SPI_DBGFS_REG("BAUDR", DW_SPI_BAUDR), 65 DW_SPI_DBGFS_REG("TXFTLR", DW_SPI_TXFTLR), 66 DW_SPI_DBGFS_REG("RXFTLR", DW_SPI_RXFTLR), 67 DW_SPI_DBGFS_REG("TXFLR", DW_SPI_TXFLR), 68 DW_SPI_DBGFS_REG("RXFLR", DW_SPI_RXFLR), 69 DW_SPI_DBGFS_REG("SR", DW_SPI_SR), 70 DW_SPI_DBGFS_REG("IMR", DW_SPI_IMR), 71 DW_SPI_DBGFS_REG("ISR", DW_SPI_ISR), 72 DW_SPI_DBGFS_REG("DMACR", DW_SPI_DMACR), 73 DW_SPI_DBGFS_REG("DMATDLR", DW_SPI_DMATDLR), 74 DW_SPI_DBGFS_REG("DMARDLR", DW_SPI_DMARDLR), 75 DW_SPI_DBGFS_REG("RX_SAMPLE_DLY", DW_SPI_RX_SAMPLE_DLY), 76 }; 77 78 static void dw_spi_debugfs_init(struct dw_spi *dws) 79 { 80 char name[32]; 81 82 snprintf(name, 32, "dw_spi%d", dws->ctlr->bus_num); 83 dws->debugfs = debugfs_create_dir(name, NULL); 84 85 dws->regset.regs = dw_spi_dbgfs_regs; 86 dws->regset.nregs = ARRAY_SIZE(dw_spi_dbgfs_regs); 87 dws->regset.base = dws->regs; 88 debugfs_create_regset32("registers", 0400, dws->debugfs, &dws->regset); 89 } 90 91 static void dw_spi_debugfs_remove(struct dw_spi *dws) 92 { 93 debugfs_remove_recursive(dws->debugfs); 94 } 95 96 #else 97 static inline void dw_spi_debugfs_init(struct dw_spi *dws) 98 { 99 } 100 101 static inline void dw_spi_debugfs_remove(struct dw_spi *dws) 102 { 103 } 104 #endif /* CONFIG_DEBUG_FS */ 105 106 void dw_spi_set_cs(struct spi_device *spi, bool enable) 107 { 108 struct dw_spi *dws = spi_controller_get_devdata(spi->controller); 109 bool cs_high = !!(spi->mode & SPI_CS_HIGH); 110 111 /* 112 * DW SPI controller demands any native CS being set in order to 113 * proceed with data transfer. So in order to activate the SPI 114 * communications we must set a corresponding bit in the Slave 115 * Enable register no matter whether the SPI core is configured to 116 * support active-high or active-low CS level. 117 */ 118 if (cs_high == enable) 119 dw_writel(dws, DW_SPI_SER, BIT(spi_get_chipselect(spi, 0))); 120 else 121 dw_writel(dws, DW_SPI_SER, 0); 122 } 123 EXPORT_SYMBOL_NS_GPL(dw_spi_set_cs, "SPI_DW_CORE"); 124 125 /* Return the max entries we can fill into tx fifo */ 126 static inline u32 dw_spi_tx_max(struct dw_spi *dws) 127 { 128 u32 tx_room, rxtx_gap; 129 130 tx_room = dws->fifo_len - dw_readl(dws, DW_SPI_TXFLR); 131 132 /* 133 * Another concern is about the tx/rx mismatch, we 134 * though to use (dws->fifo_len - rxflr - txflr) as 135 * one maximum value for tx, but it doesn't cover the 136 * data which is out of tx/rx fifo and inside the 137 * shift registers. So a control from sw point of 138 * view is taken. 139 */ 140 rxtx_gap = dws->fifo_len - (dws->rx_len - dws->tx_len); 141 142 return min3((u32)dws->tx_len, tx_room, rxtx_gap); 143 } 144 145 /* Return the max entries we should read out of rx fifo */ 146 static inline u32 dw_spi_rx_max(struct dw_spi *dws) 147 { 148 return min_t(u32, dws->rx_len, dw_readl(dws, DW_SPI_RXFLR)); 149 } 150 151 static u32 dw_writer(struct dw_spi *dws) 152 { 153 u32 max = dw_spi_tx_max(dws); 154 u32 txw = 0; 155 u32 tx = 0; 156 157 while (max--) { 158 if (dws->tx) { 159 if (dws->n_bytes == 1) 160 txw = *(u8 *)(dws->tx); 161 else if (dws->n_bytes == 2) 162 txw = *(u16 *)(dws->tx); 163 else 164 txw = *(u32 *)(dws->tx); 165 166 dws->tx += dws->n_bytes; 167 } 168 dw_write_io_reg(dws, DW_SPI_DR, txw); 169 --dws->tx_len; 170 ++tx; 171 } 172 return tx; 173 } 174 175 static u32 dw_reader(struct dw_spi *dws) 176 { 177 u32 max = dw_spi_rx_max(dws); 178 u32 rxw; 179 u32 rx = 0; 180 181 while (max--) { 182 rxw = dw_read_io_reg(dws, DW_SPI_DR); 183 if (dws->rx) { 184 if (dws->n_bytes == 1) 185 *(u8 *)(dws->rx) = rxw; 186 else if (dws->n_bytes == 2) 187 *(u16 *)(dws->rx) = rxw; 188 else 189 *(u32 *)(dws->rx) = rxw; 190 191 dws->rx += dws->n_bytes; 192 } 193 --dws->rx_len; 194 ++rx; 195 } 196 return rx; 197 } 198 199 int dw_spi_check_status(struct dw_spi *dws, bool raw) 200 { 201 u32 irq_status; 202 int ret = 0; 203 204 if (raw) 205 irq_status = dw_readl(dws, DW_SPI_RISR); 206 else 207 irq_status = dw_readl(dws, DW_SPI_ISR); 208 209 if (irq_status & DW_SPI_INT_RXOI) { 210 dev_err(&dws->ctlr->dev, "RX FIFO overflow detected\n"); 211 ret = -EIO; 212 } 213 214 if (irq_status & DW_SPI_INT_RXUI) { 215 dev_err(&dws->ctlr->dev, "RX FIFO underflow detected\n"); 216 ret = -EIO; 217 } 218 219 if (irq_status & DW_SPI_INT_TXOI) { 220 dev_err(&dws->ctlr->dev, "TX FIFO overflow detected\n"); 221 ret = -EIO; 222 } 223 224 /* Generically handle the erroneous situation */ 225 if (ret) { 226 dw_spi_reset_chip(dws); 227 if (dws->ctlr->cur_msg) 228 dws->ctlr->cur_msg->status = ret; 229 } 230 231 return ret; 232 } 233 EXPORT_SYMBOL_NS_GPL(dw_spi_check_status, "SPI_DW_CORE"); 234 235 static irqreturn_t dw_spi_irq_thread_fn(int irq, void *dev_id) 236 { 237 struct spi_controller *ctlr = dev_id; 238 struct dw_spi *dws = spi_controller_get_devdata(ctlr); 239 u32 rx, tx, imask, mask = 0; 240 bool finalize = false; 241 242 do { 243 /* 244 * Read data from the Rx FIFO every time we've got a chance executing 245 * this method. If there is nothing left to receive, terminate the 246 * procedure. Otherwise adjust the Rx FIFO Threshold level if it's a 247 * final stage of the transfer. By doing so we'll get the next IRQ 248 * right when the leftover incoming data is received. 249 */ 250 rx = dw_reader(dws); 251 if (!dws->rx_len) { 252 mask |= 0xff; 253 finalize = true; 254 } else if (dws->rx_len <= dw_readl(dws, DW_SPI_RXFTLR)) { 255 dw_writel(dws, DW_SPI_RXFTLR, dws->rx_len - 1); 256 } 257 258 /* 259 * Send data out as much as possible. The Tx FIFO Empty IRQ will be 260 * disabled after the data transmission is finished so not to 261 * have the TXE IRQ flood at the final stage of the transfer. 262 */ 263 tx = dw_writer(dws); 264 if (!dws->tx_len) 265 mask |= DW_SPI_INT_TXEI; 266 cond_resched(); 267 } while (rx != 0 || tx != 0); 268 269 imask = DW_SPI_INT_TXEI | DW_SPI_INT_TXOI | 270 DW_SPI_INT_RXUI | DW_SPI_INT_RXOI | DW_SPI_INT_RXFI; 271 imask &= ~mask; 272 dw_spi_umask_intr(dws, imask); 273 274 if (finalize) 275 spi_finalize_current_transfer(dws->ctlr); 276 277 return IRQ_HANDLED; 278 } 279 280 static irqreturn_t dw_spi_transfer_handler(struct dw_spi *dws) 281 { 282 u16 irq_status = dw_readl(dws, DW_SPI_ISR); 283 bool finalize = false; 284 285 if (dw_spi_check_status(dws, false)) { 286 spi_finalize_current_transfer(dws->ctlr); 287 return IRQ_HANDLED; 288 } 289 290 /* 291 * Read data from the Rx FIFO every time we've got a chance executing 292 * this method. If there is nothing left to receive, terminate the 293 * procedure. Otherwise adjust the Rx FIFO Threshold level if it's a 294 * final stage of the transfer. By doing so we'll get the next IRQ 295 * right when the leftover incoming data is received. 296 */ 297 dw_reader(dws); 298 if (!dws->rx_len) { 299 dw_spi_mask_intr(dws, 0xff); 300 finalize = true; 301 spi_finalize_current_transfer(dws->ctlr); 302 } else if (dws->rx_len <= dw_readl(dws, DW_SPI_RXFTLR)) { 303 dw_writel(dws, DW_SPI_RXFTLR, dws->rx_len - 1); 304 } 305 306 /* 307 * Send data out if Tx FIFO Empty IRQ is received. The IRQ will be 308 * disabled after the data transmission is finished so not to 309 * have the TXE IRQ flood at the final stage of the transfer. 310 */ 311 if (irq_status & DW_SPI_INT_TXEI) { 312 dw_writer(dws); 313 if (!dws->tx_len) 314 dw_spi_mask_intr(dws, DW_SPI_INT_TXEI); 315 } 316 317 if (spi_controller_is_target(dws->ctlr) || finalize || 318 !dws->rx_len || !dws->tx_len || 319 !spi_get_csgpiod(dws->ctlr->cur_msg->spi, 0)) { 320 return IRQ_HANDLED; 321 } 322 323 dw_spi_mask_intr(dws, 0xff); 324 return IRQ_WAKE_THREAD; 325 } 326 327 static irqreturn_t dw_spi_enh_handler(struct dw_spi *dws) 328 { 329 u16 irq_status = dw_readl(dws, DW_SPI_ISR); 330 331 if (irq_status & DW_SPI_INT_RXFI) { 332 dw_reader(dws); 333 if (dws->rx_len && dws->rx_len <= dw_readl(dws, DW_SPI_RXFTLR)) 334 dw_writel(dws, DW_SPI_RXFTLR, dws->rx_len - 1); 335 } 336 337 if (irq_status & DW_SPI_INT_TXEI) 338 dw_writer(dws); 339 340 if (!dws->tx_len && dws->rx_len) { 341 dw_spi_mask_intr(dws, DW_SPI_INT_TXEI); 342 } else if (!dws->rx_len && !dws->tx_len) { 343 dw_spi_mask_intr(dws, 0xff); 344 spi_finalize_current_transfer(dws->ctlr); 345 } 346 347 return IRQ_HANDLED; 348 } 349 350 static irqreturn_t dw_spi_irq(int irq, void *dev_id) 351 { 352 struct spi_controller *ctlr = dev_id; 353 struct dw_spi *dws = spi_controller_get_devdata(ctlr); 354 u16 irq_status = dw_readl(dws, DW_SPI_ISR); 355 356 if (!irq_status) 357 return IRQ_NONE; 358 359 if (!ctlr->cur_msg && dws->transfer_handler != dw_spi_enh_handler) { 360 dw_spi_mask_intr(dws, 0xff); 361 return IRQ_HANDLED; 362 } 363 364 return dws->transfer_handler(dws); 365 } 366 367 static u32 dw_spi_prepare_cr0(struct dw_spi *dws, struct spi_device *spi) 368 { 369 u32 cr0 = 0; 370 371 if (dw_spi_ip_is(dws, PSSI)) { 372 /* CTRLR0[ 5: 4] Frame Format */ 373 cr0 |= FIELD_PREP(DW_PSSI_CTRLR0_FRF_MASK, DW_SPI_CTRLR0_FRF_MOTO_SPI); 374 375 /* 376 * SPI mode (SCPOL|SCPH) 377 * CTRLR0[ 6] Serial Clock Phase 378 * CTRLR0[ 7] Serial Clock Polarity 379 */ 380 if (spi->mode & SPI_CPOL) 381 cr0 |= DW_PSSI_CTRLR0_SCPOL; 382 if (spi->mode & SPI_CPHA) 383 cr0 |= DW_PSSI_CTRLR0_SCPHA; 384 385 /* CTRLR0[11] Shift Register Loop */ 386 if (spi->mode & SPI_LOOP) 387 cr0 |= DW_PSSI_CTRLR0_SRL; 388 } else { 389 /* CTRLR0[ 7: 6] Frame Format */ 390 cr0 |= FIELD_PREP(DW_HSSI_CTRLR0_FRF_MASK, DW_SPI_CTRLR0_FRF_MOTO_SPI); 391 392 /* 393 * SPI mode (SCPOL|SCPH) 394 * CTRLR0[ 8] Serial Clock Phase 395 * CTRLR0[ 9] Serial Clock Polarity 396 */ 397 if (spi->mode & SPI_CPOL) 398 cr0 |= DW_HSSI_CTRLR0_SCPOL; 399 if (spi->mode & SPI_CPHA) 400 cr0 |= DW_HSSI_CTRLR0_SCPHA; 401 402 /* CTRLR0[13] Shift Register Loop */ 403 if (spi->mode & SPI_LOOP) 404 cr0 |= DW_HSSI_CTRLR0_SRL; 405 406 /* CTRLR0[31] MST */ 407 if (dw_spi_ver_is_ge(dws, HSSI, 102A)) 408 cr0 |= DW_HSSI_CTRLR0_MST; 409 } 410 411 return cr0; 412 } 413 414 static void dw_spi_update_config(struct dw_spi *dws, struct spi_device *spi, 415 struct dw_spi_cfg *cfg, struct dw_spi_enh_cfg *enh_cfg) 416 { 417 struct dw_spi_chip_data *chip = spi_get_ctldata(spi); 418 u32 cr0 = chip->cr0; 419 u32 speed_hz; 420 u16 clk_div; 421 422 /* CTRLR0[ 4/3: 0] or CTRLR0[ 20: 16] Data Frame Size */ 423 cr0 |= (cfg->dfs - 1) << dws->dfs_offset; 424 425 if (dw_spi_ip_is(dws, PSSI)) 426 /* CTRLR0[ 9:8] Transfer Mode */ 427 cr0 |= FIELD_PREP(DW_PSSI_CTRLR0_TMOD_MASK, cfg->tmode); 428 else 429 /* CTRLR0[11:10] Transfer Mode */ 430 cr0 |= FIELD_PREP(DW_HSSI_CTRLR0_TMOD_MASK, cfg->tmode); 431 432 if (dw_spi_ver_is_ge(dws, HSSI, 103A)) { 433 cr0 &= ~DW_HSSI_CTRLR0_SPI_FRF_MASK; 434 cr0 |= FIELD_PREP(DW_HSSI_CTRLR0_SPI_FRF_MASK, 435 cfg->spi_frf); 436 } else if (dw_spi_ver_is_ge(dws, PSSI, 400A)) { 437 cr0 &= ~DW_PSSI_CTRLR0_SPI_FRF_MASK; 438 cr0 |= FIELD_PREP(DW_PSSI_CTRLR0_SPI_FRF_MASK, 439 cfg->spi_frf); 440 } 441 442 dw_writel(dws, DW_SPI_CTRLR0, cr0); 443 444 if (spi_controller_is_target(dws->ctlr)) 445 return; 446 447 if (cfg->tmode == DW_SPI_CTRLR0_TMOD_EPROMREAD || 448 cfg->tmode == DW_SPI_CTRLR0_TMOD_RO) 449 dw_writel(dws, DW_SPI_CTRLR1, cfg->ndf ? cfg->ndf - 1 : 0); 450 else if (cfg->tmode == DW_SPI_CTRLR0_TMOD_TO && 451 dws->caps & DW_SPI_CAP_EMODE) 452 dw_writel(dws, DW_SPI_CTRLR1, cfg->ndf); 453 454 /* Note DW APB SSI clock divider doesn't support odd numbers */ 455 clk_div = (DIV_ROUND_UP(dws->max_freq, cfg->freq) + 1) & 0xfffe; 456 speed_hz = dws->max_freq / clk_div; 457 458 if (dws->current_freq != speed_hz) { 459 dw_spi_set_clk(dws, clk_div); 460 dws->current_freq = speed_hz; 461 } 462 463 /* Update RX sample delay if required */ 464 if (dws->cur_rx_sample_dly != chip->rx_sample_dly) { 465 dw_writel(dws, DW_SPI_RX_SAMPLE_DLY, chip->rx_sample_dly); 466 dws->cur_rx_sample_dly = chip->rx_sample_dly; 467 } 468 469 if (enh_cfg) { 470 cr0 = DW_SPI_ENH_CTRLR0_CLK_STRETCH_EN; 471 cr0 |= FIELD_PREP(DW_SPI_ENH_CTRLR0_WAIT_CYCLE_MASK, enh_cfg->wait_c); 472 cr0 |= FIELD_PREP(DW_SPI_ENH_CTRLR0_INST_L_MASK, enh_cfg->inst_l); 473 cr0 |= FIELD_PREP(DW_SPI_ENH_CTRLR0_ADDR_L_MASK, enh_cfg->addr_l); 474 cr0 |= FIELD_PREP(DW_SPI_ENH_CTRLR0_TRANS_TYPE_MASK, enh_cfg->trans_t); 475 dw_writel(dws, DW_SPI_SPI_CTRLR0, cr0); 476 } 477 } 478 479 static void dw_spi_irq_setup(struct dw_spi *dws) 480 { 481 u16 level; 482 u8 imask; 483 484 /* 485 * Originally Tx and Rx data lengths match. Rx FIFO Threshold level 486 * will be adjusted at the final stage of the IRQ-based SPI transfer 487 * execution so not to lose the leftover of the incoming data. 488 */ 489 level = min_t(unsigned int, dws->fifo_len / 2, dws->tx_len); 490 dw_writel(dws, DW_SPI_TXFTLR, level); 491 dw_writel(dws, DW_SPI_RXFTLR, level - 1); 492 493 dws->transfer_handler = dw_spi_transfer_handler; 494 495 imask = DW_SPI_INT_TXEI | DW_SPI_INT_TXOI | 496 DW_SPI_INT_RXUI | DW_SPI_INT_RXOI | DW_SPI_INT_RXFI; 497 dw_spi_umask_intr(dws, imask); 498 } 499 500 static void dw_spi_enh_irq_setup(struct dw_spi *dws) 501 { 502 u16 level; 503 u8 imask; 504 505 /* 506 * Originally Tx and Rx data lengths match. Rx FIFO Threshold level 507 * will be adjusted at the final stage of the IRQ-based SPI transfer 508 * execution so not to lose the leftover of the incoming data. 509 */ 510 level = min_t(unsigned int, dws->fifo_len / 2, dws->tx_len); 511 dw_writel(dws, DW_SPI_TXFTLR, level); 512 513 /* 514 * In enhanced mode if we are reading then tx_len is 0 as we 515 * have nothing to transmit. Calculate DW_SPI_RXFTLR with 516 * rx_len. 517 */ 518 level = min_t(unsigned int, dws->fifo_len / 2, dws->rx_len); 519 dw_writel(dws, DW_SPI_RXFTLR, level ? level - 1 : 0); 520 521 dws->transfer_handler = dw_spi_enh_handler; 522 523 imask = DW_SPI_INT_TXEI | DW_SPI_INT_RXFI; 524 dw_spi_umask_intr(dws, imask); 525 } 526 527 /* 528 * The iterative procedure of the poll-based transfer is simple: write as much 529 * as possible to the Tx FIFO, wait until the pending to receive data is ready 530 * to be read, read it from the Rx FIFO and check whether the performed 531 * procedure has been successful. 532 * 533 * Note this method the same way as the IRQ-based transfer won't work well for 534 * the SPI devices connected to the controller with native CS due to the 535 * automatic CS assertion/de-assertion. 536 */ 537 static int dw_spi_poll_transfer(struct dw_spi *dws, 538 struct spi_transfer *transfer) 539 { 540 struct spi_delay delay; 541 u16 nbits; 542 int ret; 543 544 delay.unit = SPI_DELAY_UNIT_SCK; 545 nbits = dws->n_bytes * BITS_PER_BYTE; 546 547 do { 548 dw_writer(dws); 549 550 delay.value = nbits * (dws->rx_len - dws->tx_len); 551 spi_delay_exec(&delay, transfer); 552 553 dw_reader(dws); 554 555 ret = dw_spi_check_status(dws, true); 556 if (ret) 557 return ret; 558 } while (dws->rx_len); 559 560 return 0; 561 } 562 563 static int dw_spi_transfer_one(struct spi_controller *ctlr, 564 struct spi_device *spi, 565 struct spi_transfer *transfer) 566 { 567 struct dw_spi *dws = spi_controller_get_devdata(ctlr); 568 struct dw_spi_cfg cfg = { 569 .tmode = DW_SPI_CTRLR0_TMOD_TR, 570 .dfs = transfer->bits_per_word, 571 .freq = transfer->speed_hz, 572 .spi_frf = DW_SPI_CTRLR0_SPI_FRF_STD_SPI, 573 }; 574 int ret; 575 576 dws->dma_mapped = 0; 577 dws->n_bytes = spi_bpw_to_bytes(transfer->bits_per_word); 578 dws->tx = (void *)transfer->tx_buf; 579 dws->tx_len = transfer->len / dws->n_bytes; 580 dws->rx = transfer->rx_buf; 581 dws->rx_len = dws->tx_len; 582 583 /* Ensure the data above is visible for all CPUs */ 584 smp_mb(); 585 586 dw_spi_enable_chip(dws, 0); 587 588 dw_spi_update_config(dws, spi, &cfg, NULL); 589 590 transfer->effective_speed_hz = dws->current_freq; 591 592 /* Check if current transfer is a DMA transaction */ 593 dws->dma_mapped = spi_xfer_is_dma_mapped(ctlr, spi, transfer); 594 595 /* For poll mode just disable all interrupts */ 596 dw_spi_mask_intr(dws, 0xff); 597 598 if (dws->dma_mapped) { 599 ret = dws->dma_ops->dma_setup(dws, transfer); 600 if (ret) 601 return ret; 602 } 603 604 dw_spi_enable_chip(dws, 1); 605 606 if (dws->dma_mapped) 607 return dws->dma_ops->dma_transfer(dws, transfer); 608 else if (dws->irq == IRQ_NOTCONNECTED) 609 return dw_spi_poll_transfer(dws, transfer); 610 611 dw_spi_irq_setup(dws); 612 613 return 1; 614 } 615 616 static inline void dw_spi_abort(struct spi_controller *ctlr) 617 { 618 struct dw_spi *dws = spi_controller_get_devdata(ctlr); 619 620 if (dws->dma_mapped) 621 dws->dma_ops->dma_stop(dws); 622 623 disable_irq(dws->irq); 624 dw_spi_reset_chip(dws); 625 enable_irq(dws->irq); 626 } 627 628 static void dw_spi_handle_err(struct spi_controller *ctlr, 629 struct spi_message *msg) 630 { 631 dw_spi_abort(ctlr); 632 } 633 634 static int dw_spi_target_abort(struct spi_controller *ctlr) 635 { 636 dw_spi_abort(ctlr); 637 638 return 0; 639 } 640 641 static int dw_spi_adjust_enh_mem_op_size(struct spi_mem *mem, struct spi_mem_op *op) 642 { 643 if (op->data.dir == SPI_MEM_DATA_IN) 644 op->data.nbytes = clamp_val(op->data.nbytes, 0, DW_SPI_NDF_MASK + 1); 645 else 646 op->data.nbytes = clamp_val(op->data.nbytes, 0, DW_SPI_NDF_MASK); 647 648 return 0; 649 } 650 651 static int dw_spi_adjust_mem_op_size(struct spi_mem *mem, struct spi_mem_op *op) 652 { 653 if (op->data.dir == SPI_MEM_DATA_IN) 654 op->data.nbytes = clamp_val(op->data.nbytes, 0, DW_SPI_NDF_MASK + 1); 655 656 return 0; 657 } 658 659 static bool dw_spi_supports_enh_mem_op(struct spi_mem *mem, 660 const struct spi_mem_op *op) 661 { 662 struct dw_spi *dws = spi_controller_get_devdata(mem->spi->controller); 663 664 if (op->addr.nbytes != 0 && op->addr.buswidth != 1 && 665 op->addr.buswidth != op->data.buswidth) 666 return false; 667 668 if (op->addr.nbytes >= 8) 669 return false; 670 671 if (op->cmd.buswidth != 1 && op->cmd.buswidth != op->addr.buswidth && 672 op->cmd.buswidth != op->data.buswidth) 673 return false; 674 675 if (op->dummy.nbytes && !op->dummy.buswidth) 676 return false; 677 678 if (op->dummy.nbytes != 0 && op->data.dir == SPI_MEM_DATA_OUT) 679 return false; 680 681 /* WAIT_CYCLES is a 5-bit field in SPI_CTRLR0 */ 682 if (op->dummy.nbytes != 0 && 683 op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth > 684 FIELD_MAX(DW_SPI_ENH_CTRLR0_WAIT_CYCLE_MASK)) 685 return false; 686 687 if ((dws->quirk_flags & DW_SPI_QUIRK_JHB100) && op->addr.nbytes && 688 op->addr.nbytes != 3 && op->addr.nbytes != 4) 689 return false; 690 691 return spi_mem_default_supports_op(mem, op); 692 } 693 694 static bool dw_spi_supports_mem_op(struct spi_mem *mem, 695 const struct spi_mem_op *op) 696 { 697 if (op->data.buswidth > 1 || op->addr.buswidth > 1 || 698 op->dummy.buswidth > 1 || op->cmd.buswidth > 1) 699 return false; 700 701 return spi_mem_default_supports_op(mem, op); 702 } 703 704 static int dw_spi_init_mem_buf(struct dw_spi *dws, const struct spi_mem_op *op) 705 { 706 unsigned int i, j, len; 707 u8 *out; 708 709 /* 710 * Calculate the total length of the EEPROM command transfer and 711 * either use the pre-allocated buffer or create a temporary one. 712 */ 713 len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes; 714 if (op->data.dir == SPI_MEM_DATA_OUT) 715 len += op->data.nbytes; 716 717 if (len <= DW_SPI_BUF_SIZE) { 718 out = dws->buf; 719 } else { 720 out = kzalloc(len, GFP_KERNEL); 721 if (!out) 722 return -ENOMEM; 723 } 724 725 /* 726 * Collect the operation code, address and dummy bytes into the single 727 * buffer. If it's a transfer with data to be sent, also copy it into the 728 * single buffer in order to speed the data transmission up. 729 */ 730 for (i = 0; i < op->cmd.nbytes; ++i) 731 out[i] = DW_SPI_GET_BYTE(op->cmd.opcode, op->cmd.nbytes - i - 1); 732 for (j = 0; j < op->addr.nbytes; ++i, ++j) 733 out[i] = DW_SPI_GET_BYTE(op->addr.val, op->addr.nbytes - j - 1); 734 for (j = 0; j < op->dummy.nbytes; ++i, ++j) 735 out[i] = 0x0; 736 737 if (op->data.dir == SPI_MEM_DATA_OUT) 738 memcpy(&out[i], op->data.buf.out, op->data.nbytes); 739 740 dws->n_bytes = 1; 741 dws->tx = out; 742 dws->tx_len = len; 743 if (op->data.dir == SPI_MEM_DATA_IN) { 744 dws->rx = op->data.buf.in; 745 dws->rx_len = op->data.nbytes; 746 } else { 747 dws->rx = NULL; 748 dws->rx_len = 0; 749 } 750 751 return 0; 752 } 753 754 static void dw_spi_free_mem_buf(struct dw_spi *dws) 755 { 756 if (dws->tx != dws->buf) 757 kfree(dws->tx); 758 } 759 760 static int dw_spi_write_then_read(struct dw_spi *dws, struct spi_device *spi) 761 { 762 u32 room, entries, sts; 763 unsigned int len; 764 u8 *buf; 765 766 /* 767 * At initial stage we just pre-fill the Tx FIFO in with no rush, 768 * since native CS hasn't been enabled yet and the automatic data 769 * transmission won't start til we do that. 770 */ 771 len = min(dws->fifo_len, dws->tx_len); 772 buf = dws->tx; 773 while (len--) 774 dw_write_io_reg(dws, DW_SPI_DR, *buf++); 775 776 /* 777 * After setting any bit in the SER register the transmission will 778 * start automatically. We have to keep up with that procedure 779 * otherwise the CS de-assertion will happen whereupon the memory 780 * operation will be pre-terminated. 781 */ 782 len = dws->tx_len - ((void *)buf - dws->tx); 783 dw_spi_set_cs(spi, false); 784 while (len) { 785 entries = readl_relaxed(dws->regs + DW_SPI_TXFLR); 786 if (!entries) { 787 dev_err(&dws->ctlr->dev, "CS de-assertion on Tx\n"); 788 return -EIO; 789 } 790 room = min(dws->fifo_len - entries, len); 791 for (; room; --room, --len) 792 dw_write_io_reg(dws, DW_SPI_DR, *buf++); 793 } 794 795 /* 796 * Data fetching will start automatically if the EEPROM-read mode is 797 * activated. We have to keep up with the incoming data pace to 798 * prevent the Rx FIFO overflow causing the inbound data loss. 799 */ 800 len = dws->rx_len; 801 buf = dws->rx; 802 while (len) { 803 entries = readl_relaxed(dws->regs + DW_SPI_RXFLR); 804 if (!entries) { 805 sts = readl_relaxed(dws->regs + DW_SPI_RISR); 806 if (sts & DW_SPI_INT_RXOI) { 807 dev_err(&dws->ctlr->dev, "FIFO overflow on Rx\n"); 808 return -EIO; 809 } 810 continue; 811 } 812 entries = min(entries, len); 813 for (; entries; --entries, --len) 814 *buf++ = dw_read_io_reg(dws, DW_SPI_DR); 815 } 816 817 return 0; 818 } 819 820 static inline bool dw_spi_ctlr_busy(struct dw_spi *dws) 821 { 822 return dw_readl(dws, DW_SPI_SR) & DW_SPI_SR_BUSY; 823 } 824 825 static int dw_spi_wait_mem_op_done(struct dw_spi *dws) 826 { 827 int retry = DW_SPI_WAIT_RETRIES; 828 struct spi_delay delay; 829 unsigned long ns, us; 830 u32 nents; 831 832 nents = dw_readl(dws, DW_SPI_TXFLR); 833 ns = NSEC_PER_SEC / dws->current_freq * nents; 834 ns *= dws->n_bytes * BITS_PER_BYTE; 835 if (ns <= NSEC_PER_USEC) { 836 delay.unit = SPI_DELAY_UNIT_NSECS; 837 delay.value = ns; 838 } else { 839 us = DIV_ROUND_UP(ns, NSEC_PER_USEC); 840 delay.unit = SPI_DELAY_UNIT_USECS; 841 delay.value = clamp_val(us, 0, USHRT_MAX); 842 } 843 844 while (dw_spi_ctlr_busy(dws) && retry--) 845 spi_delay_exec(&delay, NULL); 846 847 if (retry < 0) { 848 dev_err(&dws->ctlr->dev, "Mem op hanged up\n"); 849 return -EIO; 850 } 851 852 return 0; 853 } 854 855 static void dw_spi_stop_mem_op(struct dw_spi *dws, struct spi_device *spi) 856 { 857 dw_spi_enable_chip(dws, 0); 858 dw_spi_set_cs(spi, true); 859 dw_spi_enable_chip(dws, 1); 860 } 861 862 /* 863 * The SPI memory operation implementation below is the best choice for the 864 * devices, which are selected by the native chip-select lane. It's 865 * specifically developed to workaround the problem with automatic chip-select 866 * lane toggle when there is no data in the Tx FIFO buffer. Luckily the current 867 * SPI-mem core calls exec_op() callback only if the GPIO-based CS is 868 * unavailable. 869 */ 870 static int dw_spi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) 871 { 872 struct dw_spi *dws = spi_controller_get_devdata(mem->spi->controller); 873 struct dw_spi_cfg cfg = {0}; 874 unsigned long flags; 875 int ret; 876 877 /* 878 * Collect the outbound data into a single buffer to speed the 879 * transmission up at least on the initial stage. 880 */ 881 ret = dw_spi_init_mem_buf(dws, op); 882 if (ret) 883 return ret; 884 885 /* 886 * DW SPI EEPROM-read mode is required only for the SPI memory Data-IN 887 * operation. Transmit-only mode is suitable for the rest of them. 888 */ 889 cfg.dfs = 8; 890 cfg.freq = clamp(op->max_freq, 0U, dws->max_mem_freq); 891 if (op->data.dir == SPI_MEM_DATA_IN) { 892 cfg.tmode = DW_SPI_CTRLR0_TMOD_EPROMREAD; 893 cfg.ndf = op->data.nbytes; 894 } else { 895 cfg.tmode = DW_SPI_CTRLR0_TMOD_TO; 896 } 897 898 dw_spi_enable_chip(dws, 0); 899 900 dw_spi_update_config(dws, mem->spi, &cfg, NULL); 901 902 dw_spi_mask_intr(dws, 0xff); 903 904 dw_spi_enable_chip(dws, 1); 905 906 /* 907 * DW APB SSI controller has very nasty peculiarities. First originally 908 * (without any vendor-specific modifications) it doesn't provide a 909 * direct way to set and clear the native chip-select signal. Instead 910 * the controller asserts the CS lane if Tx FIFO isn't empty and a 911 * transmission is going on, and automatically de-asserts it back to 912 * the high level if the Tx FIFO doesn't have anything to be pushed 913 * out. Due to that a multi-tasking or heavy IRQs activity might be 914 * fatal, since the transfer procedure preemption may cause the Tx FIFO 915 * getting empty and sudden CS de-assertion, which in the middle of the 916 * transfer will most likely cause the data loss. Secondly the 917 * EEPROM-read or Read-only DW SPI transfer modes imply the incoming 918 * data being automatically pulled in into the Rx FIFO. So if the 919 * driver software is late in fetching the data from the FIFO before 920 * it's overflown, new incoming data will be lost. In order to make 921 * sure the executed memory operations are CS-atomic and to prevent the 922 * Rx FIFO overflow we have to disable the local interrupts so to block 923 * any preemption during the subsequent IO operations. 924 * 925 * Note. At some circumstances disabling IRQs may not help to prevent 926 * the problems described above. The CS de-assertion and Rx FIFO 927 * overflow may still happen due to the relatively slow system bus or 928 * CPU not working fast enough, so the write-then-read algo implemented 929 * here just won't keep up with the SPI bus data transfer. Such 930 * situation is highly platform specific and is supposed to be fixed by 931 * manually restricting the SPI bus frequency using the 932 * dws->max_mem_freq parameter. 933 */ 934 local_irq_save(flags); 935 preempt_disable(); 936 937 ret = dw_spi_write_then_read(dws, mem->spi); 938 939 local_irq_restore(flags); 940 preempt_enable(); 941 942 /* 943 * Wait for the operation being finished and check the controller 944 * status only if there hasn't been any run-time error detected. In the 945 * former case it's just pointless. In the later one to prevent an 946 * additional error message printing since any hw error flag being set 947 * would be due to an error detected on the data transfer. 948 */ 949 if (!ret) { 950 ret = dw_spi_wait_mem_op_done(dws); 951 if (!ret) 952 ret = dw_spi_check_status(dws, true); 953 } 954 955 dw_spi_stop_mem_op(dws, mem->spi); 956 957 dw_spi_free_mem_buf(dws); 958 959 return ret; 960 } 961 962 static void dw_spi_init_enh_mem_buf(struct dw_spi *dws, const struct spi_mem_op *op) 963 { 964 dws->n_bytes = 1; 965 if (op->data.dir == SPI_MEM_DATA_IN) { 966 dws->rx = op->data.buf.in; 967 dws->rx_len = op->data.nbytes; 968 dws->tx = NULL; 969 dws->tx_len = 0; 970 } else if (op->data.dir == SPI_MEM_DATA_OUT) { 971 dws->tx_len = op->data.nbytes; 972 dws->tx = (void *)op->data.buf.out; 973 dws->rx = NULL; 974 dws->rx_len = 0; 975 } else { 976 dws->rx = NULL; 977 dws->rx_len = 0; 978 dws->tx = NULL; 979 dws->tx_len = 0; 980 } 981 } 982 983 static void dw_spi_enh_write_cmd_addr(struct dw_spi *dws, const struct spi_mem_op *op, 984 struct spi_mem *mem) 985 { 986 if (dws->quirk_flags & DW_SPI_QUIRK_JHB100) { 987 dw_write_io_reg(dws, DW_SPI_JHB100_INST, op->cmd.opcode); 988 if (op->addr.nbytes) 989 dw_write_io_reg(dws, DW_SPI_JHB100_ADDR, op->addr.val); 990 991 dw_spi_set_cs(mem->spi, false); 992 dw_spi_enable_chip(dws, 1); 993 } else { 994 dw_spi_enable_chip(dws, 1); 995 996 /* Send cmd as 32 bit value */ 997 dw_write_io_reg(dws, DW_SPI_DR, op->cmd.opcode); 998 if (op->addr.nbytes) { 999 dw_write_io_reg(dws, DW_SPI_DR, lower_32_bits(op->addr.val)); 1000 if (op->addr.nbytes > 4) { 1001 /* address more than 32bit */ 1002 dw_write_io_reg(dws, DW_SPI_DR, upper_32_bits(op->addr.val)); 1003 } 1004 } 1005 1006 dw_spi_set_cs(mem->spi, false); 1007 } 1008 1009 /* 1010 * FIXME: The exact reason for this delay is not fully understood, 1011 * but empirical testing shows it significantly improves the stability 1012 * of read/write operations. Without this delay, occasional transfer 1013 * errors or timeouts may occur under certain conditions. 1014 * Keeping it as a safeguard based on practical validation. 1015 */ 1016 udelay(5); 1017 } 1018 1019 static int dw_spi_exec_enh_mem_op(struct spi_mem *mem, const struct spi_mem_op *op) 1020 { 1021 struct spi_controller *ctlr = mem->spi->controller; 1022 struct dw_spi *dws = spi_controller_get_devdata(ctlr); 1023 struct dw_spi_enh_cfg enh_cfg = {0}; 1024 struct dw_spi_cfg cfg = {0}; 1025 unsigned long long ms; 1026 int ret; 1027 1028 dws->dma_mapped = false; 1029 1030 switch (op->data.buswidth) { 1031 case 0: 1032 case 1: 1033 cfg.spi_frf = DW_SPI_CTRLR0_SPI_FRF_STD_SPI; 1034 break; 1035 case 2: 1036 cfg.spi_frf = DW_SPI_CTRLR0_SPI_FRF_DUAL_SPI; 1037 break; 1038 case 4: 1039 cfg.spi_frf = DW_SPI_CTRLR0_SPI_FRF_QUAD_SPI; 1040 break; 1041 case 8: 1042 cfg.spi_frf = DW_SPI_CTRLR0_SPI_FRF_OCT_SPI; 1043 break; 1044 default: 1045 return -EINVAL; 1046 } 1047 1048 dw_spi_init_enh_mem_buf(dws, op); 1049 1050 cfg.dfs = 8; 1051 cfg.freq = clamp(op->max_freq, 0U, dws->max_mem_freq); 1052 cfg.ndf = op->data.nbytes; 1053 if (op->data.dir == SPI_MEM_DATA_IN) { 1054 cfg.tmode = DW_SPI_CTRLR0_TMOD_RO; 1055 dws->tx_dir = false; 1056 } else { 1057 cfg.tmode = DW_SPI_CTRLR0_TMOD_TO; 1058 dws->tx_dir = true; 1059 } 1060 1061 if (op->data.buswidth == op->addr.buswidth && 1062 op->data.buswidth == op->cmd.buswidth) 1063 enh_cfg.trans_t = DW_SPI_ENH_CTRLR0_TRANS_TYPE_TT2; 1064 else if (op->data.buswidth == op->addr.buswidth) 1065 enh_cfg.trans_t = DW_SPI_ENH_CTRLR0_TRANS_TYPE_TT1; 1066 else 1067 enh_cfg.trans_t = DW_SPI_ENH_CTRLR0_TRANS_TYPE_TT0; 1068 1069 enh_cfg.addr_l = op->addr.nbytes << 1; 1070 if (op->cmd.nbytes == 2) 1071 enh_cfg.inst_l = DW_SPI_ENH_CTRLR0_INST_L_INST_L16; 1072 else if (op->cmd.nbytes == 1) 1073 enh_cfg.inst_l = DW_SPI_ENH_CTRLR0_INST_L_INST_L8; 1074 else 1075 enh_cfg.inst_l = DW_SPI_ENH_CTRLR0_INST_L_INST_L0; 1076 1077 if (op->dummy.buswidth) 1078 enh_cfg.wait_c = op->dummy.nbytes * BITS_PER_BYTE / op->dummy.buswidth; 1079 1080 dw_spi_enable_chip(dws, 0); 1081 1082 dw_spi_update_config(dws, mem->spi, &cfg, &enh_cfg); 1083 1084 dw_spi_mask_intr(dws, 0xff); 1085 reinit_completion(&ctlr->xfer_completion); 1086 1087 if (op->addr.nbytes && dws->set_addr_nbyte) { 1088 ret = dws->set_addr_nbyte(mem->spi, op->addr.nbytes); 1089 if (ret) { 1090 dw_spi_enable_chip(dws, 1); 1091 return ret; 1092 } 1093 } 1094 1095 if (dws->dma_nr_chans && dws->dma_ops->dma_enh_mem_get_chan && 1096 op->data.nbytes > dws->fifo_len) { 1097 ret = dws->dma_ops->dma_enh_mem_get_chan(ctlr->dev.parent, dws); 1098 if (ret) { 1099 /* fall back to the PIO/IRQ path */ 1100 dev_dbg(&ctlr->dev, "DMA enh mem channel unavailable (%d)\n", ret); 1101 } else if (dws->dma_ops->can_dma_enh_mem(ctlr)) { 1102 ret = dws->dma_ops->dma_enh_mem_setup(dws); 1103 if (ret) 1104 dev_err(&ctlr->dev, "DMA enh mem setup failed (%d)\n", ret); 1105 else 1106 dws->dma_mapped = true; 1107 } 1108 } 1109 1110 if (dws->dma_mapped) { 1111 dw_spi_enh_write_cmd_addr(dws, op, mem); 1112 1113 ret = dws->dma_ops->dma_enh_mem_transfer(mem, op); 1114 1115 dw_spi_stop_mem_op(dws, mem->spi); 1116 } else { 1117 dw_spi_enh_write_cmd_addr(dws, op, mem); 1118 1119 dw_spi_enh_irq_setup(dws); 1120 1121 /* Use timeout calculation from spi_transfer_wait() */ 1122 ms = 8LL * MSEC_PER_SEC * (dws->rx_len ? dws->rx_len : dws->tx_len); 1123 do_div(ms, dws->current_freq); 1124 1125 /* 1126 * Increase it twice and add 200 ms tolerance, use 1127 * predefined maximum in case of overflow. 1128 */ 1129 ms += ms + 200; 1130 if (ms > UINT_MAX) 1131 ms = UINT_MAX; 1132 1133 ms = wait_for_completion_timeout(&ctlr->xfer_completion, 1134 msecs_to_jiffies(ms)); 1135 if (ms == 0) { 1136 dw_spi_mask_intr(dws, 0xff); 1137 synchronize_irq(dws->irq); 1138 dws->rx = NULL; 1139 dws->tx = NULL; 1140 dws->rx_len = 0; 1141 dws->tx_len = 0; 1142 dw_spi_stop_mem_op(dws, mem->spi); 1143 return -EIO; 1144 } 1145 1146 ret = dw_spi_wait_mem_op_done(dws); 1147 1148 dw_spi_stop_mem_op(dws, mem->spi); 1149 } 1150 1151 return ret; 1152 } 1153 1154 static bool dw_spi_can_use_mem_ops(struct dw_spi *dws) 1155 { 1156 return !dws->mem_ops.exec_op && !(dws->caps & DW_SPI_CAP_CS_OVERRIDE) && 1157 !dws->set_cs; 1158 } 1159 1160 /* 1161 * Initialize the default memory operations if a glue layer hasn't specified 1162 * custom ones. Direct mapping operations will be preserved anyway since DW SPI 1163 * controller doesn't have an embedded dirmap interface. Note the memory 1164 * operations implemented in this driver is the best choice only for the DW APB 1165 * SSI controller with standard native CS functionality. If a hardware vendor 1166 * has fixed the automatic CS assertion/de-assertion peculiarity, then it will 1167 * be safer to use the normal SPI-messages-based transfers implementation. 1168 */ 1169 static void dw_spi_init_mem_ops(struct dw_spi *dws) 1170 { 1171 if (dw_spi_can_use_mem_ops(dws)) { 1172 dws->mem_ops.adjust_op_size = dw_spi_adjust_mem_op_size; 1173 if (dws->caps & DW_SPI_CAP_EMODE) { 1174 dws->mem_ops.exec_op = dw_spi_exec_enh_mem_op; 1175 dws->mem_ops.supports_op = dw_spi_supports_enh_mem_op; 1176 dws->mem_ops.adjust_op_size = dw_spi_adjust_enh_mem_op_size; 1177 } else { 1178 dws->mem_ops.exec_op = dw_spi_exec_mem_op; 1179 dws->mem_ops.supports_op = dw_spi_supports_mem_op; 1180 dws->mem_ops.adjust_op_size = dw_spi_adjust_mem_op_size; 1181 } 1182 1183 if (!dws->max_mem_freq) 1184 dws->max_mem_freq = dws->max_freq; 1185 } 1186 } 1187 1188 /* This may be called twice for each spi dev */ 1189 static int dw_spi_setup(struct spi_device *spi) 1190 { 1191 struct dw_spi *dws = spi_controller_get_devdata(spi->controller); 1192 struct dw_spi_chip_data *chip; 1193 1194 /* Only alloc on first setup */ 1195 chip = spi_get_ctldata(spi); 1196 if (!chip) { 1197 u32 rx_sample_dly_ns; 1198 1199 chip = kzalloc_obj(*chip); 1200 if (!chip) 1201 return -ENOMEM; 1202 spi_set_ctldata(spi, chip); 1203 /* 1204 * Use the per-device value the core parsed from the peripheral 1205 * node, and fall back to the controller-wide default when the 1206 * device does not ask for a delay of its own. 1207 */ 1208 rx_sample_dly_ns = spi->rx_sample_delay_ns ?: 1209 dws->def_rx_sample_dly_ns; 1210 chip->rx_sample_dly = DIV_ROUND_CLOSEST(rx_sample_dly_ns, 1211 NSEC_PER_SEC / 1212 dws->max_freq); 1213 } 1214 1215 /* 1216 * Update CR0 data each time the setup callback is invoked since 1217 * the device parameters could have been changed, for instance, by 1218 * the MMC SPI driver or something else. 1219 */ 1220 chip->cr0 = dw_spi_prepare_cr0(dws, spi); 1221 1222 return 0; 1223 } 1224 1225 static void dw_spi_cleanup(struct spi_device *spi) 1226 { 1227 struct dw_spi_chip_data *chip = spi_get_ctldata(spi); 1228 1229 kfree(chip); 1230 spi_set_ctldata(spi, NULL); 1231 } 1232 1233 static u16 detect_enh_mode(struct dw_spi *dws) 1234 { 1235 u32 tmp_spi_ctrlr0, tmp_ctrlr0; 1236 u32 tmp_val, frf_shift; 1237 u16 mode = 0; 1238 1239 if (dw_spi_ver_is_ge(dws, HSSI, 103A)) 1240 frf_shift = __bf_shf(DW_HSSI_CTRLR0_SPI_FRF_MASK); 1241 else if (dw_spi_ver_is_ge(dws, PSSI, 400A)) 1242 frf_shift = __bf_shf(DW_PSSI_CTRLR0_SPI_FRF_MASK); 1243 else 1244 return 0; 1245 1246 tmp_ctrlr0 = dw_readl(dws, DW_SPI_CTRLR0); 1247 tmp_spi_ctrlr0 = dw_readl(dws, DW_SPI_SPI_CTRLR0); 1248 dw_spi_enable_chip(dws, 0); 1249 1250 /* test dual mode */ 1251 tmp_val = DW_SPI_CTRLR0_SPI_FRF_DUAL_SPI << frf_shift; 1252 dw_writel(dws, DW_SPI_CTRLR0, tmp_val); 1253 if ((tmp_val & dw_readl(dws, DW_SPI_CTRLR0)) == tmp_val) 1254 mode |= SPI_TX_DUAL | SPI_RX_DUAL; 1255 1256 /* test quad mode */ 1257 tmp_val = DW_SPI_CTRLR0_SPI_FRF_QUAD_SPI << frf_shift; 1258 dw_writel(dws, DW_SPI_CTRLR0, tmp_val); 1259 if ((tmp_val & dw_readl(dws, DW_SPI_CTRLR0)) == tmp_val) 1260 mode |= SPI_TX_QUAD | SPI_RX_QUAD; 1261 1262 /* test octal mode */ 1263 tmp_val = DW_SPI_CTRLR0_SPI_FRF_OCT_SPI << frf_shift; 1264 dw_writel(dws, DW_SPI_CTRLR0, tmp_val); 1265 if ((tmp_val & dw_readl(dws, DW_SPI_CTRLR0)) == tmp_val) 1266 mode |= SPI_TX_OCTAL | SPI_RX_OCTAL; 1267 1268 if (!mode) 1269 goto disable_enh; 1270 1271 /* test clock stretching */ 1272 dw_writel(dws, DW_SPI_SPI_CTRLR0, DW_SPI_ENH_CTRLR0_CLK_STRETCH_EN); 1273 if ((DW_SPI_ENH_CTRLR0_CLK_STRETCH_EN & dw_readl(dws, DW_SPI_SPI_CTRLR0)) != 1274 DW_SPI_ENH_CTRLR0_CLK_STRETCH_EN) { 1275 /* 1276 * If clock stretching is not enabled then do not use 1277 * enhanced mode. 1278 */ 1279 mode = 0; 1280 goto disable_enh; 1281 } 1282 1283 dws->caps |= DW_SPI_CAP_EMODE; 1284 1285 disable_enh: 1286 dw_writel(dws, DW_SPI_CTRLR0, tmp_ctrlr0); 1287 dw_writel(dws, DW_SPI_SPI_CTRLR0, tmp_spi_ctrlr0); 1288 dw_spi_enable_chip(dws, 1); 1289 1290 return mode; 1291 } 1292 1293 /* Restart the controller, disable all interrupts, clean rx fifo */ 1294 static void dw_spi_hw_init(struct device *dev, struct dw_spi *dws) 1295 { 1296 dw_spi_reset_chip(dws); 1297 1298 /* 1299 * Retrieve the Synopsys component version if it hasn't been specified 1300 * by the platform. CoreKit version ID is encoded as a 3-chars ASCII 1301 * code enclosed with '*' (typical for the most of Synopsys IP-cores). 1302 */ 1303 if (!dws->ver) { 1304 dws->ver = dw_readl(dws, DW_SPI_VERSION); 1305 1306 dev_dbg(dev, "Synopsys DWC%sSSI v%c.%c%c\n", 1307 dw_spi_ip_is(dws, PSSI) ? " APB " : " ", 1308 DW_SPI_GET_BYTE(dws->ver, 3), DW_SPI_GET_BYTE(dws->ver, 2), 1309 DW_SPI_GET_BYTE(dws->ver, 1)); 1310 } 1311 1312 if (spi_controller_is_target(dws->ctlr)) { 1313 /* There is only one CS input signal in target mode */ 1314 dws->num_cs = 1; 1315 } else { 1316 /* 1317 * Try to detect the number of native chip-selects if the platform 1318 * driver didn't set it up. There can be up to 16 lines configured. 1319 */ 1320 if (!dws->num_cs) { 1321 u32 ser; 1322 1323 dw_writel(dws, DW_SPI_SER, 0xffff); 1324 ser = dw_readl(dws, DW_SPI_SER); 1325 dw_writel(dws, DW_SPI_SER, 0); 1326 1327 dws->num_cs = hweight16(ser); 1328 } 1329 } 1330 1331 /* 1332 * Try to detect the FIFO depth if not set by interface driver, 1333 * the depth could be from 2 to 256 from HW spec 1334 */ 1335 if (!dws->fifo_len) { 1336 u32 fifo; 1337 1338 for (fifo = 1; fifo < 256; fifo++) { 1339 dw_writel(dws, DW_SPI_TXFTLR, fifo); 1340 if (fifo != dw_readl(dws, DW_SPI_TXFTLR)) 1341 break; 1342 } 1343 dw_writel(dws, DW_SPI_TXFTLR, 0); 1344 1345 dws->fifo_len = (fifo == 1) ? 0 : fifo; 1346 dev_dbg(dev, "Detected FIFO size: %u bytes\n", dws->fifo_len); 1347 } 1348 1349 /* 1350 * Detect CTRLR0.DFS field size and offset by testing the lowest bits 1351 * writability. Note DWC SSI controller also has the extended DFS, but 1352 * with zero offset. 1353 */ 1354 if (dw_spi_ip_is(dws, PSSI)) { 1355 u32 cr0, tmp = dw_readl(dws, DW_SPI_CTRLR0); 1356 1357 dw_spi_enable_chip(dws, 0); 1358 dw_writel(dws, DW_SPI_CTRLR0, 0xffffffff); 1359 cr0 = dw_readl(dws, DW_SPI_CTRLR0); 1360 dw_writel(dws, DW_SPI_CTRLR0, tmp); 1361 dw_spi_enable_chip(dws, 1); 1362 1363 if (!(cr0 & DW_PSSI_CTRLR0_DFS_MASK)) { 1364 dws->caps |= DW_SPI_CAP_DFS32; 1365 dws->dfs_offset = __bf_shf(DW_PSSI_CTRLR0_DFS32_MASK); 1366 dev_dbg(dev, "Detected 32-bits max data frame size\n"); 1367 } 1368 } else { 1369 dws->caps |= DW_SPI_CAP_DFS32; 1370 } 1371 1372 dws->ctlr->mode_bits |= SPI_CPOL | SPI_CPHA; 1373 1374 if (!spi_controller_is_target(dws->ctlr) && dw_spi_can_use_mem_ops(dws)) 1375 dws->ctlr->mode_bits |= detect_enh_mode(dws); 1376 1377 /* enable HW fixup for explicit CS deselect for Amazon's alpine chip */ 1378 if (dws->caps & DW_SPI_CAP_CS_OVERRIDE) 1379 dw_writel(dws, DW_SPI_CS_OVERRIDE, 0xF); 1380 } 1381 1382 static const struct spi_controller_mem_caps dw_spi_mem_caps = { 1383 .per_op_freq = true, 1384 }; 1385 1386 int dw_spi_add_controller(struct device *dev, struct dw_spi *dws) 1387 { 1388 struct spi_controller *ctlr; 1389 bool target; 1390 int ret; 1391 1392 if (!dws) 1393 return -EINVAL; 1394 1395 target = device_property_read_bool(dev, "spi-slave"); 1396 if (target) 1397 ctlr = spi_alloc_target(dev, 0); 1398 else 1399 ctlr = spi_alloc_host(dev, 0); 1400 1401 if (!ctlr) 1402 return -ENOMEM; 1403 1404 dws->ctlr = ctlr; 1405 dws->dma_addr = (dma_addr_t)(dws->paddr + DW_SPI_DR); 1406 1407 spi_controller_set_devdata(ctlr, dws); 1408 1409 /* Basic HW init */ 1410 dw_spi_hw_init(dev, dws); 1411 1412 ret = request_threaded_irq(dws->irq, dw_spi_irq, dw_spi_irq_thread_fn, 1413 IRQF_SHARED, dev_name(dev), ctlr); 1414 if (ret < 0 && ret != -ENOTCONN) { 1415 dev_err(dev, "can not request IRQ\n"); 1416 goto err_free_ctlr; 1417 } 1418 1419 dw_spi_init_mem_ops(dws); 1420 1421 if (dws->caps & DW_SPI_CAP_DFS32) 1422 ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32); 1423 else 1424 ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16); 1425 ctlr->bus_num = dws->bus_num; 1426 ctlr->num_chipselect = dws->num_cs; 1427 ctlr->setup = dw_spi_setup; 1428 ctlr->cleanup = dw_spi_cleanup; 1429 ctlr->transfer_one = dw_spi_transfer_one; 1430 ctlr->handle_err = dw_spi_handle_err; 1431 ctlr->auto_runtime_pm = true; 1432 1433 if (!target) { 1434 ctlr->use_gpio_descriptors = true; 1435 ctlr->mode_bits |= SPI_LOOP; 1436 if (dws->set_cs) 1437 ctlr->set_cs = dws->set_cs; 1438 else 1439 ctlr->set_cs = dw_spi_set_cs; 1440 if (dws->mem_ops.exec_op) { 1441 ctlr->mem_ops = &dws->mem_ops; 1442 ctlr->mem_caps = &dw_spi_mem_caps; 1443 } 1444 ctlr->max_speed_hz = dws->max_freq; 1445 ctlr->flags = SPI_CONTROLLER_GPIO_SS; 1446 } else { 1447 ctlr->target_abort = dw_spi_target_abort; 1448 } 1449 1450 /* Get default rx sample delay */ 1451 device_property_read_u32(dev, "rx-sample-delay-ns", 1452 &dws->def_rx_sample_dly_ns); 1453 1454 if (dws->dma_ops && dws->dma_ops->dma_init) { 1455 ret = dws->dma_ops->dma_init(dev, dws); 1456 if (ret == -EPROBE_DEFER) { 1457 goto err_free_irq; 1458 } else if (ret) { 1459 dev_warn(dev, "DMA init failed\n"); 1460 } else if (dws->dma_nr_chans == 1) { 1461 /* Full-duplex DMA transfers need both channels */ 1462 dev_info(dev, "Single DMA channel, full-duplex DMA disabled\n"); 1463 } else { 1464 ctlr->can_dma = dws->dma_ops->can_dma; 1465 ctlr->flags |= SPI_CONTROLLER_MUST_TX; 1466 } 1467 } 1468 1469 ret = spi_register_controller(ctlr); 1470 if (ret) { 1471 dev_err_probe(dev, ret, "problem registering spi controller\n"); 1472 goto err_dma_exit; 1473 } 1474 1475 dw_spi_debugfs_init(dws); 1476 return 0; 1477 1478 err_dma_exit: 1479 if (dws->dma_ops && dws->dma_ops->dma_exit) 1480 dws->dma_ops->dma_exit(dws); 1481 dw_spi_enable_chip(dws, 0); 1482 err_free_irq: 1483 free_irq(dws->irq, ctlr); 1484 err_free_ctlr: 1485 spi_controller_put(ctlr); 1486 return ret; 1487 } 1488 EXPORT_SYMBOL_NS_GPL(dw_spi_add_controller, "SPI_DW_CORE"); 1489 1490 void dw_spi_remove_controller(struct dw_spi *dws) 1491 { 1492 dw_spi_debugfs_remove(dws); 1493 1494 spi_unregister_controller(dws->ctlr); 1495 1496 if (dws->dma_ops && dws->dma_ops->dma_exit) 1497 dws->dma_ops->dma_exit(dws); 1498 1499 dw_spi_shutdown_chip(dws); 1500 1501 free_irq(dws->irq, dws->ctlr); 1502 1503 spi_controller_put(dws->ctlr); 1504 } 1505 EXPORT_SYMBOL_NS_GPL(dw_spi_remove_controller, "SPI_DW_CORE"); 1506 1507 int dw_spi_suspend_controller(struct dw_spi *dws) 1508 { 1509 int ret; 1510 1511 ret = spi_controller_suspend(dws->ctlr); 1512 if (ret) 1513 return ret; 1514 1515 dw_spi_shutdown_chip(dws); 1516 return 0; 1517 } 1518 EXPORT_SYMBOL_NS_GPL(dw_spi_suspend_controller, "SPI_DW_CORE"); 1519 1520 int dw_spi_resume_controller(struct dw_spi *dws) 1521 { 1522 dw_spi_hw_init(&dws->ctlr->dev, dws); 1523 return spi_controller_resume(dws->ctlr); 1524 } 1525 EXPORT_SYMBOL_NS_GPL(dw_spi_resume_controller, "SPI_DW_CORE"); 1526 1527 MODULE_AUTHOR("Feng Tang <feng.tang@intel.com>"); 1528 MODULE_DESCRIPTION("Driver for DesignWare SPI controller core"); 1529 MODULE_LICENSE("GPL v2"); 1530