1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * SPI-Engine SPI controller driver 4 * Copyright 2015 Analog Devices Inc. 5 * Copyright 2024 BayLibre, SAS 6 * Author: Lars-Peter Clausen <lars@metafoo.de> 7 */ 8 9 #include <linux/bitfield.h> 10 #include <linux/bitops.h> 11 #include <linux/clk.h> 12 #include <linux/completion.h> 13 #include <linux/dmaengine.h> 14 #include <linux/fpga/adi-axi-common.h> 15 #include <linux/interrupt.h> 16 #include <linux/io.h> 17 #include <linux/of.h> 18 #include <linux/module.h> 19 #include <linux/overflow.h> 20 #include <linux/platform_device.h> 21 #include <linux/spi/offload/provider.h> 22 #include <linux/spi/spi.h> 23 #include <trace/events/spi.h> 24 25 #define SPI_ENGINE_REG_OFFLOAD_MEM_ADDR_WIDTH 0x10 26 #define SPI_ENGINE_REG_RESET 0x40 27 28 #define SPI_ENGINE_REG_INT_ENABLE 0x80 29 #define SPI_ENGINE_REG_INT_PENDING 0x84 30 #define SPI_ENGINE_REG_INT_SOURCE 0x88 31 32 #define SPI_ENGINE_REG_SYNC_ID 0xc0 33 #define SPI_ENGINE_REG_OFFLOAD_SYNC_ID 0xc4 34 35 #define SPI_ENGINE_REG_CMD_FIFO_ROOM 0xd0 36 #define SPI_ENGINE_REG_SDO_FIFO_ROOM 0xd4 37 #define SPI_ENGINE_REG_SDI_FIFO_LEVEL 0xd8 38 39 #define SPI_ENGINE_REG_CMD_FIFO 0xe0 40 #define SPI_ENGINE_REG_SDO_DATA_FIFO 0xe4 41 #define SPI_ENGINE_REG_SDI_DATA_FIFO 0xe8 42 #define SPI_ENGINE_REG_SDI_DATA_FIFO_PEEK 0xec 43 44 #define SPI_ENGINE_MAX_NUM_OFFLOADS 32 45 46 #define SPI_ENGINE_REG_OFFLOAD_CTRL(x) (0x100 + SPI_ENGINE_MAX_NUM_OFFLOADS * (x)) 47 #define SPI_ENGINE_REG_OFFLOAD_STATUS(x) (0x104 + SPI_ENGINE_MAX_NUM_OFFLOADS * (x)) 48 #define SPI_ENGINE_REG_OFFLOAD_RESET(x) (0x108 + SPI_ENGINE_MAX_NUM_OFFLOADS * (x)) 49 #define SPI_ENGINE_REG_OFFLOAD_CMD_FIFO(x) (0x110 + SPI_ENGINE_MAX_NUM_OFFLOADS * (x)) 50 #define SPI_ENGINE_REG_OFFLOAD_SDO_FIFO(x) (0x114 + SPI_ENGINE_MAX_NUM_OFFLOADS * (x)) 51 52 #define SPI_ENGINE_SPI_OFFLOAD_MEM_WIDTH_SDO GENMASK(15, 8) 53 #define SPI_ENGINE_SPI_OFFLOAD_MEM_WIDTH_CMD GENMASK(7, 0) 54 55 #define SPI_ENGINE_INT_CMD_ALMOST_EMPTY BIT(0) 56 #define SPI_ENGINE_INT_SDO_ALMOST_EMPTY BIT(1) 57 #define SPI_ENGINE_INT_SDI_ALMOST_FULL BIT(2) 58 #define SPI_ENGINE_INT_SYNC BIT(3) 59 #define SPI_ENGINE_INT_OFFLOAD_SYNC BIT(4) 60 61 #define SPI_ENGINE_OFFLOAD_CTRL_ENABLE BIT(0) 62 63 #define SPI_ENGINE_CONFIG_CPHA BIT(0) 64 #define SPI_ENGINE_CONFIG_CPOL BIT(1) 65 #define SPI_ENGINE_CONFIG_3WIRE BIT(2) 66 #define SPI_ENGINE_CONFIG_SDO_IDLE_HIGH BIT(3) 67 68 #define SPI_ENGINE_INST_TRANSFER 0x0 69 #define SPI_ENGINE_INST_ASSERT 0x1 70 #define SPI_ENGINE_INST_WRITE 0x2 71 #define SPI_ENGINE_INST_MISC 0x3 72 #define SPI_ENGINE_INST_CS_INV 0x4 73 74 #define SPI_ENGINE_CMD_REG_CLK_DIV 0x0 75 #define SPI_ENGINE_CMD_REG_CONFIG 0x1 76 #define SPI_ENGINE_CMD_REG_XFER_BITS 0x2 77 78 #define SPI_ENGINE_MISC_SYNC 0x0 79 #define SPI_ENGINE_MISC_SLEEP 0x1 80 81 #define SPI_ENGINE_TRANSFER_WRITE 0x1 82 #define SPI_ENGINE_TRANSFER_READ 0x2 83 84 /* Arbitrary sync ID for use by host->cur_msg */ 85 #define AXI_SPI_ENGINE_CUR_MSG_SYNC_ID 0x1 86 87 #define SPI_ENGINE_CMD(inst, arg1, arg2) \ 88 (((inst) << 12) | ((arg1) << 8) | (arg2)) 89 90 #define SPI_ENGINE_CMD_TRANSFER(flags, n) \ 91 SPI_ENGINE_CMD(SPI_ENGINE_INST_TRANSFER, (flags), (n)) 92 #define SPI_ENGINE_CMD_ASSERT(delay, cs) \ 93 SPI_ENGINE_CMD(SPI_ENGINE_INST_ASSERT, (delay), (cs)) 94 #define SPI_ENGINE_CMD_WRITE(reg, val) \ 95 SPI_ENGINE_CMD(SPI_ENGINE_INST_WRITE, (reg), (val)) 96 #define SPI_ENGINE_CMD_SLEEP(delay) \ 97 SPI_ENGINE_CMD(SPI_ENGINE_INST_MISC, SPI_ENGINE_MISC_SLEEP, (delay)) 98 #define SPI_ENGINE_CMD_SYNC(id) \ 99 SPI_ENGINE_CMD(SPI_ENGINE_INST_MISC, SPI_ENGINE_MISC_SYNC, (id)) 100 #define SPI_ENGINE_CMD_CS_INV(flags) \ 101 SPI_ENGINE_CMD(SPI_ENGINE_INST_CS_INV, 0, (flags)) 102 103 /* default sizes - can be changed when SPI Engine firmware is compiled */ 104 #define SPI_ENGINE_OFFLOAD_CMD_FIFO_SIZE 16 105 #define SPI_ENGINE_OFFLOAD_SDO_FIFO_SIZE 16 106 107 struct spi_engine_program { 108 unsigned int length; 109 uint16_t instructions[] __counted_by(length); 110 }; 111 112 /** 113 * struct spi_engine_message_state - SPI engine per-message state 114 */ 115 struct spi_engine_message_state { 116 /** @cmd_length: Number of elements in cmd_buf array. */ 117 unsigned cmd_length; 118 /** @cmd_buf: Array of commands not yet written to CMD FIFO. */ 119 const uint16_t *cmd_buf; 120 /** @tx_xfer: Next xfer with tx_buf not yet fully written to TX FIFO. */ 121 struct spi_transfer *tx_xfer; 122 /** @tx_length: Size of tx_buf in bytes. */ 123 unsigned int tx_length; 124 /** @tx_buf: Bytes not yet written to TX FIFO. */ 125 const uint8_t *tx_buf; 126 /** @rx_xfer: Next xfer with rx_buf not yet fully written to RX FIFO. */ 127 struct spi_transfer *rx_xfer; 128 /** @rx_length: Size of tx_buf in bytes. */ 129 unsigned int rx_length; 130 /** @rx_buf: Bytes not yet written to the RX FIFO. */ 131 uint8_t *rx_buf; 132 }; 133 134 enum { 135 SPI_ENGINE_OFFLOAD_FLAG_ASSIGNED, 136 SPI_ENGINE_OFFLOAD_FLAG_PREPARED, 137 }; 138 139 struct spi_engine_offload { 140 struct spi_engine *spi_engine; 141 unsigned long flags; 142 unsigned int offload_num; 143 }; 144 145 struct spi_engine { 146 struct clk *clk; 147 struct clk *ref_clk; 148 149 spinlock_t lock; 150 151 void __iomem *base; 152 struct spi_engine_message_state msg_state; 153 struct completion msg_complete; 154 unsigned int int_enable; 155 /* shadows hardware CS inversion flag state */ 156 u8 cs_inv; 157 158 unsigned int offload_ctrl_mem_size; 159 unsigned int offload_sdo_mem_size; 160 struct spi_offload *offload; 161 u32 offload_caps; 162 }; 163 164 static void spi_engine_program_add_cmd(struct spi_engine_program *p, 165 bool dry, uint16_t cmd) 166 { 167 p->length++; 168 169 if (!dry) 170 p->instructions[p->length - 1] = cmd; 171 } 172 173 static unsigned int spi_engine_get_config(struct spi_device *spi) 174 { 175 unsigned int config = 0; 176 177 if (spi->mode & SPI_CPOL) 178 config |= SPI_ENGINE_CONFIG_CPOL; 179 if (spi->mode & SPI_CPHA) 180 config |= SPI_ENGINE_CONFIG_CPHA; 181 if (spi->mode & SPI_3WIRE) 182 config |= SPI_ENGINE_CONFIG_3WIRE; 183 if (spi->mode & SPI_MOSI_IDLE_HIGH) 184 config |= SPI_ENGINE_CONFIG_SDO_IDLE_HIGH; 185 if (spi->mode & SPI_MOSI_IDLE_LOW) 186 config &= ~SPI_ENGINE_CONFIG_SDO_IDLE_HIGH; 187 188 return config; 189 } 190 191 static void spi_engine_gen_xfer(struct spi_engine_program *p, bool dry, 192 struct spi_transfer *xfer) 193 { 194 unsigned int len; 195 196 if (xfer->bits_per_word <= 8) 197 len = xfer->len; 198 else if (xfer->bits_per_word <= 16) 199 len = xfer->len / 2; 200 else 201 len = xfer->len / 4; 202 203 while (len) { 204 unsigned int n = min(len, 256U); 205 unsigned int flags = 0; 206 207 if (xfer->tx_buf || (xfer->offload_flags & SPI_OFFLOAD_XFER_TX_STREAM)) 208 flags |= SPI_ENGINE_TRANSFER_WRITE; 209 if (xfer->rx_buf || (xfer->offload_flags & SPI_OFFLOAD_XFER_RX_STREAM)) 210 flags |= SPI_ENGINE_TRANSFER_READ; 211 212 spi_engine_program_add_cmd(p, dry, 213 SPI_ENGINE_CMD_TRANSFER(flags, n - 1)); 214 len -= n; 215 } 216 } 217 218 static void spi_engine_gen_sleep(struct spi_engine_program *p, bool dry, 219 int delay_ns, int inst_ns, u32 sclk_hz) 220 { 221 unsigned int t; 222 223 /* 224 * Negative delay indicates error, e.g. from spi_delay_to_ns(). And if 225 * delay is less that the instruction execution time, there is no need 226 * for an extra sleep instruction since the instruction execution time 227 * will already cover the required delay. 228 */ 229 if (delay_ns < 0 || delay_ns <= inst_ns) 230 return; 231 232 t = DIV_ROUND_UP_ULL((u64)(delay_ns - inst_ns) * sclk_hz, NSEC_PER_SEC); 233 while (t) { 234 unsigned int n = min(t, 256U); 235 236 spi_engine_program_add_cmd(p, dry, SPI_ENGINE_CMD_SLEEP(n - 1)); 237 t -= n; 238 } 239 } 240 241 static void spi_engine_gen_cs(struct spi_engine_program *p, bool dry, 242 struct spi_device *spi, bool assert) 243 { 244 unsigned int mask = 0xff; 245 246 if (assert) 247 mask ^= BIT(spi_get_chipselect(spi, 0)); 248 249 spi_engine_program_add_cmd(p, dry, SPI_ENGINE_CMD_ASSERT(0, mask)); 250 } 251 252 /* 253 * Performs precompile steps on the message. 254 * 255 * The SPI core does most of the message/transfer validation and filling in 256 * fields for us via __spi_validate(). This fixes up anything remaining not 257 * done there. 258 * 259 * NB: This is separate from spi_engine_compile_message() because the latter 260 * is called twice and would otherwise result in double-evaluation. 261 * 262 * Returns 0 on success, -EINVAL on failure. 263 */ 264 static int spi_engine_precompile_message(struct spi_message *msg) 265 { 266 unsigned int clk_div, max_hz = msg->spi->controller->max_speed_hz; 267 struct spi_transfer *xfer; 268 269 list_for_each_entry(xfer, &msg->transfers, transfer_list) { 270 /* If we have an offload transfer, we can't rx to buffer */ 271 if (msg->offload && xfer->rx_buf) 272 return -EINVAL; 273 274 clk_div = DIV_ROUND_UP(max_hz, xfer->speed_hz); 275 xfer->effective_speed_hz = max_hz / min(clk_div, 256U); 276 } 277 278 return 0; 279 } 280 281 static void spi_engine_compile_message(struct spi_message *msg, bool dry, 282 struct spi_engine_program *p) 283 { 284 struct spi_device *spi = msg->spi; 285 struct spi_controller *host = spi->controller; 286 struct spi_transfer *xfer; 287 int clk_div, new_clk_div, inst_ns; 288 bool keep_cs = false; 289 u8 bits_per_word = 0; 290 291 /* 292 * Take into account instruction execution time for more accurate sleep 293 * times, especially when the delay is small. 294 */ 295 inst_ns = DIV_ROUND_UP(NSEC_PER_SEC, host->max_speed_hz); 296 297 clk_div = 1; 298 299 spi_engine_program_add_cmd(p, dry, 300 SPI_ENGINE_CMD_WRITE(SPI_ENGINE_CMD_REG_CONFIG, 301 spi_engine_get_config(spi))); 302 303 xfer = list_first_entry(&msg->transfers, struct spi_transfer, transfer_list); 304 spi_engine_gen_cs(p, dry, spi, !xfer->cs_off); 305 306 list_for_each_entry(xfer, &msg->transfers, transfer_list) { 307 new_clk_div = host->max_speed_hz / xfer->effective_speed_hz; 308 if (new_clk_div != clk_div) { 309 clk_div = new_clk_div; 310 /* actual divider used is register value + 1 */ 311 spi_engine_program_add_cmd(p, dry, 312 SPI_ENGINE_CMD_WRITE(SPI_ENGINE_CMD_REG_CLK_DIV, 313 clk_div - 1)); 314 } 315 316 if (bits_per_word != xfer->bits_per_word && xfer->len) { 317 bits_per_word = xfer->bits_per_word; 318 spi_engine_program_add_cmd(p, dry, 319 SPI_ENGINE_CMD_WRITE(SPI_ENGINE_CMD_REG_XFER_BITS, 320 bits_per_word)); 321 } 322 323 spi_engine_gen_xfer(p, dry, xfer); 324 spi_engine_gen_sleep(p, dry, spi_delay_to_ns(&xfer->delay, xfer), 325 inst_ns, xfer->effective_speed_hz); 326 327 if (xfer->cs_change) { 328 if (list_is_last(&xfer->transfer_list, &msg->transfers)) { 329 keep_cs = true; 330 } else { 331 if (!xfer->cs_off) 332 spi_engine_gen_cs(p, dry, spi, false); 333 334 spi_engine_gen_sleep(p, dry, spi_delay_to_ns( 335 &xfer->cs_change_delay, xfer), inst_ns, 336 xfer->effective_speed_hz); 337 338 if (!list_next_entry(xfer, transfer_list)->cs_off) 339 spi_engine_gen_cs(p, dry, spi, true); 340 } 341 } else if (!list_is_last(&xfer->transfer_list, &msg->transfers) && 342 xfer->cs_off != list_next_entry(xfer, transfer_list)->cs_off) { 343 spi_engine_gen_cs(p, dry, spi, xfer->cs_off); 344 } 345 } 346 347 if (!keep_cs) 348 spi_engine_gen_cs(p, dry, spi, false); 349 350 /* 351 * Restore clockdiv to default so that future gen_sleep commands don't 352 * have to be aware of the current register state. 353 */ 354 if (clk_div != 1) 355 spi_engine_program_add_cmd(p, dry, 356 SPI_ENGINE_CMD_WRITE(SPI_ENGINE_CMD_REG_CLK_DIV, 0)); 357 } 358 359 static void spi_engine_xfer_next(struct spi_message *msg, 360 struct spi_transfer **_xfer) 361 { 362 struct spi_transfer *xfer = *_xfer; 363 364 if (!xfer) { 365 xfer = list_first_entry(&msg->transfers, 366 struct spi_transfer, transfer_list); 367 } else if (list_is_last(&xfer->transfer_list, &msg->transfers)) { 368 xfer = NULL; 369 } else { 370 xfer = list_next_entry(xfer, transfer_list); 371 } 372 373 *_xfer = xfer; 374 } 375 376 static void spi_engine_tx_next(struct spi_message *msg) 377 { 378 struct spi_engine_message_state *st = msg->state; 379 struct spi_transfer *xfer = st->tx_xfer; 380 381 do { 382 spi_engine_xfer_next(msg, &xfer); 383 } while (xfer && !xfer->tx_buf); 384 385 st->tx_xfer = xfer; 386 if (xfer) { 387 st->tx_length = xfer->len; 388 st->tx_buf = xfer->tx_buf; 389 } else { 390 st->tx_buf = NULL; 391 } 392 } 393 394 static void spi_engine_rx_next(struct spi_message *msg) 395 { 396 struct spi_engine_message_state *st = msg->state; 397 struct spi_transfer *xfer = st->rx_xfer; 398 399 do { 400 spi_engine_xfer_next(msg, &xfer); 401 } while (xfer && !xfer->rx_buf); 402 403 st->rx_xfer = xfer; 404 if (xfer) { 405 st->rx_length = xfer->len; 406 st->rx_buf = xfer->rx_buf; 407 } else { 408 st->rx_buf = NULL; 409 } 410 } 411 412 static bool spi_engine_write_cmd_fifo(struct spi_engine *spi_engine, 413 struct spi_message *msg) 414 { 415 void __iomem *addr = spi_engine->base + SPI_ENGINE_REG_CMD_FIFO; 416 struct spi_engine_message_state *st = msg->state; 417 unsigned int n, m, i; 418 const uint16_t *buf; 419 420 n = readl_relaxed(spi_engine->base + SPI_ENGINE_REG_CMD_FIFO_ROOM); 421 while (n && st->cmd_length) { 422 m = min(n, st->cmd_length); 423 buf = st->cmd_buf; 424 for (i = 0; i < m; i++) 425 writel_relaxed(buf[i], addr); 426 st->cmd_buf += m; 427 st->cmd_length -= m; 428 n -= m; 429 } 430 431 return st->cmd_length != 0; 432 } 433 434 static bool spi_engine_write_tx_fifo(struct spi_engine *spi_engine, 435 struct spi_message *msg) 436 { 437 void __iomem *addr = spi_engine->base + SPI_ENGINE_REG_SDO_DATA_FIFO; 438 struct spi_engine_message_state *st = msg->state; 439 unsigned int n, m, i; 440 441 n = readl_relaxed(spi_engine->base + SPI_ENGINE_REG_SDO_FIFO_ROOM); 442 while (n && st->tx_length) { 443 if (st->tx_xfer->bits_per_word <= 8) { 444 const u8 *buf = st->tx_buf; 445 446 m = min(n, st->tx_length); 447 for (i = 0; i < m; i++) 448 writel_relaxed(buf[i], addr); 449 st->tx_buf += m; 450 st->tx_length -= m; 451 } else if (st->tx_xfer->bits_per_word <= 16) { 452 const u16 *buf = (const u16 *)st->tx_buf; 453 454 m = min(n, st->tx_length / 2); 455 for (i = 0; i < m; i++) 456 writel_relaxed(buf[i], addr); 457 st->tx_buf += m * 2; 458 st->tx_length -= m * 2; 459 } else { 460 const u32 *buf = (const u32 *)st->tx_buf; 461 462 m = min(n, st->tx_length / 4); 463 for (i = 0; i < m; i++) 464 writel_relaxed(buf[i], addr); 465 st->tx_buf += m * 4; 466 st->tx_length -= m * 4; 467 } 468 n -= m; 469 if (st->tx_length == 0) 470 spi_engine_tx_next(msg); 471 } 472 473 return st->tx_length != 0; 474 } 475 476 static bool spi_engine_read_rx_fifo(struct spi_engine *spi_engine, 477 struct spi_message *msg) 478 { 479 void __iomem *addr = spi_engine->base + SPI_ENGINE_REG_SDI_DATA_FIFO; 480 struct spi_engine_message_state *st = msg->state; 481 unsigned int n, m, i; 482 483 n = readl_relaxed(spi_engine->base + SPI_ENGINE_REG_SDI_FIFO_LEVEL); 484 while (n && st->rx_length) { 485 if (st->rx_xfer->bits_per_word <= 8) { 486 u8 *buf = st->rx_buf; 487 488 m = min(n, st->rx_length); 489 for (i = 0; i < m; i++) 490 buf[i] = readl_relaxed(addr); 491 st->rx_buf += m; 492 st->rx_length -= m; 493 } else if (st->rx_xfer->bits_per_word <= 16) { 494 u16 *buf = (u16 *)st->rx_buf; 495 496 m = min(n, st->rx_length / 2); 497 for (i = 0; i < m; i++) 498 buf[i] = readl_relaxed(addr); 499 st->rx_buf += m * 2; 500 st->rx_length -= m * 2; 501 } else { 502 u32 *buf = (u32 *)st->rx_buf; 503 504 m = min(n, st->rx_length / 4); 505 for (i = 0; i < m; i++) 506 buf[i] = readl_relaxed(addr); 507 st->rx_buf += m * 4; 508 st->rx_length -= m * 4; 509 } 510 n -= m; 511 if (st->rx_length == 0) 512 spi_engine_rx_next(msg); 513 } 514 515 return st->rx_length != 0; 516 } 517 518 static irqreturn_t spi_engine_irq(int irq, void *devid) 519 { 520 struct spi_controller *host = devid; 521 struct spi_message *msg = host->cur_msg; 522 struct spi_engine *spi_engine = spi_controller_get_devdata(host); 523 unsigned int disable_int = 0; 524 unsigned int pending; 525 int completed_id = -1; 526 527 pending = readl_relaxed(spi_engine->base + SPI_ENGINE_REG_INT_PENDING); 528 529 if (pending & SPI_ENGINE_INT_SYNC) { 530 writel_relaxed(SPI_ENGINE_INT_SYNC, 531 spi_engine->base + SPI_ENGINE_REG_INT_PENDING); 532 completed_id = readl_relaxed( 533 spi_engine->base + SPI_ENGINE_REG_SYNC_ID); 534 } 535 536 spin_lock(&spi_engine->lock); 537 538 if (pending & SPI_ENGINE_INT_CMD_ALMOST_EMPTY) { 539 if (!spi_engine_write_cmd_fifo(spi_engine, msg)) 540 disable_int |= SPI_ENGINE_INT_CMD_ALMOST_EMPTY; 541 } 542 543 if (pending & SPI_ENGINE_INT_SDO_ALMOST_EMPTY) { 544 if (!spi_engine_write_tx_fifo(spi_engine, msg)) 545 disable_int |= SPI_ENGINE_INT_SDO_ALMOST_EMPTY; 546 } 547 548 if (pending & (SPI_ENGINE_INT_SDI_ALMOST_FULL | SPI_ENGINE_INT_SYNC)) { 549 if (!spi_engine_read_rx_fifo(spi_engine, msg)) 550 disable_int |= SPI_ENGINE_INT_SDI_ALMOST_FULL; 551 } 552 553 if (pending & SPI_ENGINE_INT_SYNC && msg) { 554 if (completed_id == AXI_SPI_ENGINE_CUR_MSG_SYNC_ID) { 555 msg->status = 0; 556 msg->actual_length = msg->frame_length; 557 complete(&spi_engine->msg_complete); 558 disable_int |= SPI_ENGINE_INT_SYNC; 559 } 560 } 561 562 if (disable_int) { 563 spi_engine->int_enable &= ~disable_int; 564 writel_relaxed(spi_engine->int_enable, 565 spi_engine->base + SPI_ENGINE_REG_INT_ENABLE); 566 } 567 568 spin_unlock(&spi_engine->lock); 569 570 return IRQ_HANDLED; 571 } 572 573 static int spi_engine_offload_prepare(struct spi_message *msg) 574 { 575 struct spi_controller *host = msg->spi->controller; 576 struct spi_engine *spi_engine = spi_controller_get_devdata(host); 577 struct spi_engine_program *p = msg->opt_state; 578 struct spi_engine_offload *priv = msg->offload->priv; 579 struct spi_transfer *xfer; 580 void __iomem *cmd_addr; 581 void __iomem *sdo_addr; 582 size_t tx_word_count = 0; 583 unsigned int i; 584 585 if (p->length > spi_engine->offload_ctrl_mem_size) 586 return -EINVAL; 587 588 /* count total number of tx words in message */ 589 list_for_each_entry(xfer, &msg->transfers, transfer_list) { 590 /* no support for reading to rx_buf */ 591 if (xfer->rx_buf) 592 return -EINVAL; 593 594 if (!xfer->tx_buf) 595 continue; 596 597 if (xfer->bits_per_word <= 8) 598 tx_word_count += xfer->len; 599 else if (xfer->bits_per_word <= 16) 600 tx_word_count += xfer->len / 2; 601 else 602 tx_word_count += xfer->len / 4; 603 } 604 605 if (tx_word_count && !(spi_engine->offload_caps & SPI_OFFLOAD_CAP_TX_STATIC_DATA)) 606 return -EINVAL; 607 608 if (tx_word_count > spi_engine->offload_sdo_mem_size) 609 return -EINVAL; 610 611 /* 612 * This protects against calling spi_optimize_message() with an offload 613 * that has already been prepared with a different message. 614 */ 615 if (test_and_set_bit_lock(SPI_ENGINE_OFFLOAD_FLAG_PREPARED, &priv->flags)) 616 return -EBUSY; 617 618 cmd_addr = spi_engine->base + 619 SPI_ENGINE_REG_OFFLOAD_CMD_FIFO(priv->offload_num); 620 sdo_addr = spi_engine->base + 621 SPI_ENGINE_REG_OFFLOAD_SDO_FIFO(priv->offload_num); 622 623 list_for_each_entry(xfer, &msg->transfers, transfer_list) { 624 if (!xfer->tx_buf) 625 continue; 626 627 if (xfer->bits_per_word <= 8) { 628 const u8 *buf = xfer->tx_buf; 629 630 for (i = 0; i < xfer->len; i++) 631 writel_relaxed(buf[i], sdo_addr); 632 } else if (xfer->bits_per_word <= 16) { 633 const u16 *buf = xfer->tx_buf; 634 635 for (i = 0; i < xfer->len / 2; i++) 636 writel_relaxed(buf[i], sdo_addr); 637 } else { 638 const u32 *buf = xfer->tx_buf; 639 640 for (i = 0; i < xfer->len / 4; i++) 641 writel_relaxed(buf[i], sdo_addr); 642 } 643 } 644 645 for (i = 0; i < p->length; i++) 646 writel_relaxed(p->instructions[i], cmd_addr); 647 648 return 0; 649 } 650 651 static void spi_engine_offload_unprepare(struct spi_offload *offload) 652 { 653 struct spi_engine_offload *priv = offload->priv; 654 struct spi_engine *spi_engine = priv->spi_engine; 655 656 writel_relaxed(1, spi_engine->base + 657 SPI_ENGINE_REG_OFFLOAD_RESET(priv->offload_num)); 658 writel_relaxed(0, spi_engine->base + 659 SPI_ENGINE_REG_OFFLOAD_RESET(priv->offload_num)); 660 661 clear_bit_unlock(SPI_ENGINE_OFFLOAD_FLAG_PREPARED, &priv->flags); 662 } 663 664 static int spi_engine_optimize_message(struct spi_message *msg) 665 { 666 struct spi_engine_program p_dry, *p; 667 int ret; 668 669 ret = spi_engine_precompile_message(msg); 670 if (ret) 671 return ret; 672 673 p_dry.length = 0; 674 spi_engine_compile_message(msg, true, &p_dry); 675 676 p = kzalloc(struct_size(p, instructions, p_dry.length + 1), GFP_KERNEL); 677 if (!p) 678 return -ENOMEM; 679 680 spi_engine_compile_message(msg, false, p); 681 682 spi_engine_program_add_cmd(p, false, SPI_ENGINE_CMD_SYNC( 683 msg->offload ? 0 : AXI_SPI_ENGINE_CUR_MSG_SYNC_ID)); 684 685 msg->opt_state = p; 686 687 if (msg->offload) { 688 ret = spi_engine_offload_prepare(msg); 689 if (ret) { 690 msg->opt_state = NULL; 691 kfree(p); 692 return ret; 693 } 694 } 695 696 return 0; 697 } 698 699 static int spi_engine_unoptimize_message(struct spi_message *msg) 700 { 701 if (msg->offload) 702 spi_engine_offload_unprepare(msg->offload); 703 704 kfree(msg->opt_state); 705 706 return 0; 707 } 708 709 static struct spi_offload 710 *spi_engine_get_offload(struct spi_device *spi, 711 const struct spi_offload_config *config) 712 { 713 struct spi_controller *host = spi->controller; 714 struct spi_engine *spi_engine = spi_controller_get_devdata(host); 715 struct spi_engine_offload *priv; 716 717 if (!spi_engine->offload) 718 return ERR_PTR(-ENODEV); 719 720 if (config->capability_flags & ~spi_engine->offload_caps) 721 return ERR_PTR(-EINVAL); 722 723 priv = spi_engine->offload->priv; 724 725 if (test_and_set_bit_lock(SPI_ENGINE_OFFLOAD_FLAG_ASSIGNED, &priv->flags)) 726 return ERR_PTR(-EBUSY); 727 728 return spi_engine->offload; 729 } 730 731 static void spi_engine_put_offload(struct spi_offload *offload) 732 { 733 struct spi_engine_offload *priv = offload->priv; 734 735 clear_bit_unlock(SPI_ENGINE_OFFLOAD_FLAG_ASSIGNED, &priv->flags); 736 } 737 738 static int spi_engine_setup(struct spi_device *device) 739 { 740 struct spi_controller *host = device->controller; 741 struct spi_engine *spi_engine = spi_controller_get_devdata(host); 742 743 if (device->mode & SPI_CS_HIGH) 744 spi_engine->cs_inv |= BIT(spi_get_chipselect(device, 0)); 745 else 746 spi_engine->cs_inv &= ~BIT(spi_get_chipselect(device, 0)); 747 748 writel_relaxed(SPI_ENGINE_CMD_CS_INV(spi_engine->cs_inv), 749 spi_engine->base + SPI_ENGINE_REG_CMD_FIFO); 750 751 /* 752 * In addition to setting the flags, we have to do a CS assert command 753 * to make the new setting actually take effect. 754 */ 755 writel_relaxed(SPI_ENGINE_CMD_ASSERT(0, 0xff), 756 spi_engine->base + SPI_ENGINE_REG_CMD_FIFO); 757 758 return 0; 759 } 760 761 static int spi_engine_transfer_one_message(struct spi_controller *host, 762 struct spi_message *msg) 763 { 764 struct spi_engine *spi_engine = spi_controller_get_devdata(host); 765 struct spi_engine_message_state *st = &spi_engine->msg_state; 766 struct spi_engine_program *p = msg->opt_state; 767 unsigned int int_enable = 0; 768 unsigned long flags; 769 770 if (msg->offload) { 771 dev_err(&host->dev, "Single transfer offload not supported\n"); 772 msg->status = -EOPNOTSUPP; 773 goto out; 774 } 775 776 /* reinitialize message state for this transfer */ 777 memset(st, 0, sizeof(*st)); 778 st->cmd_buf = p->instructions; 779 st->cmd_length = p->length; 780 msg->state = st; 781 782 reinit_completion(&spi_engine->msg_complete); 783 784 if (trace_spi_transfer_start_enabled()) { 785 struct spi_transfer *xfer; 786 787 list_for_each_entry(xfer, &msg->transfers, transfer_list) 788 trace_spi_transfer_start(msg, xfer); 789 } 790 791 spin_lock_irqsave(&spi_engine->lock, flags); 792 793 if (spi_engine_write_cmd_fifo(spi_engine, msg)) 794 int_enable |= SPI_ENGINE_INT_CMD_ALMOST_EMPTY; 795 796 spi_engine_tx_next(msg); 797 if (spi_engine_write_tx_fifo(spi_engine, msg)) 798 int_enable |= SPI_ENGINE_INT_SDO_ALMOST_EMPTY; 799 800 spi_engine_rx_next(msg); 801 if (st->rx_length != 0) 802 int_enable |= SPI_ENGINE_INT_SDI_ALMOST_FULL; 803 804 int_enable |= SPI_ENGINE_INT_SYNC; 805 806 writel_relaxed(int_enable, 807 spi_engine->base + SPI_ENGINE_REG_INT_ENABLE); 808 spi_engine->int_enable = int_enable; 809 spin_unlock_irqrestore(&spi_engine->lock, flags); 810 811 if (!wait_for_completion_timeout(&spi_engine->msg_complete, 812 msecs_to_jiffies(5000))) { 813 dev_err(&host->dev, 814 "Timeout occurred while waiting for transfer to complete. Hardware is probably broken.\n"); 815 msg->status = -ETIMEDOUT; 816 } 817 818 if (trace_spi_transfer_stop_enabled()) { 819 struct spi_transfer *xfer; 820 821 list_for_each_entry(xfer, &msg->transfers, transfer_list) 822 trace_spi_transfer_stop(msg, xfer); 823 } 824 825 out: 826 spi_finalize_current_message(host); 827 828 return msg->status; 829 } 830 831 static int spi_engine_trigger_enable(struct spi_offload *offload) 832 { 833 struct spi_engine_offload *priv = offload->priv; 834 struct spi_engine *spi_engine = priv->spi_engine; 835 unsigned int reg; 836 837 reg = readl_relaxed(spi_engine->base + 838 SPI_ENGINE_REG_OFFLOAD_CTRL(priv->offload_num)); 839 reg |= SPI_ENGINE_OFFLOAD_CTRL_ENABLE; 840 writel_relaxed(reg, spi_engine->base + 841 SPI_ENGINE_REG_OFFLOAD_CTRL(priv->offload_num)); 842 return 0; 843 } 844 845 static void spi_engine_trigger_disable(struct spi_offload *offload) 846 { 847 struct spi_engine_offload *priv = offload->priv; 848 struct spi_engine *spi_engine = priv->spi_engine; 849 unsigned int reg; 850 851 reg = readl_relaxed(spi_engine->base + 852 SPI_ENGINE_REG_OFFLOAD_CTRL(priv->offload_num)); 853 reg &= ~SPI_ENGINE_OFFLOAD_CTRL_ENABLE; 854 writel_relaxed(reg, spi_engine->base + 855 SPI_ENGINE_REG_OFFLOAD_CTRL(priv->offload_num)); 856 } 857 858 static struct dma_chan 859 *spi_engine_tx_stream_request_dma_chan(struct spi_offload *offload) 860 { 861 struct spi_engine_offload *priv = offload->priv; 862 char name[16]; 863 864 snprintf(name, sizeof(name), "offload%u-tx", priv->offload_num); 865 866 return dma_request_chan(offload->provider_dev, name); 867 } 868 869 static struct dma_chan 870 *spi_engine_rx_stream_request_dma_chan(struct spi_offload *offload) 871 { 872 struct spi_engine_offload *priv = offload->priv; 873 char name[16]; 874 875 snprintf(name, sizeof(name), "offload%u-rx", priv->offload_num); 876 877 return dma_request_chan(offload->provider_dev, name); 878 } 879 880 static const struct spi_offload_ops spi_engine_offload_ops = { 881 .trigger_enable = spi_engine_trigger_enable, 882 .trigger_disable = spi_engine_trigger_disable, 883 .tx_stream_request_dma_chan = spi_engine_tx_stream_request_dma_chan, 884 .rx_stream_request_dma_chan = spi_engine_rx_stream_request_dma_chan, 885 }; 886 887 static void spi_engine_release_hw(void *p) 888 { 889 struct spi_engine *spi_engine = p; 890 891 writel_relaxed(0xff, spi_engine->base + SPI_ENGINE_REG_INT_PENDING); 892 writel_relaxed(0x00, spi_engine->base + SPI_ENGINE_REG_INT_ENABLE); 893 writel_relaxed(0x01, spi_engine->base + SPI_ENGINE_REG_RESET); 894 } 895 896 static int spi_engine_probe(struct platform_device *pdev) 897 { 898 struct spi_engine *spi_engine; 899 struct spi_controller *host; 900 unsigned int version; 901 int irq, ret; 902 903 irq = platform_get_irq(pdev, 0); 904 if (irq < 0) 905 return irq; 906 907 host = devm_spi_alloc_host(&pdev->dev, sizeof(*spi_engine)); 908 if (!host) 909 return -ENOMEM; 910 911 spi_engine = spi_controller_get_devdata(host); 912 913 spin_lock_init(&spi_engine->lock); 914 init_completion(&spi_engine->msg_complete); 915 916 /* 917 * REVISIT: for now, all SPI Engines only have one offload. In the 918 * future, this should be read from a memory mapped register to 919 * determine the number of offloads enabled at HDL compile time. For 920 * now, we can tell if an offload is present if there is a trigger 921 * source wired up to it. 922 */ 923 if (device_property_present(&pdev->dev, "trigger-sources")) { 924 struct spi_engine_offload *priv; 925 926 spi_engine->offload = 927 devm_spi_offload_alloc(&pdev->dev, 928 sizeof(struct spi_engine_offload)); 929 if (IS_ERR(spi_engine->offload)) 930 return PTR_ERR(spi_engine->offload); 931 932 priv = spi_engine->offload->priv; 933 priv->spi_engine = spi_engine; 934 priv->offload_num = 0; 935 936 spi_engine->offload->ops = &spi_engine_offload_ops; 937 spi_engine->offload_caps = SPI_OFFLOAD_CAP_TRIGGER; 938 939 if (device_property_match_string(&pdev->dev, "dma-names", "offload0-rx") >= 0) { 940 spi_engine->offload_caps |= SPI_OFFLOAD_CAP_RX_STREAM_DMA; 941 spi_engine->offload->xfer_flags |= SPI_OFFLOAD_XFER_RX_STREAM; 942 } 943 944 if (device_property_match_string(&pdev->dev, "dma-names", "offload0-tx") >= 0) { 945 spi_engine->offload_caps |= SPI_OFFLOAD_CAP_TX_STREAM_DMA; 946 spi_engine->offload->xfer_flags |= SPI_OFFLOAD_XFER_TX_STREAM; 947 } else { 948 /* 949 * HDL compile option to enable TX DMA stream also disables 950 * the SDO memory, so can't do both at the same time. 951 */ 952 spi_engine->offload_caps |= SPI_OFFLOAD_CAP_TX_STATIC_DATA; 953 } 954 } 955 956 spi_engine->clk = devm_clk_get_enabled(&pdev->dev, "s_axi_aclk"); 957 if (IS_ERR(spi_engine->clk)) 958 return PTR_ERR(spi_engine->clk); 959 960 spi_engine->ref_clk = devm_clk_get_enabled(&pdev->dev, "spi_clk"); 961 if (IS_ERR(spi_engine->ref_clk)) 962 return PTR_ERR(spi_engine->ref_clk); 963 964 spi_engine->base = devm_platform_ioremap_resource(pdev, 0); 965 if (IS_ERR(spi_engine->base)) 966 return PTR_ERR(spi_engine->base); 967 968 version = readl(spi_engine->base + ADI_AXI_REG_VERSION); 969 if (ADI_AXI_PCORE_VER_MAJOR(version) != 1) { 970 dev_err(&pdev->dev, "Unsupported peripheral version %u.%u.%u\n", 971 ADI_AXI_PCORE_VER_MAJOR(version), 972 ADI_AXI_PCORE_VER_MINOR(version), 973 ADI_AXI_PCORE_VER_PATCH(version)); 974 return -ENODEV; 975 } 976 977 if (ADI_AXI_PCORE_VER_MINOR(version) >= 1) { 978 unsigned int sizes = readl(spi_engine->base + 979 SPI_ENGINE_REG_OFFLOAD_MEM_ADDR_WIDTH); 980 981 spi_engine->offload_ctrl_mem_size = 1 << 982 FIELD_GET(SPI_ENGINE_SPI_OFFLOAD_MEM_WIDTH_CMD, sizes); 983 spi_engine->offload_sdo_mem_size = 1 << 984 FIELD_GET(SPI_ENGINE_SPI_OFFLOAD_MEM_WIDTH_SDO, sizes); 985 } else { 986 spi_engine->offload_ctrl_mem_size = SPI_ENGINE_OFFLOAD_CMD_FIFO_SIZE; 987 spi_engine->offload_sdo_mem_size = SPI_ENGINE_OFFLOAD_SDO_FIFO_SIZE; 988 } 989 990 writel_relaxed(0x00, spi_engine->base + SPI_ENGINE_REG_RESET); 991 writel_relaxed(0xff, spi_engine->base + SPI_ENGINE_REG_INT_PENDING); 992 writel_relaxed(0x00, spi_engine->base + SPI_ENGINE_REG_INT_ENABLE); 993 994 ret = devm_add_action_or_reset(&pdev->dev, spi_engine_release_hw, 995 spi_engine); 996 if (ret) 997 return ret; 998 999 ret = devm_request_irq(&pdev->dev, irq, spi_engine_irq, 0, pdev->name, 1000 host); 1001 if (ret) 1002 return ret; 1003 1004 host->dev.of_node = pdev->dev.of_node; 1005 host->mode_bits = SPI_CPOL | SPI_CPHA | SPI_3WIRE; 1006 host->bits_per_word_mask = SPI_BPW_RANGE_MASK(1, 32); 1007 host->max_speed_hz = clk_get_rate(spi_engine->ref_clk) / 2; 1008 host->transfer_one_message = spi_engine_transfer_one_message; 1009 host->optimize_message = spi_engine_optimize_message; 1010 host->unoptimize_message = spi_engine_unoptimize_message; 1011 host->get_offload = spi_engine_get_offload; 1012 host->put_offload = spi_engine_put_offload; 1013 host->num_chipselect = 8; 1014 1015 /* Some features depend of the IP core version. */ 1016 if (ADI_AXI_PCORE_VER_MAJOR(version) >= 1) { 1017 if (ADI_AXI_PCORE_VER_MINOR(version) >= 2) { 1018 host->mode_bits |= SPI_CS_HIGH; 1019 host->setup = spi_engine_setup; 1020 } 1021 if (ADI_AXI_PCORE_VER_MINOR(version) >= 3) 1022 host->mode_bits |= SPI_MOSI_IDLE_LOW | SPI_MOSI_IDLE_HIGH; 1023 } 1024 1025 if (host->max_speed_hz == 0) 1026 return dev_err_probe(&pdev->dev, -EINVAL, "spi_clk rate is 0"); 1027 1028 return devm_spi_register_controller(&pdev->dev, host); 1029 } 1030 1031 static const struct of_device_id spi_engine_match_table[] = { 1032 { .compatible = "adi,axi-spi-engine-1.00.a" }, 1033 { }, 1034 }; 1035 MODULE_DEVICE_TABLE(of, spi_engine_match_table); 1036 1037 static struct platform_driver spi_engine_driver = { 1038 .probe = spi_engine_probe, 1039 .driver = { 1040 .name = "spi-engine", 1041 .of_match_table = spi_engine_match_table, 1042 }, 1043 }; 1044 module_platform_driver(spi_engine_driver); 1045 1046 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>"); 1047 MODULE_DESCRIPTION("Analog Devices SPI engine peripheral driver"); 1048 MODULE_LICENSE("GPL"); 1049