1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2018 Exceet Electronics GmbH 4 * Copyright (C) 2018 Bootlin 5 * 6 * Author: Boris Brezillon <boris.brezillon@bootlin.com> 7 */ 8 #include <linux/dmaengine.h> 9 #include <linux/iopoll.h> 10 #include <linux/pm_runtime.h> 11 #include <linux/spi/spi.h> 12 #include <linux/spi/spi-mem.h> 13 #include <linux/sched/task_stack.h> 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/spi-mem.h> 17 18 #include "internals.h" 19 20 #define SPI_MEM_MAX_BUSWIDTH 8 21 22 /** 23 * spi_controller_dma_map_mem_op_data() - DMA-map the buffer attached to a 24 * memory operation 25 * @ctlr: the SPI controller requesting this dma_map() 26 * @op: the memory operation containing the buffer to map 27 * @sgt: a pointer to a non-initialized sg_table that will be filled by this 28 * function 29 * 30 * Some controllers might want to do DMA on the data buffer embedded in @op. 31 * This helper prepares everything for you and provides a ready-to-use 32 * sg_table. This function is not intended to be called from spi drivers. 33 * Only SPI controller drivers should use it. 34 * Note that the caller must ensure the memory region pointed by 35 * op->data.buf.{in,out} is DMA-able before calling this function. 36 * 37 * Return: 0 in case of success, a negative error code otherwise. 38 */ 39 int spi_controller_dma_map_mem_op_data(struct spi_controller *ctlr, 40 const struct spi_mem_op *op, 41 struct sg_table *sgt) 42 { 43 struct device *dmadev; 44 45 if (!op->data.nbytes) 46 return -EINVAL; 47 48 if (op->data.dir == SPI_MEM_DATA_OUT && ctlr->dma_tx) 49 dmadev = dmaengine_get_dma_device(ctlr->dma_tx); 50 else if (op->data.dir == SPI_MEM_DATA_IN && ctlr->dma_rx) 51 dmadev = dmaengine_get_dma_device(ctlr->dma_rx); 52 else 53 dmadev = ctlr->dev.parent; 54 55 if (!dmadev) 56 return -EINVAL; 57 58 return spi_map_buf(ctlr, dmadev, sgt, op->data.buf.in, op->data.nbytes, 59 op->data.dir == SPI_MEM_DATA_IN ? 60 DMA_FROM_DEVICE : DMA_TO_DEVICE); 61 } 62 EXPORT_SYMBOL_GPL(spi_controller_dma_map_mem_op_data); 63 64 /** 65 * spi_controller_dma_unmap_mem_op_data() - DMA-unmap the buffer attached to a 66 * memory operation 67 * @ctlr: the SPI controller requesting this dma_unmap() 68 * @op: the memory operation containing the buffer to unmap 69 * @sgt: a pointer to an sg_table previously initialized by 70 * spi_controller_dma_map_mem_op_data() 71 * 72 * Some controllers might want to do DMA on the data buffer embedded in @op. 73 * This helper prepares things so that the CPU can access the 74 * op->data.buf.{in,out} buffer again. 75 * 76 * This function is not intended to be called from SPI drivers. Only SPI 77 * controller drivers should use it. 78 * 79 * This function should be called after the DMA operation has finished and is 80 * only valid if the previous spi_controller_dma_map_mem_op_data() call 81 * returned 0. 82 * 83 * Return: 0 in case of success, a negative error code otherwise. 84 */ 85 void spi_controller_dma_unmap_mem_op_data(struct spi_controller *ctlr, 86 const struct spi_mem_op *op, 87 struct sg_table *sgt) 88 { 89 struct device *dmadev; 90 91 if (!op->data.nbytes) 92 return; 93 94 if (op->data.dir == SPI_MEM_DATA_OUT && ctlr->dma_tx) 95 dmadev = dmaengine_get_dma_device(ctlr->dma_tx); 96 else if (op->data.dir == SPI_MEM_DATA_IN && ctlr->dma_rx) 97 dmadev = dmaengine_get_dma_device(ctlr->dma_rx); 98 else 99 dmadev = ctlr->dev.parent; 100 101 spi_unmap_buf(ctlr, dmadev, sgt, 102 op->data.dir == SPI_MEM_DATA_IN ? 103 DMA_FROM_DEVICE : DMA_TO_DEVICE); 104 } 105 EXPORT_SYMBOL_GPL(spi_controller_dma_unmap_mem_op_data); 106 107 static int spi_check_buswidth_req(struct spi_mem *mem, u8 buswidth, bool tx) 108 { 109 u32 mode = mem->spi->mode; 110 111 switch (buswidth) { 112 case 1: 113 return 0; 114 115 case 2: 116 if ((tx && 117 (mode & (SPI_TX_DUAL | SPI_TX_QUAD | SPI_TX_OCTAL))) || 118 (!tx && 119 (mode & (SPI_RX_DUAL | SPI_RX_QUAD | SPI_RX_OCTAL)))) 120 return 0; 121 122 break; 123 124 case 4: 125 if ((tx && (mode & (SPI_TX_QUAD | SPI_TX_OCTAL))) || 126 (!tx && (mode & (SPI_RX_QUAD | SPI_RX_OCTAL)))) 127 return 0; 128 129 break; 130 131 case 8: 132 if ((tx && (mode & SPI_TX_OCTAL)) || 133 (!tx && (mode & SPI_RX_OCTAL))) 134 return 0; 135 136 break; 137 138 default: 139 break; 140 } 141 142 return -ENOTSUPP; 143 } 144 145 static bool spi_mem_check_buswidth(struct spi_mem *mem, 146 const struct spi_mem_op *op) 147 { 148 if (spi_check_buswidth_req(mem, op->cmd.buswidth, true)) 149 return false; 150 151 if (op->addr.nbytes && 152 spi_check_buswidth_req(mem, op->addr.buswidth, true)) 153 return false; 154 155 if (op->dummy.nbytes && 156 spi_check_buswidth_req(mem, op->dummy.buswidth, true)) 157 return false; 158 159 if (op->data.dir != SPI_MEM_NO_DATA && 160 spi_check_buswidth_req(mem, op->data.buswidth, 161 op->data.dir == SPI_MEM_DATA_OUT)) 162 return false; 163 164 return true; 165 } 166 167 bool spi_mem_default_supports_op(struct spi_mem *mem, 168 const struct spi_mem_op *op) 169 { 170 struct spi_controller *ctlr = mem->spi->controller; 171 bool op_is_dtr = 172 op->cmd.dtr || op->addr.dtr || op->dummy.dtr || op->data.dtr; 173 174 if (op_is_dtr) { 175 if (!spi_mem_controller_is_capable(ctlr, dtr) && !ctlr->dtr_caps) 176 return false; 177 178 if (op->data.swap16 && !spi_mem_controller_is_capable(ctlr, swap16)) 179 return false; 180 181 /* Extra 8D-8D-8D limitations */ 182 if (op->cmd.dtr && op->cmd.buswidth == 8) { 183 if (op->cmd.nbytes != 2) 184 return false; 185 186 if ((op->addr.nbytes % 2) || 187 (op->dummy.nbytes % 2) || 188 (op->data.nbytes % 2)) { 189 dev_err(&ctlr->dev, 190 "Even byte numbers not allowed in octal DTR operations\n"); 191 return false; 192 } 193 } 194 } else { 195 if (op->cmd.nbytes != 1) 196 return false; 197 } 198 199 if (op->data.ecc) { 200 if (!spi_mem_controller_is_capable(ctlr, ecc)) 201 return false; 202 } 203 204 if (op->max_freq && mem->spi->controller->min_speed_hz && 205 op->max_freq < mem->spi->controller->min_speed_hz) 206 return false; 207 208 if (op->max_freq && 209 op->max_freq < mem->spi->max_speed_hz) { 210 if (!spi_mem_controller_is_capable(ctlr, per_op_freq)) 211 return false; 212 } 213 214 return spi_mem_check_buswidth(mem, op); 215 } 216 EXPORT_SYMBOL_GPL(spi_mem_default_supports_op); 217 218 static bool spi_mem_buswidth_is_valid(u8 buswidth) 219 { 220 if (hweight8(buswidth) > 1 || buswidth > SPI_MEM_MAX_BUSWIDTH) 221 return false; 222 223 return true; 224 } 225 226 static int spi_mem_check_op(const struct spi_mem_op *op) 227 { 228 if (!op->cmd.buswidth || !op->cmd.nbytes) 229 return -EINVAL; 230 231 if ((op->addr.nbytes && !op->addr.buswidth) || 232 (op->dummy.nbytes && !op->dummy.buswidth) || 233 (op->data.nbytes && !op->data.buswidth)) 234 return -EINVAL; 235 236 if (!spi_mem_buswidth_is_valid(op->cmd.buswidth) || 237 !spi_mem_buswidth_is_valid(op->addr.buswidth) || 238 !spi_mem_buswidth_is_valid(op->dummy.buswidth) || 239 !spi_mem_buswidth_is_valid(op->data.buswidth)) 240 return -EINVAL; 241 242 /* Buffers must be DMA-able. */ 243 if (WARN_ON_ONCE(op->data.dir == SPI_MEM_DATA_IN && 244 object_is_on_stack(op->data.buf.in))) 245 return -EINVAL; 246 247 if (WARN_ON_ONCE(op->data.dir == SPI_MEM_DATA_OUT && 248 object_is_on_stack(op->data.buf.out))) 249 return -EINVAL; 250 251 return 0; 252 } 253 254 static bool spi_mem_internal_supports_op(struct spi_mem *mem, 255 const struct spi_mem_op *op) 256 { 257 struct spi_controller *ctlr = mem->spi->controller; 258 259 if (ctlr->mem_ops && ctlr->mem_ops->supports_op) 260 return ctlr->mem_ops->supports_op(mem, op); 261 262 return spi_mem_default_supports_op(mem, op); 263 } 264 265 /** 266 * spi_mem_supports_op() - Check if a memory device and the controller it is 267 * connected to support a specific memory operation 268 * @mem: the SPI memory 269 * @op: the memory operation to check 270 * 271 * Some controllers are only supporting Single or Dual IOs, others might only 272 * support specific opcodes, or it can even be that the controller and device 273 * both support Quad IOs but the hardware prevents you from using it because 274 * only 2 IO lines are connected. 275 * 276 * This function checks whether a specific operation is supported. 277 * 278 * Return: true if @op is supported, false otherwise. 279 */ 280 bool spi_mem_supports_op(struct spi_mem *mem, const struct spi_mem_op *op) 281 { 282 struct spi_mem_op eval_op = *op; 283 284 /* 285 * Work on a local copy; this is a pure capability check and must 286 * not modify the caller's op. Stored templates with max_freq == 0 287 * must remain unset so their frequency is always re-capped to the 288 * current device maximum at execution time. 289 */ 290 spi_mem_adjust_op_freq(mem, &eval_op); 291 292 if (spi_mem_check_op(&eval_op)) 293 return false; 294 295 return spi_mem_internal_supports_op(mem, &eval_op); 296 } 297 EXPORT_SYMBOL_GPL(spi_mem_supports_op); 298 299 static int spi_mem_access_start(struct spi_mem *mem) 300 { 301 struct spi_controller *ctlr = mem->spi->controller; 302 303 /* 304 * Flush the message queue before executing our SPI memory 305 * operation to prevent preemption of regular SPI transfers. 306 */ 307 spi_flush_queue(ctlr); 308 309 if (ctlr->auto_runtime_pm) { 310 int ret; 311 312 ret = pm_runtime_resume_and_get(ctlr->dev.parent); 313 if (ret < 0) { 314 dev_err(&ctlr->dev, "Failed to power device: %d\n", 315 ret); 316 return ret; 317 } 318 } 319 320 mutex_lock(&ctlr->bus_lock_mutex); 321 mutex_lock(&ctlr->io_mutex); 322 323 return 0; 324 } 325 326 static void spi_mem_access_end(struct spi_mem *mem) 327 { 328 struct spi_controller *ctlr = mem->spi->controller; 329 330 mutex_unlock(&ctlr->io_mutex); 331 mutex_unlock(&ctlr->bus_lock_mutex); 332 333 if (ctlr->auto_runtime_pm) 334 pm_runtime_put(ctlr->dev.parent); 335 } 336 337 static void spi_mem_add_op_stats(struct spi_statistics __percpu *pcpu_stats, 338 const struct spi_mem_op *op, int exec_op_ret) 339 { 340 struct spi_statistics *stats; 341 u64 len, l2len; 342 343 get_cpu(); 344 stats = this_cpu_ptr(pcpu_stats); 345 u64_stats_update_begin(&stats->syncp); 346 347 /* 348 * We do not have the concept of messages or transfers. Let's consider 349 * that one operation is equivalent to one message and one transfer. 350 */ 351 u64_stats_inc(&stats->messages); 352 u64_stats_inc(&stats->transfers); 353 354 /* Use the sum of all lengths as bytes count and histogram value. */ 355 len = op->cmd.nbytes + op->addr.nbytes; 356 len += op->dummy.nbytes + op->data.nbytes; 357 u64_stats_add(&stats->bytes, len); 358 l2len = min(fls(len), SPI_STATISTICS_HISTO_SIZE) - 1; 359 u64_stats_inc(&stats->transfer_bytes_histo[l2len]); 360 361 /* Only account for data bytes as transferred bytes. */ 362 if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_OUT) 363 u64_stats_add(&stats->bytes_tx, op->data.nbytes); 364 if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_IN) 365 u64_stats_add(&stats->bytes_rx, op->data.nbytes); 366 367 /* 368 * A timeout is not an error, following the same behavior as 369 * spi_transfer_one_message(). 370 */ 371 if (exec_op_ret == -ETIMEDOUT) 372 u64_stats_inc(&stats->timedout); 373 else if (exec_op_ret) 374 u64_stats_inc(&stats->errors); 375 376 u64_stats_update_end(&stats->syncp); 377 put_cpu(); 378 } 379 380 /** 381 * spi_mem_exec_op() - Execute a memory operation 382 * @mem: the SPI memory 383 * @op: the memory operation to execute 384 * 385 * Executes a memory operation. 386 * 387 * This function first checks that @op is supported and then tries to execute 388 * it. 389 * 390 * Return: 0 in case of success, a negative error code otherwise. 391 */ 392 int spi_mem_exec_op(struct spi_mem *mem, const struct spi_mem_op *op) 393 { 394 unsigned int tmpbufsize, xferpos = 0, totalxferlen = 0; 395 struct spi_controller *ctlr = mem->spi->controller; 396 struct spi_transfer xfers[4] = { }; 397 struct spi_message msg; 398 u8 *tmpbuf; 399 int ret; 400 401 /* Make sure the operation frequency is correct before going futher */ 402 spi_mem_adjust_op_freq(mem, (struct spi_mem_op *)op); 403 404 dev_vdbg(&mem->spi->dev, "[cmd: 0x%02x][%dB addr: %#8llx][%2dB dummy][%4dB data %s] %d%c-%d%c-%d%c-%d%c @ %uHz\n", 405 op->cmd.opcode, 406 op->addr.nbytes, (op->addr.nbytes ? op->addr.val : 0), 407 op->dummy.nbytes, 408 op->data.nbytes, (op->data.nbytes ? (op->data.dir == SPI_MEM_DATA_IN ? " read" : "write") : " "), 409 op->cmd.buswidth, op->cmd.dtr ? 'D' : 'S', 410 op->addr.buswidth, op->addr.dtr ? 'D' : 'S', 411 op->dummy.buswidth, op->dummy.dtr ? 'D' : 'S', 412 op->data.buswidth, op->data.dtr ? 'D' : 'S', 413 op->max_freq ? op->max_freq : mem->spi->max_speed_hz); 414 415 ret = spi_mem_check_op(op); 416 if (ret) 417 return ret; 418 419 if (!spi_mem_internal_supports_op(mem, op)) 420 return -EOPNOTSUPP; 421 422 if (ctlr->mem_ops && ctlr->mem_ops->exec_op && !spi_get_csgpiod(mem->spi, 0)) { 423 ret = spi_mem_access_start(mem); 424 if (ret) 425 return ret; 426 427 trace_spi_mem_start_op(mem, op); 428 ret = ctlr->mem_ops->exec_op(mem, op); 429 trace_spi_mem_stop_op(mem, op); 430 431 spi_mem_access_end(mem); 432 433 /* 434 * Some controllers only optimize specific paths (typically the 435 * read path) and expect the core to use the regular SPI 436 * interface in other cases. 437 */ 438 if (!ret || (ret != -ENOTSUPP && ret != -EOPNOTSUPP)) { 439 spi_mem_add_op_stats(ctlr->pcpu_statistics, op, ret); 440 spi_mem_add_op_stats(mem->spi->pcpu_statistics, op, ret); 441 442 return ret; 443 } 444 } 445 446 tmpbufsize = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes; 447 448 /* 449 * Allocate a buffer to transmit the CMD, ADDR cycles with kmalloc() so 450 * we're guaranteed that this buffer is DMA-able, as required by the 451 * SPI layer. 452 */ 453 tmpbuf = kzalloc(tmpbufsize, GFP_KERNEL | GFP_DMA); 454 if (!tmpbuf) 455 return -ENOMEM; 456 457 spi_message_init(&msg); 458 459 tmpbuf[0] = op->cmd.opcode; 460 xfers[xferpos].tx_buf = tmpbuf; 461 xfers[xferpos].len = op->cmd.nbytes; 462 xfers[xferpos].tx_nbits = op->cmd.buswidth; 463 xfers[xferpos].speed_hz = op->max_freq; 464 xfers[xferpos].dtr_mode = op->cmd.dtr; 465 spi_message_add_tail(&xfers[xferpos], &msg); 466 xferpos++; 467 totalxferlen++; 468 469 if (op->addr.nbytes) { 470 int i; 471 472 for (i = 0; i < op->addr.nbytes; i++) 473 tmpbuf[i + 1] = op->addr.val >> 474 (8 * (op->addr.nbytes - i - 1)); 475 476 xfers[xferpos].tx_buf = tmpbuf + 1; 477 xfers[xferpos].len = op->addr.nbytes; 478 xfers[xferpos].tx_nbits = op->addr.buswidth; 479 xfers[xferpos].speed_hz = op->max_freq; 480 xfers[xferpos].dtr_mode = op->addr.dtr; 481 spi_message_add_tail(&xfers[xferpos], &msg); 482 xferpos++; 483 totalxferlen += op->addr.nbytes; 484 } 485 486 if (op->dummy.nbytes) { 487 memset(tmpbuf + op->addr.nbytes + 1, 0xff, op->dummy.nbytes); 488 xfers[xferpos].tx_buf = tmpbuf + op->addr.nbytes + 1; 489 xfers[xferpos].len = op->dummy.nbytes; 490 xfers[xferpos].tx_nbits = op->dummy.buswidth; 491 xfers[xferpos].dummy_data = 1; 492 xfers[xferpos].speed_hz = op->max_freq; 493 xfers[xferpos].dtr_mode = op->dummy.dtr; 494 spi_message_add_tail(&xfers[xferpos], &msg); 495 xferpos++; 496 totalxferlen += op->dummy.nbytes; 497 } 498 499 if (op->data.nbytes) { 500 if (op->data.dir == SPI_MEM_DATA_IN) { 501 xfers[xferpos].rx_buf = op->data.buf.in; 502 xfers[xferpos].rx_nbits = op->data.buswidth; 503 } else { 504 xfers[xferpos].tx_buf = op->data.buf.out; 505 xfers[xferpos].tx_nbits = op->data.buswidth; 506 } 507 508 xfers[xferpos].len = op->data.nbytes; 509 xfers[xferpos].speed_hz = op->max_freq; 510 xfers[xferpos].dtr_mode = op->data.dtr; 511 spi_message_add_tail(&xfers[xferpos], &msg); 512 xferpos++; 513 totalxferlen += op->data.nbytes; 514 } 515 516 ret = spi_sync(mem->spi, &msg); 517 518 kfree(tmpbuf); 519 520 if (ret) 521 return ret; 522 523 if (msg.actual_length != totalxferlen) 524 return -EIO; 525 526 return 0; 527 } 528 EXPORT_SYMBOL_GPL(spi_mem_exec_op); 529 530 /** 531 * spi_mem_get_name() - Return the SPI mem device name to be used by the 532 * upper layer if necessary 533 * @mem: the SPI memory 534 * 535 * This function allows SPI mem users to retrieve the SPI mem device name. 536 * It is useful if the upper layer needs to expose a custom name for 537 * compatibility reasons. 538 * 539 * Return: a string containing the name of the memory device to be used 540 * by the SPI mem user 541 */ 542 const char *spi_mem_get_name(struct spi_mem *mem) 543 { 544 return mem->name; 545 } 546 EXPORT_SYMBOL_GPL(spi_mem_get_name); 547 548 /** 549 * spi_mem_set_dqs() - Mark DQS as being available 550 * @mem: the SPI memory 551 * 552 * When reading at high frequencies (> 100MHz), especially when DTR is enabled, 553 * transfer speed is limited due to clock skews. In particular, the controller 554 * does not know the board propagation delay nor the memory chip internal delay 555 * (clock in to data out) and thus cannot optimize its sampling points. 556 * Mitigating this limitation is possible with the addition of a data strobe 557 * signal, commonly named DQS. 558 * 559 * Set the DQS boolean if the feature is available and configured at the chip 560 * level. Controllers may query this value. 561 */ 562 void spi_mem_set_dqs(struct spi_mem *mem) 563 { 564 mem->dqs = true; 565 } 566 EXPORT_SYMBOL_GPL(spi_mem_set_dqs); 567 568 /** 569 * spi_mem_has_dqs() - Query whether the DQS is available or not 570 * @mem: the SPI memory 571 * 572 * Return: a boolean indicating whether the DQS signal is available or not. 573 */ 574 bool spi_mem_has_dqs(struct spi_mem *mem) 575 { 576 return mem->dqs; 577 } 578 EXPORT_SYMBOL_GPL(spi_mem_has_dqs); 579 580 /** 581 * spi_mem_adjust_op_size() - Adjust the data size of a SPI mem operation to 582 * match controller limitations 583 * @mem: the SPI memory 584 * @op: the operation to adjust 585 * 586 * Some controllers have FIFO limitations and must split a data transfer 587 * operation into multiple ones, others require a specific alignment for 588 * optimized accesses. This function allows SPI mem drivers to split a single 589 * operation into multiple sub-operations when required. 590 * 591 * Return: a negative error code if the controller can't properly adjust @op, 592 * 0 otherwise. Note that @op->data.nbytes will be updated if @op 593 * can't be handled in a single step. 594 */ 595 int spi_mem_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op) 596 { 597 struct spi_controller *ctlr = mem->spi->controller; 598 size_t len; 599 600 if (ctlr->mem_ops && ctlr->mem_ops->adjust_op_size) 601 return ctlr->mem_ops->adjust_op_size(mem, op); 602 603 if (!ctlr->mem_ops || !ctlr->mem_ops->exec_op) { 604 len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes; 605 606 if (len > spi_max_transfer_size(mem->spi)) 607 return -EINVAL; 608 609 op->data.nbytes = min3((size_t)op->data.nbytes, 610 spi_max_transfer_size(mem->spi), 611 spi_max_message_size(mem->spi) - 612 len); 613 if (!op->data.nbytes) 614 return -EINVAL; 615 } 616 617 return 0; 618 } 619 EXPORT_SYMBOL_GPL(spi_mem_adjust_op_size); 620 621 /** 622 * spi_mem_adjust_op_freq() - Adjust the frequency of a SPI mem operation to 623 * match controller, PCB and chip limitations 624 * @mem: the SPI memory 625 * @op: the operation to adjust 626 * 627 * Some chips have per-op frequency limitations and must adapt the maximum 628 * speed. This function allows SPI mem drivers to set @op->max_freq to the 629 * maximum supported value. 630 */ 631 void spi_mem_adjust_op_freq(struct spi_mem *mem, struct spi_mem_op *op) 632 { 633 if (!op->max_freq || op->max_freq > mem->spi->max_speed_hz) 634 op->max_freq = mem->spi->max_speed_hz; 635 } 636 EXPORT_SYMBOL_GPL(spi_mem_adjust_op_freq); 637 638 /** 639 * spi_mem_calc_op_duration() - Derives the theoretical length (in ns) of an 640 * operation. This helps finding the best variant 641 * among a list of possible choices. 642 * @mem: the SPI memory 643 * @op: the operation to benchmark 644 * 645 * Some chips have per-op frequency limitations, PCBs usually have their own 646 * limitations as well, and controllers can support dual, quad or even octal 647 * modes, sometimes in DTR. All these combinations make it impossible to 648 * statically list the best combination for all situations. If we want something 649 * accurate, all these combinations should be rated (eg. with a time estimate) 650 * and the best pick should be taken based on these calculations. 651 * 652 * Returns a ns estimate for the time this op would take, except if no 653 * frequency limit has been set, in this case we return the number of 654 * cycles nevertheless to allow callers to distinguish which operation 655 * would be the fastest at iso-frequency. 656 */ 657 u64 spi_mem_calc_op_duration(struct spi_mem *mem, struct spi_mem_op *op) 658 { 659 u64 ncycles = 0; 660 u64 ps_per_cycles, duration; 661 662 spi_mem_adjust_op_freq(mem, op); 663 664 if (op->max_freq) { 665 ps_per_cycles = 1000000000000ULL; 666 do_div(ps_per_cycles, op->max_freq); 667 } else { 668 /* In this case, the unit is no longer a time unit */ 669 ps_per_cycles = 1; 670 } 671 672 ncycles += ((op->cmd.nbytes * 8) / op->cmd.buswidth) / (op->cmd.dtr ? 2 : 1); 673 ncycles += ((op->addr.nbytes * 8) / op->addr.buswidth) / (op->addr.dtr ? 2 : 1); 674 675 /* Dummy bytes are optional for some SPI flash memory operations */ 676 if (op->dummy.nbytes) 677 ncycles += ((op->dummy.nbytes * 8) / op->dummy.buswidth) / (op->dummy.dtr ? 2 : 1); 678 679 ncycles += ((op->data.nbytes * 8) / op->data.buswidth) / (op->data.dtr ? 2 : 1); 680 681 /* Derive the duration in ps */ 682 duration = ncycles * ps_per_cycles; 683 /* Convert into ns */ 684 do_div(duration, 1000); 685 686 return duration; 687 } 688 EXPORT_SYMBOL_GPL(spi_mem_calc_op_duration); 689 690 static ssize_t spi_mem_no_dirmap_read(struct spi_mem_dirmap_desc *desc, 691 u64 offs, size_t len, void *buf) 692 { 693 struct spi_mem_op op = *desc->info.op_tmpl; 694 int ret; 695 696 op.addr.val = desc->info.offset + offs; 697 op.data.buf.in = buf; 698 op.data.nbytes = len; 699 ret = spi_mem_adjust_op_size(desc->mem, &op); 700 if (ret) 701 return ret; 702 703 ret = spi_mem_exec_op(desc->mem, &op); 704 if (ret) 705 return ret; 706 707 return op.data.nbytes; 708 } 709 710 static ssize_t spi_mem_no_dirmap_write(struct spi_mem_dirmap_desc *desc, 711 u64 offs, size_t len, const void *buf) 712 { 713 struct spi_mem_op op = *desc->info.op_tmpl; 714 int ret; 715 716 op.addr.val = desc->info.offset + offs; 717 op.data.buf.out = buf; 718 op.data.nbytes = len; 719 ret = spi_mem_adjust_op_size(desc->mem, &op); 720 if (ret) 721 return ret; 722 723 ret = spi_mem_exec_op(desc->mem, &op); 724 if (ret) 725 return ret; 726 727 return op.data.nbytes; 728 } 729 730 /** 731 * spi_mem_dirmap_create() - Create a direct mapping descriptor 732 * @mem: SPI mem device this direct mapping should be created for 733 * @info: direct mapping information 734 * 735 * This function is creating a direct mapping descriptor which can then be used 736 * to access the memory using spi_mem_dirmap_read() or spi_mem_dirmap_write(). 737 * If the SPI controller driver does not support direct mapping, this function 738 * falls back to an implementation using spi_mem_exec_op(), so that the caller 739 * doesn't have to bother implementing a fallback on his own. 740 * 741 * Return: a valid pointer in case of success, and ERR_PTR() otherwise. 742 */ 743 struct spi_mem_dirmap_desc * 744 spi_mem_dirmap_create(struct spi_mem *mem, 745 const struct spi_mem_dirmap_info *info) 746 { 747 struct spi_controller *ctlr = mem->spi->controller; 748 struct spi_mem_dirmap_desc *desc; 749 int ret = -ENOTSUPP; 750 751 /* Make sure the number of address cycles is between 1 and 8 bytes. */ 752 if (!info->primary_op_tmpl.addr.nbytes || info->primary_op_tmpl.addr.nbytes > 8) 753 return ERR_PTR(-EINVAL); 754 755 /* data.dir should either be SPI_MEM_DATA_IN or SPI_MEM_DATA_OUT. */ 756 if (info->primary_op_tmpl.data.dir == SPI_MEM_NO_DATA) 757 return ERR_PTR(-EINVAL); 758 759 /* Apply similar constraints to the secondary template */ 760 if (info->secondary_op_tmpl.cmd.opcode) { 761 if (!info->secondary_op_tmpl.addr.nbytes || 762 info->secondary_op_tmpl.addr.nbytes > 8) 763 return ERR_PTR(-EINVAL); 764 765 if (info->secondary_op_tmpl.data.dir == SPI_MEM_NO_DATA) 766 return ERR_PTR(-EINVAL); 767 768 if (!spi_mem_supports_op(mem, &info->secondary_op_tmpl)) 769 return ERR_PTR(-EOPNOTSUPP); 770 771 if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create && 772 !spi_mem_controller_is_capable(ctlr, secondary_op_tmpl)) 773 return ERR_PTR(-EOPNOTSUPP); 774 } 775 776 desc = kzalloc_obj(*desc); 777 if (!desc) 778 return ERR_PTR(-ENOMEM); 779 780 desc->mem = mem; 781 desc->info = *info; 782 desc->info.op_tmpl = &desc->info.primary_op_tmpl; 783 if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create) { 784 ret = spi_mem_access_start(mem); 785 if (ret) { 786 kfree(desc); 787 return ERR_PTR(ret); 788 } 789 790 ret = ctlr->mem_ops->dirmap_create(desc); 791 792 spi_mem_access_end(mem); 793 } 794 795 if (ret) { 796 desc->nodirmap = true; 797 if (!spi_mem_supports_op(desc->mem, &desc->info.primary_op_tmpl)) 798 ret = -EOPNOTSUPP; 799 else 800 ret = 0; 801 } 802 803 if (ret) { 804 kfree(desc); 805 return ERR_PTR(ret); 806 } 807 808 return desc; 809 } 810 EXPORT_SYMBOL_GPL(spi_mem_dirmap_create); 811 812 /** 813 * spi_mem_dirmap_destroy() - Destroy a direct mapping descriptor 814 * @desc: the direct mapping descriptor to destroy 815 * 816 * This function destroys a direct mapping descriptor previously created by 817 * spi_mem_dirmap_create(). 818 */ 819 void spi_mem_dirmap_destroy(struct spi_mem_dirmap_desc *desc) 820 { 821 struct spi_controller *ctlr = desc->mem->spi->controller; 822 823 if (!desc->nodirmap && ctlr->mem_ops && ctlr->mem_ops->dirmap_destroy) 824 ctlr->mem_ops->dirmap_destroy(desc); 825 826 kfree(desc); 827 } 828 EXPORT_SYMBOL_GPL(spi_mem_dirmap_destroy); 829 830 static void devm_spi_mem_dirmap_release(struct device *dev, void *res) 831 { 832 struct spi_mem_dirmap_desc *desc = *(struct spi_mem_dirmap_desc **)res; 833 834 spi_mem_dirmap_destroy(desc); 835 } 836 837 /** 838 * devm_spi_mem_dirmap_create() - Create a direct mapping descriptor and attach 839 * it to a device 840 * @dev: device the dirmap desc will be attached to 841 * @mem: SPI mem device this direct mapping should be created for 842 * @info: direct mapping information 843 * 844 * devm_ variant of the spi_mem_dirmap_create() function. See 845 * spi_mem_dirmap_create() for more details. 846 * 847 * Return: a valid pointer in case of success, and ERR_PTR() otherwise. 848 */ 849 struct spi_mem_dirmap_desc * 850 devm_spi_mem_dirmap_create(struct device *dev, struct spi_mem *mem, 851 const struct spi_mem_dirmap_info *info) 852 { 853 struct spi_mem_dirmap_desc **ptr, *desc; 854 855 ptr = devres_alloc(devm_spi_mem_dirmap_release, sizeof(*ptr), 856 GFP_KERNEL); 857 if (!ptr) 858 return ERR_PTR(-ENOMEM); 859 860 desc = spi_mem_dirmap_create(mem, info); 861 if (IS_ERR(desc)) { 862 devres_free(ptr); 863 } else { 864 *ptr = desc; 865 devres_add(dev, ptr); 866 } 867 868 return desc; 869 } 870 EXPORT_SYMBOL_GPL(devm_spi_mem_dirmap_create); 871 872 static int devm_spi_mem_dirmap_match(struct device *dev, void *res, void *data) 873 { 874 struct spi_mem_dirmap_desc **ptr = res; 875 876 if (WARN_ON(!ptr || !*ptr)) 877 return 0; 878 879 return *ptr == data; 880 } 881 882 /** 883 * devm_spi_mem_dirmap_destroy() - Destroy a direct mapping descriptor attached 884 * to a device 885 * @dev: device the dirmap desc is attached to 886 * @desc: the direct mapping descriptor to destroy 887 * 888 * devm_ variant of the spi_mem_dirmap_destroy() function. See 889 * spi_mem_dirmap_destroy() for more details. 890 */ 891 void devm_spi_mem_dirmap_destroy(struct device *dev, 892 struct spi_mem_dirmap_desc *desc) 893 { 894 devres_release(dev, devm_spi_mem_dirmap_release, 895 devm_spi_mem_dirmap_match, desc); 896 } 897 EXPORT_SYMBOL_GPL(devm_spi_mem_dirmap_destroy); 898 899 /** 900 * spi_mem_dirmap_read() - Read data through a direct mapping 901 * @desc: direct mapping descriptor 902 * @offs: offset to start reading from. Note that this is not an absolute 903 * offset, but the offset within the direct mapping which already has 904 * its own offset 905 * @len: length in bytes 906 * @buf: destination buffer. This buffer must be DMA-able 907 * 908 * This function reads data from a memory device using a direct mapping 909 * previously instantiated with spi_mem_dirmap_create(). 910 * 911 * Return: the amount of data read from the memory device or a negative error 912 * code. Note that the returned size might be smaller than @len, and the caller 913 * is responsible for calling spi_mem_dirmap_read() again when that happens. 914 */ 915 ssize_t spi_mem_dirmap_read(struct spi_mem_dirmap_desc *desc, 916 u64 offs, size_t len, void *buf) 917 { 918 struct spi_controller *ctlr = desc->mem->spi->controller; 919 ssize_t ret; 920 921 if (desc->info.op_tmpl->data.dir != SPI_MEM_DATA_IN) 922 return -EINVAL; 923 924 if (!len) 925 return 0; 926 927 if (desc->nodirmap) { 928 ret = spi_mem_no_dirmap_read(desc, offs, len, buf); 929 } else if (ctlr->mem_ops && ctlr->mem_ops->dirmap_read) { 930 ret = spi_mem_access_start(desc->mem); 931 if (ret) 932 return ret; 933 934 ret = ctlr->mem_ops->dirmap_read(desc, offs, len, buf); 935 936 spi_mem_access_end(desc->mem); 937 } else { 938 ret = -ENOTSUPP; 939 } 940 941 return ret; 942 } 943 EXPORT_SYMBOL_GPL(spi_mem_dirmap_read); 944 945 /** 946 * spi_mem_dirmap_write() - Write data through a direct mapping 947 * @desc: direct mapping descriptor 948 * @offs: offset to start writing from. Note that this is not an absolute 949 * offset, but the offset within the direct mapping which already has 950 * its own offset 951 * @len: length in bytes 952 * @buf: source buffer. This buffer must be DMA-able 953 * 954 * This function writes data to a memory device using a direct mapping 955 * previously instantiated with spi_mem_dirmap_create(). 956 * 957 * Return: the amount of data written to the memory device or a negative error 958 * code. Note that the returned size might be smaller than @len, and the caller 959 * is responsible for calling spi_mem_dirmap_write() again when that happens. 960 */ 961 ssize_t spi_mem_dirmap_write(struct spi_mem_dirmap_desc *desc, 962 u64 offs, size_t len, const void *buf) 963 { 964 struct spi_controller *ctlr = desc->mem->spi->controller; 965 ssize_t ret; 966 967 if (desc->info.op_tmpl->data.dir != SPI_MEM_DATA_OUT) 968 return -EINVAL; 969 970 if (!len) 971 return 0; 972 973 if (desc->nodirmap) { 974 ret = spi_mem_no_dirmap_write(desc, offs, len, buf); 975 } else if (ctlr->mem_ops && ctlr->mem_ops->dirmap_write) { 976 ret = spi_mem_access_start(desc->mem); 977 if (ret) 978 return ret; 979 980 ret = ctlr->mem_ops->dirmap_write(desc, offs, len, buf); 981 982 spi_mem_access_end(desc->mem); 983 } else { 984 ret = -ENOTSUPP; 985 } 986 987 return ret; 988 } 989 EXPORT_SYMBOL_GPL(spi_mem_dirmap_write); 990 991 static inline struct spi_mem_driver *to_spi_mem_drv(struct device_driver *drv) 992 { 993 return container_of(drv, struct spi_mem_driver, spidrv.driver); 994 } 995 996 static int spi_mem_read_status(struct spi_mem *mem, 997 const struct spi_mem_op *op, 998 u16 *status) 999 { 1000 const u8 *bytes = (u8 *)op->data.buf.in; 1001 int ret; 1002 1003 ret = spi_mem_exec_op(mem, op); 1004 if (ret) 1005 return ret; 1006 1007 if (op->data.nbytes > 1) 1008 *status = ((u16)bytes[0] << 8) | bytes[1]; 1009 else 1010 *status = bytes[0]; 1011 1012 return 0; 1013 } 1014 1015 /** 1016 * spi_mem_poll_status() - Poll memory device status 1017 * @mem: SPI memory device 1018 * @op: the memory operation to execute 1019 * @mask: status bitmask to ckeck 1020 * @match: (status & mask) expected value 1021 * @initial_delay_us: delay in us before starting to poll 1022 * @polling_delay_us: time to sleep between reads in us 1023 * @timeout_ms: timeout in milliseconds 1024 * 1025 * This function polls a status register and returns when 1026 * (status & mask) == match or when the timeout has expired. 1027 * 1028 * Return: 0 in case of success, -ETIMEDOUT in case of error, 1029 * -EOPNOTSUPP if not supported. 1030 */ 1031 int spi_mem_poll_status(struct spi_mem *mem, 1032 const struct spi_mem_op *op, 1033 u16 mask, u16 match, 1034 unsigned long initial_delay_us, 1035 unsigned long polling_delay_us, 1036 u16 timeout_ms) 1037 { 1038 struct spi_controller *ctlr = mem->spi->controller; 1039 int ret = -EOPNOTSUPP; 1040 int read_status_ret; 1041 u16 status; 1042 1043 if (op->data.nbytes < 1 || op->data.nbytes > 2 || 1044 op->data.dir != SPI_MEM_DATA_IN) 1045 return -EINVAL; 1046 1047 if (ctlr->mem_ops && ctlr->mem_ops->poll_status && !spi_get_csgpiod(mem->spi, 0)) { 1048 ret = spi_mem_access_start(mem); 1049 if (ret) 1050 return ret; 1051 1052 ret = ctlr->mem_ops->poll_status(mem, op, mask, match, 1053 initial_delay_us, polling_delay_us, 1054 timeout_ms); 1055 1056 spi_mem_access_end(mem); 1057 } 1058 1059 if (ret == -EOPNOTSUPP) { 1060 if (!spi_mem_supports_op(mem, op)) 1061 return ret; 1062 1063 if (initial_delay_us < 10) 1064 udelay(initial_delay_us); 1065 else 1066 usleep_range((initial_delay_us >> 2) + 1, 1067 initial_delay_us); 1068 1069 ret = read_poll_timeout(spi_mem_read_status, read_status_ret, 1070 (read_status_ret || ((status) & mask) == match), 1071 polling_delay_us, timeout_ms * 1000, false, mem, 1072 op, &status); 1073 if (read_status_ret) 1074 return read_status_ret; 1075 } 1076 1077 return ret; 1078 } 1079 EXPORT_SYMBOL_GPL(spi_mem_poll_status); 1080 1081 static int spi_mem_probe(struct spi_device *spi) 1082 { 1083 struct spi_mem_driver *memdrv = to_spi_mem_drv(spi->dev.driver); 1084 struct spi_controller *ctlr = spi->controller; 1085 struct spi_mem *mem; 1086 1087 mem = devm_kzalloc(&spi->dev, sizeof(*mem), GFP_KERNEL); 1088 if (!mem) 1089 return -ENOMEM; 1090 1091 mem->spi = spi; 1092 1093 if (ctlr->mem_ops && ctlr->mem_ops->get_name) 1094 mem->name = ctlr->mem_ops->get_name(mem); 1095 else 1096 mem->name = dev_name(&spi->dev); 1097 1098 if (IS_ERR_OR_NULL(mem->name)) 1099 return PTR_ERR_OR_ZERO(mem->name); 1100 1101 spi_set_drvdata(spi, mem); 1102 1103 return memdrv->probe(mem); 1104 } 1105 1106 static void spi_mem_remove(struct spi_device *spi) 1107 { 1108 struct spi_mem_driver *memdrv = to_spi_mem_drv(spi->dev.driver); 1109 struct spi_mem *mem = spi_get_drvdata(spi); 1110 1111 if (memdrv->remove) 1112 memdrv->remove(mem); 1113 } 1114 1115 static void spi_mem_shutdown(struct spi_device *spi) 1116 { 1117 struct spi_mem_driver *memdrv = to_spi_mem_drv(spi->dev.driver); 1118 struct spi_mem *mem = spi_get_drvdata(spi); 1119 1120 if (memdrv->shutdown) 1121 memdrv->shutdown(mem); 1122 } 1123 1124 /** 1125 * spi_mem_driver_register_with_owner() - Register a SPI memory driver 1126 * @memdrv: the SPI memory driver to register 1127 * @owner: the owner of this driver 1128 * 1129 * Registers a SPI memory driver. 1130 * 1131 * Return: 0 in case of success, a negative error core otherwise. 1132 */ 1133 1134 int spi_mem_driver_register_with_owner(struct spi_mem_driver *memdrv, 1135 struct module *owner) 1136 { 1137 memdrv->spidrv.probe = spi_mem_probe; 1138 memdrv->spidrv.remove = spi_mem_remove; 1139 memdrv->spidrv.shutdown = spi_mem_shutdown; 1140 1141 return __spi_register_driver(owner, &memdrv->spidrv); 1142 } 1143 EXPORT_SYMBOL_GPL(spi_mem_driver_register_with_owner); 1144 1145 /** 1146 * spi_mem_driver_unregister() - Unregister a SPI memory driver 1147 * @memdrv: the SPI memory driver to unregister 1148 * 1149 * Unregisters a SPI memory driver. 1150 */ 1151 void spi_mem_driver_unregister(struct spi_mem_driver *memdrv) 1152 { 1153 spi_unregister_driver(&memdrv->spidrv); 1154 } 1155 EXPORT_SYMBOL_GPL(spi_mem_driver_unregister); 1156