1 // SPDX-License-Identifier: GPL-2.0+
12 #include <linux/spi/spi-mem.h>
20  * spi_controller_dma_map_mem_op_data() - DMA-map the buffer attached to a
21  *					  memory operation
23  * @op: the memory operation containing the buffer to map
24  * @sgt: a pointer to a non-initialized sg_table that will be filled by this
28  * This helper prepares everything for you and provides a ready-to-use
31  * Note that the caller must ensure the memory region pointed by
32  * op->data.buf.{in,out} is DMA-able before calling this function.
42 	if (!op->data.nbytes)  in spi_controller_dma_map_mem_op_data()
43 		return -EINVAL;  in spi_controller_dma_map_mem_op_data()
45 	if (op->data.dir == SPI_MEM_DATA_OUT && ctlr->dma_tx)  in spi_controller_dma_map_mem_op_data()
46 		dmadev = ctlr->dma_tx->device->dev;  in spi_controller_dma_map_mem_op_data()
47 	else if (op->data.dir == SPI_MEM_DATA_IN && ctlr->dma_rx)  in spi_controller_dma_map_mem_op_data()
48 		dmadev = ctlr->dma_rx->device->dev;  in spi_controller_dma_map_mem_op_data()
50 		dmadev = ctlr->dev.parent;  in spi_controller_dma_map_mem_op_data()
53 		return -EINVAL;  in spi_controller_dma_map_mem_op_data()
55 	return spi_map_buf(ctlr, dmadev, sgt, op->data.buf.in, op->data.nbytes,  in spi_controller_dma_map_mem_op_data()
56 			   op->data.dir == SPI_MEM_DATA_IN ?  in spi_controller_dma_map_mem_op_data()
62  * spi_controller_dma_unmap_mem_op_data() - DMA-unmap the buffer attached to a
63  *					    memory operation
65  * @op: the memory operation containing the buffer to unmap
71  * op->data.buf.{in,out} buffer again.
88 	if (!op->data.nbytes)  in spi_controller_dma_unmap_mem_op_data()
91 	if (op->data.dir == SPI_MEM_DATA_OUT && ctlr->dma_tx)  in spi_controller_dma_unmap_mem_op_data()
92 		dmadev = ctlr->dma_tx->device->dev;  in spi_controller_dma_unmap_mem_op_data()
93 	else if (op->data.dir == SPI_MEM_DATA_IN && ctlr->dma_rx)  in spi_controller_dma_unmap_mem_op_data()
94 		dmadev = ctlr->dma_rx->device->dev;  in spi_controller_dma_unmap_mem_op_data()
96 		dmadev = ctlr->dev.parent;  in spi_controller_dma_unmap_mem_op_data()
99 		      op->data.dir == SPI_MEM_DATA_IN ?  in spi_controller_dma_unmap_mem_op_data()
106 	u32 mode = mem->spi->mode;  in spi_check_buswidth_req()
139 	return -ENOTSUPP;  in spi_check_buswidth_req()
145 	if (spi_check_buswidth_req(mem, op->cmd.buswidth, true))  in spi_mem_check_buswidth()
148 	if (op->addr.nbytes &&  in spi_mem_check_buswidth()
149 	    spi_check_buswidth_req(mem, op->addr.buswidth, true))  in spi_mem_check_buswidth()
152 	if (op->dummy.nbytes &&  in spi_mem_check_buswidth()
153 	    spi_check_buswidth_req(mem, op->dummy.buswidth, true))  in spi_mem_check_buswidth()
156 	if (op->data.dir != SPI_MEM_NO_DATA &&  in spi_mem_check_buswidth()
157 	    spi_check_buswidth_req(mem, op->data.buswidth,  in spi_mem_check_buswidth()
158 				   op->data.dir == SPI_MEM_DATA_OUT))  in spi_mem_check_buswidth()
167 	struct spi_controller *ctlr = mem->spi->controller;  in spi_mem_default_supports_op()
169 		op->cmd.dtr || op->addr.dtr || op->dummy.dtr || op->data.dtr;  in spi_mem_default_supports_op()
175 		if (op->data.swap16 && !spi_mem_controller_is_capable(ctlr, swap16))  in spi_mem_default_supports_op()
178 		if (op->cmd.nbytes != 2)  in spi_mem_default_supports_op()
181 		if (op->cmd.nbytes != 1)  in spi_mem_default_supports_op()
185 	if (op->data.ecc) {  in spi_mem_default_supports_op()
190 	if (op->max_freq && mem->spi->controller->min_speed_hz &&  in spi_mem_default_supports_op()
191 	    op->max_freq < mem->spi->controller->min_speed_hz)  in spi_mem_default_supports_op()
194 	if (op->max_freq &&  in spi_mem_default_supports_op()
195 	    op->max_freq < mem->spi->max_speed_hz) {  in spi_mem_default_supports_op()
214 	if (!op->cmd.buswidth || !op->cmd.nbytes)  in spi_mem_check_op()
215 		return -EINVAL;  in spi_mem_check_op()
217 	if ((op->addr.nbytes && !op->addr.buswidth) ||  in spi_mem_check_op()
218 	    (op->dummy.nbytes && !op->dummy.buswidth) ||  in spi_mem_check_op()
219 	    (op->data.nbytes && !op->data.buswidth))  in spi_mem_check_op()
220 		return -EINVAL;  in spi_mem_check_op()
222 	if (!spi_mem_buswidth_is_valid(op->cmd.buswidth) ||  in spi_mem_check_op()
223 	    !spi_mem_buswidth_is_valid(op->addr.buswidth) ||  in spi_mem_check_op()
224 	    !spi_mem_buswidth_is_valid(op->dummy.buswidth) ||  in spi_mem_check_op()
225 	    !spi_mem_buswidth_is_valid(op->data.buswidth))  in spi_mem_check_op()
226 		return -EINVAL;  in spi_mem_check_op()
228 	/* Buffers must be DMA-able. */  in spi_mem_check_op()
229 	if (WARN_ON_ONCE(op->data.dir == SPI_MEM_DATA_IN &&  in spi_mem_check_op()
230 			 object_is_on_stack(op->data.buf.in)))  in spi_mem_check_op()
231 		return -EINVAL;  in spi_mem_check_op()
233 	if (WARN_ON_ONCE(op->data.dir == SPI_MEM_DATA_OUT &&  in spi_mem_check_op()
234 			 object_is_on_stack(op->data.buf.out)))  in spi_mem_check_op()
235 		return -EINVAL;  in spi_mem_check_op()
243 	struct spi_controller *ctlr = mem->spi->controller;  in spi_mem_internal_supports_op()
245 	if (ctlr->mem_ops && ctlr->mem_ops->supports_op)  in spi_mem_internal_supports_op()
246 		return ctlr->mem_ops->supports_op(mem, op);  in spi_mem_internal_supports_op()
252  * spi_mem_supports_op() - Check if a memory device and the controller it is
253  *			   connected to support a specific memory operation
254  * @mem: the SPI memory
255  * @op: the memory operation to check
280 	struct spi_controller *ctlr = mem->spi->controller;  in spi_mem_access_start()
283 	 * Flush the message queue before executing our SPI memory  in spi_mem_access_start()
288 	if (ctlr->auto_runtime_pm) {  in spi_mem_access_start()
291 		ret = pm_runtime_resume_and_get(ctlr->dev.parent);  in spi_mem_access_start()
293 			dev_err(&ctlr->dev, "Failed to power device: %d\n",  in spi_mem_access_start()
299 	mutex_lock(&ctlr->bus_lock_mutex);  in spi_mem_access_start()
300 	mutex_lock(&ctlr->io_mutex);  in spi_mem_access_start()
307 	struct spi_controller *ctlr = mem->spi->controller;  in spi_mem_access_end()
309 	mutex_unlock(&ctlr->io_mutex);  in spi_mem_access_end()
310 	mutex_unlock(&ctlr->bus_lock_mutex);  in spi_mem_access_end()
312 	if (ctlr->auto_runtime_pm)  in spi_mem_access_end()
313 		pm_runtime_put(ctlr->dev.parent);  in spi_mem_access_end()
324 	u64_stats_update_begin(&stats->syncp);  in spi_mem_add_op_stats()
330 	u64_stats_inc(&stats->messages);  in spi_mem_add_op_stats()
331 	u64_stats_inc(&stats->transfers);  in spi_mem_add_op_stats()
334 	len = op->cmd.nbytes + op->addr.nbytes;  in spi_mem_add_op_stats()
335 	len += op->dummy.nbytes + op->data.nbytes;  in spi_mem_add_op_stats()
336 	u64_stats_add(&stats->bytes, len);  in spi_mem_add_op_stats()
337 	l2len = min(fls(len), SPI_STATISTICS_HISTO_SIZE) - 1;  in spi_mem_add_op_stats()
338 	u64_stats_inc(&stats->transfer_bytes_histo[l2len]);  in spi_mem_add_op_stats()
341 	if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_OUT)  in spi_mem_add_op_stats()
342 		u64_stats_add(&stats->bytes_tx, op->data.nbytes);  in spi_mem_add_op_stats()
343 	if (op->data.nbytes && op->data.dir == SPI_MEM_DATA_IN)  in spi_mem_add_op_stats()
344 		u64_stats_add(&stats->bytes_rx, op->data.nbytes);  in spi_mem_add_op_stats()
350 	if (exec_op_ret == -ETIMEDOUT)  in spi_mem_add_op_stats()
351 		u64_stats_inc(&stats->timedout);  in spi_mem_add_op_stats()
353 		u64_stats_inc(&stats->errors);  in spi_mem_add_op_stats()
355 	u64_stats_update_end(&stats->syncp);  in spi_mem_add_op_stats()
360  * spi_mem_exec_op() - Execute a memory operation
361  * @mem: the SPI memory
362  * @op: the memory operation to execute
364  * Executes a memory operation.
374 	struct spi_controller *ctlr = mem->spi->controller;  in spi_mem_exec_op()
383 …dev_vdbg(&mem->spi->dev, "[cmd: 0x%02x][%dB addr: %#8llx][%2dB dummy][%4dB data %s] %d%c-%d%c-%d%c…  in spi_mem_exec_op()
384 		 op->cmd.opcode,  in spi_mem_exec_op()
385 		 op->addr.nbytes, (op->addr.nbytes ? op->addr.val : 0),  in spi_mem_exec_op()
386 		 op->dummy.nbytes,  in spi_mem_exec_op()
387 …op->data.nbytes, (op->data.nbytes ? (op->data.dir == SPI_MEM_DATA_IN ? " read" : "write") : "     …  in spi_mem_exec_op()
388 		 op->cmd.buswidth, op->cmd.dtr ? 'D' : 'S',  in spi_mem_exec_op()
389 		 op->addr.buswidth, op->addr.dtr ? 'D' : 'S',  in spi_mem_exec_op()
390 		 op->dummy.buswidth, op->dummy.dtr ? 'D' : 'S',  in spi_mem_exec_op()
391 		 op->data.buswidth, op->data.dtr ? 'D' : 'S',  in spi_mem_exec_op()
392 		 op->max_freq ? op->max_freq : mem->spi->max_speed_hz);  in spi_mem_exec_op()
399 		return -EOPNOTSUPP;  in spi_mem_exec_op()
401 	if (ctlr->mem_ops && ctlr->mem_ops->exec_op && !spi_get_csgpiod(mem->spi, 0)) {  in spi_mem_exec_op()
406 		ret = ctlr->mem_ops->exec_op(mem, op);  in spi_mem_exec_op()
415 		if (!ret || (ret != -ENOTSUPP && ret != -EOPNOTSUPP)) {  in spi_mem_exec_op()
416 			spi_mem_add_op_stats(ctlr->pcpu_statistics, op, ret);  in spi_mem_exec_op()
417 			spi_mem_add_op_stats(mem->spi->pcpu_statistics, op, ret);  in spi_mem_exec_op()
423 	tmpbufsize = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;  in spi_mem_exec_op()
427 	 * we're guaranteed that this buffer is DMA-able, as required by the  in spi_mem_exec_op()
432 		return -ENOMEM;  in spi_mem_exec_op()
436 	tmpbuf[0] = op->cmd.opcode;  in spi_mem_exec_op()
438 	xfers[xferpos].len = op->cmd.nbytes;  in spi_mem_exec_op()
439 	xfers[xferpos].tx_nbits = op->cmd.buswidth;  in spi_mem_exec_op()
440 	xfers[xferpos].speed_hz = op->max_freq;  in spi_mem_exec_op()
445 	if (op->addr.nbytes) {  in spi_mem_exec_op()
448 		for (i = 0; i < op->addr.nbytes; i++)  in spi_mem_exec_op()
449 			tmpbuf[i + 1] = op->addr.val >>  in spi_mem_exec_op()
450 					(8 * (op->addr.nbytes - i - 1));  in spi_mem_exec_op()
453 		xfers[xferpos].len = op->addr.nbytes;  in spi_mem_exec_op()
454 		xfers[xferpos].tx_nbits = op->addr.buswidth;  in spi_mem_exec_op()
455 		xfers[xferpos].speed_hz = op->max_freq;  in spi_mem_exec_op()
458 		totalxferlen += op->addr.nbytes;  in spi_mem_exec_op()
461 	if (op->dummy.nbytes) {  in spi_mem_exec_op()
462 		memset(tmpbuf + op->addr.nbytes + 1, 0xff, op->dummy.nbytes);  in spi_mem_exec_op()
463 		xfers[xferpos].tx_buf = tmpbuf + op->addr.nbytes + 1;  in spi_mem_exec_op()
464 		xfers[xferpos].len = op->dummy.nbytes;  in spi_mem_exec_op()
465 		xfers[xferpos].tx_nbits = op->dummy.buswidth;  in spi_mem_exec_op()
467 		xfers[xferpos].speed_hz = op->max_freq;  in spi_mem_exec_op()
470 		totalxferlen += op->dummy.nbytes;  in spi_mem_exec_op()
473 	if (op->data.nbytes) {  in spi_mem_exec_op()
474 		if (op->data.dir == SPI_MEM_DATA_IN) {  in spi_mem_exec_op()
475 			xfers[xferpos].rx_buf = op->data.buf.in;  in spi_mem_exec_op()
476 			xfers[xferpos].rx_nbits = op->data.buswidth;  in spi_mem_exec_op()
478 			xfers[xferpos].tx_buf = op->data.buf.out;  in spi_mem_exec_op()
479 			xfers[xferpos].tx_nbits = op->data.buswidth;  in spi_mem_exec_op()
482 		xfers[xferpos].len = op->data.nbytes;  in spi_mem_exec_op()
483 		xfers[xferpos].speed_hz = op->max_freq;  in spi_mem_exec_op()
486 		totalxferlen += op->data.nbytes;  in spi_mem_exec_op()
489 	ret = spi_sync(mem->spi, &msg);  in spi_mem_exec_op()
497 		return -EIO;  in spi_mem_exec_op()
504  * spi_mem_get_name() - Return the SPI mem device name to be used by the
506  * @mem: the SPI memory
512  * Return: a string containing the name of the memory device to be used
517 	return mem->name;  in spi_mem_get_name()
522  * spi_mem_adjust_op_size() - Adjust the data size of a SPI mem operation to
524  * @mem: the SPI memory
530  * operation into multiple sub-operations when required.
533  *	   0 otherwise. Note that @op->data.nbytes will be updated if @op
538 	struct spi_controller *ctlr = mem->spi->controller;  in spi_mem_adjust_op_size()
541 	if (ctlr->mem_ops && ctlr->mem_ops->adjust_op_size)  in spi_mem_adjust_op_size()
542 		return ctlr->mem_ops->adjust_op_size(mem, op);  in spi_mem_adjust_op_size()
544 	if (!ctlr->mem_ops || !ctlr->mem_ops->exec_op) {  in spi_mem_adjust_op_size()
545 		len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;  in spi_mem_adjust_op_size()
547 		if (len > spi_max_transfer_size(mem->spi))  in spi_mem_adjust_op_size()
548 			return -EINVAL;  in spi_mem_adjust_op_size()
550 		op->data.nbytes = min3((size_t)op->data.nbytes,  in spi_mem_adjust_op_size()
551 				       spi_max_transfer_size(mem->spi),  in spi_mem_adjust_op_size()
552 				       spi_max_message_size(mem->spi) -  in spi_mem_adjust_op_size()
554 		if (!op->data.nbytes)  in spi_mem_adjust_op_size()
555 			return -EINVAL;  in spi_mem_adjust_op_size()
563  * spi_mem_adjust_op_freq() - Adjust the frequency of a SPI mem operation to
565  * @mem: the SPI memory
568  * Some chips have per-op frequency limitations and must adapt the maximum
569  * speed. This function allows SPI mem drivers to set @op->max_freq to the
574 	if (!op->max_freq || op->max_freq > mem->spi->max_speed_hz)  in spi_mem_adjust_op_freq()
575 		op->max_freq = mem->spi->max_speed_hz;  in spi_mem_adjust_op_freq()
580  * spi_mem_calc_op_duration() - Derives the theoretical length (in ns) of an
583  * @mem: the SPI memory
586  * Some chips have per-op frequency limitations, PCBs usually have their own
596  * would be the fastest at iso-frequency.
605 	if (op->max_freq) {  in spi_mem_calc_op_duration()
607 		do_div(ps_per_cycles, op->max_freq);  in spi_mem_calc_op_duration()
613 	ncycles += ((op->cmd.nbytes * 8) / op->cmd.buswidth) / (op->cmd.dtr ? 2 : 1);  in spi_mem_calc_op_duration()
614 	ncycles += ((op->addr.nbytes * 8) / op->addr.buswidth) / (op->addr.dtr ? 2 : 1);  in spi_mem_calc_op_duration()
616 	/* Dummy bytes are optional for some SPI flash memory operations */  in spi_mem_calc_op_duration()
617 	if (op->dummy.nbytes)  in spi_mem_calc_op_duration()
618 		ncycles += ((op->dummy.nbytes * 8) / op->dummy.buswidth) / (op->dummy.dtr ? 2 : 1);  in spi_mem_calc_op_duration()
620 	ncycles += ((op->data.nbytes * 8) / op->data.buswidth) / (op->data.dtr ? 2 : 1);  in spi_mem_calc_op_duration()
634 	struct spi_mem_op op = desc->info.op_tmpl;  in spi_mem_no_dirmap_read()
637 	op.addr.val = desc->info.offset + offs;  in spi_mem_no_dirmap_read()
640 	ret = spi_mem_adjust_op_size(desc->mem, &op);  in spi_mem_no_dirmap_read()
644 	ret = spi_mem_exec_op(desc->mem, &op);  in spi_mem_no_dirmap_read()
654 	struct spi_mem_op op = desc->info.op_tmpl;  in spi_mem_no_dirmap_write()
657 	op.addr.val = desc->info.offset + offs;  in spi_mem_no_dirmap_write()
660 	ret = spi_mem_adjust_op_size(desc->mem, &op);  in spi_mem_no_dirmap_write()
664 	ret = spi_mem_exec_op(desc->mem, &op);  in spi_mem_no_dirmap_write()
672  * spi_mem_dirmap_create() - Create a direct mapping descriptor
673  * @mem: SPI mem device this direct mapping should be created for
674  * @info: direct mapping information
676  * This function is creating a direct mapping descriptor which can then be used
677  * to access the memory using spi_mem_dirmap_read() or spi_mem_dirmap_write().
678  * If the SPI controller driver does not support direct mapping, this function
688 	struct spi_controller *ctlr = mem->spi->controller;  in spi_mem_dirmap_create()
690 	int ret = -ENOTSUPP;  in spi_mem_dirmap_create()
693 	if (!info->op_tmpl.addr.nbytes || info->op_tmpl.addr.nbytes > 8)  in spi_mem_dirmap_create()
694 		return ERR_PTR(-EINVAL);  in spi_mem_dirmap_create()
697 	if (info->op_tmpl.data.dir == SPI_MEM_NO_DATA)  in spi_mem_dirmap_create()
698 		return ERR_PTR(-EINVAL);  in spi_mem_dirmap_create()
702 		return ERR_PTR(-ENOMEM);  in spi_mem_dirmap_create()
704 	desc->mem = mem;  in spi_mem_dirmap_create()
705 	desc->info = *info;  in spi_mem_dirmap_create()
706 	if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create)  in spi_mem_dirmap_create()
707 		ret = ctlr->mem_ops->dirmap_create(desc);  in spi_mem_dirmap_create()
710 		desc->nodirmap = true;  in spi_mem_dirmap_create()
711 		if (!spi_mem_supports_op(desc->mem, &desc->info.op_tmpl))  in spi_mem_dirmap_create()
712 			ret = -EOPNOTSUPP;  in spi_mem_dirmap_create()
727  * spi_mem_dirmap_destroy() - Destroy a direct mapping descriptor
728  * @desc: the direct mapping descriptor to destroy
730  * This function destroys a direct mapping descriptor previously created by
735 	struct spi_controller *ctlr = desc->mem->spi->controller;  in spi_mem_dirmap_destroy()
737 	if (!desc->nodirmap && ctlr->mem_ops && ctlr->mem_ops->dirmap_destroy)  in spi_mem_dirmap_destroy()
738 		ctlr->mem_ops->dirmap_destroy(desc);  in spi_mem_dirmap_destroy()
752  * devm_spi_mem_dirmap_create() - Create a direct mapping descriptor and attach
755  * @mem: SPI mem device this direct mapping should be created for
756  * @info: direct mapping information
772 		return ERR_PTR(-ENOMEM);  in devm_spi_mem_dirmap_create()
797  * devm_spi_mem_dirmap_destroy() - Destroy a direct mapping descriptor attached
800  * @desc: the direct mapping descriptor to destroy
814  * spi_mem_dirmap_read() - Read data through a direct mapping
815  * @desc: direct mapping descriptor
817  *	  offset, but the offset within the direct mapping which already has
820  * @buf: destination buffer. This buffer must be DMA-able
822  * This function reads data from a memory device using a direct mapping
825  * Return: the amount of data read from the memory device or a negative error
832 	struct spi_controller *ctlr = desc->mem->spi->controller;  in spi_mem_dirmap_read()
835 	if (desc->info.op_tmpl.data.dir != SPI_MEM_DATA_IN)  in spi_mem_dirmap_read()
836 		return -EINVAL;  in spi_mem_dirmap_read()
841 	if (desc->nodirmap) {  in spi_mem_dirmap_read()
843 	} else if (ctlr->mem_ops && ctlr->mem_ops->dirmap_read) {  in spi_mem_dirmap_read()
844 		ret = spi_mem_access_start(desc->mem);  in spi_mem_dirmap_read()
848 		ret = ctlr->mem_ops->dirmap_read(desc, offs, len, buf);  in spi_mem_dirmap_read()
850 		spi_mem_access_end(desc->mem);  in spi_mem_dirmap_read()
852 		ret = -ENOTSUPP;  in spi_mem_dirmap_read()
860  * spi_mem_dirmap_write() - Write data through a direct mapping
861  * @desc: direct mapping descriptor
863  *	  offset, but the offset within the direct mapping which already has
866  * @buf: source buffer. This buffer must be DMA-able
868  * This function writes data to a memory device using a direct mapping
871  * Return: the amount of data written to the memory device or a negative error
878 	struct spi_controller *ctlr = desc->mem->spi->controller;  in spi_mem_dirmap_write()
881 	if (desc->info.op_tmpl.data.dir != SPI_MEM_DATA_OUT)  in spi_mem_dirmap_write()
882 		return -EINVAL;  in spi_mem_dirmap_write()
887 	if (desc->nodirmap) {  in spi_mem_dirmap_write()
889 	} else if (ctlr->mem_ops && ctlr->mem_ops->dirmap_write) {  in spi_mem_dirmap_write()
890 		ret = spi_mem_access_start(desc->mem);  in spi_mem_dirmap_write()
894 		ret = ctlr->mem_ops->dirmap_write(desc, offs, len, buf);  in spi_mem_dirmap_write()
896 		spi_mem_access_end(desc->mem);  in spi_mem_dirmap_write()
898 		ret = -ENOTSUPP;  in spi_mem_dirmap_write()
914 	const u8 *bytes = (u8 *)op->data.buf.in;  in spi_mem_read_status()
921 	if (op->data.nbytes > 1)  in spi_mem_read_status()
930  * spi_mem_poll_status() - Poll memory device status
931  * @mem: SPI memory device
932  * @op: the memory operation to execute
942  * Return: 0 in case of success, -ETIMEDOUT in case of error,
943  *         -EOPNOTSUPP if not supported.
952 	struct spi_controller *ctlr = mem->spi->controller;  in spi_mem_poll_status()
953 	int ret = -EOPNOTSUPP;  in spi_mem_poll_status()
957 	if (op->data.nbytes < 1 || op->data.nbytes > 2 ||  in spi_mem_poll_status()
958 	    op->data.dir != SPI_MEM_DATA_IN)  in spi_mem_poll_status()
959 		return -EINVAL;  in spi_mem_poll_status()
961 	if (ctlr->mem_ops && ctlr->mem_ops->poll_status && !spi_get_csgpiod(mem->spi, 0)) {  in spi_mem_poll_status()
966 		ret = ctlr->mem_ops->poll_status(mem, op, mask, match,  in spi_mem_poll_status()
973 	if (ret == -EOPNOTSUPP) {  in spi_mem_poll_status()
997 	struct spi_mem_driver *memdrv = to_spi_mem_drv(spi->dev.driver);  in spi_mem_probe()
998 	struct spi_controller *ctlr = spi->controller;  in spi_mem_probe()
1001 	mem = devm_kzalloc(&spi->dev, sizeof(*mem), GFP_KERNEL);  in spi_mem_probe()
1003 		return -ENOMEM;  in spi_mem_probe()
1005 	mem->spi = spi;  in spi_mem_probe()
1007 	if (ctlr->mem_ops && ctlr->mem_ops->get_name)  in spi_mem_probe()
1008 		mem->name = ctlr->mem_ops->get_name(mem);  in spi_mem_probe()
1010 		mem->name = dev_name(&spi->dev);  in spi_mem_probe()
1012 	if (IS_ERR_OR_NULL(mem->name))  in spi_mem_probe()
1013 		return PTR_ERR_OR_ZERO(mem->name);  in spi_mem_probe()
1017 	return memdrv->probe(mem);  in spi_mem_probe()
1022 	struct spi_mem_driver *memdrv = to_spi_mem_drv(spi->dev.driver);  in spi_mem_remove()
1025 	if (memdrv->remove)  in spi_mem_remove()
1026 		memdrv->remove(mem);  in spi_mem_remove()
1031 	struct spi_mem_driver *memdrv = to_spi_mem_drv(spi->dev.driver);  in spi_mem_shutdown()
1034 	if (memdrv->shutdown)  in spi_mem_shutdown()
1035 		memdrv->shutdown(mem);  in spi_mem_shutdown()
1039  * spi_mem_driver_register_with_owner() - Register a SPI memory driver
1040  * @memdrv: the SPI memory driver to register
1043  * Registers a SPI memory driver.
1051 	memdrv->spidrv.probe = spi_mem_probe;  in spi_mem_driver_register_with_owner()
1052 	memdrv->spidrv.remove = spi_mem_remove;  in spi_mem_driver_register_with_owner()
1053 	memdrv->spidrv.shutdown = spi_mem_shutdown;  in spi_mem_driver_register_with_owner()
1055 	return __spi_register_driver(owner, &memdrv->spidrv);  in spi_mem_driver_register_with_owner()
1060  * spi_mem_driver_unregister() - Unregister a SPI memory driver
1061  * @memdrv: the SPI memory driver to unregister
1063  * Unregisters a SPI memory driver.
1067 	spi_unregister_driver(&memdrv->spidrv);  in spi_mem_driver_unregister()