314 					      struct spi_transfer *xfer,  in spi_statistics_add_transfer_stats()  argument
317 	int l2len = min(fls(xfer->len), SPI_STATISTICS_HISTO_SIZE) - 1;  in spi_statistics_add_transfer_stats()
330 	u64_stats_add(&stats->bytes, xfer->len);  in spi_statistics_add_transfer_stats()
331 	if (spi_valid_txbuf(msg, xfer))  in spi_statistics_add_transfer_stats()
332 		u64_stats_add(&stats->bytes_tx, xfer->len);  in spi_statistics_add_transfer_stats()
333 	if (spi_valid_rxbuf(msg, xfer))  in spi_statistics_add_transfer_stats()
334 		u64_stats_add(&stats->bytes_rx, xfer->len);  in spi_statistics_add_transfer_stats()
1235 	struct spi_transfer *xfer;  in __spi_map_msg()  local
1256 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in __spi_map_msg()
1260 		if (!ctlr->can_dma(ctlr, msg->spi, xfer))  in __spi_map_msg()
1263 		if (xfer->tx_buf != NULL) {  in __spi_map_msg()
1264 			ret = spi_map_buf_attrs(ctlr, tx_dev, &xfer->tx_sg,  in __spi_map_msg()
1265 						(void *)xfer->tx_buf,  in __spi_map_msg()
1266 						xfer->len, DMA_TO_DEVICE,  in __spi_map_msg()
1271 			xfer->tx_sg_mapped = true;  in __spi_map_msg()
1274 		if (xfer->rx_buf != NULL) {  in __spi_map_msg()
1275 			ret = spi_map_buf_attrs(ctlr, rx_dev, &xfer->rx_sg,  in __spi_map_msg()
1276 						xfer->rx_buf, xfer->len,  in __spi_map_msg()
1280 						&xfer->tx_sg, DMA_TO_DEVICE,  in __spi_map_msg()
1286 			xfer->rx_sg_mapped = true;  in __spi_map_msg()
1303 	struct spi_transfer *xfer;  in __spi_unmap_msg()  local
1305 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in __spi_unmap_msg()
1309 		if (xfer->rx_sg_mapped)  in __spi_unmap_msg()
1310 			spi_unmap_buf_attrs(ctlr, rx_dev, &xfer->rx_sg,  in __spi_unmap_msg()
1312 		xfer->rx_sg_mapped = false;  in __spi_unmap_msg()
1314 		if (xfer->tx_sg_mapped)  in __spi_unmap_msg()
1315 			spi_unmap_buf_attrs(ctlr, tx_dev, &xfer->tx_sg,  in __spi_unmap_msg()
1317 		xfer->tx_sg_mapped = false;  in __spi_unmap_msg()
1324 				    struct spi_transfer *xfer)  in spi_dma_sync_for_device()  argument
1329 	if (xfer->tx_sg_mapped)  in spi_dma_sync_for_device()
1330 		dma_sync_sgtable_for_device(tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);  in spi_dma_sync_for_device()
1331 	if (xfer->rx_sg_mapped)  in spi_dma_sync_for_device()
1332 		dma_sync_sgtable_for_device(rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);  in spi_dma_sync_for_device()
1336 				 struct spi_transfer *xfer)  in spi_dma_sync_for_cpu()  argument
1341 	if (xfer->rx_sg_mapped)  in spi_dma_sync_for_cpu()
1342 		dma_sync_sgtable_for_cpu(rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);  in spi_dma_sync_for_cpu()
1343 	if (xfer->tx_sg_mapped)  in spi_dma_sync_for_cpu()
1344 		dma_sync_sgtable_for_cpu(tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);  in spi_dma_sync_for_cpu()
1360 				    struct spi_transfer *xfer)  in spi_dma_sync_for_device()  argument
1365 				 struct spi_transfer *xfer)  in spi_dma_sync_for_cpu()  argument
1373 	struct spi_transfer *xfer;  in spi_unmap_msg()  local
1375 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in spi_unmap_msg()
1380 		if (xfer->tx_buf == ctlr->dummy_tx)  in spi_unmap_msg()
1381 			xfer->tx_buf = NULL;  in spi_unmap_msg()
1382 		if (xfer->rx_buf == ctlr->dummy_rx)  in spi_unmap_msg()
1383 			xfer->rx_buf = NULL;  in spi_unmap_msg()
1391 	struct spi_transfer *xfer;  in spi_map_msg()  local
1400 		list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in spi_map_msg()
1402 			    !xfer->tx_buf)  in spi_map_msg()
1403 				max_tx = max(xfer->len, max_tx);  in spi_map_msg()
1405 			    !xfer->rx_buf)  in spi_map_msg()
1406 				max_rx = max(xfer->len, max_rx);  in spi_map_msg()
1426 			list_for_each_entry(xfer, &msg->transfers,  in spi_map_msg()
1428 				if (!xfer->len)  in spi_map_msg()
1430 				if (!xfer->tx_buf)  in spi_map_msg()
1431 					xfer->tx_buf = ctlr->dummy_tx;  in spi_map_msg()
1432 				if (!xfer->rx_buf)  in spi_map_msg()
1433 					xfer->rx_buf = ctlr->dummy_rx;  in spi_map_msg()
1443 			     struct spi_transfer *xfer)  in spi_transfer_wait()  argument
1447 	u32 speed_hz = xfer->speed_hz;  in spi_transfer_wait()
1465 		ms = 8LL * MSEC_PER_SEC * xfer->len;  in spi_transfer_wait()
1487 		if (xfer->error & SPI_TRANS_FAIL_IO)  in spi_transfer_wait()
1510 int spi_delay_to_ns(struct spi_delay *_delay, struct spi_transfer *xfer)  in spi_delay_to_ns()  argument
1528 		if (!xfer)  in spi_delay_to_ns()
1534 		hz = xfer->effective_speed_hz ?: xfer->speed_hz / 2;  in spi_delay_to_ns()
1549 int spi_delay_exec(struct spi_delay *_delay, struct spi_transfer *xfer)  in spi_delay_exec()  argument
1558 	delay = spi_delay_to_ns(_delay, xfer);  in spi_delay_exec()
1569 					  struct spi_transfer *xfer)  in _spi_transfer_cs_change_delay()  argument
1572 	u32 delay = xfer->cs_change_delay.value;  in _spi_transfer_cs_change_delay()
1573 	u32 unit = xfer->cs_change_delay.unit;  in _spi_transfer_cs_change_delay()
1583 	ret = spi_delay_exec(&xfer->cs_change_delay, xfer);  in _spi_transfer_cs_change_delay()
1593 						  struct spi_transfer *xfer)  in spi_transfer_cs_change_delay_exec()  argument
1595 	_spi_transfer_cs_change_delay(msg, xfer);  in spi_transfer_cs_change_delay_exec()
1609 	struct spi_transfer *xfer;  in spi_transfer_one_message()  local
1615 	xfer = list_first_entry(&msg->transfers, struct spi_transfer, transfer_list);  in spi_transfer_one_message()
1616 	spi_set_cs(msg->spi, !xfer->cs_off, false);  in spi_transfer_one_message()
1621 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in spi_transfer_one_message()
1622 		trace_spi_transfer_start(msg, xfer);  in spi_transfer_one_message()
1624 		spi_statistics_add_transfer_stats(statm, xfer, msg);  in spi_transfer_one_message()
1625 		spi_statistics_add_transfer_stats(stats, xfer, msg);  in spi_transfer_one_message()
1628 			xfer->ptp_sts_word_pre = 0;  in spi_transfer_one_message()
1629 			ptp_read_system_prets(xfer->ptp_sts);  in spi_transfer_one_message()
1632 		if ((xfer->tx_buf || xfer->rx_buf) && xfer->len) {  in spi_transfer_one_message()
1636 			spi_dma_sync_for_device(ctlr, xfer);  in spi_transfer_one_message()
1637 			ret = ctlr->transfer_one(ctlr, msg->spi, xfer);  in spi_transfer_one_message()
1639 				spi_dma_sync_for_cpu(ctlr, xfer);  in spi_transfer_one_message()
1641 				if ((xfer->tx_sg_mapped || xfer->rx_sg_mapped) &&  in spi_transfer_one_message()
1642 				    (xfer->error & SPI_TRANS_FAIL_NO_START)) {  in spi_transfer_one_message()
1645 					xfer->error &= ~SPI_TRANS_FAIL_NO_START;  in spi_transfer_one_message()
1659 				ret = spi_transfer_wait(ctlr, msg, xfer);  in spi_transfer_one_message()
1664 			spi_dma_sync_for_cpu(ctlr, xfer);  in spi_transfer_one_message()
1666 			if (xfer->len)  in spi_transfer_one_message()
1669 					xfer->len);  in spi_transfer_one_message()
1673 			ptp_read_system_postts(xfer->ptp_sts);  in spi_transfer_one_message()
1674 			xfer->ptp_sts_word_post = xfer->len;  in spi_transfer_one_message()
1677 		trace_spi_transfer_stop(msg, xfer);  in spi_transfer_one_message()
1682 		spi_transfer_delay_exec(xfer);  in spi_transfer_one_message()
1684 		if (xfer->cs_change) {  in spi_transfer_one_message()
1685 			if (list_is_last(&xfer->transfer_list,  in spi_transfer_one_message()
1689 				if (!xfer->cs_off)  in spi_transfer_one_message()
1691 				_spi_transfer_cs_change_delay(msg, xfer);  in spi_transfer_one_message()
1692 				if (!list_next_entry(xfer, transfer_list)->cs_off)  in spi_transfer_one_message()
1695 		} else if (!list_is_last(&xfer->transfer_list, &msg->transfers) &&  in spi_transfer_one_message()
1696 			   xfer->cs_off != list_next_entry(xfer, transfer_list)->cs_off) {  in spi_transfer_one_message()
1697 			spi_set_cs(msg->spi, xfer->cs_off, false);  in spi_transfer_one_message()
1700 		msg->actual_length += xfer->len;  in spi_transfer_one_message()
1743 	struct spi_transfer *xfer;  in __spi_pump_transfer_message()  local
1802 		list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in __spi_pump_transfer_message()
1803 			xfer->ptp_sts_word_pre = 0;  in __spi_pump_transfer_message()
1804 			ptp_read_system_prets(xfer->ptp_sts);  in __spi_pump_transfer_message()
1972 			    struct spi_transfer *xfer,  in spi_take_timestamp_pre()  argument
1975 	if (!xfer->ptp_sts)  in spi_take_timestamp_pre()
1978 	if (xfer->timestamped)  in spi_take_timestamp_pre()
1981 	if (progress > xfer->ptp_sts_word_pre)  in spi_take_timestamp_pre()
1985 	xfer->ptp_sts_word_pre = progress;  in spi_take_timestamp_pre()
1992 	ptp_read_system_prets(xfer->ptp_sts);  in spi_take_timestamp_pre()
2009 			     struct spi_transfer *xfer,  in spi_take_timestamp_post()  argument
2012 	if (!xfer->ptp_sts)  in spi_take_timestamp_post()
2015 	if (xfer->timestamped)  in spi_take_timestamp_post()
2018 	if (progress < xfer->ptp_sts_word_post)  in spi_take_timestamp_post()
2021 	ptp_read_system_postts(xfer->ptp_sts);  in spi_take_timestamp_post()
2029 	xfer->ptp_sts_word_post = progress;  in spi_take_timestamp_post()
2031 	xfer->timestamped = 1;  in spi_take_timestamp_post()
2156 	struct spi_transfer *xfer;  in spi_finalize_current_message()  local
2163 		list_for_each_entry(xfer, &mesg->transfers, transfer_list) {  in spi_finalize_current_message()
2164 			ptp_read_system_postts(xfer->ptp_sts);  in spi_finalize_current_message()
2165 			xfer->ptp_sts_word_post = xfer->len;  in spi_finalize_current_message()
2170 		list_for_each_entry(xfer, &mesg->transfers, transfer_list)  in spi_finalize_current_message()
2171 			WARN_ON_ONCE(xfer->ptp_sts && !xfer->timestamped);  in spi_finalize_current_message()
3590 	struct spi_transfer *xfer;  in spi_replace_transfers()  local
3653 		xfer = &rxfer->inserted_transfers[insert - 1 - i];  in spi_replace_transfers()
3656 		memcpy(xfer, xfer_first, sizeof(*xfer));  in spi_replace_transfers()
3659 		list_add(&xfer->transfer_list, rxfer->replaced_after);  in spi_replace_transfers()
3663 			xfer->cs_change = false;  in spi_replace_transfers()
3664 			xfer->delay.value = 0;  in spi_replace_transfers()
3682 	struct spi_transfer *xfer = *xferp, *xfers;  in __spi_split_transfer_maxsize()  local
3688 	count = DIV_ROUND_UP(xfer->len, maxsize);  in __spi_split_transfer_maxsize()
3691 	srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, GFP_KERNEL);  in __spi_split_transfer_maxsize()
3708 	xfers[0].len = min_t(size_t, maxsize, xfer[0].len);  in __spi_split_transfer_maxsize()
3755 	struct spi_transfer *xfer;  in spi_split_transfers_maxsize()  local
3765 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in spi_split_transfers_maxsize()
3766 		if (xfer->len > maxsize) {  in spi_split_transfers_maxsize()
3767 			ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer,  in spi_split_transfers_maxsize()
3797 	struct spi_transfer *xfer;  in spi_split_transfers_maxwords()  local
3806 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in spi_split_transfers_maxwords()
3810 		maxsize = maxwords * roundup_pow_of_two(BITS_TO_BYTES(xfer->bits_per_word));  in spi_split_transfers_maxwords()
3811 		if (xfer->len > maxsize) {  in spi_split_transfers_maxwords()
3812 			ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer,  in spi_split_transfers_maxwords()
4029 static int _spi_xfer_word_delay_update(struct spi_transfer *xfer,  in _spi_xfer_word_delay_update()  argument
4034 	delay1 = spi_delay_to_ns(&xfer->word_delay, xfer);  in _spi_xfer_word_delay_update()
4038 	delay2 = spi_delay_to_ns(&spi->word_delay, xfer);  in _spi_xfer_word_delay_update()
4043 		memcpy(&xfer->word_delay, &spi->word_delay,  in _spi_xfer_word_delay_update()
4044 		       sizeof(xfer->word_delay));  in _spi_xfer_word_delay_update()
4052 	struct spi_transfer *xfer;  in __spi_validate()  local
4070 		list_for_each_entry(xfer, &message->transfers, transfer_list) {  in __spi_validate()
4071 			if (xfer->rx_buf && xfer->tx_buf)  in __spi_validate()
4073 			if ((flags & SPI_CONTROLLER_NO_TX) && xfer->tx_buf)  in __spi_validate()
4075 			if ((flags & SPI_CONTROLLER_NO_RX) && xfer->rx_buf)  in __spi_validate()
4089 	list_for_each_entry(xfer, &message->transfers, transfer_list) {  in __spi_validate()
4090 		xfer->effective_speed_hz = 0;  in __spi_validate()
4091 		message->frame_length += xfer->len;  in __spi_validate()
4092 		if (!xfer->bits_per_word)  in __spi_validate()
4093 			xfer->bits_per_word = spi->bits_per_word;  in __spi_validate()
4095 		if (!xfer->speed_hz)  in __spi_validate()
4096 			xfer->speed_hz = spi->max_speed_hz;  in __spi_validate()
4098 		if (ctlr->max_speed_hz && xfer->speed_hz > ctlr->max_speed_hz)  in __spi_validate()
4099 			xfer->speed_hz = ctlr->max_speed_hz;  in __spi_validate()
4101 		if (__spi_validate_bits_per_word(ctlr, xfer->bits_per_word))  in __spi_validate()
4108 		if (xfer->bits_per_word <= 8)  in __spi_validate()
4110 		else if (xfer->bits_per_word <= 16)  in __spi_validate()
4116 		if (xfer->len % w_size)  in __spi_validate()
4119 		if (xfer->speed_hz && ctlr->min_speed_hz &&  in __spi_validate()
4120 		    xfer->speed_hz < ctlr->min_speed_hz)  in __spi_validate()
4123 		if (xfer->tx_buf && !xfer->tx_nbits)  in __spi_validate()
4124 			xfer->tx_nbits = SPI_NBITS_SINGLE;  in __spi_validate()
4125 		if (xfer->rx_buf && !xfer->rx_nbits)  in __spi_validate()
4126 			xfer->rx_nbits = SPI_NBITS_SINGLE;  in __spi_validate()
4132 		if (xfer->tx_buf) {  in __spi_validate()
4135 			if (xfer->tx_nbits != SPI_NBITS_SINGLE &&  in __spi_validate()
4136 				xfer->tx_nbits != SPI_NBITS_DUAL &&  in __spi_validate()
4137 				xfer->tx_nbits != SPI_NBITS_QUAD &&  in __spi_validate()
4138 				xfer->tx_nbits != SPI_NBITS_OCTAL)  in __spi_validate()
4140 			if ((xfer->tx_nbits == SPI_NBITS_DUAL) &&  in __spi_validate()
4143 			if ((xfer->tx_nbits == SPI_NBITS_QUAD) &&  in __spi_validate()
4148 		if (xfer->rx_buf) {  in __spi_validate()
4151 			if (xfer->rx_nbits != SPI_NBITS_SINGLE &&  in __spi_validate()
4152 				xfer->rx_nbits != SPI_NBITS_DUAL &&  in __spi_validate()
4153 				xfer->rx_nbits != SPI_NBITS_QUAD &&  in __spi_validate()
4154 				xfer->rx_nbits != SPI_NBITS_OCTAL)  in __spi_validate()
4156 			if ((xfer->rx_nbits == SPI_NBITS_DUAL) &&  in __spi_validate()
4159 			if ((xfer->rx_nbits == SPI_NBITS_QUAD) &&  in __spi_validate()
4164 		if (_spi_xfer_word_delay_update(xfer, spi))  in __spi_validate()
4191 	struct spi_transfer *xfer;  in spi_split_transfers()  local
4207 		list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in spi_split_transfers()
4209 			if (list_is_last(&xfer->transfer_list, &msg->transfers))  in spi_split_transfers()
4212 			xfer->cs_change = 1;  in spi_split_transfers()
4352 	struct spi_transfer *xfer;  in __spi_async()  local
4367 		list_for_each_entry(xfer, &message->transfers, transfer_list) {  in __spi_async()
4368 			xfer->ptp_sts_word_pre = 0;  in __spi_async()
4369 			ptp_read_system_prets(xfer->ptp_sts);  in __spi_async()