182 static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl,  in qspi_buswidth_to_iomode()  argument
193 		dev_warn_once(ctrl->dev,  in qspi_buswidth_to_iomode()
199 static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl)  in qcom_qspi_pio_xfer_cfg()  argument
205 	xfer = &ctrl->xfer;  in qcom_qspi_pio_xfer_cfg()
206 	pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG);  in qcom_qspi_pio_xfer_cfg()
214 	iomode = qspi_buswidth_to_iomode(ctrl, xfer->buswidth);  in qcom_qspi_pio_xfer_cfg()
217 	writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG);  in qcom_qspi_pio_xfer_cfg()
220 static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl)  in qcom_qspi_pio_xfer_ctrl()  argument
224 	pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL);  in qcom_qspi_pio_xfer_ctrl()
226 	pio_xfer_ctrl |= ctrl->xfer.rem_bytes;  in qcom_qspi_pio_xfer_ctrl()
227 	writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL);  in qcom_qspi_pio_xfer_ctrl()
230 static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl)  in qcom_qspi_pio_xfer()  argument
234 	qcom_qspi_pio_xfer_cfg(ctrl);  in qcom_qspi_pio_xfer()
237 	writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS);  in qcom_qspi_pio_xfer()
240 	if (ctrl->xfer.dir == QSPI_WRITE)  in qcom_qspi_pio_xfer()
244 	writel(ints, ctrl->base + MSTR_INT_EN);  in qcom_qspi_pio_xfer()
247 	qcom_qspi_pio_xfer_ctrl(ctrl);  in qcom_qspi_pio_xfer()
254 	struct qcom_qspi *ctrl = spi_controller_get_devdata(host);  in qcom_qspi_handle_err()  local
258 	spin_lock_irqsave(&ctrl->lock, flags);  in qcom_qspi_handle_err()
259 	writel(0, ctrl->base + MSTR_INT_EN);  in qcom_qspi_handle_err()
260 	int_status = readl(ctrl->base + MSTR_INT_STATUS);  in qcom_qspi_handle_err()
261 	writel(int_status, ctrl->base + MSTR_INT_STATUS);  in qcom_qspi_handle_err()
262 	ctrl->xfer.rem_bytes = 0;  in qcom_qspi_handle_err()
265 	for (i = 0; i < ctrl->n_cmd_desc; i++)  in qcom_qspi_handle_err()
266 		dma_pool_free(ctrl->dma_cmd_pool, ctrl->virt_cmd_desc[i],  in qcom_qspi_handle_err()
267 				  ctrl->dma_cmd_desc[i]);  in qcom_qspi_handle_err()
268 	ctrl->n_cmd_desc = 0;  in qcom_qspi_handle_err()
269 	spin_unlock_irqrestore(&ctrl->lock, flags);  in qcom_qspi_handle_err()
272 static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)  in qcom_qspi_set_speed()  argument
277 	if (speed_hz == ctrl->last_speed)  in qcom_qspi_set_speed()
281 	ret = dev_pm_opp_set_rate(ctrl->dev, speed_hz * 4);  in qcom_qspi_set_speed()
283 		dev_err(ctrl->dev, "Failed to set core clk %d\n", ret);  in qcom_qspi_set_speed()
292 	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, avg_bw_cpu, avg_bw_cpu);  in qcom_qspi_set_speed()
294 		dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",  in qcom_qspi_set_speed()
299 	ctrl->last_speed = speed_hz;  in qcom_qspi_set_speed()
304 static int qcom_qspi_alloc_desc(struct qcom_qspi *ctrl, dma_addr_t dma_ptr,  in qcom_qspi_alloc_desc()  argument
311 	virt_cmd_desc = dma_pool_alloc(ctrl->dma_cmd_pool, GFP_ATOMIC | __GFP_ZERO, &dma_cmd_desc);  in qcom_qspi_alloc_desc()
313 		dev_warn_once(ctrl->dev, "Couldn't find memory for descriptor\n");  in qcom_qspi_alloc_desc()
317 	ctrl->virt_cmd_desc[ctrl->n_cmd_desc] = virt_cmd_desc;  in qcom_qspi_alloc_desc()
318 	ctrl->dma_cmd_desc[ctrl->n_cmd_desc] = dma_cmd_desc;  in qcom_qspi_alloc_desc()
319 	ctrl->n_cmd_desc++;  in qcom_qspi_alloc_desc()
323 	virt_cmd_desc->direction = ctrl->xfer.dir;  in qcom_qspi_alloc_desc()
324 	virt_cmd_desc->multi_io_mode = qspi_buswidth_to_iomode(ctrl, ctrl->xfer.buswidth);  in qcom_qspi_alloc_desc()
325 	virt_cmd_desc->fragment = !ctrl->xfer.is_last;  in qcom_qspi_alloc_desc()
329 	if (ctrl->n_cmd_desc >= 2) {  in qcom_qspi_alloc_desc()
330 		prev = (ctrl->virt_cmd_desc)[ctrl->n_cmd_desc - 2];  in qcom_qspi_alloc_desc()
338 static int qcom_qspi_setup_dma_desc(struct qcom_qspi *ctrl,  in qcom_qspi_setup_dma_desc()  argument
347 	if (ctrl->n_cmd_desc) {  in qcom_qspi_setup_dma_desc()
348 		dev_err(ctrl->dev, "Remnant dma buffers n_cmd_desc-%d\n", ctrl->n_cmd_desc);  in qcom_qspi_setup_dma_desc()
352 	sgt = (ctrl->xfer.dir == QSPI_READ) ? &xfer->rx_sg : &xfer->tx_sg;  in qcom_qspi_setup_dma_desc()
354 		dev_warn_once(ctrl->dev, "Cannot handle %d entries in scatter list\n", sgt->nents);  in qcom_qspi_setup_dma_desc()
362 			dev_warn_once(ctrl->dev, "dma_address not aligned to %d\n", QSPI_ALIGN_REQ);  in qcom_qspi_setup_dma_desc()
371 		if (ctrl->xfer.dir == QSPI_READ && (dma_len_sg & 0x03)) {  in qcom_qspi_setup_dma_desc()
372 			dev_warn_once(ctrl->dev, "fallback to PIO for read of size %#010x\n",  in qcom_qspi_setup_dma_desc()
382 		ret = qcom_qspi_alloc_desc(ctrl, dma_ptr_sg, dma_len_sg);  in qcom_qspi_setup_dma_desc()
389 	for (i = 0; i < ctrl->n_cmd_desc; i++)  in qcom_qspi_setup_dma_desc()
390 		dma_pool_free(ctrl->dma_cmd_pool, ctrl->virt_cmd_desc[i],  in qcom_qspi_setup_dma_desc()
391 				  ctrl->dma_cmd_desc[i]);  in qcom_qspi_setup_dma_desc()
392 	ctrl->n_cmd_desc = 0;  in qcom_qspi_setup_dma_desc()
396 static void qcom_qspi_dma_xfer(struct qcom_qspi *ctrl)  in qcom_qspi_dma_xfer()  argument
399 	writel(DMA_CHAIN_DONE, ctrl->base + MSTR_INT_EN);  in qcom_qspi_dma_xfer()
402 	writel((u32)((ctrl->dma_cmd_desc)[0]), ctrl->base + NEXT_DMA_DESC_ADDR);  in qcom_qspi_dma_xfer()
418 	struct qcom_qspi *ctrl = spi_controller_get_devdata(host);  in qcom_qspi_transfer_one()  local
428 	ret = qcom_qspi_set_speed(ctrl, speed_hz);  in qcom_qspi_transfer_one()
432 	spin_lock_irqsave(&ctrl->lock, flags);  in qcom_qspi_transfer_one()
433 	mstr_cfg = readl(ctrl->base + MSTR_CONFIG);  in qcom_qspi_transfer_one()
437 		ctrl->xfer.dir = QSPI_READ;  in qcom_qspi_transfer_one()
438 		ctrl->xfer.buswidth = xfer->rx_nbits;  in qcom_qspi_transfer_one()
439 		ctrl->xfer.rx_buf = xfer->rx_buf;  in qcom_qspi_transfer_one()
441 		ctrl->xfer.dir = QSPI_WRITE;  in qcom_qspi_transfer_one()
442 		ctrl->xfer.buswidth = xfer->tx_nbits;  in qcom_qspi_transfer_one()
443 		ctrl->xfer.tx_buf = xfer->tx_buf;  in qcom_qspi_transfer_one()
445 	ctrl->xfer.is_last = list_is_last(&xfer->transfer_list,  in qcom_qspi_transfer_one()
447 	ctrl->xfer.rem_bytes = xfer->len;  in qcom_qspi_transfer_one()
453 			writel(mstr_cfg, ctrl->base + MSTR_CONFIG);  in qcom_qspi_transfer_one()
456 		ret = qcom_qspi_setup_dma_desc(ctrl, xfer);  in qcom_qspi_transfer_one()
460 				qcom_qspi_dma_xfer(ctrl);  in qcom_qspi_transfer_one()
464 		dev_warn_once(ctrl->dev, "DMA failure, falling back to PIO\n");  in qcom_qspi_transfer_one()
470 		writel(mstr_cfg, ctrl->base + MSTR_CONFIG);  in qcom_qspi_transfer_one()
472 	qcom_qspi_pio_xfer(ctrl);  in qcom_qspi_transfer_one()
475 	spin_unlock_irqrestore(&ctrl->lock, flags);  in qcom_qspi_transfer_one()
488 	struct qcom_qspi *ctrl;  in qcom_qspi_prepare_message()  local
493 	ctrl = spi_controller_get_devdata(host);  in qcom_qspi_prepare_message()
494 	spin_lock_irqsave(&ctrl->lock, flags);  in qcom_qspi_prepare_message()
496 	mstr_cfg = readl(ctrl->base + MSTR_CONFIG);  in qcom_qspi_prepare_message()
508 	writel(mstr_cfg, ctrl->base + MSTR_CONFIG);  in qcom_qspi_prepare_message()
509 	spin_unlock_irqrestore(&ctrl->lock, flags);  in qcom_qspi_prepare_message()
514 static int qcom_qspi_alloc_dma(struct qcom_qspi *ctrl)  in qcom_qspi_alloc_dma()  argument
516 	ctrl->dma_cmd_pool = dmam_pool_create("qspi cmd desc pool",  in qcom_qspi_alloc_dma()
517 		ctrl->dev, sizeof(struct qspi_cmd_desc), 0, 0);  in qcom_qspi_alloc_dma()
518 	if (!ctrl->dma_cmd_pool)  in qcom_qspi_alloc_dma()
524 static irqreturn_t pio_read(struct qcom_qspi *ctrl)  in pio_read()  argument
535 	rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS);  in pio_read()
538 		dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status);  in pio_read()
543 	wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes);  in pio_read()
549 		word_buf = ctrl->xfer.rx_buf;  in pio_read()
550 		ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD;  in pio_read()
551 		ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read);  in pio_read()
552 		ctrl->xfer.rx_buf = word_buf + words_to_read;  in pio_read()
556 		byte_buf = ctrl->xfer.rx_buf;  in pio_read()
557 		rd_fifo = readl(ctrl->base + RD_FIFO);  in pio_read()
558 		ctrl->xfer.rem_bytes -= bytes_to_read;  in pio_read()
561 		ctrl->xfer.rx_buf = byte_buf;  in pio_read()
567 static irqreturn_t pio_write(struct qcom_qspi *ctrl)  in pio_write()  argument
569 	const void *xfer_buf = ctrl->xfer.tx_buf;  in pio_write()
577 	wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS);  in pio_write()
580 	if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) {  in pio_write()
582 		wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes);  in pio_write()
583 		ctrl->xfer.rem_bytes -= wr_size;  in pio_write()
588 			       ctrl->base + PIO_DATAOUT_1B);  in pio_write()
589 		ctrl->xfer.tx_buf = byte_buf;  in pio_write()
596 		rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD;  in pio_write()
600 		ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD;  in pio_write()
603 		iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size);  in pio_write()
604 		ctrl->xfer.tx_buf = word_buf + wr_size;  in pio_write()
614 	struct qcom_qspi *ctrl = dev_id;  in qcom_qspi_irq()  local
617 	spin_lock(&ctrl->lock);  in qcom_qspi_irq()
619 	int_status = readl(ctrl->base + MSTR_INT_STATUS);  in qcom_qspi_irq()
620 	writel(int_status, ctrl->base + MSTR_INT_STATUS);  in qcom_qspi_irq()
623 	int_status &= readl(ctrl->base + MSTR_INT_EN);  in qcom_qspi_irq()
626 	if (ctrl->xfer.dir == QSPI_WRITE) {  in qcom_qspi_irq()
628 			ret = pio_write(ctrl);  in qcom_qspi_irq()
631 			ret = pio_read(ctrl);  in qcom_qspi_irq()
636 			dev_err(ctrl->dev, "IRQ error: FIFO underrun\n");  in qcom_qspi_irq()
638 			dev_err(ctrl->dev, "IRQ error: FIFO overrun\n");  in qcom_qspi_irq()
640 			dev_err(ctrl->dev, "IRQ error: NOC response error\n");  in qcom_qspi_irq()
644 	if (!ctrl->xfer.rem_bytes) {  in qcom_qspi_irq()
645 		writel(0, ctrl->base + MSTR_INT_EN);  in qcom_qspi_irq()
646 		spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));  in qcom_qspi_irq()
653 		writel(0, ctrl->base + MSTR_INT_EN);  in qcom_qspi_irq()
654 		ctrl->xfer.rem_bytes = 0;  in qcom_qspi_irq()
656 		for (i = 0; i < ctrl->n_cmd_desc; i++)  in qcom_qspi_irq()
657 			dma_pool_free(ctrl->dma_cmd_pool, ctrl->virt_cmd_desc[i],  in qcom_qspi_irq()
658 					  ctrl->dma_cmd_desc[i]);  in qcom_qspi_irq()
659 		ctrl->n_cmd_desc = 0;  in qcom_qspi_irq()
662 		spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));  in qcom_qspi_irq()
665 	spin_unlock(&ctrl->lock);  in qcom_qspi_irq()
698 	struct qcom_qspi *ctrl;  in qcom_qspi_probe()  local
702 	host = devm_spi_alloc_host(dev, sizeof(*ctrl));  in qcom_qspi_probe()
708 	ctrl = spi_controller_get_devdata(host);  in qcom_qspi_probe()
710 	spin_lock_init(&ctrl->lock);  in qcom_qspi_probe()
711 	ctrl->dev = dev;  in qcom_qspi_probe()
712 	ctrl->base = devm_platform_ioremap_resource(pdev, 0);  in qcom_qspi_probe()
713 	if (IS_ERR(ctrl->base))  in qcom_qspi_probe()
714 		return PTR_ERR(ctrl->base);  in qcom_qspi_probe()
716 	ctrl->clks = devm_kcalloc(dev, QSPI_NUM_CLKS,  in qcom_qspi_probe()
717 				  sizeof(*ctrl->clks), GFP_KERNEL);  in qcom_qspi_probe()
718 	if (!ctrl->clks)  in qcom_qspi_probe()
721 	ctrl->clks[QSPI_CLK_CORE].id = "core";  in qcom_qspi_probe()
722 	ctrl->clks[QSPI_CLK_IFACE].id = "iface";  in qcom_qspi_probe()
723 	ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks);  in qcom_qspi_probe()
727 	ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");  in qcom_qspi_probe()
728 	if (IS_ERR(ctrl->icc_path_cpu_to_qspi))  in qcom_qspi_probe()
729 		return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),  in qcom_qspi_probe()
733 	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),  in qcom_qspi_probe()
736 		dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",  in qcom_qspi_probe()
741 	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);  in qcom_qspi_probe()
743 		dev_err(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",  in qcom_qspi_probe()
751 	ret = devm_request_irq(dev, ret, qcom_qspi_irq, 0, dev_name(dev), ctrl);  in qcom_qspi_probe()
789 	ret = qcom_qspi_alloc_dma(ctrl);  in qcom_qspi_probe()
819 	struct qcom_qspi *ctrl = spi_controller_get_devdata(host);  in qcom_qspi_runtime_suspend()  local
824 	clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);  in qcom_qspi_runtime_suspend()
826 	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);  in qcom_qspi_runtime_suspend()
828 		dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",  in qcom_qspi_runtime_suspend()
841 	struct qcom_qspi *ctrl = spi_controller_get_devdata(host);  in qcom_qspi_runtime_resume()  local
846 	ret = icc_enable(ctrl->icc_path_cpu_to_qspi);  in qcom_qspi_runtime_resume()
848 		dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",  in qcom_qspi_runtime_resume()
853 	ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);  in qcom_qspi_runtime_resume()
857 	return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);  in qcom_qspi_runtime_resume()