1 // SPDX-License-Identifier: GPL-2.0-only
6  * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
19 	if (!sgiter->sg)  in mv_cesa_req_dma_iter_next_transfer()
22 	sgiter->op_offset += len;  in mv_cesa_req_dma_iter_next_transfer()
23 	sgiter->offset += len;  in mv_cesa_req_dma_iter_next_transfer()
24 	if (sgiter->offset == sg_dma_len(sgiter->sg)) {  in mv_cesa_req_dma_iter_next_transfer()
25 		if (sg_is_last(sgiter->sg))  in mv_cesa_req_dma_iter_next_transfer()
27 		sgiter->offset = 0;  in mv_cesa_req_dma_iter_next_transfer()
28 		sgiter->sg = sg_next(sgiter->sg);  in mv_cesa_req_dma_iter_next_transfer()
31 	if (sgiter->op_offset == iter->op_len)  in mv_cesa_req_dma_iter_next_transfer()
39 	struct mv_cesa_engine *engine = dreq->engine;  in mv_cesa_dma_step()  local
41 	writel_relaxed(0, engine->regs + CESA_SA_CFG);  in mv_cesa_dma_step()
43 	mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE);  in mv_cesa_dma_step()
46 		       engine->regs + CESA_TDMA_CONTROL);  in mv_cesa_dma_step()
50 		       engine->regs + CESA_SA_CFG);  in mv_cesa_dma_step()
51 	writel_relaxed(dreq->chain.first->cur_dma,  in mv_cesa_dma_step()
52 		       engine->regs + CESA_TDMA_NEXT_ADDR);  in mv_cesa_dma_step()
53 	WARN_ON(readl(engine->regs + CESA_SA_CMD) &  in mv_cesa_dma_step()
55 	writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD);  in mv_cesa_dma_step()
62 	for (tdma = dreq->chain.first; tdma;) {  in mv_cesa_dma_cleanup()
64 		u32 type = tdma->flags & CESA_TDMA_TYPE_MSK;  in mv_cesa_dma_cleanup()
67 			dma_pool_free(cesa_dev->dma->op_pool, tdma->op,  in mv_cesa_dma_cleanup()
68 				      le32_to_cpu(tdma->src));  in mv_cesa_dma_cleanup()
70 		tdma = tdma->next;  in mv_cesa_dma_cleanup()
71 		dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma,  in mv_cesa_dma_cleanup()
72 			      old_tdma->cur_dma);  in mv_cesa_dma_cleanup()
75 	dreq->chain.first = NULL;  in mv_cesa_dma_cleanup()
76 	dreq->chain.last = NULL;  in mv_cesa_dma_cleanup()
80 			 struct mv_cesa_engine *engine)  in mv_cesa_dma_prepare()  argument
84 	for (tdma = dreq->chain.first; tdma; tdma = tdma->next) {  in mv_cesa_dma_prepare()
85 		if (tdma->flags & CESA_TDMA_DST_IN_SRAM)  in mv_cesa_dma_prepare()
86 			tdma->dst = cpu_to_le32(tdma->dst_dma + engine->sram_dma);  in mv_cesa_dma_prepare()
88 		if (tdma->flags & CESA_TDMA_SRC_IN_SRAM)  in mv_cesa_dma_prepare()
89 			tdma->src = cpu_to_le32(tdma->src_dma + engine->sram_dma);  in mv_cesa_dma_prepare()
91 		if ((tdma->flags & CESA_TDMA_TYPE_MSK) == CESA_TDMA_OP)  in mv_cesa_dma_prepare()
92 			mv_cesa_adjust_op(engine, tdma->op);  in mv_cesa_dma_prepare()
96 void mv_cesa_tdma_chain(struct mv_cesa_engine *engine,  in mv_cesa_tdma_chain()  argument
99 	if (engine->chain.first == NULL && engine->chain.last == NULL) {  in mv_cesa_tdma_chain()
100 		engine->chain.first = dreq->chain.first;  in mv_cesa_tdma_chain()
101 		engine->chain.last  = dreq->chain.last;  in mv_cesa_tdma_chain()
105 		last = engine->chain.last;  in mv_cesa_tdma_chain()
106 		last->next = dreq->chain.first;  in mv_cesa_tdma_chain()
107 		engine->chain.last = dreq->chain.last;  in mv_cesa_tdma_chain()
115 		if (!(last->flags & CESA_TDMA_BREAK_CHAIN) &&  in mv_cesa_tdma_chain()
116 		    !(dreq->chain.first->flags & CESA_TDMA_SET_STATE))  in mv_cesa_tdma_chain()
117 			last->next_dma = cpu_to_le32(dreq->chain.first->cur_dma);  in mv_cesa_tdma_chain()
121 int mv_cesa_tdma_process(struct mv_cesa_engine *engine, u32 status)  in mv_cesa_tdma_process()  argument
128 	tdma_cur = readl(engine->regs + CESA_TDMA_CUR);  in mv_cesa_tdma_process()
130 	for (tdma = engine->chain.first; tdma; tdma = next) {  in mv_cesa_tdma_process()
131 		spin_lock_bh(&engine->lock);  in mv_cesa_tdma_process()
132 		next = tdma->next;  in mv_cesa_tdma_process()
133 		spin_unlock_bh(&engine->lock);  in mv_cesa_tdma_process()
135 		if (tdma->flags & CESA_TDMA_END_OF_REQ) {  in mv_cesa_tdma_process()
140 			spin_lock_bh(&engine->lock);  in mv_cesa_tdma_process()
142 			 * if req is NULL, this means we're processing the  in mv_cesa_tdma_process()
143 			 * request in engine->req.  in mv_cesa_tdma_process()
146 				req = engine->req;  in mv_cesa_tdma_process()
148 				req = mv_cesa_dequeue_req_locked(engine,  in mv_cesa_tdma_process()
151 			/* Re-chaining to the next request */  in mv_cesa_tdma_process()
152 			engine->chain.first = tdma->next;  in mv_cesa_tdma_process()
153 			tdma->next = NULL;  in mv_cesa_tdma_process()
156 			if (engine->chain.first == NULL)  in mv_cesa_tdma_process()
157 				engine->chain.last  = NULL;  in mv_cesa_tdma_process()
158 			spin_unlock_bh(&engine->lock);  in mv_cesa_tdma_process()
160 			ctx = crypto_tfm_ctx(req->tfm);  in mv_cesa_tdma_process()
161 			current_status = (tdma->cur_dma == tdma_cur) ?  in mv_cesa_tdma_process()
163 			res = ctx->ops->process(req, current_status);  in mv_cesa_tdma_process()
164 			ctx->ops->complete(req);  in mv_cesa_tdma_process()
167 				mv_cesa_engine_enqueue_complete_request(engine,  in mv_cesa_tdma_process()
171 				crypto_request_complete(backlog, -EINPROGRESS);  in mv_cesa_tdma_process()
174 		if (res || tdma->cur_dma == tdma_cur)  in mv_cesa_tdma_process()
179 	 * Save the last request in error to engine->req, so that the core  in mv_cesa_tdma_process()
183 		spin_lock_bh(&engine->lock);  in mv_cesa_tdma_process()
184 		engine->req = req;  in mv_cesa_tdma_process()
185 		spin_unlock_bh(&engine->lock);  in mv_cesa_tdma_process()
197 	new_tdma = dma_pool_zalloc(cesa_dev->dma->tdma_desc_pool, flags,  in mv_cesa_dma_add_desc()
200 		return ERR_PTR(-ENOMEM);  in mv_cesa_dma_add_desc()
202 	new_tdma->cur_dma = dma_handle;  in mv_cesa_dma_add_desc()
203 	if (chain->last) {  in mv_cesa_dma_add_desc()
204 		chain->last->next_dma = cpu_to_le32(dma_handle);  in mv_cesa_dma_add_desc()
205 		chain->last->next = new_tdma;  in mv_cesa_dma_add_desc()
207 		chain->first = new_tdma;  in mv_cesa_dma_add_desc()
210 	chain->last = new_tdma;  in mv_cesa_dma_add_desc()
224 	/* We re-use an existing op_desc object to retrieve the context  in mv_cesa_dma_add_result_op()
229 	for (op_desc = chain->first; op_desc; op_desc = op_desc->next) {  in mv_cesa_dma_add_result_op()
230 		u32 type = op_desc->flags & CESA_TDMA_TYPE_MSK;  in mv_cesa_dma_add_result_op()
237 		return -EIO;  in mv_cesa_dma_add_result_op()
239 	tdma->byte_cnt = cpu_to_le32(size | BIT(31));  in mv_cesa_dma_add_result_op()
240 	tdma->src_dma = src;  in mv_cesa_dma_add_result_op()
241 	tdma->dst_dma = op_desc->src_dma;  in mv_cesa_dma_add_result_op()
242 	tdma->op = op_desc->op;  in mv_cesa_dma_add_result_op()
245 	tdma->flags = flags | CESA_TDMA_RESULT;  in mv_cesa_dma_add_result_op()
263 	op = dma_pool_alloc(cesa_dev->dma->op_pool, flags, &dma_handle);  in mv_cesa_dma_add_op()
265 		return ERR_PTR(-ENOMEM);  in mv_cesa_dma_add_op()
269 	size = skip_ctx ? sizeof(op->desc) : sizeof(*op);  in mv_cesa_dma_add_op()
271 	tdma = chain->last;  in mv_cesa_dma_add_op()
272 	tdma->op = op;  in mv_cesa_dma_add_op()
273 	tdma->byte_cnt = cpu_to_le32(size | BIT(31));  in mv_cesa_dma_add_op()
274 	tdma->src = cpu_to_le32(dma_handle);  in mv_cesa_dma_add_op()
275 	tdma->dst_dma = CESA_SA_CFG_SRAM_OFFSET;  in mv_cesa_dma_add_op()
276 	tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP;  in mv_cesa_dma_add_op()
291 	tdma->byte_cnt = cpu_to_le32(size | BIT(31));  in mv_cesa_dma_add_data_transfer()
292 	tdma->src_dma = src;  in mv_cesa_dma_add_data_transfer()
293 	tdma->dst_dma = dst;  in mv_cesa_dma_add_data_transfer()
296 	tdma->flags = flags | CESA_TDMA_DATA;  in mv_cesa_dma_add_data_transfer()
317 	tdma->byte_cnt = cpu_to_le32(BIT(31));  in mv_cesa_dma_add_dummy_end()
327 	u32 flags = sgiter->dir == DMA_TO_DEVICE ?  in mv_cesa_dma_add_op_transfers()
336 		if (sgiter->dir == DMA_TO_DEVICE) {  in mv_cesa_dma_add_op_transfers()
337 			dst = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset;  in mv_cesa_dma_add_op_transfers()
338 			src = sg_dma_address(sgiter->sg) + sgiter->offset;  in mv_cesa_dma_add_op_transfers()
340 			dst = sg_dma_address(sgiter->sg) + sgiter->offset;  in mv_cesa_dma_add_op_transfers()
341 			src = CESA_SA_DATA_SRAM_OFFSET + sgiter->op_offset;  in mv_cesa_dma_add_op_transfers()
354 size_t mv_cesa_sg_copy(struct mv_cesa_engine *engine,  in mv_cesa_sg_copy()  argument
376 		len = min(miter.length, buflen - offset);  in mv_cesa_sg_copy()
379 			if (engine->pool)  in mv_cesa_sg_copy()
380 				memcpy(engine->sram_pool + sram_off + offset,  in mv_cesa_sg_copy()
383 				memcpy_toio(engine->sram + sram_off + offset,  in mv_cesa_sg_copy()
386 			if (engine->pool)  in mv_cesa_sg_copy()
388 				       engine->sram_pool + sram_off + offset,  in mv_cesa_sg_copy()
392 					      engine->sram + sram_off + offset,  in mv_cesa_sg_copy()