52 static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp)  in starfive_aes_wait_busy()  argument
56 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,  in starfive_aes_wait_busy()
60 static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp)  in starfive_aes_wait_keydone()  argument
64 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,  in starfive_aes_wait_keydone()
68 static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp)  in starfive_aes_wait_gcmdone()  argument
72 	return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,  in starfive_aes_wait_gcmdone()
76 static inline int is_gcm(struct starfive_cryp_dev *cryp)  in is_gcm()  argument
78 	return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM;  in is_gcm()
81 static inline bool is_encrypt(struct starfive_cryp_dev *cryp)  in is_encrypt()  argument
83 	return cryp->flags & FLG_ENCRYPT;  in is_encrypt()
88 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_aead_hw_start()  local
93 		value = readl(ctx->cryp->base + STARFIVE_AES_CSR);  in starfive_aes_aead_hw_start()
95 		writel(value, cryp->base + STARFIVE_AES_CSR);  in starfive_aes_aead_hw_start()
96 		starfive_aes_wait_gcmdone(cryp);  in starfive_aes_aead_hw_start()
99 		value = readl(ctx->cryp->base + STARFIVE_AES_CSR);  in starfive_aes_aead_hw_start()
101 		writel(value, cryp->base + STARFIVE_AES_CSR);  in starfive_aes_aead_hw_start()
108 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_set_alen()  local
110 	writel(upper_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN0);  in starfive_aes_set_alen()
111 	writel(lower_32_bits(cryp->assoclen), cryp->base + STARFIVE_AES_ALEN1);  in starfive_aes_set_alen()
116 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_set_mlen()  local
118 	writel(upper_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN0);  in starfive_aes_set_mlen()
119 	writel(lower_32_bits(cryp->total_in), cryp->base + STARFIVE_AES_MLEN1);  in starfive_aes_set_mlen()
133 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_write_iv()  local
135 	writel(iv[0], cryp->base + STARFIVE_AES_IV0);  in starfive_aes_write_iv()
136 	writel(iv[1], cryp->base + STARFIVE_AES_IV1);  in starfive_aes_write_iv()
137 	writel(iv[2], cryp->base + STARFIVE_AES_IV2);  in starfive_aes_write_iv()
139 	if (is_gcm(cryp)) {  in starfive_aes_write_iv()
140 		if (starfive_aes_wait_gcmdone(cryp))  in starfive_aes_write_iv()
146 	writel(iv[3], cryp->base + STARFIVE_AES_IV3);  in starfive_aes_write_iv()
151 static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv)  in starfive_aes_get_iv()  argument
153 	iv[0] = readl(cryp->base + STARFIVE_AES_IV0);  in starfive_aes_get_iv()
154 	iv[1] = readl(cryp->base + STARFIVE_AES_IV1);  in starfive_aes_get_iv()
155 	iv[2] = readl(cryp->base + STARFIVE_AES_IV2);  in starfive_aes_get_iv()
156 	iv[3] = readl(cryp->base + STARFIVE_AES_IV3);  in starfive_aes_get_iv()
161 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_write_nonce()  local
163 	writel(nonce[0], cryp->base + STARFIVE_AES_NONCE0);  in starfive_aes_write_nonce()
164 	writel(nonce[1], cryp->base + STARFIVE_AES_NONCE1);  in starfive_aes_write_nonce()
165 	writel(nonce[2], cryp->base + STARFIVE_AES_NONCE2);  in starfive_aes_write_nonce()
166 	writel(nonce[3], cryp->base + STARFIVE_AES_NONCE3);  in starfive_aes_write_nonce()
171 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_write_key()  local
175 		writel(key[0], cryp->base + STARFIVE_AES_KEY0);  in starfive_aes_write_key()
176 		writel(key[1], cryp->base + STARFIVE_AES_KEY1);  in starfive_aes_write_key()
177 		writel(key[2], cryp->base + STARFIVE_AES_KEY2);  in starfive_aes_write_key()
178 		writel(key[3], cryp->base + STARFIVE_AES_KEY3);  in starfive_aes_write_key()
182 		writel(key[4], cryp->base + STARFIVE_AES_KEY4);  in starfive_aes_write_key()
183 		writel(key[5], cryp->base + STARFIVE_AES_KEY5);  in starfive_aes_write_key()
187 		writel(key[6], cryp->base + STARFIVE_AES_KEY6);  in starfive_aes_write_key()
188 		writel(key[7], cryp->base + STARFIVE_AES_KEY7);  in starfive_aes_write_key()
191 	if (starfive_aes_wait_keydone(cryp))  in starfive_aes_write_key()
199 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_ccm_init()  local
203 	memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE);  in starfive_aes_ccm_init()
209 	b0[0] |= (8 * ((cryp->authsize - 2) / 2));  in starfive_aes_ccm_init()
211 	if (cryp->assoclen)  in starfive_aes_ccm_init()
214 	textlen = cryp->total_in;  in starfive_aes_ccm_init()
227 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_hw_init()  local
233 	writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);  in starfive_aes_hw_init()
236 	hw_mode = cryp->flags & FLG_MODE_MASK;  in starfive_aes_hw_init()
253 	rctx->csr.aes.cmode = !is_encrypt(cryp);  in starfive_aes_hw_init()
256 	if (cryp->side_chan) {  in starfive_aes_hw_init()
261 	writel(rctx->csr.aes.v, cryp->base + STARFIVE_AES_CSR);  in starfive_aes_hw_init()
263 	cryp->err = starfive_aes_write_key(ctx);  in starfive_aes_hw_init()
264 	if (cryp->err)  in starfive_aes_hw_init()
265 		return cryp->err;  in starfive_aes_hw_init()
271 		writel(GCM_AES_IV_SIZE, cryp->base + STARFIVE_AES_IVLEN);  in starfive_aes_hw_init()
273 		starfive_aes_write_iv(ctx, (void *)cryp->req.areq->iv);  in starfive_aes_hw_init()
283 		starfive_aes_write_iv(ctx, (void *)cryp->req.sreq->iv);  in starfive_aes_hw_init()
289 	return cryp->err;  in starfive_aes_hw_init()
294 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_read_authtag()  local
298 	if (starfive_aes_wait_busy(cryp))  in starfive_aes_read_authtag()
299 		return dev_err_probe(cryp->dev, -ETIMEDOUT,  in starfive_aes_read_authtag()
302 	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM) {  in starfive_aes_read_authtag()
303 		cryp->tag_out[0] = readl(cryp->base + STARFIVE_AES_NONCE0);  in starfive_aes_read_authtag()
304 		cryp->tag_out[1] = readl(cryp->base + STARFIVE_AES_NONCE1);  in starfive_aes_read_authtag()
305 		cryp->tag_out[2] = readl(cryp->base + STARFIVE_AES_NONCE2);  in starfive_aes_read_authtag()
306 		cryp->tag_out[3] = readl(cryp->base + STARFIVE_AES_NONCE3);  in starfive_aes_read_authtag()
309 			cryp->tag_out[i] = readl(cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_read_authtag()
312 	if (is_encrypt(cryp)) {  in starfive_aes_read_authtag()
313 		scatterwalk_map_and_copy(cryp->tag_out, rctx->out_sg,  in starfive_aes_read_authtag()
314 					 cryp->total_in, cryp->authsize, 1);  in starfive_aes_read_authtag()
316 		if (crypto_memneq(cryp->tag_in, cryp->tag_out, cryp->authsize))  in starfive_aes_read_authtag()
325 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_finish_req()  local
326 	int err = cryp->err;  in starfive_aes_finish_req()
328 	if (!err && cryp->authsize)  in starfive_aes_finish_req()
331 	if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC ||  in starfive_aes_finish_req()
332 		     (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR))  in starfive_aes_finish_req()
333 		starfive_aes_get_iv(cryp, (void *)cryp->req.sreq->iv);  in starfive_aes_finish_req()
335 	if (cryp->authsize)  in starfive_aes_finish_req()
336 		crypto_finalize_aead_request(cryp->engine, cryp->req.areq, err);  in starfive_aes_finish_req()
338 		crypto_finalize_skcipher_request(cryp->engine, cryp->req.sreq,  in starfive_aes_finish_req()
344 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_gcm_write_adata()  local
349 	total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int);  in starfive_aes_gcm_write_adata()
353 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE0);  in starfive_aes_gcm_write_adata()
355 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE1);  in starfive_aes_gcm_write_adata()
357 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE2);  in starfive_aes_gcm_write_adata()
359 		writel(*buffer, cryp->base + STARFIVE_AES_NONCE3);  in starfive_aes_gcm_write_adata()
363 	if (starfive_aes_wait_gcmdone(cryp))  in starfive_aes_gcm_write_adata()
364 		return dev_err_probe(cryp->dev, -ETIMEDOUT,  in starfive_aes_gcm_write_adata()
372 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_ccm_write_adata()  local
378 	total_len = cryp->assoclen;  in starfive_aes_ccm_write_adata()
381 	writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_ccm_write_adata()
383 	writeb(*ci, cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_ccm_write_adata()
389 		writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_ccm_write_adata()
395 			writel(*buffer, cryp->base + STARFIVE_AES_AESDIO0R);  in starfive_aes_ccm_write_adata()
400 	if (starfive_aes_wait_busy(cryp))  in starfive_aes_ccm_write_adata()
401 		return dev_err_probe(cryp->dev, -ETIMEDOUT,  in starfive_aes_ccm_write_adata()
409 	struct starfive_cryp_dev *cryp = param;  in starfive_aes_dma_done()  local
411 	complete(&cryp->dma_done);  in starfive_aes_dma_done()
414 static void starfive_aes_dma_init(struct starfive_cryp_dev *cryp)  in starfive_aes_dma_init()  argument
416 	cryp->cfg_in.direction = DMA_MEM_TO_DEV;  in starfive_aes_dma_init()
417 	cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES;  in starfive_aes_dma_init()
418 	cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;  in starfive_aes_dma_init()
419 	cryp->cfg_in.src_maxburst = cryp->dma_maxburst;  in starfive_aes_dma_init()
420 	cryp->cfg_in.dst_maxburst = cryp->dma_maxburst;  in starfive_aes_dma_init()
421 	cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET;  in starfive_aes_dma_init()
423 	dmaengine_slave_config(cryp->tx, &cryp->cfg_in);  in starfive_aes_dma_init()
425 	cryp->cfg_out.direction = DMA_DEV_TO_MEM;  in starfive_aes_dma_init()
426 	cryp->cfg_out.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;  in starfive_aes_dma_init()
427 	cryp->cfg_out.dst_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES;  in starfive_aes_dma_init()
428 	cryp->cfg_out.src_maxburst = 4;  in starfive_aes_dma_init()
429 	cryp->cfg_out.dst_maxburst = 4;  in starfive_aes_dma_init()
430 	cryp->cfg_out.src_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET;  in starfive_aes_dma_init()
432 	dmaengine_slave_config(cryp->rx, &cryp->cfg_out);  in starfive_aes_dma_init()
434 	init_completion(&cryp->dma_done);  in starfive_aes_dma_init()
437 static int starfive_aes_dma_xfer(struct starfive_cryp_dev *cryp,  in starfive_aes_dma_xfer()  argument
449 	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);  in starfive_aes_dma_xfer()
454 	writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET);  in starfive_aes_dma_xfer()
455 	writel(ALIGN(len, AES_BLOCK_SIZE), cryp->base + STARFIVE_DMA_OUT_LEN_OFFSET);  in starfive_aes_dma_xfer()
460 	out_desc = dmaengine_prep_slave_sg(cryp->rx, dst, 1, DMA_DEV_TO_MEM,  in starfive_aes_dma_xfer()
468 	out_desc->callback_param = cryp;  in starfive_aes_dma_xfer()
470 	reinit_completion(&cryp->dma_done);  in starfive_aes_dma_xfer()
472 	dma_async_issue_pending(cryp->rx);  in starfive_aes_dma_xfer()
474 	in_desc = dmaengine_prep_slave_sg(cryp->tx, src, 1, DMA_MEM_TO_DEV,  in starfive_aes_dma_xfer()
482 	dma_async_issue_pending(cryp->tx);  in starfive_aes_dma_xfer()
484 	if (!wait_for_completion_timeout(&cryp->dma_done,  in starfive_aes_dma_xfer()
494 	writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);  in starfive_aes_dma_xfer()
499 static int starfive_aes_map_sg(struct starfive_cryp_dev *cryp,  in starfive_aes_map_sg()  argument
505 	unsigned int remain = cryp->total_in;  in starfive_aes_map_sg()
512 			src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL);  in starfive_aes_map_sg()
514 				return dev_err_probe(cryp->dev, -ENOMEM,  in starfive_aes_map_sg()
520 			ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len);  in starfive_aes_map_sg()
521 			dma_unmap_sg(cryp->dev, stsg, 1, DMA_BIDIRECTIONAL);  in starfive_aes_map_sg()
529 			src_nents = dma_map_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE);  in starfive_aes_map_sg()
531 				return dev_err_probe(cryp->dev, -ENOMEM,  in starfive_aes_map_sg()
534 			dst_nents = dma_map_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE);  in starfive_aes_map_sg()
536 				return dev_err_probe(cryp->dev, -ENOMEM,  in starfive_aes_map_sg()
542 			ret = starfive_aes_dma_xfer(cryp, stsg, dtsg, len);  in starfive_aes_map_sg()
543 			dma_unmap_sg(cryp->dev, stsg, 1, DMA_TO_DEVICE);  in starfive_aes_map_sg()
544 			dma_unmap_sg(cryp->dev, dtsg, 1, DMA_FROM_DEVICE);  in starfive_aes_map_sg()
575 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_do_one_req()  local
578 	cryp->req.sreq = req;  in starfive_aes_do_one_req()
579 	cryp->total_in = req->cryptlen;  in starfive_aes_do_one_req()
580 	cryp->total_out = req->cryptlen;  in starfive_aes_do_one_req()
581 	cryp->assoclen = 0;  in starfive_aes_do_one_req()
582 	cryp->authsize = 0;  in starfive_aes_do_one_req()
593 	if (!cryp->total_in)  in starfive_aes_do_one_req()
596 	starfive_aes_dma_init(cryp);  in starfive_aes_do_one_req()
598 	ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg);  in starfive_aes_do_one_req()
613 	ctx->cryp = starfive_cryp_find_dev(ctx);  in starfive_aes_init_tfm()
614 	if (!ctx->cryp)  in starfive_aes_init_tfm()
620 		return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->skcipher_fbk),  in starfive_aes_init_tfm()
643 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_aead_do_one_req()  local
648 	cryp->req.areq = req;  in starfive_aes_aead_do_one_req()
649 	cryp->assoclen = req->assoclen;  in starfive_aes_aead_do_one_req()
650 	cryp->authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));  in starfive_aes_aead_do_one_req()
652 	rctx->in_sg = scatterwalk_ffwd(_src, req->src, cryp->assoclen);  in starfive_aes_aead_do_one_req()
656 		rctx->out_sg = scatterwalk_ffwd(_dst, req->dst, cryp->assoclen);  in starfive_aes_aead_do_one_req()
658 	if (is_encrypt(cryp)) {  in starfive_aes_aead_do_one_req()
659 		cryp->total_in = req->cryptlen;  in starfive_aes_aead_do_one_req()
660 		cryp->total_out = req->cryptlen;  in starfive_aes_aead_do_one_req()
662 		cryp->total_in = req->cryptlen - cryp->authsize;  in starfive_aes_aead_do_one_req()
663 		cryp->total_out = cryp->total_in;  in starfive_aes_aead_do_one_req()
664 		scatterwalk_map_and_copy(cryp->tag_in, req->src,  in starfive_aes_aead_do_one_req()
665 					 cryp->total_in + cryp->assoclen,  in starfive_aes_aead_do_one_req()
666 					 cryp->authsize, 0);  in starfive_aes_aead_do_one_req()
669 	if (cryp->assoclen) {  in starfive_aes_aead_do_one_req()
670 		rctx->adata = kzalloc(cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL);  in starfive_aes_aead_do_one_req()
672 			return dev_err_probe(cryp->dev, -ENOMEM,  in starfive_aes_aead_do_one_req()
675 		if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen),  in starfive_aes_aead_do_one_req()
676 				      rctx->adata, cryp->assoclen) != cryp->assoclen)  in starfive_aes_aead_do_one_req()
680 	if (cryp->total_in)  in starfive_aes_aead_do_one_req()
682 			       sg_dma_len(rctx->in_sg) - cryp->total_in,  in starfive_aes_aead_do_one_req()
683 			       cryp->total_in);  in starfive_aes_aead_do_one_req()
691 	if (!cryp->assoclen)  in starfive_aes_aead_do_one_req()
694 	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM)  in starfive_aes_aead_do_one_req()
705 	if (!cryp->total_in)  in starfive_aes_aead_do_one_req()
708 	starfive_aes_dma_init(cryp);  in starfive_aes_aead_do_one_req()
710 	ret = starfive_aes_map_sg(cryp, rctx->in_sg, rctx->out_sg);  in starfive_aes_aead_do_one_req()
724 	ctx->cryp = starfive_cryp_find_dev(ctx);  in starfive_aes_aead_init_tfm()
725 	if (!ctx->cryp)  in starfive_aes_aead_init_tfm()
731 		return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->aead_fbk),  in starfive_aes_aead_init_tfm()
748 static bool starfive_aes_check_unaligned(struct starfive_cryp_dev *cryp,  in starfive_aes_check_unaligned()  argument
792 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_crypt()  local
795 	cryp->flags = flags;  in starfive_aes_crypt()
797 	if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB ||  in starfive_aes_crypt()
798 	    (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC)  in starfive_aes_crypt()
802 	if (starfive_aes_check_unaligned(cryp, req->src, req->dst))  in starfive_aes_crypt()
803 		return starfive_aes_do_fallback(req, is_encrypt(cryp));  in starfive_aes_crypt()
805 	return crypto_transfer_skcipher_request_to_engine(cryp->engine, req);  in starfive_aes_crypt()
829 	struct starfive_cryp_dev *cryp = ctx->cryp;  in starfive_aes_aead_crypt()  local
832 	cryp->flags = flags;  in starfive_aes_aead_crypt()
837 	if (((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM) &&  in starfive_aes_aead_crypt()
838 	    !is_encrypt(cryp))  in starfive_aes_aead_crypt()
848 	if (starfive_aes_check_unaligned(cryp, src, dst))  in starfive_aes_aead_crypt()
849 		return starfive_aes_aead_do_fallback(req, is_encrypt(cryp));  in starfive_aes_aead_crypt()
851 	return crypto_transfer_aead_request_to_engine(cryp->engine, req);  in starfive_aes_aead_crypt()