29 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
32 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)  in omap_aes_gcm_finish_req()  argument
34 	struct aead_request *req = dd->aead_req;  in omap_aes_gcm_finish_req()
36 	dd->in_sg = NULL;  in omap_aes_gcm_finish_req()
37 	dd->out_sg = NULL;  in omap_aes_gcm_finish_req()
39 	crypto_finalize_aead_request(dd->engine, req, ret);  in omap_aes_gcm_finish_req()
41 	pm_runtime_mark_last_busy(dd->dev);  in omap_aes_gcm_finish_req()
42 	pm_runtime_put_autosuspend(dd->dev);  in omap_aes_gcm_finish_req()
45 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)  in omap_aes_gcm_done_task()  argument
51 	alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);  in omap_aes_gcm_done_task()
52 	clen = ALIGN(dd->total, AES_BLOCK_SIZE);  in omap_aes_gcm_done_task()
53 	rctx = aead_request_ctx(dd->aead_req);  in omap_aes_gcm_done_task()
55 	nsg = !!(dd->assoc_len && dd->total);  in omap_aes_gcm_done_task()
57 	dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_aes_gcm_done_task()
59 	dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);  in omap_aes_gcm_done_task()
60 	dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);  in omap_aes_gcm_done_task()
61 	omap_aes_crypt_dma_stop(dd);  in omap_aes_gcm_done_task()
63 	omap_crypto_cleanup(dd->out_sg, dd->orig_out,  in omap_aes_gcm_done_task()
64 			    dd->aead_req->assoclen, dd->total,  in omap_aes_gcm_done_task()
65 			    FLAGS_OUT_DATA_ST_SHIFT, dd->flags);  in omap_aes_gcm_done_task()
67 	if (dd->flags & FLAGS_ENCRYPT)  in omap_aes_gcm_done_task()
69 					 dd->aead_req->dst,  in omap_aes_gcm_done_task()
70 					 dd->total + dd->aead_req->assoclen,  in omap_aes_gcm_done_task()
71 					 dd->authsize, 1);  in omap_aes_gcm_done_task()
73 	omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,  in omap_aes_gcm_done_task()
74 			    FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);  in omap_aes_gcm_done_task()
76 	omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,  in omap_aes_gcm_done_task()
77 			    FLAGS_IN_DATA_ST_SHIFT, dd->flags);  in omap_aes_gcm_done_task()
79 	if (!(dd->flags & FLAGS_ENCRYPT)) {  in omap_aes_gcm_done_task()
81 		for (i = 0; i < dd->authsize; i++) {  in omap_aes_gcm_done_task()
88 	omap_aes_gcm_finish_req(dd, ret);  in omap_aes_gcm_done_task()
91 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,  in omap_aes_gcm_copy_buffers()  argument
104 	if (dd->flags & FLAGS_RFC4106_GCM)  in omap_aes_gcm_copy_buffers()
107 	if (!(dd->flags & FLAGS_ENCRYPT))  in omap_aes_gcm_copy_buffers()
115 	omap_aes_clear_copy_flags(dd);  in omap_aes_gcm_copy_buffers()
117 	sg_init_table(dd->in_sgl, nsg + 1);  in omap_aes_gcm_copy_buffers()
121 					   AES_BLOCK_SIZE, dd->in_sgl,  in omap_aes_gcm_copy_buffers()
126 					   &dd->flags);  in omap_aes_gcm_copy_buffers()
135 			sg_unmark_end(dd->in_sgl);  in omap_aes_gcm_copy_buffers()
138 					   AES_BLOCK_SIZE, &dd->in_sgl[nsg],  in omap_aes_gcm_copy_buffers()
143 					   &dd->flags);  in omap_aes_gcm_copy_buffers()
148 	dd->in_sg = dd->in_sgl;  in omap_aes_gcm_copy_buffers()
149 	dd->total = cryptlen;  in omap_aes_gcm_copy_buffers()
150 	dd->assoc_len = assoclen;  in omap_aes_gcm_copy_buffers()
151 	dd->authsize = authlen;  in omap_aes_gcm_copy_buffers()
153 	dd->out_sg = req->dst;  in omap_aes_gcm_copy_buffers()
154 	dd->orig_out = req->dst;  in omap_aes_gcm_copy_buffers()
156 	dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, req->assoclen);  in omap_aes_gcm_copy_buffers()
159 	if (req->src == req->dst || dd->out_sg == sg_arr)  in omap_aes_gcm_copy_buffers()
163 		ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,  in omap_aes_gcm_copy_buffers()
164 					   AES_BLOCK_SIZE, &dd->out_sgl,  in omap_aes_gcm_copy_buffers()
166 					   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);  in omap_aes_gcm_copy_buffers()
171 	dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);  in omap_aes_gcm_copy_buffers()
172 	dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);  in omap_aes_gcm_copy_buffers()
187 	struct omap_aes_dev *dd = data;  in omap_aes_gcm_dma_out_callback()  local
192 	if (!(dd->flags & FLAGS_ENCRYPT))  in omap_aes_gcm_dma_out_callback()
193 		scatterwalk_map_and_copy(tag, dd->aead_req->src,  in omap_aes_gcm_dma_out_callback()
194 					 dd->total + dd->aead_req->assoclen,  in omap_aes_gcm_dma_out_callback()
195 					 dd->authsize, 0);  in omap_aes_gcm_dma_out_callback()
197 	rctx = aead_request_ctx(dd->aead_req);  in omap_aes_gcm_dma_out_callback()
200 		val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));  in omap_aes_gcm_dma_out_callback()
202 		if (!(dd->flags & FLAGS_ENCRYPT))  in omap_aes_gcm_dma_out_callback()
206 	omap_aes_gcm_done_task(dd);  in omap_aes_gcm_dma_out_callback()
209 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,  in omap_aes_gcm_handle_queue()  argument
213 		return crypto_transfer_aead_request_to_engine(dd->engine, req);  in omap_aes_gcm_handle_queue()
219 				    struct omap_aes_dev *dd)  in omap_aes_gcm_prepare_req()  argument
225 	dd->aead_req = req;  in omap_aes_gcm_prepare_req()
228 	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;  in omap_aes_gcm_prepare_req()
230 	err = omap_aes_gcm_copy_buffers(dd, req);  in omap_aes_gcm_prepare_req()
234 	dd->ctx = &ctx->octx;  in omap_aes_gcm_prepare_req()
236 	return omap_aes_write_ctrl(dd);  in omap_aes_gcm_prepare_req()
244 	struct omap_aes_dev *dd;  in omap_aes_gcm_crypt()  local
265 	dd = omap_aes_find_dev(rctx);  in omap_aes_gcm_crypt()
266 	if (!dd)  in omap_aes_gcm_crypt()
270 	return omap_aes_gcm_handle_queue(dd, req);  in omap_aes_gcm_crypt()
365 	struct omap_aes_dev *dd = rctx->dd;  in omap_aes_gcm_crypt_req()  local
368 	if (!dd)  in omap_aes_gcm_crypt_req()
371 	ret = omap_aes_gcm_prepare_req(req, dd);  in omap_aes_gcm_crypt_req()
375 	if (dd->in_sg_len)  in omap_aes_gcm_crypt_req()
376 		ret = omap_aes_crypt_dma_start(dd);  in omap_aes_gcm_crypt_req()
378 		omap_aes_gcm_dma_out_callback(dd);  in omap_aes_gcm_crypt_req()