xref: /linux/drivers/crypto/ccree/cc_hash.c (revision ec63e2a4897075e427c121d863bd89c44578094f)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
3 
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <crypto/algapi.h>
7 #include <crypto/hash.h>
8 #include <crypto/md5.h>
9 #include <crypto/sm3.h>
10 #include <crypto/internal/hash.h>
11 
12 #include "cc_driver.h"
13 #include "cc_request_mgr.h"
14 #include "cc_buffer_mgr.h"
15 #include "cc_hash.h"
16 #include "cc_sram_mgr.h"
17 
18 #define CC_MAX_HASH_SEQ_LEN 12
19 #define CC_MAX_OPAD_KEYS_SIZE CC_MAX_HASH_BLCK_SIZE
20 #define CC_SM3_HASH_LEN_SIZE 8
21 
22 struct cc_hash_handle {
23 	cc_sram_addr_t digest_len_sram_addr; /* const value in SRAM*/
24 	cc_sram_addr_t larval_digest_sram_addr;   /* const value in SRAM */
25 	struct list_head hash_list;
26 };
27 
28 static const u32 digest_len_init[] = {
29 	0x00000040, 0x00000000, 0x00000000, 0x00000000 };
30 static const u32 md5_init[] = {
31 	SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
32 static const u32 sha1_init[] = {
33 	SHA1_H4, SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
34 static const u32 sha224_init[] = {
35 	SHA224_H7, SHA224_H6, SHA224_H5, SHA224_H4,
36 	SHA224_H3, SHA224_H2, SHA224_H1, SHA224_H0 };
37 static const u32 sha256_init[] = {
38 	SHA256_H7, SHA256_H6, SHA256_H5, SHA256_H4,
39 	SHA256_H3, SHA256_H2, SHA256_H1, SHA256_H0 };
40 static const u32 digest_len_sha512_init[] = {
41 	0x00000080, 0x00000000, 0x00000000, 0x00000000 };
42 static u64 sha384_init[] = {
43 	SHA384_H7, SHA384_H6, SHA384_H5, SHA384_H4,
44 	SHA384_H3, SHA384_H2, SHA384_H1, SHA384_H0 };
45 static u64 sha512_init[] = {
46 	SHA512_H7, SHA512_H6, SHA512_H5, SHA512_H4,
47 	SHA512_H3, SHA512_H2, SHA512_H1, SHA512_H0 };
48 static const u32 sm3_init[] = {
49 	SM3_IVH, SM3_IVG, SM3_IVF, SM3_IVE,
50 	SM3_IVD, SM3_IVC, SM3_IVB, SM3_IVA };
51 
52 static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[],
53 			  unsigned int *seq_size);
54 
55 static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[],
56 			  unsigned int *seq_size);
57 
58 static const void *cc_larval_digest(struct device *dev, u32 mode);
59 
60 struct cc_hash_alg {
61 	struct list_head entry;
62 	int hash_mode;
63 	int hw_mode;
64 	int inter_digestsize;
65 	struct cc_drvdata *drvdata;
66 	struct ahash_alg ahash_alg;
67 };
68 
69 struct hash_key_req_ctx {
70 	u32 keylen;
71 	dma_addr_t key_dma_addr;
72 };
73 
74 /* hash per-session context */
75 struct cc_hash_ctx {
76 	struct cc_drvdata *drvdata;
77 	/* holds the origin digest; the digest after "setkey" if HMAC,*
78 	 * the initial digest if HASH.
79 	 */
80 	u8 digest_buff[CC_MAX_HASH_DIGEST_SIZE]  ____cacheline_aligned;
81 	u8 opad_tmp_keys_buff[CC_MAX_OPAD_KEYS_SIZE]  ____cacheline_aligned;
82 
83 	dma_addr_t opad_tmp_keys_dma_addr  ____cacheline_aligned;
84 	dma_addr_t digest_buff_dma_addr;
85 	/* use for hmac with key large then mode block size */
86 	struct hash_key_req_ctx key_params;
87 	int hash_mode;
88 	int hw_mode;
89 	int inter_digestsize;
90 	unsigned int hash_len;
91 	struct completion setkey_comp;
92 	bool is_hmac;
93 };
94 
95 static void cc_set_desc(struct ahash_req_ctx *areq_ctx, struct cc_hash_ctx *ctx,
96 			unsigned int flow_mode, struct cc_hw_desc desc[],
97 			bool is_not_last_data, unsigned int *seq_size);
98 
99 static void cc_set_endianity(u32 mode, struct cc_hw_desc *desc)
100 {
101 	if (mode == DRV_HASH_MD5 || mode == DRV_HASH_SHA384 ||
102 	    mode == DRV_HASH_SHA512) {
103 		set_bytes_swap(desc, 1);
104 	} else {
105 		set_cipher_config0(desc, HASH_DIGEST_RESULT_LITTLE_ENDIAN);
106 	}
107 }
108 
109 static int cc_map_result(struct device *dev, struct ahash_req_ctx *state,
110 			 unsigned int digestsize)
111 {
112 	state->digest_result_dma_addr =
113 		dma_map_single(dev, state->digest_result_buff,
114 			       digestsize, DMA_BIDIRECTIONAL);
115 	if (dma_mapping_error(dev, state->digest_result_dma_addr)) {
116 		dev_err(dev, "Mapping digest result buffer %u B for DMA failed\n",
117 			digestsize);
118 		return -ENOMEM;
119 	}
120 	dev_dbg(dev, "Mapped digest result buffer %u B at va=%pK to dma=%pad\n",
121 		digestsize, state->digest_result_buff,
122 		&state->digest_result_dma_addr);
123 
124 	return 0;
125 }
126 
127 static void cc_init_req(struct device *dev, struct ahash_req_ctx *state,
128 			struct cc_hash_ctx *ctx)
129 {
130 	bool is_hmac = ctx->is_hmac;
131 
132 	memset(state, 0, sizeof(*state));
133 
134 	if (is_hmac) {
135 		if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC &&
136 		    ctx->hw_mode != DRV_CIPHER_CMAC) {
137 			dma_sync_single_for_cpu(dev, ctx->digest_buff_dma_addr,
138 						ctx->inter_digestsize,
139 						DMA_BIDIRECTIONAL);
140 
141 			memcpy(state->digest_buff, ctx->digest_buff,
142 			       ctx->inter_digestsize);
143 			if (ctx->hash_mode == DRV_HASH_SHA512 ||
144 			    ctx->hash_mode == DRV_HASH_SHA384)
145 				memcpy(state->digest_bytes_len,
146 				       digest_len_sha512_init,
147 				       ctx->hash_len);
148 			else
149 				memcpy(state->digest_bytes_len, digest_len_init,
150 				       ctx->hash_len);
151 		}
152 
153 		if (ctx->hash_mode != DRV_HASH_NULL) {
154 			dma_sync_single_for_cpu(dev,
155 						ctx->opad_tmp_keys_dma_addr,
156 						ctx->inter_digestsize,
157 						DMA_BIDIRECTIONAL);
158 			memcpy(state->opad_digest_buff,
159 			       ctx->opad_tmp_keys_buff, ctx->inter_digestsize);
160 		}
161 	} else { /*hash*/
162 		/* Copy the initial digests if hash flow. */
163 		const void *larval = cc_larval_digest(dev, ctx->hash_mode);
164 
165 		memcpy(state->digest_buff, larval, ctx->inter_digestsize);
166 	}
167 }
168 
169 static int cc_map_req(struct device *dev, struct ahash_req_ctx *state,
170 		      struct cc_hash_ctx *ctx)
171 {
172 	bool is_hmac = ctx->is_hmac;
173 
174 	state->digest_buff_dma_addr =
175 		dma_map_single(dev, state->digest_buff,
176 			       ctx->inter_digestsize, DMA_BIDIRECTIONAL);
177 	if (dma_mapping_error(dev, state->digest_buff_dma_addr)) {
178 		dev_err(dev, "Mapping digest len %d B at va=%pK for DMA failed\n",
179 			ctx->inter_digestsize, state->digest_buff);
180 		return -EINVAL;
181 	}
182 	dev_dbg(dev, "Mapped digest %d B at va=%pK to dma=%pad\n",
183 		ctx->inter_digestsize, state->digest_buff,
184 		&state->digest_buff_dma_addr);
185 
186 	if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC) {
187 		state->digest_bytes_len_dma_addr =
188 			dma_map_single(dev, state->digest_bytes_len,
189 				       HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);
190 		if (dma_mapping_error(dev, state->digest_bytes_len_dma_addr)) {
191 			dev_err(dev, "Mapping digest len %u B at va=%pK for DMA failed\n",
192 				HASH_MAX_LEN_SIZE, state->digest_bytes_len);
193 			goto unmap_digest_buf;
194 		}
195 		dev_dbg(dev, "Mapped digest len %u B at va=%pK to dma=%pad\n",
196 			HASH_MAX_LEN_SIZE, state->digest_bytes_len,
197 			&state->digest_bytes_len_dma_addr);
198 	}
199 
200 	if (is_hmac && ctx->hash_mode != DRV_HASH_NULL) {
201 		state->opad_digest_dma_addr =
202 			dma_map_single(dev, state->opad_digest_buff,
203 				       ctx->inter_digestsize,
204 				       DMA_BIDIRECTIONAL);
205 		if (dma_mapping_error(dev, state->opad_digest_dma_addr)) {
206 			dev_err(dev, "Mapping opad digest %d B at va=%pK for DMA failed\n",
207 				ctx->inter_digestsize,
208 				state->opad_digest_buff);
209 			goto unmap_digest_len;
210 		}
211 		dev_dbg(dev, "Mapped opad digest %d B at va=%pK to dma=%pad\n",
212 			ctx->inter_digestsize, state->opad_digest_buff,
213 			&state->opad_digest_dma_addr);
214 	}
215 
216 	return 0;
217 
218 unmap_digest_len:
219 	if (state->digest_bytes_len_dma_addr) {
220 		dma_unmap_single(dev, state->digest_bytes_len_dma_addr,
221 				 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);
222 		state->digest_bytes_len_dma_addr = 0;
223 	}
224 unmap_digest_buf:
225 	if (state->digest_buff_dma_addr) {
226 		dma_unmap_single(dev, state->digest_buff_dma_addr,
227 				 ctx->inter_digestsize, DMA_BIDIRECTIONAL);
228 		state->digest_buff_dma_addr = 0;
229 	}
230 
231 	return -EINVAL;
232 }
233 
234 static void cc_unmap_req(struct device *dev, struct ahash_req_ctx *state,
235 			 struct cc_hash_ctx *ctx)
236 {
237 	if (state->digest_buff_dma_addr) {
238 		dma_unmap_single(dev, state->digest_buff_dma_addr,
239 				 ctx->inter_digestsize, DMA_BIDIRECTIONAL);
240 		dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n",
241 			&state->digest_buff_dma_addr);
242 		state->digest_buff_dma_addr = 0;
243 	}
244 	if (state->digest_bytes_len_dma_addr) {
245 		dma_unmap_single(dev, state->digest_bytes_len_dma_addr,
246 				 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);
247 		dev_dbg(dev, "Unmapped digest-bytes-len buffer: digest_bytes_len_dma_addr=%pad\n",
248 			&state->digest_bytes_len_dma_addr);
249 		state->digest_bytes_len_dma_addr = 0;
250 	}
251 	if (state->opad_digest_dma_addr) {
252 		dma_unmap_single(dev, state->opad_digest_dma_addr,
253 				 ctx->inter_digestsize, DMA_BIDIRECTIONAL);
254 		dev_dbg(dev, "Unmapped opad-digest: opad_digest_dma_addr=%pad\n",
255 			&state->opad_digest_dma_addr);
256 		state->opad_digest_dma_addr = 0;
257 	}
258 }
259 
260 static void cc_unmap_result(struct device *dev, struct ahash_req_ctx *state,
261 			    unsigned int digestsize, u8 *result)
262 {
263 	if (state->digest_result_dma_addr) {
264 		dma_unmap_single(dev, state->digest_result_dma_addr, digestsize,
265 				 DMA_BIDIRECTIONAL);
266 		dev_dbg(dev, "unmpa digest result buffer va (%pK) pa (%pad) len %u\n",
267 			state->digest_result_buff,
268 			&state->digest_result_dma_addr, digestsize);
269 		memcpy(result, state->digest_result_buff, digestsize);
270 	}
271 	state->digest_result_dma_addr = 0;
272 }
273 
274 static void cc_update_complete(struct device *dev, void *cc_req, int err)
275 {
276 	struct ahash_request *req = (struct ahash_request *)cc_req;
277 	struct ahash_req_ctx *state = ahash_request_ctx(req);
278 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
279 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
280 
281 	dev_dbg(dev, "req=%pK\n", req);
282 
283 	cc_unmap_hash_request(dev, state, req->src, false);
284 	cc_unmap_req(dev, state, ctx);
285 	req->base.complete(&req->base, err);
286 }
287 
288 static void cc_digest_complete(struct device *dev, void *cc_req, int err)
289 {
290 	struct ahash_request *req = (struct ahash_request *)cc_req;
291 	struct ahash_req_ctx *state = ahash_request_ctx(req);
292 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
293 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
294 	u32 digestsize = crypto_ahash_digestsize(tfm);
295 
296 	dev_dbg(dev, "req=%pK\n", req);
297 
298 	cc_unmap_hash_request(dev, state, req->src, false);
299 	cc_unmap_result(dev, state, digestsize, req->result);
300 	cc_unmap_req(dev, state, ctx);
301 	req->base.complete(&req->base, err);
302 }
303 
304 static void cc_hash_complete(struct device *dev, void *cc_req, int err)
305 {
306 	struct ahash_request *req = (struct ahash_request *)cc_req;
307 	struct ahash_req_ctx *state = ahash_request_ctx(req);
308 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
309 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
310 	u32 digestsize = crypto_ahash_digestsize(tfm);
311 
312 	dev_dbg(dev, "req=%pK\n", req);
313 
314 	cc_unmap_hash_request(dev, state, req->src, false);
315 	cc_unmap_result(dev, state, digestsize, req->result);
316 	cc_unmap_req(dev, state, ctx);
317 	req->base.complete(&req->base, err);
318 }
319 
320 static int cc_fin_result(struct cc_hw_desc *desc, struct ahash_request *req,
321 			 int idx)
322 {
323 	struct ahash_req_ctx *state = ahash_request_ctx(req);
324 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
325 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
326 	u32 digestsize = crypto_ahash_digestsize(tfm);
327 
328 	/* Get final MAC result */
329 	hw_desc_init(&desc[idx]);
330 	set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
331 	/* TODO */
332 	set_dout_dlli(&desc[idx], state->digest_result_dma_addr, digestsize,
333 		      NS_BIT, 1);
334 	set_queue_last_ind(ctx->drvdata, &desc[idx]);
335 	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
336 	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
337 	set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
338 	cc_set_endianity(ctx->hash_mode, &desc[idx]);
339 	idx++;
340 
341 	return idx;
342 }
343 
344 static int cc_fin_hmac(struct cc_hw_desc *desc, struct ahash_request *req,
345 		       int idx)
346 {
347 	struct ahash_req_ctx *state = ahash_request_ctx(req);
348 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
349 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
350 	u32 digestsize = crypto_ahash_digestsize(tfm);
351 
352 	/* store the hash digest result in the context */
353 	hw_desc_init(&desc[idx]);
354 	set_cipher_mode(&desc[idx], ctx->hw_mode);
355 	set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, digestsize,
356 		      NS_BIT, 0);
357 	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
358 	cc_set_endianity(ctx->hash_mode, &desc[idx]);
359 	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
360 	idx++;
361 
362 	/* Loading hash opad xor key state */
363 	hw_desc_init(&desc[idx]);
364 	set_cipher_mode(&desc[idx], ctx->hw_mode);
365 	set_din_type(&desc[idx], DMA_DLLI, state->opad_digest_dma_addr,
366 		     ctx->inter_digestsize, NS_BIT);
367 	set_flow_mode(&desc[idx], S_DIN_to_HASH);
368 	set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
369 	idx++;
370 
371 	/* Load the hash current length */
372 	hw_desc_init(&desc[idx]);
373 	set_cipher_mode(&desc[idx], ctx->hw_mode);
374 	set_din_sram(&desc[idx],
375 		     cc_digest_len_addr(ctx->drvdata, ctx->hash_mode),
376 		     ctx->hash_len);
377 	set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
378 	set_flow_mode(&desc[idx], S_DIN_to_HASH);
379 	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
380 	idx++;
381 
382 	/* Memory Barrier: wait for IPAD/OPAD axi write to complete */
383 	hw_desc_init(&desc[idx]);
384 	set_din_no_dma(&desc[idx], 0, 0xfffff0);
385 	set_dout_no_dma(&desc[idx], 0, 0, 1);
386 	idx++;
387 
388 	/* Perform HASH update */
389 	hw_desc_init(&desc[idx]);
390 	set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
391 		     digestsize, NS_BIT);
392 	set_flow_mode(&desc[idx], DIN_HASH);
393 	idx++;
394 
395 	return idx;
396 }
397 
398 static int cc_hash_digest(struct ahash_request *req)
399 {
400 	struct ahash_req_ctx *state = ahash_request_ctx(req);
401 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
402 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
403 	u32 digestsize = crypto_ahash_digestsize(tfm);
404 	struct scatterlist *src = req->src;
405 	unsigned int nbytes = req->nbytes;
406 	u8 *result = req->result;
407 	struct device *dev = drvdata_to_dev(ctx->drvdata);
408 	bool is_hmac = ctx->is_hmac;
409 	struct cc_crypto_req cc_req = {};
410 	struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
411 	cc_sram_addr_t larval_digest_addr =
412 		cc_larval_digest_addr(ctx->drvdata, ctx->hash_mode);
413 	int idx = 0;
414 	int rc = 0;
415 	gfp_t flags = cc_gfp_flags(&req->base);
416 
417 	dev_dbg(dev, "===== %s-digest (%d) ====\n", is_hmac ? "hmac" : "hash",
418 		nbytes);
419 
420 	cc_init_req(dev, state, ctx);
421 
422 	if (cc_map_req(dev, state, ctx)) {
423 		dev_err(dev, "map_ahash_source() failed\n");
424 		return -ENOMEM;
425 	}
426 
427 	if (cc_map_result(dev, state, digestsize)) {
428 		dev_err(dev, "map_ahash_digest() failed\n");
429 		cc_unmap_req(dev, state, ctx);
430 		return -ENOMEM;
431 	}
432 
433 	if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 1,
434 				      flags)) {
435 		dev_err(dev, "map_ahash_request_final() failed\n");
436 		cc_unmap_result(dev, state, digestsize, result);
437 		cc_unmap_req(dev, state, ctx);
438 		return -ENOMEM;
439 	}
440 
441 	/* Setup request structure */
442 	cc_req.user_cb = cc_digest_complete;
443 	cc_req.user_arg = req;
444 
445 	/* If HMAC then load hash IPAD xor key, if HASH then load initial
446 	 * digest
447 	 */
448 	hw_desc_init(&desc[idx]);
449 	set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
450 	if (is_hmac) {
451 		set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
452 			     ctx->inter_digestsize, NS_BIT);
453 	} else {
454 		set_din_sram(&desc[idx], larval_digest_addr,
455 			     ctx->inter_digestsize);
456 	}
457 	set_flow_mode(&desc[idx], S_DIN_to_HASH);
458 	set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
459 	idx++;
460 
461 	/* Load the hash current length */
462 	hw_desc_init(&desc[idx]);
463 	set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
464 
465 	if (is_hmac) {
466 		set_din_type(&desc[idx], DMA_DLLI,
467 			     state->digest_bytes_len_dma_addr,
468 			     ctx->hash_len, NS_BIT);
469 	} else {
470 		set_din_const(&desc[idx], 0, ctx->hash_len);
471 		if (nbytes)
472 			set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
473 		else
474 			set_cipher_do(&desc[idx], DO_PAD);
475 	}
476 	set_flow_mode(&desc[idx], S_DIN_to_HASH);
477 	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
478 	idx++;
479 
480 	cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx);
481 
482 	if (is_hmac) {
483 		/* HW last hash block padding (aka. "DO_PAD") */
484 		hw_desc_init(&desc[idx]);
485 		set_cipher_mode(&desc[idx], ctx->hw_mode);
486 		set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
487 			      ctx->hash_len, NS_BIT, 0);
488 		set_flow_mode(&desc[idx], S_HASH_to_DOUT);
489 		set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
490 		set_cipher_do(&desc[idx], DO_PAD);
491 		idx++;
492 
493 		idx = cc_fin_hmac(desc, req, idx);
494 	}
495 
496 	idx = cc_fin_result(desc, req, idx);
497 
498 	rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
499 	if (rc != -EINPROGRESS && rc != -EBUSY) {
500 		dev_err(dev, "send_request() failed (rc=%d)\n", rc);
501 		cc_unmap_hash_request(dev, state, src, true);
502 		cc_unmap_result(dev, state, digestsize, result);
503 		cc_unmap_req(dev, state, ctx);
504 	}
505 	return rc;
506 }
507 
508 static int cc_restore_hash(struct cc_hw_desc *desc, struct cc_hash_ctx *ctx,
509 			   struct ahash_req_ctx *state, unsigned int idx)
510 {
511 	/* Restore hash digest */
512 	hw_desc_init(&desc[idx]);
513 	set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
514 	set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
515 		     ctx->inter_digestsize, NS_BIT);
516 	set_flow_mode(&desc[idx], S_DIN_to_HASH);
517 	set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
518 	idx++;
519 
520 	/* Restore hash current length */
521 	hw_desc_init(&desc[idx]);
522 	set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
523 	set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
524 	set_din_type(&desc[idx], DMA_DLLI, state->digest_bytes_len_dma_addr,
525 		     ctx->hash_len, NS_BIT);
526 	set_flow_mode(&desc[idx], S_DIN_to_HASH);
527 	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
528 	idx++;
529 
530 	cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx);
531 
532 	return idx;
533 }
534 
535 static int cc_hash_update(struct ahash_request *req)
536 {
537 	struct ahash_req_ctx *state = ahash_request_ctx(req);
538 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
539 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
540 	unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base);
541 	struct scatterlist *src = req->src;
542 	unsigned int nbytes = req->nbytes;
543 	struct device *dev = drvdata_to_dev(ctx->drvdata);
544 	struct cc_crypto_req cc_req = {};
545 	struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
546 	u32 idx = 0;
547 	int rc;
548 	gfp_t flags = cc_gfp_flags(&req->base);
549 
550 	dev_dbg(dev, "===== %s-update (%d) ====\n", ctx->is_hmac ?
551 		"hmac" : "hash", nbytes);
552 
553 	if (nbytes == 0) {
554 		/* no real updates required */
555 		return 0;
556 	}
557 
558 	rc = cc_map_hash_request_update(ctx->drvdata, state, src, nbytes,
559 					block_size, flags);
560 	if (rc) {
561 		if (rc == 1) {
562 			dev_dbg(dev, " data size not require HW update %x\n",
563 				nbytes);
564 			/* No hardware updates are required */
565 			return 0;
566 		}
567 		dev_err(dev, "map_ahash_request_update() failed\n");
568 		return -ENOMEM;
569 	}
570 
571 	if (cc_map_req(dev, state, ctx)) {
572 		dev_err(dev, "map_ahash_source() failed\n");
573 		cc_unmap_hash_request(dev, state, src, true);
574 		return -EINVAL;
575 	}
576 
577 	/* Setup request structure */
578 	cc_req.user_cb = cc_update_complete;
579 	cc_req.user_arg = req;
580 
581 	idx = cc_restore_hash(desc, ctx, state, idx);
582 
583 	/* store the hash digest result in context */
584 	hw_desc_init(&desc[idx]);
585 	set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
586 	set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
587 		      ctx->inter_digestsize, NS_BIT, 0);
588 	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
589 	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
590 	idx++;
591 
592 	/* store current hash length in context */
593 	hw_desc_init(&desc[idx]);
594 	set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
595 	set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr,
596 		      ctx->hash_len, NS_BIT, 1);
597 	set_queue_last_ind(ctx->drvdata, &desc[idx]);
598 	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
599 	set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
600 	idx++;
601 
602 	rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
603 	if (rc != -EINPROGRESS && rc != -EBUSY) {
604 		dev_err(dev, "send_request() failed (rc=%d)\n", rc);
605 		cc_unmap_hash_request(dev, state, src, true);
606 		cc_unmap_req(dev, state, ctx);
607 	}
608 	return rc;
609 }
610 
611 static int cc_do_finup(struct ahash_request *req, bool update)
612 {
613 	struct ahash_req_ctx *state = ahash_request_ctx(req);
614 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
615 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
616 	u32 digestsize = crypto_ahash_digestsize(tfm);
617 	struct scatterlist *src = req->src;
618 	unsigned int nbytes = req->nbytes;
619 	u8 *result = req->result;
620 	struct device *dev = drvdata_to_dev(ctx->drvdata);
621 	bool is_hmac = ctx->is_hmac;
622 	struct cc_crypto_req cc_req = {};
623 	struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
624 	unsigned int idx = 0;
625 	int rc;
626 	gfp_t flags = cc_gfp_flags(&req->base);
627 
628 	dev_dbg(dev, "===== %s-%s (%d) ====\n", is_hmac ? "hmac" : "hash",
629 		update ? "finup" : "final", nbytes);
630 
631 	if (cc_map_req(dev, state, ctx)) {
632 		dev_err(dev, "map_ahash_source() failed\n");
633 		return -EINVAL;
634 	}
635 
636 	if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, update,
637 				      flags)) {
638 		dev_err(dev, "map_ahash_request_final() failed\n");
639 		cc_unmap_req(dev, state, ctx);
640 		return -ENOMEM;
641 	}
642 	if (cc_map_result(dev, state, digestsize)) {
643 		dev_err(dev, "map_ahash_digest() failed\n");
644 		cc_unmap_hash_request(dev, state, src, true);
645 		cc_unmap_req(dev, state, ctx);
646 		return -ENOMEM;
647 	}
648 
649 	/* Setup request structure */
650 	cc_req.user_cb = cc_hash_complete;
651 	cc_req.user_arg = req;
652 
653 	idx = cc_restore_hash(desc, ctx, state, idx);
654 
655 	/* Pad the hash */
656 	hw_desc_init(&desc[idx]);
657 	set_cipher_do(&desc[idx], DO_PAD);
658 	set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
659 	set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr,
660 		      ctx->hash_len, NS_BIT, 0);
661 	set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
662 	set_flow_mode(&desc[idx], S_HASH_to_DOUT);
663 	idx++;
664 
665 	if (is_hmac)
666 		idx = cc_fin_hmac(desc, req, idx);
667 
668 	idx = cc_fin_result(desc, req, idx);
669 
670 	rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
671 	if (rc != -EINPROGRESS && rc != -EBUSY) {
672 		dev_err(dev, "send_request() failed (rc=%d)\n", rc);
673 		cc_unmap_hash_request(dev, state, src, true);
674 		cc_unmap_result(dev, state, digestsize, result);
675 		cc_unmap_req(dev, state, ctx);
676 	}
677 	return rc;
678 }
679 
680 static int cc_hash_finup(struct ahash_request *req)
681 {
682 	return cc_do_finup(req, true);
683 }
684 
685 
686 static int cc_hash_final(struct ahash_request *req)
687 {
688 	return cc_do_finup(req, false);
689 }
690 
691 static int cc_hash_init(struct ahash_request *req)
692 {
693 	struct ahash_req_ctx *state = ahash_request_ctx(req);
694 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
695 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
696 	struct device *dev = drvdata_to_dev(ctx->drvdata);
697 
698 	dev_dbg(dev, "===== init (%d) ====\n", req->nbytes);
699 
700 	cc_init_req(dev, state, ctx);
701 
702 	return 0;
703 }
704 
705 static int cc_hash_setkey(struct crypto_ahash *ahash, const u8 *key,
706 			  unsigned int keylen)
707 {
708 	unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
709 	struct cc_crypto_req cc_req = {};
710 	struct cc_hash_ctx *ctx = NULL;
711 	int blocksize = 0;
712 	int digestsize = 0;
713 	int i, idx = 0, rc = 0;
714 	struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
715 	cc_sram_addr_t larval_addr;
716 	struct device *dev;
717 
718 	ctx = crypto_ahash_ctx(ahash);
719 	dev = drvdata_to_dev(ctx->drvdata);
720 	dev_dbg(dev, "start keylen: %d", keylen);
721 
722 	blocksize = crypto_tfm_alg_blocksize(&ahash->base);
723 	digestsize = crypto_ahash_digestsize(ahash);
724 
725 	larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->hash_mode);
726 
727 	/* The keylen value distinguishes HASH in case keylen is ZERO bytes,
728 	 * any NON-ZERO value utilizes HMAC flow
729 	 */
730 	ctx->key_params.keylen = keylen;
731 	ctx->key_params.key_dma_addr = 0;
732 	ctx->is_hmac = true;
733 
734 	if (keylen) {
735 		ctx->key_params.key_dma_addr =
736 			dma_map_single(dev, (void *)key, keylen, DMA_TO_DEVICE);
737 		if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) {
738 			dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
739 				key, keylen);
740 			return -ENOMEM;
741 		}
742 		dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n",
743 			&ctx->key_params.key_dma_addr, ctx->key_params.keylen);
744 
745 		if (keylen > blocksize) {
746 			/* Load hash initial state */
747 			hw_desc_init(&desc[idx]);
748 			set_cipher_mode(&desc[idx], ctx->hw_mode);
749 			set_din_sram(&desc[idx], larval_addr,
750 				     ctx->inter_digestsize);
751 			set_flow_mode(&desc[idx], S_DIN_to_HASH);
752 			set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
753 			idx++;
754 
755 			/* Load the hash current length*/
756 			hw_desc_init(&desc[idx]);
757 			set_cipher_mode(&desc[idx], ctx->hw_mode);
758 			set_din_const(&desc[idx], 0, ctx->hash_len);
759 			set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
760 			set_flow_mode(&desc[idx], S_DIN_to_HASH);
761 			set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
762 			idx++;
763 
764 			hw_desc_init(&desc[idx]);
765 			set_din_type(&desc[idx], DMA_DLLI,
766 				     ctx->key_params.key_dma_addr, keylen,
767 				     NS_BIT);
768 			set_flow_mode(&desc[idx], DIN_HASH);
769 			idx++;
770 
771 			/* Get hashed key */
772 			hw_desc_init(&desc[idx]);
773 			set_cipher_mode(&desc[idx], ctx->hw_mode);
774 			set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,
775 				      digestsize, NS_BIT, 0);
776 			set_flow_mode(&desc[idx], S_HASH_to_DOUT);
777 			set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
778 			set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
779 			cc_set_endianity(ctx->hash_mode, &desc[idx]);
780 			idx++;
781 
782 			hw_desc_init(&desc[idx]);
783 			set_din_const(&desc[idx], 0, (blocksize - digestsize));
784 			set_flow_mode(&desc[idx], BYPASS);
785 			set_dout_dlli(&desc[idx],
786 				      (ctx->opad_tmp_keys_dma_addr +
787 				       digestsize),
788 				      (blocksize - digestsize), NS_BIT, 0);
789 			idx++;
790 		} else {
791 			hw_desc_init(&desc[idx]);
792 			set_din_type(&desc[idx], DMA_DLLI,
793 				     ctx->key_params.key_dma_addr, keylen,
794 				     NS_BIT);
795 			set_flow_mode(&desc[idx], BYPASS);
796 			set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,
797 				      keylen, NS_BIT, 0);
798 			idx++;
799 
800 			if ((blocksize - keylen)) {
801 				hw_desc_init(&desc[idx]);
802 				set_din_const(&desc[idx], 0,
803 					      (blocksize - keylen));
804 				set_flow_mode(&desc[idx], BYPASS);
805 				set_dout_dlli(&desc[idx],
806 					      (ctx->opad_tmp_keys_dma_addr +
807 					       keylen), (blocksize - keylen),
808 					      NS_BIT, 0);
809 				idx++;
810 			}
811 		}
812 	} else {
813 		hw_desc_init(&desc[idx]);
814 		set_din_const(&desc[idx], 0, blocksize);
815 		set_flow_mode(&desc[idx], BYPASS);
816 		set_dout_dlli(&desc[idx], (ctx->opad_tmp_keys_dma_addr),
817 			      blocksize, NS_BIT, 0);
818 		idx++;
819 	}
820 
821 	rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
822 	if (rc) {
823 		dev_err(dev, "send_request() failed (rc=%d)\n", rc);
824 		goto out;
825 	}
826 
827 	/* calc derived HMAC key */
828 	for (idx = 0, i = 0; i < 2; i++) {
829 		/* Load hash initial state */
830 		hw_desc_init(&desc[idx]);
831 		set_cipher_mode(&desc[idx], ctx->hw_mode);
832 		set_din_sram(&desc[idx], larval_addr, ctx->inter_digestsize);
833 		set_flow_mode(&desc[idx], S_DIN_to_HASH);
834 		set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
835 		idx++;
836 
837 		/* Load the hash current length*/
838 		hw_desc_init(&desc[idx]);
839 		set_cipher_mode(&desc[idx], ctx->hw_mode);
840 		set_din_const(&desc[idx], 0, ctx->hash_len);
841 		set_flow_mode(&desc[idx], S_DIN_to_HASH);
842 		set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
843 		idx++;
844 
845 		/* Prepare ipad key */
846 		hw_desc_init(&desc[idx]);
847 		set_xor_val(&desc[idx], hmac_pad_const[i]);
848 		set_cipher_mode(&desc[idx], ctx->hw_mode);
849 		set_flow_mode(&desc[idx], S_DIN_to_HASH);
850 		set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
851 		idx++;
852 
853 		/* Perform HASH update */
854 		hw_desc_init(&desc[idx]);
855 		set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr,
856 			     blocksize, NS_BIT);
857 		set_cipher_mode(&desc[idx], ctx->hw_mode);
858 		set_xor_active(&desc[idx]);
859 		set_flow_mode(&desc[idx], DIN_HASH);
860 		idx++;
861 
862 		/* Get the IPAD/OPAD xor key (Note, IPAD is the initial digest
863 		 * of the first HASH "update" state)
864 		 */
865 		hw_desc_init(&desc[idx]);
866 		set_cipher_mode(&desc[idx], ctx->hw_mode);
867 		if (i > 0) /* Not first iteration */
868 			set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,
869 				      ctx->inter_digestsize, NS_BIT, 0);
870 		else /* First iteration */
871 			set_dout_dlli(&desc[idx], ctx->digest_buff_dma_addr,
872 				      ctx->inter_digestsize, NS_BIT, 0);
873 		set_flow_mode(&desc[idx], S_HASH_to_DOUT);
874 		set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
875 		idx++;
876 	}
877 
878 	rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
879 
880 out:
881 	if (rc)
882 		crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
883 
884 	if (ctx->key_params.key_dma_addr) {
885 		dma_unmap_single(dev, ctx->key_params.key_dma_addr,
886 				 ctx->key_params.keylen, DMA_TO_DEVICE);
887 		dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n",
888 			&ctx->key_params.key_dma_addr, ctx->key_params.keylen);
889 	}
890 	return rc;
891 }
892 
893 static int cc_xcbc_setkey(struct crypto_ahash *ahash,
894 			  const u8 *key, unsigned int keylen)
895 {
896 	struct cc_crypto_req cc_req = {};
897 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
898 	struct device *dev = drvdata_to_dev(ctx->drvdata);
899 	int rc = 0;
900 	unsigned int idx = 0;
901 	struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
902 
903 	dev_dbg(dev, "===== setkey (%d) ====\n", keylen);
904 
905 	switch (keylen) {
906 	case AES_KEYSIZE_128:
907 	case AES_KEYSIZE_192:
908 	case AES_KEYSIZE_256:
909 		break;
910 	default:
911 		return -EINVAL;
912 	}
913 
914 	ctx->key_params.keylen = keylen;
915 
916 	ctx->key_params.key_dma_addr =
917 		dma_map_single(dev, (void *)key, keylen, DMA_TO_DEVICE);
918 	if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) {
919 		dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
920 			key, keylen);
921 		return -ENOMEM;
922 	}
923 	dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n",
924 		&ctx->key_params.key_dma_addr, ctx->key_params.keylen);
925 
926 	ctx->is_hmac = true;
927 	/* 1. Load the AES key */
928 	hw_desc_init(&desc[idx]);
929 	set_din_type(&desc[idx], DMA_DLLI, ctx->key_params.key_dma_addr,
930 		     keylen, NS_BIT);
931 	set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
932 	set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
933 	set_key_size_aes(&desc[idx], keylen);
934 	set_flow_mode(&desc[idx], S_DIN_to_AES);
935 	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
936 	idx++;
937 
938 	hw_desc_init(&desc[idx]);
939 	set_din_const(&desc[idx], 0x01010101, CC_AES_128_BIT_KEY_SIZE);
940 	set_flow_mode(&desc[idx], DIN_AES_DOUT);
941 	set_dout_dlli(&desc[idx],
942 		      (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET),
943 		      CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);
944 	idx++;
945 
946 	hw_desc_init(&desc[idx]);
947 	set_din_const(&desc[idx], 0x02020202, CC_AES_128_BIT_KEY_SIZE);
948 	set_flow_mode(&desc[idx], DIN_AES_DOUT);
949 	set_dout_dlli(&desc[idx],
950 		      (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET),
951 		      CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);
952 	idx++;
953 
954 	hw_desc_init(&desc[idx]);
955 	set_din_const(&desc[idx], 0x03030303, CC_AES_128_BIT_KEY_SIZE);
956 	set_flow_mode(&desc[idx], DIN_AES_DOUT);
957 	set_dout_dlli(&desc[idx],
958 		      (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET),
959 		      CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);
960 	idx++;
961 
962 	rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
963 
964 	if (rc)
965 		crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
966 
967 	dma_unmap_single(dev, ctx->key_params.key_dma_addr,
968 			 ctx->key_params.keylen, DMA_TO_DEVICE);
969 	dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n",
970 		&ctx->key_params.key_dma_addr, ctx->key_params.keylen);
971 
972 	return rc;
973 }
974 
975 static int cc_cmac_setkey(struct crypto_ahash *ahash,
976 			  const u8 *key, unsigned int keylen)
977 {
978 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
979 	struct device *dev = drvdata_to_dev(ctx->drvdata);
980 
981 	dev_dbg(dev, "===== setkey (%d) ====\n", keylen);
982 
983 	ctx->is_hmac = true;
984 
985 	switch (keylen) {
986 	case AES_KEYSIZE_128:
987 	case AES_KEYSIZE_192:
988 	case AES_KEYSIZE_256:
989 		break;
990 	default:
991 		return -EINVAL;
992 	}
993 
994 	ctx->key_params.keylen = keylen;
995 
996 	/* STAT_PHASE_1: Copy key to ctx */
997 
998 	dma_sync_single_for_cpu(dev, ctx->opad_tmp_keys_dma_addr,
999 				keylen, DMA_TO_DEVICE);
1000 
1001 	memcpy(ctx->opad_tmp_keys_buff, key, keylen);
1002 	if (keylen == 24) {
1003 		memset(ctx->opad_tmp_keys_buff + 24, 0,
1004 		       CC_AES_KEY_SIZE_MAX - 24);
1005 	}
1006 
1007 	dma_sync_single_for_device(dev, ctx->opad_tmp_keys_dma_addr,
1008 				   keylen, DMA_TO_DEVICE);
1009 
1010 	ctx->key_params.keylen = keylen;
1011 
1012 	return 0;
1013 }
1014 
1015 static void cc_free_ctx(struct cc_hash_ctx *ctx)
1016 {
1017 	struct device *dev = drvdata_to_dev(ctx->drvdata);
1018 
1019 	if (ctx->digest_buff_dma_addr) {
1020 		dma_unmap_single(dev, ctx->digest_buff_dma_addr,
1021 				 sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL);
1022 		dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n",
1023 			&ctx->digest_buff_dma_addr);
1024 		ctx->digest_buff_dma_addr = 0;
1025 	}
1026 	if (ctx->opad_tmp_keys_dma_addr) {
1027 		dma_unmap_single(dev, ctx->opad_tmp_keys_dma_addr,
1028 				 sizeof(ctx->opad_tmp_keys_buff),
1029 				 DMA_BIDIRECTIONAL);
1030 		dev_dbg(dev, "Unmapped opad-digest: opad_tmp_keys_dma_addr=%pad\n",
1031 			&ctx->opad_tmp_keys_dma_addr);
1032 		ctx->opad_tmp_keys_dma_addr = 0;
1033 	}
1034 
1035 	ctx->key_params.keylen = 0;
1036 }
1037 
1038 static int cc_alloc_ctx(struct cc_hash_ctx *ctx)
1039 {
1040 	struct device *dev = drvdata_to_dev(ctx->drvdata);
1041 
1042 	ctx->key_params.keylen = 0;
1043 
1044 	ctx->digest_buff_dma_addr =
1045 		dma_map_single(dev, (void *)ctx->digest_buff,
1046 			       sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL);
1047 	if (dma_mapping_error(dev, ctx->digest_buff_dma_addr)) {
1048 		dev_err(dev, "Mapping digest len %zu B at va=%pK for DMA failed\n",
1049 			sizeof(ctx->digest_buff), ctx->digest_buff);
1050 		goto fail;
1051 	}
1052 	dev_dbg(dev, "Mapped digest %zu B at va=%pK to dma=%pad\n",
1053 		sizeof(ctx->digest_buff), ctx->digest_buff,
1054 		&ctx->digest_buff_dma_addr);
1055 
1056 	ctx->opad_tmp_keys_dma_addr =
1057 		dma_map_single(dev, (void *)ctx->opad_tmp_keys_buff,
1058 			       sizeof(ctx->opad_tmp_keys_buff),
1059 			       DMA_BIDIRECTIONAL);
1060 	if (dma_mapping_error(dev, ctx->opad_tmp_keys_dma_addr)) {
1061 		dev_err(dev, "Mapping opad digest %zu B at va=%pK for DMA failed\n",
1062 			sizeof(ctx->opad_tmp_keys_buff),
1063 			ctx->opad_tmp_keys_buff);
1064 		goto fail;
1065 	}
1066 	dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%pK to dma=%pad\n",
1067 		sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff,
1068 		&ctx->opad_tmp_keys_dma_addr);
1069 
1070 	ctx->is_hmac = false;
1071 	return 0;
1072 
1073 fail:
1074 	cc_free_ctx(ctx);
1075 	return -ENOMEM;
1076 }
1077 
1078 static int cc_get_hash_len(struct crypto_tfm *tfm)
1079 {
1080 	struct cc_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1081 
1082 	if (ctx->hash_mode == DRV_HASH_SM3)
1083 		return CC_SM3_HASH_LEN_SIZE;
1084 	else
1085 		return cc_get_default_hash_len(ctx->drvdata);
1086 }
1087 
1088 static int cc_cra_init(struct crypto_tfm *tfm)
1089 {
1090 	struct cc_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1091 	struct hash_alg_common *hash_alg_common =
1092 		container_of(tfm->__crt_alg, struct hash_alg_common, base);
1093 	struct ahash_alg *ahash_alg =
1094 		container_of(hash_alg_common, struct ahash_alg, halg);
1095 	struct cc_hash_alg *cc_alg =
1096 			container_of(ahash_alg, struct cc_hash_alg, ahash_alg);
1097 
1098 	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1099 				 sizeof(struct ahash_req_ctx));
1100 
1101 	ctx->hash_mode = cc_alg->hash_mode;
1102 	ctx->hw_mode = cc_alg->hw_mode;
1103 	ctx->inter_digestsize = cc_alg->inter_digestsize;
1104 	ctx->drvdata = cc_alg->drvdata;
1105 	ctx->hash_len = cc_get_hash_len(tfm);
1106 	return cc_alloc_ctx(ctx);
1107 }
1108 
1109 static void cc_cra_exit(struct crypto_tfm *tfm)
1110 {
1111 	struct cc_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1112 	struct device *dev = drvdata_to_dev(ctx->drvdata);
1113 
1114 	dev_dbg(dev, "cc_cra_exit");
1115 	cc_free_ctx(ctx);
1116 }
1117 
1118 static int cc_mac_update(struct ahash_request *req)
1119 {
1120 	struct ahash_req_ctx *state = ahash_request_ctx(req);
1121 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1122 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1123 	struct device *dev = drvdata_to_dev(ctx->drvdata);
1124 	unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base);
1125 	struct cc_crypto_req cc_req = {};
1126 	struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1127 	int rc;
1128 	u32 idx = 0;
1129 	gfp_t flags = cc_gfp_flags(&req->base);
1130 
1131 	if (req->nbytes == 0) {
1132 		/* no real updates required */
1133 		return 0;
1134 	}
1135 
1136 	state->xcbc_count++;
1137 
1138 	rc = cc_map_hash_request_update(ctx->drvdata, state, req->src,
1139 					req->nbytes, block_size, flags);
1140 	if (rc) {
1141 		if (rc == 1) {
1142 			dev_dbg(dev, " data size not require HW update %x\n",
1143 				req->nbytes);
1144 			/* No hardware updates are required */
1145 			return 0;
1146 		}
1147 		dev_err(dev, "map_ahash_request_update() failed\n");
1148 		return -ENOMEM;
1149 	}
1150 
1151 	if (cc_map_req(dev, state, ctx)) {
1152 		dev_err(dev, "map_ahash_source() failed\n");
1153 		return -EINVAL;
1154 	}
1155 
1156 	if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC)
1157 		cc_setup_xcbc(req, desc, &idx);
1158 	else
1159 		cc_setup_cmac(req, desc, &idx);
1160 
1161 	cc_set_desc(state, ctx, DIN_AES_DOUT, desc, true, &idx);
1162 
1163 	/* store the hash digest result in context */
1164 	hw_desc_init(&desc[idx]);
1165 	set_cipher_mode(&desc[idx], ctx->hw_mode);
1166 	set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
1167 		      ctx->inter_digestsize, NS_BIT, 1);
1168 	set_queue_last_ind(ctx->drvdata, &desc[idx]);
1169 	set_flow_mode(&desc[idx], S_AES_to_DOUT);
1170 	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1171 	idx++;
1172 
1173 	/* Setup request structure */
1174 	cc_req.user_cb = (void *)cc_update_complete;
1175 	cc_req.user_arg = (void *)req;
1176 
1177 	rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1178 	if (rc != -EINPROGRESS && rc != -EBUSY) {
1179 		dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1180 		cc_unmap_hash_request(dev, state, req->src, true);
1181 		cc_unmap_req(dev, state, ctx);
1182 	}
1183 	return rc;
1184 }
1185 
1186 static int cc_mac_final(struct ahash_request *req)
1187 {
1188 	struct ahash_req_ctx *state = ahash_request_ctx(req);
1189 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1190 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1191 	struct device *dev = drvdata_to_dev(ctx->drvdata);
1192 	struct cc_crypto_req cc_req = {};
1193 	struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1194 	int idx = 0;
1195 	int rc = 0;
1196 	u32 key_size, key_len;
1197 	u32 digestsize = crypto_ahash_digestsize(tfm);
1198 	gfp_t flags = cc_gfp_flags(&req->base);
1199 	u32 rem_cnt = *cc_hash_buf_cnt(state);
1200 
1201 	if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
1202 		key_size = CC_AES_128_BIT_KEY_SIZE;
1203 		key_len  = CC_AES_128_BIT_KEY_SIZE;
1204 	} else {
1205 		key_size = (ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE :
1206 			ctx->key_params.keylen;
1207 		key_len =  ctx->key_params.keylen;
1208 	}
1209 
1210 	dev_dbg(dev, "===== final  xcbc reminder (%d) ====\n", rem_cnt);
1211 
1212 	if (cc_map_req(dev, state, ctx)) {
1213 		dev_err(dev, "map_ahash_source() failed\n");
1214 		return -EINVAL;
1215 	}
1216 
1217 	if (cc_map_hash_request_final(ctx->drvdata, state, req->src,
1218 				      req->nbytes, 0, flags)) {
1219 		dev_err(dev, "map_ahash_request_final() failed\n");
1220 		cc_unmap_req(dev, state, ctx);
1221 		return -ENOMEM;
1222 	}
1223 
1224 	if (cc_map_result(dev, state, digestsize)) {
1225 		dev_err(dev, "map_ahash_digest() failed\n");
1226 		cc_unmap_hash_request(dev, state, req->src, true);
1227 		cc_unmap_req(dev, state, ctx);
1228 		return -ENOMEM;
1229 	}
1230 
1231 	/* Setup request structure */
1232 	cc_req.user_cb = (void *)cc_hash_complete;
1233 	cc_req.user_arg = (void *)req;
1234 
1235 	if (state->xcbc_count && rem_cnt == 0) {
1236 		/* Load key for ECB decryption */
1237 		hw_desc_init(&desc[idx]);
1238 		set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
1239 		set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_DECRYPT);
1240 		set_din_type(&desc[idx], DMA_DLLI,
1241 			     (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET),
1242 			     key_size, NS_BIT);
1243 		set_key_size_aes(&desc[idx], key_len);
1244 		set_flow_mode(&desc[idx], S_DIN_to_AES);
1245 		set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1246 		idx++;
1247 
1248 		/* Initiate decryption of block state to previous
1249 		 * block_state-XOR-M[n]
1250 		 */
1251 		hw_desc_init(&desc[idx]);
1252 		set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
1253 			     CC_AES_BLOCK_SIZE, NS_BIT);
1254 		set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
1255 			      CC_AES_BLOCK_SIZE, NS_BIT, 0);
1256 		set_flow_mode(&desc[idx], DIN_AES_DOUT);
1257 		idx++;
1258 
1259 		/* Memory Barrier: wait for axi write to complete */
1260 		hw_desc_init(&desc[idx]);
1261 		set_din_no_dma(&desc[idx], 0, 0xfffff0);
1262 		set_dout_no_dma(&desc[idx], 0, 0, 1);
1263 		idx++;
1264 	}
1265 
1266 	if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC)
1267 		cc_setup_xcbc(req, desc, &idx);
1268 	else
1269 		cc_setup_cmac(req, desc, &idx);
1270 
1271 	if (state->xcbc_count == 0) {
1272 		hw_desc_init(&desc[idx]);
1273 		set_cipher_mode(&desc[idx], ctx->hw_mode);
1274 		set_key_size_aes(&desc[idx], key_len);
1275 		set_cmac_size0_mode(&desc[idx]);
1276 		set_flow_mode(&desc[idx], S_DIN_to_AES);
1277 		idx++;
1278 	} else if (rem_cnt > 0) {
1279 		cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);
1280 	} else {
1281 		hw_desc_init(&desc[idx]);
1282 		set_din_const(&desc[idx], 0x00, CC_AES_BLOCK_SIZE);
1283 		set_flow_mode(&desc[idx], DIN_AES_DOUT);
1284 		idx++;
1285 	}
1286 
1287 	/* Get final MAC result */
1288 	hw_desc_init(&desc[idx]);
1289 	/* TODO */
1290 	set_dout_dlli(&desc[idx], state->digest_result_dma_addr,
1291 		      digestsize, NS_BIT, 1);
1292 	set_queue_last_ind(ctx->drvdata, &desc[idx]);
1293 	set_flow_mode(&desc[idx], S_AES_to_DOUT);
1294 	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1295 	set_cipher_mode(&desc[idx], ctx->hw_mode);
1296 	idx++;
1297 
1298 	rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1299 	if (rc != -EINPROGRESS && rc != -EBUSY) {
1300 		dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1301 		cc_unmap_hash_request(dev, state, req->src, true);
1302 		cc_unmap_result(dev, state, digestsize, req->result);
1303 		cc_unmap_req(dev, state, ctx);
1304 	}
1305 	return rc;
1306 }
1307 
1308 static int cc_mac_finup(struct ahash_request *req)
1309 {
1310 	struct ahash_req_ctx *state = ahash_request_ctx(req);
1311 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1312 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1313 	struct device *dev = drvdata_to_dev(ctx->drvdata);
1314 	struct cc_crypto_req cc_req = {};
1315 	struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1316 	int idx = 0;
1317 	int rc = 0;
1318 	u32 key_len = 0;
1319 	u32 digestsize = crypto_ahash_digestsize(tfm);
1320 	gfp_t flags = cc_gfp_flags(&req->base);
1321 
1322 	dev_dbg(dev, "===== finup xcbc(%d) ====\n", req->nbytes);
1323 	if (state->xcbc_count > 0 && req->nbytes == 0) {
1324 		dev_dbg(dev, "No data to update. Call to fdx_mac_final\n");
1325 		return cc_mac_final(req);
1326 	}
1327 
1328 	if (cc_map_req(dev, state, ctx)) {
1329 		dev_err(dev, "map_ahash_source() failed\n");
1330 		return -EINVAL;
1331 	}
1332 
1333 	if (cc_map_hash_request_final(ctx->drvdata, state, req->src,
1334 				      req->nbytes, 1, flags)) {
1335 		dev_err(dev, "map_ahash_request_final() failed\n");
1336 		cc_unmap_req(dev, state, ctx);
1337 		return -ENOMEM;
1338 	}
1339 	if (cc_map_result(dev, state, digestsize)) {
1340 		dev_err(dev, "map_ahash_digest() failed\n");
1341 		cc_unmap_hash_request(dev, state, req->src, true);
1342 		cc_unmap_req(dev, state, ctx);
1343 		return -ENOMEM;
1344 	}
1345 
1346 	/* Setup request structure */
1347 	cc_req.user_cb = (void *)cc_hash_complete;
1348 	cc_req.user_arg = (void *)req;
1349 
1350 	if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
1351 		key_len = CC_AES_128_BIT_KEY_SIZE;
1352 		cc_setup_xcbc(req, desc, &idx);
1353 	} else {
1354 		key_len = ctx->key_params.keylen;
1355 		cc_setup_cmac(req, desc, &idx);
1356 	}
1357 
1358 	if (req->nbytes == 0) {
1359 		hw_desc_init(&desc[idx]);
1360 		set_cipher_mode(&desc[idx], ctx->hw_mode);
1361 		set_key_size_aes(&desc[idx], key_len);
1362 		set_cmac_size0_mode(&desc[idx]);
1363 		set_flow_mode(&desc[idx], S_DIN_to_AES);
1364 		idx++;
1365 	} else {
1366 		cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);
1367 	}
1368 
1369 	/* Get final MAC result */
1370 	hw_desc_init(&desc[idx]);
1371 	/* TODO */
1372 	set_dout_dlli(&desc[idx], state->digest_result_dma_addr,
1373 		      digestsize, NS_BIT, 1);
1374 	set_queue_last_ind(ctx->drvdata, &desc[idx]);
1375 	set_flow_mode(&desc[idx], S_AES_to_DOUT);
1376 	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1377 	set_cipher_mode(&desc[idx], ctx->hw_mode);
1378 	idx++;
1379 
1380 	rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1381 	if (rc != -EINPROGRESS && rc != -EBUSY) {
1382 		dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1383 		cc_unmap_hash_request(dev, state, req->src, true);
1384 		cc_unmap_result(dev, state, digestsize, req->result);
1385 		cc_unmap_req(dev, state, ctx);
1386 	}
1387 	return rc;
1388 }
1389 
1390 static int cc_mac_digest(struct ahash_request *req)
1391 {
1392 	struct ahash_req_ctx *state = ahash_request_ctx(req);
1393 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1394 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
1395 	struct device *dev = drvdata_to_dev(ctx->drvdata);
1396 	u32 digestsize = crypto_ahash_digestsize(tfm);
1397 	struct cc_crypto_req cc_req = {};
1398 	struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1399 	u32 key_len;
1400 	unsigned int idx = 0;
1401 	int rc;
1402 	gfp_t flags = cc_gfp_flags(&req->base);
1403 
1404 	dev_dbg(dev, "===== -digest mac (%d) ====\n",  req->nbytes);
1405 
1406 	cc_init_req(dev, state, ctx);
1407 
1408 	if (cc_map_req(dev, state, ctx)) {
1409 		dev_err(dev, "map_ahash_source() failed\n");
1410 		return -ENOMEM;
1411 	}
1412 	if (cc_map_result(dev, state, digestsize)) {
1413 		dev_err(dev, "map_ahash_digest() failed\n");
1414 		cc_unmap_req(dev, state, ctx);
1415 		return -ENOMEM;
1416 	}
1417 
1418 	if (cc_map_hash_request_final(ctx->drvdata, state, req->src,
1419 				      req->nbytes, 1, flags)) {
1420 		dev_err(dev, "map_ahash_request_final() failed\n");
1421 		cc_unmap_req(dev, state, ctx);
1422 		return -ENOMEM;
1423 	}
1424 
1425 	/* Setup request structure */
1426 	cc_req.user_cb = (void *)cc_digest_complete;
1427 	cc_req.user_arg = (void *)req;
1428 
1429 	if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
1430 		key_len = CC_AES_128_BIT_KEY_SIZE;
1431 		cc_setup_xcbc(req, desc, &idx);
1432 	} else {
1433 		key_len = ctx->key_params.keylen;
1434 		cc_setup_cmac(req, desc, &idx);
1435 	}
1436 
1437 	if (req->nbytes == 0) {
1438 		hw_desc_init(&desc[idx]);
1439 		set_cipher_mode(&desc[idx], ctx->hw_mode);
1440 		set_key_size_aes(&desc[idx], key_len);
1441 		set_cmac_size0_mode(&desc[idx]);
1442 		set_flow_mode(&desc[idx], S_DIN_to_AES);
1443 		idx++;
1444 	} else {
1445 		cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);
1446 	}
1447 
1448 	/* Get final MAC result */
1449 	hw_desc_init(&desc[idx]);
1450 	set_dout_dlli(&desc[idx], state->digest_result_dma_addr,
1451 		      CC_AES_BLOCK_SIZE, NS_BIT, 1);
1452 	set_queue_last_ind(ctx->drvdata, &desc[idx]);
1453 	set_flow_mode(&desc[idx], S_AES_to_DOUT);
1454 	set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1455 	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1456 	set_cipher_mode(&desc[idx], ctx->hw_mode);
1457 	idx++;
1458 
1459 	rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1460 	if (rc != -EINPROGRESS && rc != -EBUSY) {
1461 		dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1462 		cc_unmap_hash_request(dev, state, req->src, true);
1463 		cc_unmap_result(dev, state, digestsize, req->result);
1464 		cc_unmap_req(dev, state, ctx);
1465 	}
1466 	return rc;
1467 }
1468 
1469 static int cc_hash_export(struct ahash_request *req, void *out)
1470 {
1471 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1472 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1473 	struct ahash_req_ctx *state = ahash_request_ctx(req);
1474 	u8 *curr_buff = cc_hash_buf(state);
1475 	u32 curr_buff_cnt = *cc_hash_buf_cnt(state);
1476 	const u32 tmp = CC_EXPORT_MAGIC;
1477 
1478 	memcpy(out, &tmp, sizeof(u32));
1479 	out += sizeof(u32);
1480 
1481 	memcpy(out, state->digest_buff, ctx->inter_digestsize);
1482 	out += ctx->inter_digestsize;
1483 
1484 	memcpy(out, state->digest_bytes_len, ctx->hash_len);
1485 	out += ctx->hash_len;
1486 
1487 	memcpy(out, &curr_buff_cnt, sizeof(u32));
1488 	out += sizeof(u32);
1489 
1490 	memcpy(out, curr_buff, curr_buff_cnt);
1491 
1492 	return 0;
1493 }
1494 
1495 static int cc_hash_import(struct ahash_request *req, const void *in)
1496 {
1497 	struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1498 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1499 	struct device *dev = drvdata_to_dev(ctx->drvdata);
1500 	struct ahash_req_ctx *state = ahash_request_ctx(req);
1501 	u32 tmp;
1502 
1503 	memcpy(&tmp, in, sizeof(u32));
1504 	if (tmp != CC_EXPORT_MAGIC)
1505 		return -EINVAL;
1506 	in += sizeof(u32);
1507 
1508 	cc_init_req(dev, state, ctx);
1509 
1510 	memcpy(state->digest_buff, in, ctx->inter_digestsize);
1511 	in += ctx->inter_digestsize;
1512 
1513 	memcpy(state->digest_bytes_len, in, ctx->hash_len);
1514 	in += ctx->hash_len;
1515 
1516 	/* Sanity check the data as much as possible */
1517 	memcpy(&tmp, in, sizeof(u32));
1518 	if (tmp > CC_MAX_HASH_BLCK_SIZE)
1519 		return -EINVAL;
1520 	in += sizeof(u32);
1521 
1522 	state->buf_cnt[0] = tmp;
1523 	memcpy(state->buffers[0], in, tmp);
1524 
1525 	return 0;
1526 }
1527 
1528 struct cc_hash_template {
1529 	char name[CRYPTO_MAX_ALG_NAME];
1530 	char driver_name[CRYPTO_MAX_ALG_NAME];
1531 	char mac_name[CRYPTO_MAX_ALG_NAME];
1532 	char mac_driver_name[CRYPTO_MAX_ALG_NAME];
1533 	unsigned int blocksize;
1534 	bool is_mac;
1535 	bool synchronize;
1536 	struct ahash_alg template_ahash;
1537 	int hash_mode;
1538 	int hw_mode;
1539 	int inter_digestsize;
1540 	struct cc_drvdata *drvdata;
1541 	u32 min_hw_rev;
1542 	enum cc_std_body std_body;
1543 };
1544 
1545 #define CC_STATE_SIZE(_x) \
1546 	((_x) + HASH_MAX_LEN_SIZE + CC_MAX_HASH_BLCK_SIZE + (2 * sizeof(u32)))
1547 
1548 /* hash descriptors */
1549 static struct cc_hash_template driver_hash[] = {
1550 	//Asynchronize hash template
1551 	{
1552 		.name = "sha1",
1553 		.driver_name = "sha1-ccree",
1554 		.mac_name = "hmac(sha1)",
1555 		.mac_driver_name = "hmac-sha1-ccree",
1556 		.blocksize = SHA1_BLOCK_SIZE,
1557 		.is_mac = true,
1558 		.synchronize = false,
1559 		.template_ahash = {
1560 			.init = cc_hash_init,
1561 			.update = cc_hash_update,
1562 			.final = cc_hash_final,
1563 			.finup = cc_hash_finup,
1564 			.digest = cc_hash_digest,
1565 			.export = cc_hash_export,
1566 			.import = cc_hash_import,
1567 			.setkey = cc_hash_setkey,
1568 			.halg = {
1569 				.digestsize = SHA1_DIGEST_SIZE,
1570 				.statesize = CC_STATE_SIZE(SHA1_DIGEST_SIZE),
1571 			},
1572 		},
1573 		.hash_mode = DRV_HASH_SHA1,
1574 		.hw_mode = DRV_HASH_HW_SHA1,
1575 		.inter_digestsize = SHA1_DIGEST_SIZE,
1576 		.min_hw_rev = CC_HW_REV_630,
1577 		.std_body = CC_STD_NIST,
1578 	},
1579 	{
1580 		.name = "sha256",
1581 		.driver_name = "sha256-ccree",
1582 		.mac_name = "hmac(sha256)",
1583 		.mac_driver_name = "hmac-sha256-ccree",
1584 		.blocksize = SHA256_BLOCK_SIZE,
1585 		.is_mac = true,
1586 		.template_ahash = {
1587 			.init = cc_hash_init,
1588 			.update = cc_hash_update,
1589 			.final = cc_hash_final,
1590 			.finup = cc_hash_finup,
1591 			.digest = cc_hash_digest,
1592 			.export = cc_hash_export,
1593 			.import = cc_hash_import,
1594 			.setkey = cc_hash_setkey,
1595 			.halg = {
1596 				.digestsize = SHA256_DIGEST_SIZE,
1597 				.statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE)
1598 			},
1599 		},
1600 		.hash_mode = DRV_HASH_SHA256,
1601 		.hw_mode = DRV_HASH_HW_SHA256,
1602 		.inter_digestsize = SHA256_DIGEST_SIZE,
1603 		.min_hw_rev = CC_HW_REV_630,
1604 		.std_body = CC_STD_NIST,
1605 	},
1606 	{
1607 		.name = "sha224",
1608 		.driver_name = "sha224-ccree",
1609 		.mac_name = "hmac(sha224)",
1610 		.mac_driver_name = "hmac-sha224-ccree",
1611 		.blocksize = SHA224_BLOCK_SIZE,
1612 		.is_mac = true,
1613 		.template_ahash = {
1614 			.init = cc_hash_init,
1615 			.update = cc_hash_update,
1616 			.final = cc_hash_final,
1617 			.finup = cc_hash_finup,
1618 			.digest = cc_hash_digest,
1619 			.export = cc_hash_export,
1620 			.import = cc_hash_import,
1621 			.setkey = cc_hash_setkey,
1622 			.halg = {
1623 				.digestsize = SHA224_DIGEST_SIZE,
1624 				.statesize = CC_STATE_SIZE(SHA224_DIGEST_SIZE),
1625 			},
1626 		},
1627 		.hash_mode = DRV_HASH_SHA224,
1628 		.hw_mode = DRV_HASH_HW_SHA256,
1629 		.inter_digestsize = SHA256_DIGEST_SIZE,
1630 		.min_hw_rev = CC_HW_REV_630,
1631 		.std_body = CC_STD_NIST,
1632 	},
1633 	{
1634 		.name = "sha384",
1635 		.driver_name = "sha384-ccree",
1636 		.mac_name = "hmac(sha384)",
1637 		.mac_driver_name = "hmac-sha384-ccree",
1638 		.blocksize = SHA384_BLOCK_SIZE,
1639 		.is_mac = true,
1640 		.template_ahash = {
1641 			.init = cc_hash_init,
1642 			.update = cc_hash_update,
1643 			.final = cc_hash_final,
1644 			.finup = cc_hash_finup,
1645 			.digest = cc_hash_digest,
1646 			.export = cc_hash_export,
1647 			.import = cc_hash_import,
1648 			.setkey = cc_hash_setkey,
1649 			.halg = {
1650 				.digestsize = SHA384_DIGEST_SIZE,
1651 				.statesize = CC_STATE_SIZE(SHA384_DIGEST_SIZE),
1652 			},
1653 		},
1654 		.hash_mode = DRV_HASH_SHA384,
1655 		.hw_mode = DRV_HASH_HW_SHA512,
1656 		.inter_digestsize = SHA512_DIGEST_SIZE,
1657 		.min_hw_rev = CC_HW_REV_712,
1658 		.std_body = CC_STD_NIST,
1659 	},
1660 	{
1661 		.name = "sha512",
1662 		.driver_name = "sha512-ccree",
1663 		.mac_name = "hmac(sha512)",
1664 		.mac_driver_name = "hmac-sha512-ccree",
1665 		.blocksize = SHA512_BLOCK_SIZE,
1666 		.is_mac = true,
1667 		.template_ahash = {
1668 			.init = cc_hash_init,
1669 			.update = cc_hash_update,
1670 			.final = cc_hash_final,
1671 			.finup = cc_hash_finup,
1672 			.digest = cc_hash_digest,
1673 			.export = cc_hash_export,
1674 			.import = cc_hash_import,
1675 			.setkey = cc_hash_setkey,
1676 			.halg = {
1677 				.digestsize = SHA512_DIGEST_SIZE,
1678 				.statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE),
1679 			},
1680 		},
1681 		.hash_mode = DRV_HASH_SHA512,
1682 		.hw_mode = DRV_HASH_HW_SHA512,
1683 		.inter_digestsize = SHA512_DIGEST_SIZE,
1684 		.min_hw_rev = CC_HW_REV_712,
1685 		.std_body = CC_STD_NIST,
1686 	},
1687 	{
1688 		.name = "md5",
1689 		.driver_name = "md5-ccree",
1690 		.mac_name = "hmac(md5)",
1691 		.mac_driver_name = "hmac-md5-ccree",
1692 		.blocksize = MD5_HMAC_BLOCK_SIZE,
1693 		.is_mac = true,
1694 		.template_ahash = {
1695 			.init = cc_hash_init,
1696 			.update = cc_hash_update,
1697 			.final = cc_hash_final,
1698 			.finup = cc_hash_finup,
1699 			.digest = cc_hash_digest,
1700 			.export = cc_hash_export,
1701 			.import = cc_hash_import,
1702 			.setkey = cc_hash_setkey,
1703 			.halg = {
1704 				.digestsize = MD5_DIGEST_SIZE,
1705 				.statesize = CC_STATE_SIZE(MD5_DIGEST_SIZE),
1706 			},
1707 		},
1708 		.hash_mode = DRV_HASH_MD5,
1709 		.hw_mode = DRV_HASH_HW_MD5,
1710 		.inter_digestsize = MD5_DIGEST_SIZE,
1711 		.min_hw_rev = CC_HW_REV_630,
1712 		.std_body = CC_STD_NIST,
1713 	},
1714 	{
1715 		.name = "sm3",
1716 		.driver_name = "sm3-ccree",
1717 		.blocksize = SM3_BLOCK_SIZE,
1718 		.is_mac = false,
1719 		.template_ahash = {
1720 			.init = cc_hash_init,
1721 			.update = cc_hash_update,
1722 			.final = cc_hash_final,
1723 			.finup = cc_hash_finup,
1724 			.digest = cc_hash_digest,
1725 			.export = cc_hash_export,
1726 			.import = cc_hash_import,
1727 			.setkey = cc_hash_setkey,
1728 			.halg = {
1729 				.digestsize = SM3_DIGEST_SIZE,
1730 				.statesize = CC_STATE_SIZE(SM3_DIGEST_SIZE),
1731 			},
1732 		},
1733 		.hash_mode = DRV_HASH_SM3,
1734 		.hw_mode = DRV_HASH_HW_SM3,
1735 		.inter_digestsize = SM3_DIGEST_SIZE,
1736 		.min_hw_rev = CC_HW_REV_713,
1737 		.std_body = CC_STD_OSCCA,
1738 	},
1739 	{
1740 		.mac_name = "xcbc(aes)",
1741 		.mac_driver_name = "xcbc-aes-ccree",
1742 		.blocksize = AES_BLOCK_SIZE,
1743 		.is_mac = true,
1744 		.template_ahash = {
1745 			.init = cc_hash_init,
1746 			.update = cc_mac_update,
1747 			.final = cc_mac_final,
1748 			.finup = cc_mac_finup,
1749 			.digest = cc_mac_digest,
1750 			.setkey = cc_xcbc_setkey,
1751 			.export = cc_hash_export,
1752 			.import = cc_hash_import,
1753 			.halg = {
1754 				.digestsize = AES_BLOCK_SIZE,
1755 				.statesize = CC_STATE_SIZE(AES_BLOCK_SIZE),
1756 			},
1757 		},
1758 		.hash_mode = DRV_HASH_NULL,
1759 		.hw_mode = DRV_CIPHER_XCBC_MAC,
1760 		.inter_digestsize = AES_BLOCK_SIZE,
1761 		.min_hw_rev = CC_HW_REV_630,
1762 		.std_body = CC_STD_NIST,
1763 	},
1764 	{
1765 		.mac_name = "cmac(aes)",
1766 		.mac_driver_name = "cmac-aes-ccree",
1767 		.blocksize = AES_BLOCK_SIZE,
1768 		.is_mac = true,
1769 		.template_ahash = {
1770 			.init = cc_hash_init,
1771 			.update = cc_mac_update,
1772 			.final = cc_mac_final,
1773 			.finup = cc_mac_finup,
1774 			.digest = cc_mac_digest,
1775 			.setkey = cc_cmac_setkey,
1776 			.export = cc_hash_export,
1777 			.import = cc_hash_import,
1778 			.halg = {
1779 				.digestsize = AES_BLOCK_SIZE,
1780 				.statesize = CC_STATE_SIZE(AES_BLOCK_SIZE),
1781 			},
1782 		},
1783 		.hash_mode = DRV_HASH_NULL,
1784 		.hw_mode = DRV_CIPHER_CMAC,
1785 		.inter_digestsize = AES_BLOCK_SIZE,
1786 		.min_hw_rev = CC_HW_REV_630,
1787 		.std_body = CC_STD_NIST,
1788 	},
1789 };
1790 
1791 static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template,
1792 					     struct device *dev, bool keyed)
1793 {
1794 	struct cc_hash_alg *t_crypto_alg;
1795 	struct crypto_alg *alg;
1796 	struct ahash_alg *halg;
1797 
1798 	t_crypto_alg = kzalloc(sizeof(*t_crypto_alg), GFP_KERNEL);
1799 	if (!t_crypto_alg)
1800 		return ERR_PTR(-ENOMEM);
1801 
1802 	t_crypto_alg->ahash_alg = template->template_ahash;
1803 	halg = &t_crypto_alg->ahash_alg;
1804 	alg = &halg->halg.base;
1805 
1806 	if (keyed) {
1807 		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1808 			 template->mac_name);
1809 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1810 			 template->mac_driver_name);
1811 	} else {
1812 		halg->setkey = NULL;
1813 		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1814 			 template->name);
1815 		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1816 			 template->driver_name);
1817 	}
1818 	alg->cra_module = THIS_MODULE;
1819 	alg->cra_ctxsize = sizeof(struct cc_hash_ctx);
1820 	alg->cra_priority = CC_CRA_PRIO;
1821 	alg->cra_blocksize = template->blocksize;
1822 	alg->cra_alignmask = 0;
1823 	alg->cra_exit = cc_cra_exit;
1824 
1825 	alg->cra_init = cc_cra_init;
1826 	alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
1827 
1828 	t_crypto_alg->hash_mode = template->hash_mode;
1829 	t_crypto_alg->hw_mode = template->hw_mode;
1830 	t_crypto_alg->inter_digestsize = template->inter_digestsize;
1831 
1832 	return t_crypto_alg;
1833 }
1834 
1835 int cc_init_hash_sram(struct cc_drvdata *drvdata)
1836 {
1837 	struct cc_hash_handle *hash_handle = drvdata->hash_handle;
1838 	cc_sram_addr_t sram_buff_ofs = hash_handle->digest_len_sram_addr;
1839 	unsigned int larval_seq_len = 0;
1840 	struct cc_hw_desc larval_seq[CC_DIGEST_SIZE_MAX / sizeof(u32)];
1841 	bool large_sha_supported = (drvdata->hw_rev >= CC_HW_REV_712);
1842 	bool sm3_supported = (drvdata->hw_rev >= CC_HW_REV_713);
1843 	int rc = 0;
1844 
1845 	/* Copy-to-sram digest-len */
1846 	cc_set_sram_desc(digest_len_init, sram_buff_ofs,
1847 			 ARRAY_SIZE(digest_len_init), larval_seq,
1848 			 &larval_seq_len);
1849 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1850 	if (rc)
1851 		goto init_digest_const_err;
1852 
1853 	sram_buff_ofs += sizeof(digest_len_init);
1854 	larval_seq_len = 0;
1855 
1856 	if (large_sha_supported) {
1857 		/* Copy-to-sram digest-len for sha384/512 */
1858 		cc_set_sram_desc(digest_len_sha512_init, sram_buff_ofs,
1859 				 ARRAY_SIZE(digest_len_sha512_init),
1860 				 larval_seq, &larval_seq_len);
1861 		rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1862 		if (rc)
1863 			goto init_digest_const_err;
1864 
1865 		sram_buff_ofs += sizeof(digest_len_sha512_init);
1866 		larval_seq_len = 0;
1867 	}
1868 
1869 	/* The initial digests offset */
1870 	hash_handle->larval_digest_sram_addr = sram_buff_ofs;
1871 
1872 	/* Copy-to-sram initial SHA* digests */
1873 	cc_set_sram_desc(md5_init, sram_buff_ofs, ARRAY_SIZE(md5_init),
1874 			 larval_seq, &larval_seq_len);
1875 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1876 	if (rc)
1877 		goto init_digest_const_err;
1878 	sram_buff_ofs += sizeof(md5_init);
1879 	larval_seq_len = 0;
1880 
1881 	cc_set_sram_desc(sha1_init, sram_buff_ofs,
1882 			 ARRAY_SIZE(sha1_init), larval_seq,
1883 			 &larval_seq_len);
1884 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1885 	if (rc)
1886 		goto init_digest_const_err;
1887 	sram_buff_ofs += sizeof(sha1_init);
1888 	larval_seq_len = 0;
1889 
1890 	cc_set_sram_desc(sha224_init, sram_buff_ofs,
1891 			 ARRAY_SIZE(sha224_init), larval_seq,
1892 			 &larval_seq_len);
1893 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1894 	if (rc)
1895 		goto init_digest_const_err;
1896 	sram_buff_ofs += sizeof(sha224_init);
1897 	larval_seq_len = 0;
1898 
1899 	cc_set_sram_desc(sha256_init, sram_buff_ofs,
1900 			 ARRAY_SIZE(sha256_init), larval_seq,
1901 			 &larval_seq_len);
1902 	rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1903 	if (rc)
1904 		goto init_digest_const_err;
1905 	sram_buff_ofs += sizeof(sha256_init);
1906 	larval_seq_len = 0;
1907 
1908 	if (sm3_supported) {
1909 		cc_set_sram_desc(sm3_init, sram_buff_ofs,
1910 				 ARRAY_SIZE(sm3_init), larval_seq,
1911 				 &larval_seq_len);
1912 		rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1913 		if (rc)
1914 			goto init_digest_const_err;
1915 		sram_buff_ofs += sizeof(sm3_init);
1916 		larval_seq_len = 0;
1917 	}
1918 
1919 	if (large_sha_supported) {
1920 		cc_set_sram_desc((u32 *)sha384_init, sram_buff_ofs,
1921 				 (ARRAY_SIZE(sha384_init) * 2), larval_seq,
1922 				 &larval_seq_len);
1923 		rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1924 		if (rc)
1925 			goto init_digest_const_err;
1926 		sram_buff_ofs += sizeof(sha384_init);
1927 		larval_seq_len = 0;
1928 
1929 		cc_set_sram_desc((u32 *)sha512_init, sram_buff_ofs,
1930 				 (ARRAY_SIZE(sha512_init) * 2), larval_seq,
1931 				 &larval_seq_len);
1932 		rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1933 		if (rc)
1934 			goto init_digest_const_err;
1935 	}
1936 
1937 init_digest_const_err:
1938 	return rc;
1939 }
1940 
1941 static void __init cc_swap_dwords(u32 *buf, unsigned long size)
1942 {
1943 	int i;
1944 	u32 tmp;
1945 
1946 	for (i = 0; i < size; i += 2) {
1947 		tmp = buf[i];
1948 		buf[i] = buf[i + 1];
1949 		buf[i + 1] = tmp;
1950 	}
1951 }
1952 
1953 /*
1954  * Due to the way the HW works we need to swap every
1955  * double word in the SHA384 and SHA512 larval hashes
1956  */
1957 void __init cc_hash_global_init(void)
1958 {
1959 	cc_swap_dwords((u32 *)&sha384_init, (ARRAY_SIZE(sha384_init) * 2));
1960 	cc_swap_dwords((u32 *)&sha512_init, (ARRAY_SIZE(sha512_init) * 2));
1961 }
1962 
1963 int cc_hash_alloc(struct cc_drvdata *drvdata)
1964 {
1965 	struct cc_hash_handle *hash_handle;
1966 	cc_sram_addr_t sram_buff;
1967 	u32 sram_size_to_alloc;
1968 	struct device *dev = drvdata_to_dev(drvdata);
1969 	int rc = 0;
1970 	int alg;
1971 
1972 	hash_handle = kzalloc(sizeof(*hash_handle), GFP_KERNEL);
1973 	if (!hash_handle)
1974 		return -ENOMEM;
1975 
1976 	INIT_LIST_HEAD(&hash_handle->hash_list);
1977 	drvdata->hash_handle = hash_handle;
1978 
1979 	sram_size_to_alloc = sizeof(digest_len_init) +
1980 			sizeof(md5_init) +
1981 			sizeof(sha1_init) +
1982 			sizeof(sha224_init) +
1983 			sizeof(sha256_init);
1984 
1985 	if (drvdata->hw_rev >= CC_HW_REV_713)
1986 		sram_size_to_alloc += sizeof(sm3_init);
1987 
1988 	if (drvdata->hw_rev >= CC_HW_REV_712)
1989 		sram_size_to_alloc += sizeof(digest_len_sha512_init) +
1990 			sizeof(sha384_init) + sizeof(sha512_init);
1991 
1992 	sram_buff = cc_sram_alloc(drvdata, sram_size_to_alloc);
1993 	if (sram_buff == NULL_SRAM_ADDR) {
1994 		dev_err(dev, "SRAM pool exhausted\n");
1995 		rc = -ENOMEM;
1996 		goto fail;
1997 	}
1998 
1999 	/* The initial digest-len offset */
2000 	hash_handle->digest_len_sram_addr = sram_buff;
2001 
2002 	/*must be set before the alg registration as it is being used there*/
2003 	rc = cc_init_hash_sram(drvdata);
2004 	if (rc) {
2005 		dev_err(dev, "Init digest CONST failed (rc=%d)\n", rc);
2006 		goto fail;
2007 	}
2008 
2009 	/* ahash registration */
2010 	for (alg = 0; alg < ARRAY_SIZE(driver_hash); alg++) {
2011 		struct cc_hash_alg *t_alg;
2012 		int hw_mode = driver_hash[alg].hw_mode;
2013 
2014 		/* Check that the HW revision and variants are suitable */
2015 		if ((driver_hash[alg].min_hw_rev > drvdata->hw_rev) ||
2016 		    !(drvdata->std_bodies & driver_hash[alg].std_body))
2017 			continue;
2018 
2019 		if (driver_hash[alg].is_mac) {
2020 			/* register hmac version */
2021 			t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, true);
2022 			if (IS_ERR(t_alg)) {
2023 				rc = PTR_ERR(t_alg);
2024 				dev_err(dev, "%s alg allocation failed\n",
2025 					driver_hash[alg].driver_name);
2026 				goto fail;
2027 			}
2028 			t_alg->drvdata = drvdata;
2029 
2030 			rc = crypto_register_ahash(&t_alg->ahash_alg);
2031 			if (rc) {
2032 				dev_err(dev, "%s alg registration failed\n",
2033 					driver_hash[alg].driver_name);
2034 				kfree(t_alg);
2035 				goto fail;
2036 			} else {
2037 				list_add_tail(&t_alg->entry,
2038 					      &hash_handle->hash_list);
2039 			}
2040 		}
2041 		if (hw_mode == DRV_CIPHER_XCBC_MAC ||
2042 		    hw_mode == DRV_CIPHER_CMAC)
2043 			continue;
2044 
2045 		/* register hash version */
2046 		t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, false);
2047 		if (IS_ERR(t_alg)) {
2048 			rc = PTR_ERR(t_alg);
2049 			dev_err(dev, "%s alg allocation failed\n",
2050 				driver_hash[alg].driver_name);
2051 			goto fail;
2052 		}
2053 		t_alg->drvdata = drvdata;
2054 
2055 		rc = crypto_register_ahash(&t_alg->ahash_alg);
2056 		if (rc) {
2057 			dev_err(dev, "%s alg registration failed\n",
2058 				driver_hash[alg].driver_name);
2059 			kfree(t_alg);
2060 			goto fail;
2061 		} else {
2062 			list_add_tail(&t_alg->entry, &hash_handle->hash_list);
2063 		}
2064 	}
2065 
2066 	return 0;
2067 
2068 fail:
2069 	kfree(drvdata->hash_handle);
2070 	drvdata->hash_handle = NULL;
2071 	return rc;
2072 }
2073 
2074 int cc_hash_free(struct cc_drvdata *drvdata)
2075 {
2076 	struct cc_hash_alg *t_hash_alg, *hash_n;
2077 	struct cc_hash_handle *hash_handle = drvdata->hash_handle;
2078 
2079 	if (hash_handle) {
2080 		list_for_each_entry_safe(t_hash_alg, hash_n,
2081 					 &hash_handle->hash_list, entry) {
2082 			crypto_unregister_ahash(&t_hash_alg->ahash_alg);
2083 			list_del(&t_hash_alg->entry);
2084 			kfree(t_hash_alg);
2085 		}
2086 
2087 		kfree(hash_handle);
2088 		drvdata->hash_handle = NULL;
2089 	}
2090 	return 0;
2091 }
2092 
2093 static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[],
2094 			  unsigned int *seq_size)
2095 {
2096 	unsigned int idx = *seq_size;
2097 	struct ahash_req_ctx *state = ahash_request_ctx(areq);
2098 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2099 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
2100 
2101 	/* Setup XCBC MAC K1 */
2102 	hw_desc_init(&desc[idx]);
2103 	set_din_type(&desc[idx], DMA_DLLI, (ctx->opad_tmp_keys_dma_addr +
2104 					    XCBC_MAC_K1_OFFSET),
2105 		     CC_AES_128_BIT_KEY_SIZE, NS_BIT);
2106 	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
2107 	set_hash_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC, ctx->hash_mode);
2108 	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2109 	set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2110 	set_flow_mode(&desc[idx], S_DIN_to_AES);
2111 	idx++;
2112 
2113 	/* Setup XCBC MAC K2 */
2114 	hw_desc_init(&desc[idx]);
2115 	set_din_type(&desc[idx], DMA_DLLI,
2116 		     (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET),
2117 		     CC_AES_128_BIT_KEY_SIZE, NS_BIT);
2118 	set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
2119 	set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
2120 	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2121 	set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2122 	set_flow_mode(&desc[idx], S_DIN_to_AES);
2123 	idx++;
2124 
2125 	/* Setup XCBC MAC K3 */
2126 	hw_desc_init(&desc[idx]);
2127 	set_din_type(&desc[idx], DMA_DLLI,
2128 		     (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET),
2129 		     CC_AES_128_BIT_KEY_SIZE, NS_BIT);
2130 	set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);
2131 	set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
2132 	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2133 	set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2134 	set_flow_mode(&desc[idx], S_DIN_to_AES);
2135 	idx++;
2136 
2137 	/* Loading MAC state */
2138 	hw_desc_init(&desc[idx]);
2139 	set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
2140 		     CC_AES_BLOCK_SIZE, NS_BIT);
2141 	set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
2142 	set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
2143 	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2144 	set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2145 	set_flow_mode(&desc[idx], S_DIN_to_AES);
2146 	idx++;
2147 	*seq_size = idx;
2148 }
2149 
2150 static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[],
2151 			  unsigned int *seq_size)
2152 {
2153 	unsigned int idx = *seq_size;
2154 	struct ahash_req_ctx *state = ahash_request_ctx(areq);
2155 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2156 	struct cc_hash_ctx *ctx = crypto_ahash_ctx(tfm);
2157 
2158 	/* Setup CMAC Key */
2159 	hw_desc_init(&desc[idx]);
2160 	set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr,
2161 		     ((ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE :
2162 		      ctx->key_params.keylen), NS_BIT);
2163 	set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
2164 	set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);
2165 	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2166 	set_key_size_aes(&desc[idx], ctx->key_params.keylen);
2167 	set_flow_mode(&desc[idx], S_DIN_to_AES);
2168 	idx++;
2169 
2170 	/* Load MAC state */
2171 	hw_desc_init(&desc[idx]);
2172 	set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
2173 		     CC_AES_BLOCK_SIZE, NS_BIT);
2174 	set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
2175 	set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);
2176 	set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2177 	set_key_size_aes(&desc[idx], ctx->key_params.keylen);
2178 	set_flow_mode(&desc[idx], S_DIN_to_AES);
2179 	idx++;
2180 	*seq_size = idx;
2181 }
2182 
2183 static void cc_set_desc(struct ahash_req_ctx *areq_ctx,
2184 			struct cc_hash_ctx *ctx, unsigned int flow_mode,
2185 			struct cc_hw_desc desc[], bool is_not_last_data,
2186 			unsigned int *seq_size)
2187 {
2188 	unsigned int idx = *seq_size;
2189 	struct device *dev = drvdata_to_dev(ctx->drvdata);
2190 
2191 	if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_DLLI) {
2192 		hw_desc_init(&desc[idx]);
2193 		set_din_type(&desc[idx], DMA_DLLI,
2194 			     sg_dma_address(areq_ctx->curr_sg),
2195 			     areq_ctx->curr_sg->length, NS_BIT);
2196 		set_flow_mode(&desc[idx], flow_mode);
2197 		idx++;
2198 	} else {
2199 		if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) {
2200 			dev_dbg(dev, " NULL mode\n");
2201 			/* nothing to build */
2202 			return;
2203 		}
2204 		/* bypass */
2205 		hw_desc_init(&desc[idx]);
2206 		set_din_type(&desc[idx], DMA_DLLI,
2207 			     areq_ctx->mlli_params.mlli_dma_addr,
2208 			     areq_ctx->mlli_params.mlli_len, NS_BIT);
2209 		set_dout_sram(&desc[idx], ctx->drvdata->mlli_sram_addr,
2210 			      areq_ctx->mlli_params.mlli_len);
2211 		set_flow_mode(&desc[idx], BYPASS);
2212 		idx++;
2213 		/* process */
2214 		hw_desc_init(&desc[idx]);
2215 		set_din_type(&desc[idx], DMA_MLLI,
2216 			     ctx->drvdata->mlli_sram_addr,
2217 			     areq_ctx->mlli_nents, NS_BIT);
2218 		set_flow_mode(&desc[idx], flow_mode);
2219 		idx++;
2220 	}
2221 	if (is_not_last_data)
2222 		set_din_not_last_indication(&desc[(idx - 1)]);
2223 	/* return updated desc sequence size */
2224 	*seq_size = idx;
2225 }
2226 
2227 static const void *cc_larval_digest(struct device *dev, u32 mode)
2228 {
2229 	switch (mode) {
2230 	case DRV_HASH_MD5:
2231 		return md5_init;
2232 	case DRV_HASH_SHA1:
2233 		return sha1_init;
2234 	case DRV_HASH_SHA224:
2235 		return sha224_init;
2236 	case DRV_HASH_SHA256:
2237 		return sha256_init;
2238 	case DRV_HASH_SHA384:
2239 		return sha384_init;
2240 	case DRV_HASH_SHA512:
2241 		return sha512_init;
2242 	case DRV_HASH_SM3:
2243 		return sm3_init;
2244 	default:
2245 		dev_err(dev, "Invalid hash mode (%d)\n", mode);
2246 		return md5_init;
2247 	}
2248 }
2249 
2250 /*!
2251  * Gets the address of the initial digest in SRAM
2252  * according to the given hash mode
2253  *
2254  * \param drvdata
2255  * \param mode The Hash mode. Supported modes: MD5/SHA1/SHA224/SHA256
2256  *
2257  * \return u32 The address of the initial digest in SRAM
2258  */
2259 cc_sram_addr_t cc_larval_digest_addr(void *drvdata, u32 mode)
2260 {
2261 	struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata;
2262 	struct cc_hash_handle *hash_handle = _drvdata->hash_handle;
2263 	struct device *dev = drvdata_to_dev(_drvdata);
2264 	bool sm3_supported = (_drvdata->hw_rev >= CC_HW_REV_713);
2265 	cc_sram_addr_t addr;
2266 
2267 	switch (mode) {
2268 	case DRV_HASH_NULL:
2269 		break; /*Ignore*/
2270 	case DRV_HASH_MD5:
2271 		return (hash_handle->larval_digest_sram_addr);
2272 	case DRV_HASH_SHA1:
2273 		return (hash_handle->larval_digest_sram_addr +
2274 			sizeof(md5_init));
2275 	case DRV_HASH_SHA224:
2276 		return (hash_handle->larval_digest_sram_addr +
2277 			sizeof(md5_init) +
2278 			sizeof(sha1_init));
2279 	case DRV_HASH_SHA256:
2280 		return (hash_handle->larval_digest_sram_addr +
2281 			sizeof(md5_init) +
2282 			sizeof(sha1_init) +
2283 			sizeof(sha224_init));
2284 	case DRV_HASH_SM3:
2285 		return (hash_handle->larval_digest_sram_addr +
2286 			sizeof(md5_init) +
2287 			sizeof(sha1_init) +
2288 			sizeof(sha224_init) +
2289 			sizeof(sha256_init));
2290 	case DRV_HASH_SHA384:
2291 		addr = (hash_handle->larval_digest_sram_addr +
2292 			sizeof(md5_init) +
2293 			sizeof(sha1_init) +
2294 			sizeof(sha224_init) +
2295 			sizeof(sha256_init));
2296 		if (sm3_supported)
2297 			addr += sizeof(sm3_init);
2298 		return addr;
2299 	case DRV_HASH_SHA512:
2300 		addr = (hash_handle->larval_digest_sram_addr +
2301 			sizeof(md5_init) +
2302 			sizeof(sha1_init) +
2303 			sizeof(sha224_init) +
2304 			sizeof(sha256_init) +
2305 			sizeof(sha384_init));
2306 		if (sm3_supported)
2307 			addr += sizeof(sm3_init);
2308 		return addr;
2309 	default:
2310 		dev_err(dev, "Invalid hash mode (%d)\n", mode);
2311 	}
2312 
2313 	/*This is valid wrong value to avoid kernel crash*/
2314 	return hash_handle->larval_digest_sram_addr;
2315 }
2316 
2317 cc_sram_addr_t
2318 cc_digest_len_addr(void *drvdata, u32 mode)
2319 {
2320 	struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata;
2321 	struct cc_hash_handle *hash_handle = _drvdata->hash_handle;
2322 	cc_sram_addr_t digest_len_addr = hash_handle->digest_len_sram_addr;
2323 
2324 	switch (mode) {
2325 	case DRV_HASH_SHA1:
2326 	case DRV_HASH_SHA224:
2327 	case DRV_HASH_SHA256:
2328 	case DRV_HASH_MD5:
2329 		return digest_len_addr;
2330 #if (CC_DEV_SHA_MAX > 256)
2331 	case DRV_HASH_SHA384:
2332 	case DRV_HASH_SHA512:
2333 		return  digest_len_addr + sizeof(digest_len_init);
2334 #endif
2335 	default:
2336 		return digest_len_addr; /*to avoid kernel crash*/
2337 	}
2338 }
2339