1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
3
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <crypto/algapi.h>
7 #include <crypto/hash.h>
8 #include <crypto/md5.h>
9 #include <crypto/sm3.h>
10 #include <crypto/internal/hash.h>
11
12 #include "cc_driver.h"
13 #include "cc_request_mgr.h"
14 #include "cc_buffer_mgr.h"
15 #include "cc_hash.h"
16 #include "cc_sram_mgr.h"
17
18 #define CC_MAX_HASH_SEQ_LEN 12
19 #define CC_MAX_OPAD_KEYS_SIZE CC_MAX_HASH_BLCK_SIZE
20 #define CC_SM3_HASH_LEN_SIZE 8
21
22 struct cc_hash_handle {
23 u32 digest_len_sram_addr; /* const value in SRAM*/
24 u32 larval_digest_sram_addr; /* const value in SRAM */
25 struct list_head hash_list;
26 };
27
28 static const u32 cc_digest_len_init[] = {
29 0x00000040, 0x00000000, 0x00000000, 0x00000000 };
30 static const u32 cc_md5_init[] = {
31 SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
32 static const u32 cc_sha1_init[] = {
33 SHA1_H4, SHA1_H3, SHA1_H2, SHA1_H1, SHA1_H0 };
34 static const u32 cc_sha224_init[] = {
35 SHA224_H7, SHA224_H6, SHA224_H5, SHA224_H4,
36 SHA224_H3, SHA224_H2, SHA224_H1, SHA224_H0 };
37 static const u32 cc_sha256_init[] = {
38 SHA256_H7, SHA256_H6, SHA256_H5, SHA256_H4,
39 SHA256_H3, SHA256_H2, SHA256_H1, SHA256_H0 };
40 static const u32 cc_digest_len_sha512_init[] = {
41 0x00000080, 0x00000000, 0x00000000, 0x00000000 };
42
43 /*
44 * Due to the way the HW works, every double word in the SHA384 and SHA512
45 * larval hashes must be stored in hi/lo order
46 */
47 #define hilo(x) upper_32_bits(x), lower_32_bits(x)
48 static const u32 cc_sha384_init[] = {
49 hilo(SHA384_H7), hilo(SHA384_H6), hilo(SHA384_H5), hilo(SHA384_H4),
50 hilo(SHA384_H3), hilo(SHA384_H2), hilo(SHA384_H1), hilo(SHA384_H0) };
51 static const u32 cc_sha512_init[] = {
52 hilo(SHA512_H7), hilo(SHA512_H6), hilo(SHA512_H5), hilo(SHA512_H4),
53 hilo(SHA512_H3), hilo(SHA512_H2), hilo(SHA512_H1), hilo(SHA512_H0) };
54
55 static const u32 cc_sm3_init[] = {
56 SM3_IVH, SM3_IVG, SM3_IVF, SM3_IVE,
57 SM3_IVD, SM3_IVC, SM3_IVB, SM3_IVA };
58
59 static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[],
60 unsigned int *seq_size);
61
62 static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[],
63 unsigned int *seq_size);
64
65 static const void *cc_larval_digest(struct device *dev, u32 mode);
66
67 struct cc_hash_alg {
68 struct list_head entry;
69 int hash_mode;
70 int hw_mode;
71 int inter_digestsize;
72 struct cc_drvdata *drvdata;
73 struct ahash_alg ahash_alg;
74 };
75
76 struct hash_key_req_ctx {
77 u32 keylen;
78 dma_addr_t key_dma_addr;
79 u8 *key;
80 };
81
82 /* hash per-session context */
83 struct cc_hash_ctx {
84 struct cc_drvdata *drvdata;
85 /* holds the origin digest; the digest after "setkey" if HMAC,*
86 * the initial digest if HASH.
87 */
88 u8 digest_buff[CC_MAX_HASH_DIGEST_SIZE] ____cacheline_aligned;
89 u8 opad_tmp_keys_buff[CC_MAX_OPAD_KEYS_SIZE] ____cacheline_aligned;
90
91 dma_addr_t opad_tmp_keys_dma_addr ____cacheline_aligned;
92 dma_addr_t digest_buff_dma_addr;
93 /* use for hmac with key large then mode block size */
94 struct hash_key_req_ctx key_params;
95 int hash_mode;
96 int hw_mode;
97 int inter_digestsize;
98 unsigned int hash_len;
99 struct completion setkey_comp;
100 bool is_hmac;
101 };
102
103 static void cc_set_desc(struct ahash_req_ctx *areq_ctx, struct cc_hash_ctx *ctx,
104 unsigned int flow_mode, struct cc_hw_desc desc[],
105 bool is_not_last_data, unsigned int *seq_size);
106
cc_set_endianity(u32 mode,struct cc_hw_desc * desc)107 static void cc_set_endianity(u32 mode, struct cc_hw_desc *desc)
108 {
109 if (mode == DRV_HASH_MD5 || mode == DRV_HASH_SHA384 ||
110 mode == DRV_HASH_SHA512) {
111 set_bytes_swap(desc, 1);
112 } else {
113 set_cipher_config0(desc, HASH_DIGEST_RESULT_LITTLE_ENDIAN);
114 }
115 }
116
cc_map_result(struct device * dev,struct ahash_req_ctx * state,unsigned int digestsize)117 static int cc_map_result(struct device *dev, struct ahash_req_ctx *state,
118 unsigned int digestsize)
119 {
120 state->digest_result_dma_addr =
121 dma_map_single(dev, state->digest_result_buff,
122 digestsize, DMA_BIDIRECTIONAL);
123 if (dma_mapping_error(dev, state->digest_result_dma_addr)) {
124 dev_err(dev, "Mapping digest result buffer %u B for DMA failed\n",
125 digestsize);
126 return -ENOMEM;
127 }
128 dev_dbg(dev, "Mapped digest result buffer %u B at va=%pK to dma=%pad\n",
129 digestsize, state->digest_result_buff,
130 &state->digest_result_dma_addr);
131
132 return 0;
133 }
134
cc_init_req(struct device * dev,struct ahash_req_ctx * state,struct cc_hash_ctx * ctx)135 static void cc_init_req(struct device *dev, struct ahash_req_ctx *state,
136 struct cc_hash_ctx *ctx)
137 {
138 bool is_hmac = ctx->is_hmac;
139
140 memset(state, 0, sizeof(*state));
141
142 if (is_hmac) {
143 if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC &&
144 ctx->hw_mode != DRV_CIPHER_CMAC) {
145 dma_sync_single_for_cpu(dev, ctx->digest_buff_dma_addr,
146 ctx->inter_digestsize,
147 DMA_BIDIRECTIONAL);
148
149 memcpy(state->digest_buff, ctx->digest_buff,
150 ctx->inter_digestsize);
151 if (ctx->hash_mode == DRV_HASH_SHA512 ||
152 ctx->hash_mode == DRV_HASH_SHA384)
153 memcpy(state->digest_bytes_len,
154 cc_digest_len_sha512_init,
155 ctx->hash_len);
156 else
157 memcpy(state->digest_bytes_len,
158 cc_digest_len_init,
159 ctx->hash_len);
160 }
161
162 if (ctx->hash_mode != DRV_HASH_NULL) {
163 dma_sync_single_for_cpu(dev,
164 ctx->opad_tmp_keys_dma_addr,
165 ctx->inter_digestsize,
166 DMA_BIDIRECTIONAL);
167 memcpy(state->opad_digest_buff,
168 ctx->opad_tmp_keys_buff, ctx->inter_digestsize);
169 }
170 } else { /*hash*/
171 /* Copy the initial digests if hash flow. */
172 const void *larval = cc_larval_digest(dev, ctx->hash_mode);
173
174 memcpy(state->digest_buff, larval, ctx->inter_digestsize);
175 }
176 }
177
cc_map_req(struct device * dev,struct ahash_req_ctx * state,struct cc_hash_ctx * ctx)178 static int cc_map_req(struct device *dev, struct ahash_req_ctx *state,
179 struct cc_hash_ctx *ctx)
180 {
181 bool is_hmac = ctx->is_hmac;
182
183 state->digest_buff_dma_addr =
184 dma_map_single(dev, state->digest_buff,
185 ctx->inter_digestsize, DMA_BIDIRECTIONAL);
186 if (dma_mapping_error(dev, state->digest_buff_dma_addr)) {
187 dev_err(dev, "Mapping digest len %d B at va=%pK for DMA failed\n",
188 ctx->inter_digestsize, state->digest_buff);
189 return -EINVAL;
190 }
191 dev_dbg(dev, "Mapped digest %d B at va=%pK to dma=%pad\n",
192 ctx->inter_digestsize, state->digest_buff,
193 &state->digest_buff_dma_addr);
194
195 if (ctx->hw_mode != DRV_CIPHER_XCBC_MAC) {
196 state->digest_bytes_len_dma_addr =
197 dma_map_single(dev, state->digest_bytes_len,
198 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);
199 if (dma_mapping_error(dev, state->digest_bytes_len_dma_addr)) {
200 dev_err(dev, "Mapping digest len %u B at va=%pK for DMA failed\n",
201 HASH_MAX_LEN_SIZE, state->digest_bytes_len);
202 goto unmap_digest_buf;
203 }
204 dev_dbg(dev, "Mapped digest len %u B at va=%pK to dma=%pad\n",
205 HASH_MAX_LEN_SIZE, state->digest_bytes_len,
206 &state->digest_bytes_len_dma_addr);
207 }
208
209 if (is_hmac && ctx->hash_mode != DRV_HASH_NULL) {
210 state->opad_digest_dma_addr =
211 dma_map_single(dev, state->opad_digest_buff,
212 ctx->inter_digestsize,
213 DMA_BIDIRECTIONAL);
214 if (dma_mapping_error(dev, state->opad_digest_dma_addr)) {
215 dev_err(dev, "Mapping opad digest %d B at va=%pK for DMA failed\n",
216 ctx->inter_digestsize,
217 state->opad_digest_buff);
218 goto unmap_digest_len;
219 }
220 dev_dbg(dev, "Mapped opad digest %d B at va=%pK to dma=%pad\n",
221 ctx->inter_digestsize, state->opad_digest_buff,
222 &state->opad_digest_dma_addr);
223 }
224
225 return 0;
226
227 unmap_digest_len:
228 if (state->digest_bytes_len_dma_addr) {
229 dma_unmap_single(dev, state->digest_bytes_len_dma_addr,
230 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);
231 state->digest_bytes_len_dma_addr = 0;
232 }
233 unmap_digest_buf:
234 if (state->digest_buff_dma_addr) {
235 dma_unmap_single(dev, state->digest_buff_dma_addr,
236 ctx->inter_digestsize, DMA_BIDIRECTIONAL);
237 state->digest_buff_dma_addr = 0;
238 }
239
240 return -EINVAL;
241 }
242
cc_unmap_req(struct device * dev,struct ahash_req_ctx * state,struct cc_hash_ctx * ctx)243 static void cc_unmap_req(struct device *dev, struct ahash_req_ctx *state,
244 struct cc_hash_ctx *ctx)
245 {
246 if (state->digest_buff_dma_addr) {
247 dma_unmap_single(dev, state->digest_buff_dma_addr,
248 ctx->inter_digestsize, DMA_BIDIRECTIONAL);
249 dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n",
250 &state->digest_buff_dma_addr);
251 state->digest_buff_dma_addr = 0;
252 }
253 if (state->digest_bytes_len_dma_addr) {
254 dma_unmap_single(dev, state->digest_bytes_len_dma_addr,
255 HASH_MAX_LEN_SIZE, DMA_BIDIRECTIONAL);
256 dev_dbg(dev, "Unmapped digest-bytes-len buffer: digest_bytes_len_dma_addr=%pad\n",
257 &state->digest_bytes_len_dma_addr);
258 state->digest_bytes_len_dma_addr = 0;
259 }
260 if (state->opad_digest_dma_addr) {
261 dma_unmap_single(dev, state->opad_digest_dma_addr,
262 ctx->inter_digestsize, DMA_BIDIRECTIONAL);
263 dev_dbg(dev, "Unmapped opad-digest: opad_digest_dma_addr=%pad\n",
264 &state->opad_digest_dma_addr);
265 state->opad_digest_dma_addr = 0;
266 }
267 }
268
cc_unmap_result(struct device * dev,struct ahash_req_ctx * state,unsigned int digestsize,u8 * result)269 static void cc_unmap_result(struct device *dev, struct ahash_req_ctx *state,
270 unsigned int digestsize, u8 *result)
271 {
272 if (state->digest_result_dma_addr) {
273 dma_unmap_single(dev, state->digest_result_dma_addr, digestsize,
274 DMA_BIDIRECTIONAL);
275 dev_dbg(dev, "unmpa digest result buffer va (%pK) pa (%pad) len %u\n",
276 state->digest_result_buff,
277 &state->digest_result_dma_addr, digestsize);
278 memcpy(result, state->digest_result_buff, digestsize);
279 }
280 state->digest_result_dma_addr = 0;
281 }
282
cc_update_complete(struct device * dev,void * cc_req,int err)283 static void cc_update_complete(struct device *dev, void *cc_req, int err)
284 {
285 struct ahash_request *req = (struct ahash_request *)cc_req;
286 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
287 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
288 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
289
290 dev_dbg(dev, "req=%pK\n", req);
291
292 if (err != -EINPROGRESS) {
293 /* Not a BACKLOG notification */
294 cc_unmap_hash_request(dev, state, req->src, false);
295 cc_unmap_req(dev, state, ctx);
296 }
297
298 ahash_request_complete(req, err);
299 }
300
cc_digest_complete(struct device * dev,void * cc_req,int err)301 static void cc_digest_complete(struct device *dev, void *cc_req, int err)
302 {
303 struct ahash_request *req = (struct ahash_request *)cc_req;
304 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
305 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
306 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
307 u32 digestsize = crypto_ahash_digestsize(tfm);
308
309 dev_dbg(dev, "req=%pK\n", req);
310
311 if (err != -EINPROGRESS) {
312 /* Not a BACKLOG notification */
313 cc_unmap_hash_request(dev, state, req->src, false);
314 cc_unmap_result(dev, state, digestsize, req->result);
315 cc_unmap_req(dev, state, ctx);
316 }
317
318 ahash_request_complete(req, err);
319 }
320
cc_hash_complete(struct device * dev,void * cc_req,int err)321 static void cc_hash_complete(struct device *dev, void *cc_req, int err)
322 {
323 struct ahash_request *req = (struct ahash_request *)cc_req;
324 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
325 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
326 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
327 u32 digestsize = crypto_ahash_digestsize(tfm);
328
329 dev_dbg(dev, "req=%pK\n", req);
330
331 if (err != -EINPROGRESS) {
332 /* Not a BACKLOG notification */
333 cc_unmap_hash_request(dev, state, req->src, false);
334 cc_unmap_result(dev, state, digestsize, req->result);
335 cc_unmap_req(dev, state, ctx);
336 }
337
338 ahash_request_complete(req, err);
339 }
340
cc_fin_result(struct cc_hw_desc * desc,struct ahash_request * req,int idx)341 static int cc_fin_result(struct cc_hw_desc *desc, struct ahash_request *req,
342 int idx)
343 {
344 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
345 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
346 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
347 u32 digestsize = crypto_ahash_digestsize(tfm);
348
349 /* Get final MAC result */
350 hw_desc_init(&desc[idx]);
351 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
352 set_dout_dlli(&desc[idx], state->digest_result_dma_addr, digestsize,
353 NS_BIT, 1);
354 set_queue_last_ind(ctx->drvdata, &desc[idx]);
355 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
356 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
357 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
358 cc_set_endianity(ctx->hash_mode, &desc[idx]);
359 idx++;
360
361 return idx;
362 }
363
cc_fin_hmac(struct cc_hw_desc * desc,struct ahash_request * req,int idx)364 static int cc_fin_hmac(struct cc_hw_desc *desc, struct ahash_request *req,
365 int idx)
366 {
367 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
368 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
369 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
370 u32 digestsize = crypto_ahash_digestsize(tfm);
371
372 /* store the hash digest result in the context */
373 hw_desc_init(&desc[idx]);
374 set_cipher_mode(&desc[idx], ctx->hw_mode);
375 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr, digestsize,
376 NS_BIT, 0);
377 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
378 cc_set_endianity(ctx->hash_mode, &desc[idx]);
379 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
380 idx++;
381
382 /* Loading hash opad xor key state */
383 hw_desc_init(&desc[idx]);
384 set_cipher_mode(&desc[idx], ctx->hw_mode);
385 set_din_type(&desc[idx], DMA_DLLI, state->opad_digest_dma_addr,
386 ctx->inter_digestsize, NS_BIT);
387 set_flow_mode(&desc[idx], S_DIN_to_HASH);
388 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
389 idx++;
390
391 /* Load the hash current length */
392 hw_desc_init(&desc[idx]);
393 set_cipher_mode(&desc[idx], ctx->hw_mode);
394 set_din_sram(&desc[idx],
395 cc_digest_len_addr(ctx->drvdata, ctx->hash_mode),
396 ctx->hash_len);
397 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
398 set_flow_mode(&desc[idx], S_DIN_to_HASH);
399 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
400 idx++;
401
402 /* Memory Barrier: wait for IPAD/OPAD axi write to complete */
403 hw_desc_init(&desc[idx]);
404 set_din_no_dma(&desc[idx], 0, 0xfffff0);
405 set_dout_no_dma(&desc[idx], 0, 0, 1);
406 idx++;
407
408 /* Perform HASH update */
409 hw_desc_init(&desc[idx]);
410 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
411 digestsize, NS_BIT);
412 set_flow_mode(&desc[idx], DIN_HASH);
413 idx++;
414
415 return idx;
416 }
417
cc_hash_digest(struct ahash_request * req)418 static int cc_hash_digest(struct ahash_request *req)
419 {
420 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
421 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
422 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
423 u32 digestsize = crypto_ahash_digestsize(tfm);
424 struct scatterlist *src = req->src;
425 unsigned int nbytes = req->nbytes;
426 u8 *result = req->result;
427 struct device *dev = drvdata_to_dev(ctx->drvdata);
428 bool is_hmac = ctx->is_hmac;
429 struct cc_crypto_req cc_req = {};
430 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
431 u32 larval_digest_addr;
432 int idx = 0;
433 int rc = 0;
434 gfp_t flags = cc_gfp_flags(&req->base);
435
436 dev_dbg(dev, "===== %s-digest (%d) ====\n", is_hmac ? "hmac" : "hash",
437 nbytes);
438
439 cc_init_req(dev, state, ctx);
440
441 if (cc_map_req(dev, state, ctx)) {
442 dev_err(dev, "map_ahash_source() failed\n");
443 return -ENOMEM;
444 }
445
446 if (cc_map_result(dev, state, digestsize)) {
447 dev_err(dev, "map_ahash_digest() failed\n");
448 cc_unmap_req(dev, state, ctx);
449 return -ENOMEM;
450 }
451
452 if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, 1,
453 flags)) {
454 dev_err(dev, "map_ahash_request_final() failed\n");
455 cc_unmap_result(dev, state, digestsize, result);
456 cc_unmap_req(dev, state, ctx);
457 return -ENOMEM;
458 }
459
460 /* Setup request structure */
461 cc_req.user_cb = cc_digest_complete;
462 cc_req.user_arg = req;
463
464 /* If HMAC then load hash IPAD xor key, if HASH then load initial
465 * digest
466 */
467 hw_desc_init(&desc[idx]);
468 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
469 if (is_hmac) {
470 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
471 ctx->inter_digestsize, NS_BIT);
472 } else {
473 larval_digest_addr = cc_larval_digest_addr(ctx->drvdata,
474 ctx->hash_mode);
475 set_din_sram(&desc[idx], larval_digest_addr,
476 ctx->inter_digestsize);
477 }
478 set_flow_mode(&desc[idx], S_DIN_to_HASH);
479 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
480 idx++;
481
482 /* Load the hash current length */
483 hw_desc_init(&desc[idx]);
484 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
485
486 if (is_hmac) {
487 set_din_type(&desc[idx], DMA_DLLI,
488 state->digest_bytes_len_dma_addr,
489 ctx->hash_len, NS_BIT);
490 } else {
491 set_din_const(&desc[idx], 0, ctx->hash_len);
492 if (nbytes)
493 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
494 else
495 set_cipher_do(&desc[idx], DO_PAD);
496 }
497 set_flow_mode(&desc[idx], S_DIN_to_HASH);
498 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
499 idx++;
500
501 cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx);
502
503 if (is_hmac) {
504 /* HW last hash block padding (aka. "DO_PAD") */
505 hw_desc_init(&desc[idx]);
506 set_cipher_mode(&desc[idx], ctx->hw_mode);
507 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
508 ctx->hash_len, NS_BIT, 0);
509 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
510 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
511 set_cipher_do(&desc[idx], DO_PAD);
512 idx++;
513
514 idx = cc_fin_hmac(desc, req, idx);
515 }
516
517 idx = cc_fin_result(desc, req, idx);
518
519 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
520 if (rc != -EINPROGRESS && rc != -EBUSY) {
521 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
522 cc_unmap_hash_request(dev, state, src, true);
523 cc_unmap_result(dev, state, digestsize, result);
524 cc_unmap_req(dev, state, ctx);
525 }
526 return rc;
527 }
528
cc_restore_hash(struct cc_hw_desc * desc,struct cc_hash_ctx * ctx,struct ahash_req_ctx * state,unsigned int idx)529 static int cc_restore_hash(struct cc_hw_desc *desc, struct cc_hash_ctx *ctx,
530 struct ahash_req_ctx *state, unsigned int idx)
531 {
532 /* Restore hash digest */
533 hw_desc_init(&desc[idx]);
534 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
535 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
536 ctx->inter_digestsize, NS_BIT);
537 set_flow_mode(&desc[idx], S_DIN_to_HASH);
538 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
539 idx++;
540
541 /* Restore hash current length */
542 hw_desc_init(&desc[idx]);
543 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
544 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
545 set_din_type(&desc[idx], DMA_DLLI, state->digest_bytes_len_dma_addr,
546 ctx->hash_len, NS_BIT);
547 set_flow_mode(&desc[idx], S_DIN_to_HASH);
548 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
549 idx++;
550
551 cc_set_desc(state, ctx, DIN_HASH, desc, false, &idx);
552
553 return idx;
554 }
555
cc_hash_update(struct ahash_request * req)556 static int cc_hash_update(struct ahash_request *req)
557 {
558 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
559 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
560 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
561 unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base);
562 struct scatterlist *src = req->src;
563 unsigned int nbytes = req->nbytes;
564 struct device *dev = drvdata_to_dev(ctx->drvdata);
565 struct cc_crypto_req cc_req = {};
566 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
567 u32 idx = 0;
568 int rc;
569 gfp_t flags = cc_gfp_flags(&req->base);
570
571 dev_dbg(dev, "===== %s-update (%d) ====\n", ctx->is_hmac ?
572 "hmac" : "hash", nbytes);
573
574 if (nbytes == 0) {
575 /* no real updates required */
576 return 0;
577 }
578
579 rc = cc_map_hash_request_update(ctx->drvdata, state, src, nbytes,
580 block_size, flags);
581 if (rc) {
582 if (rc == 1) {
583 dev_dbg(dev, " data size not require HW update %x\n",
584 nbytes);
585 /* No hardware updates are required */
586 return 0;
587 }
588 dev_err(dev, "map_ahash_request_update() failed\n");
589 return -ENOMEM;
590 }
591
592 if (cc_map_req(dev, state, ctx)) {
593 dev_err(dev, "map_ahash_source() failed\n");
594 cc_unmap_hash_request(dev, state, src, true);
595 return -EINVAL;
596 }
597
598 /* Setup request structure */
599 cc_req.user_cb = cc_update_complete;
600 cc_req.user_arg = req;
601
602 idx = cc_restore_hash(desc, ctx, state, idx);
603
604 /* store the hash digest result in context */
605 hw_desc_init(&desc[idx]);
606 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
607 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
608 ctx->inter_digestsize, NS_BIT, 0);
609 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
610 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
611 idx++;
612
613 /* store current hash length in context */
614 hw_desc_init(&desc[idx]);
615 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
616 set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr,
617 ctx->hash_len, NS_BIT, 1);
618 set_queue_last_ind(ctx->drvdata, &desc[idx]);
619 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
620 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
621 idx++;
622
623 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
624 if (rc != -EINPROGRESS && rc != -EBUSY) {
625 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
626 cc_unmap_hash_request(dev, state, src, true);
627 cc_unmap_req(dev, state, ctx);
628 }
629 return rc;
630 }
631
cc_do_finup(struct ahash_request * req,bool update)632 static int cc_do_finup(struct ahash_request *req, bool update)
633 {
634 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
635 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
636 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
637 u32 digestsize = crypto_ahash_digestsize(tfm);
638 struct scatterlist *src = req->src;
639 unsigned int nbytes = req->nbytes;
640 u8 *result = req->result;
641 struct device *dev = drvdata_to_dev(ctx->drvdata);
642 bool is_hmac = ctx->is_hmac;
643 struct cc_crypto_req cc_req = {};
644 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
645 unsigned int idx = 0;
646 int rc;
647 gfp_t flags = cc_gfp_flags(&req->base);
648
649 dev_dbg(dev, "===== %s-%s (%d) ====\n", is_hmac ? "hmac" : "hash",
650 update ? "finup" : "final", nbytes);
651
652 if (cc_map_req(dev, state, ctx)) {
653 dev_err(dev, "map_ahash_source() failed\n");
654 return -EINVAL;
655 }
656
657 if (cc_map_hash_request_final(ctx->drvdata, state, src, nbytes, update,
658 flags)) {
659 dev_err(dev, "map_ahash_request_final() failed\n");
660 cc_unmap_req(dev, state, ctx);
661 return -ENOMEM;
662 }
663 if (cc_map_result(dev, state, digestsize)) {
664 dev_err(dev, "map_ahash_digest() failed\n");
665 cc_unmap_hash_request(dev, state, src, true);
666 cc_unmap_req(dev, state, ctx);
667 return -ENOMEM;
668 }
669
670 /* Setup request structure */
671 cc_req.user_cb = cc_hash_complete;
672 cc_req.user_arg = req;
673
674 idx = cc_restore_hash(desc, ctx, state, idx);
675
676 /* Pad the hash */
677 hw_desc_init(&desc[idx]);
678 set_cipher_do(&desc[idx], DO_PAD);
679 set_hash_cipher_mode(&desc[idx], ctx->hw_mode, ctx->hash_mode);
680 set_dout_dlli(&desc[idx], state->digest_bytes_len_dma_addr,
681 ctx->hash_len, NS_BIT, 0);
682 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
683 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
684 idx++;
685
686 if (is_hmac)
687 idx = cc_fin_hmac(desc, req, idx);
688
689 idx = cc_fin_result(desc, req, idx);
690
691 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
692 if (rc != -EINPROGRESS && rc != -EBUSY) {
693 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
694 cc_unmap_hash_request(dev, state, src, true);
695 cc_unmap_result(dev, state, digestsize, result);
696 cc_unmap_req(dev, state, ctx);
697 }
698 return rc;
699 }
700
cc_hash_finup(struct ahash_request * req)701 static int cc_hash_finup(struct ahash_request *req)
702 {
703 return cc_do_finup(req, true);
704 }
705
706
cc_hash_final(struct ahash_request * req)707 static int cc_hash_final(struct ahash_request *req)
708 {
709 return cc_do_finup(req, false);
710 }
711
cc_hash_init(struct ahash_request * req)712 static int cc_hash_init(struct ahash_request *req)
713 {
714 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
715 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
716 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
717 struct device *dev = drvdata_to_dev(ctx->drvdata);
718
719 dev_dbg(dev, "===== init (%d) ====\n", req->nbytes);
720
721 cc_init_req(dev, state, ctx);
722
723 return 0;
724 }
725
cc_hash_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)726 static int cc_hash_setkey(struct crypto_ahash *ahash, const u8 *key,
727 unsigned int keylen)
728 {
729 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
730 struct cc_crypto_req cc_req = {};
731 struct cc_hash_ctx *ctx = NULL;
732 int blocksize = 0;
733 int digestsize = 0;
734 int i, idx = 0, rc = 0;
735 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
736 u32 larval_addr;
737 struct device *dev;
738
739 ctx = crypto_ahash_ctx_dma(ahash);
740 dev = drvdata_to_dev(ctx->drvdata);
741 dev_dbg(dev, "start keylen: %d", keylen);
742
743 blocksize = crypto_tfm_alg_blocksize(&ahash->base);
744 digestsize = crypto_ahash_digestsize(ahash);
745
746 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->hash_mode);
747
748 /* The keylen value distinguishes HASH in case keylen is ZERO bytes,
749 * any NON-ZERO value utilizes HMAC flow
750 */
751 ctx->key_params.keylen = keylen;
752 ctx->key_params.key_dma_addr = 0;
753 ctx->is_hmac = true;
754 ctx->key_params.key = NULL;
755
756 if (keylen) {
757 ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL);
758 if (!ctx->key_params.key)
759 return -ENOMEM;
760
761 ctx->key_params.key_dma_addr =
762 dma_map_single(dev, ctx->key_params.key, keylen,
763 DMA_TO_DEVICE);
764 if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) {
765 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
766 ctx->key_params.key, keylen);
767 kfree_sensitive(ctx->key_params.key);
768 return -ENOMEM;
769 }
770 dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n",
771 &ctx->key_params.key_dma_addr, ctx->key_params.keylen);
772
773 if (keylen > blocksize) {
774 /* Load hash initial state */
775 hw_desc_init(&desc[idx]);
776 set_cipher_mode(&desc[idx], ctx->hw_mode);
777 set_din_sram(&desc[idx], larval_addr,
778 ctx->inter_digestsize);
779 set_flow_mode(&desc[idx], S_DIN_to_HASH);
780 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
781 idx++;
782
783 /* Load the hash current length*/
784 hw_desc_init(&desc[idx]);
785 set_cipher_mode(&desc[idx], ctx->hw_mode);
786 set_din_const(&desc[idx], 0, ctx->hash_len);
787 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
788 set_flow_mode(&desc[idx], S_DIN_to_HASH);
789 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
790 idx++;
791
792 hw_desc_init(&desc[idx]);
793 set_din_type(&desc[idx], DMA_DLLI,
794 ctx->key_params.key_dma_addr, keylen,
795 NS_BIT);
796 set_flow_mode(&desc[idx], DIN_HASH);
797 idx++;
798
799 /* Get hashed key */
800 hw_desc_init(&desc[idx]);
801 set_cipher_mode(&desc[idx], ctx->hw_mode);
802 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,
803 digestsize, NS_BIT, 0);
804 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
805 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
806 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
807 cc_set_endianity(ctx->hash_mode, &desc[idx]);
808 idx++;
809
810 hw_desc_init(&desc[idx]);
811 set_din_const(&desc[idx], 0, (blocksize - digestsize));
812 set_flow_mode(&desc[idx], BYPASS);
813 set_dout_dlli(&desc[idx],
814 (ctx->opad_tmp_keys_dma_addr +
815 digestsize),
816 (blocksize - digestsize), NS_BIT, 0);
817 idx++;
818 } else {
819 hw_desc_init(&desc[idx]);
820 set_din_type(&desc[idx], DMA_DLLI,
821 ctx->key_params.key_dma_addr, keylen,
822 NS_BIT);
823 set_flow_mode(&desc[idx], BYPASS);
824 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,
825 keylen, NS_BIT, 0);
826 idx++;
827
828 if ((blocksize - keylen)) {
829 hw_desc_init(&desc[idx]);
830 set_din_const(&desc[idx], 0,
831 (blocksize - keylen));
832 set_flow_mode(&desc[idx], BYPASS);
833 set_dout_dlli(&desc[idx],
834 (ctx->opad_tmp_keys_dma_addr +
835 keylen), (blocksize - keylen),
836 NS_BIT, 0);
837 idx++;
838 }
839 }
840 } else {
841 hw_desc_init(&desc[idx]);
842 set_din_const(&desc[idx], 0, blocksize);
843 set_flow_mode(&desc[idx], BYPASS);
844 set_dout_dlli(&desc[idx], (ctx->opad_tmp_keys_dma_addr),
845 blocksize, NS_BIT, 0);
846 idx++;
847 }
848
849 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
850 if (rc) {
851 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
852 goto out;
853 }
854
855 /* calc derived HMAC key */
856 for (idx = 0, i = 0; i < 2; i++) {
857 /* Load hash initial state */
858 hw_desc_init(&desc[idx]);
859 set_cipher_mode(&desc[idx], ctx->hw_mode);
860 set_din_sram(&desc[idx], larval_addr, ctx->inter_digestsize);
861 set_flow_mode(&desc[idx], S_DIN_to_HASH);
862 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
863 idx++;
864
865 /* Load the hash current length*/
866 hw_desc_init(&desc[idx]);
867 set_cipher_mode(&desc[idx], ctx->hw_mode);
868 set_din_const(&desc[idx], 0, ctx->hash_len);
869 set_flow_mode(&desc[idx], S_DIN_to_HASH);
870 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
871 idx++;
872
873 /* Prepare ipad key */
874 hw_desc_init(&desc[idx]);
875 set_xor_val(&desc[idx], hmac_pad_const[i]);
876 set_cipher_mode(&desc[idx], ctx->hw_mode);
877 set_flow_mode(&desc[idx], S_DIN_to_HASH);
878 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
879 idx++;
880
881 /* Perform HASH update */
882 hw_desc_init(&desc[idx]);
883 set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr,
884 blocksize, NS_BIT);
885 set_cipher_mode(&desc[idx], ctx->hw_mode);
886 set_xor_active(&desc[idx]);
887 set_flow_mode(&desc[idx], DIN_HASH);
888 idx++;
889
890 /* Get the IPAD/OPAD xor key (Note, IPAD is the initial digest
891 * of the first HASH "update" state)
892 */
893 hw_desc_init(&desc[idx]);
894 set_cipher_mode(&desc[idx], ctx->hw_mode);
895 if (i > 0) /* Not first iteration */
896 set_dout_dlli(&desc[idx], ctx->opad_tmp_keys_dma_addr,
897 ctx->inter_digestsize, NS_BIT, 0);
898 else /* First iteration */
899 set_dout_dlli(&desc[idx], ctx->digest_buff_dma_addr,
900 ctx->inter_digestsize, NS_BIT, 0);
901 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
902 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
903 idx++;
904 }
905
906 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
907
908 out:
909 if (ctx->key_params.key_dma_addr) {
910 dma_unmap_single(dev, ctx->key_params.key_dma_addr,
911 ctx->key_params.keylen, DMA_TO_DEVICE);
912 dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n",
913 &ctx->key_params.key_dma_addr, ctx->key_params.keylen);
914 }
915
916 kfree_sensitive(ctx->key_params.key);
917
918 return rc;
919 }
920
cc_xcbc_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)921 static int cc_xcbc_setkey(struct crypto_ahash *ahash,
922 const u8 *key, unsigned int keylen)
923 {
924 struct cc_crypto_req cc_req = {};
925 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
926 struct device *dev = drvdata_to_dev(ctx->drvdata);
927 int rc = 0;
928 unsigned int idx = 0;
929 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
930
931 dev_dbg(dev, "===== setkey (%d) ====\n", keylen);
932
933 switch (keylen) {
934 case AES_KEYSIZE_128:
935 case AES_KEYSIZE_192:
936 case AES_KEYSIZE_256:
937 break;
938 default:
939 return -EINVAL;
940 }
941
942 ctx->key_params.keylen = keylen;
943
944 ctx->key_params.key = kmemdup(key, keylen, GFP_KERNEL);
945 if (!ctx->key_params.key)
946 return -ENOMEM;
947
948 ctx->key_params.key_dma_addr =
949 dma_map_single(dev, ctx->key_params.key, keylen, DMA_TO_DEVICE);
950 if (dma_mapping_error(dev, ctx->key_params.key_dma_addr)) {
951 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
952 key, keylen);
953 kfree_sensitive(ctx->key_params.key);
954 return -ENOMEM;
955 }
956 dev_dbg(dev, "mapping key-buffer: key_dma_addr=%pad keylen=%u\n",
957 &ctx->key_params.key_dma_addr, ctx->key_params.keylen);
958
959 ctx->is_hmac = true;
960 /* 1. Load the AES key */
961 hw_desc_init(&desc[idx]);
962 set_din_type(&desc[idx], DMA_DLLI, ctx->key_params.key_dma_addr,
963 keylen, NS_BIT);
964 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
965 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
966 set_key_size_aes(&desc[idx], keylen);
967 set_flow_mode(&desc[idx], S_DIN_to_AES);
968 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
969 idx++;
970
971 hw_desc_init(&desc[idx]);
972 set_din_const(&desc[idx], 0x01010101, CC_AES_128_BIT_KEY_SIZE);
973 set_flow_mode(&desc[idx], DIN_AES_DOUT);
974 set_dout_dlli(&desc[idx],
975 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET),
976 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);
977 idx++;
978
979 hw_desc_init(&desc[idx]);
980 set_din_const(&desc[idx], 0x02020202, CC_AES_128_BIT_KEY_SIZE);
981 set_flow_mode(&desc[idx], DIN_AES_DOUT);
982 set_dout_dlli(&desc[idx],
983 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET),
984 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);
985 idx++;
986
987 hw_desc_init(&desc[idx]);
988 set_din_const(&desc[idx], 0x03030303, CC_AES_128_BIT_KEY_SIZE);
989 set_flow_mode(&desc[idx], DIN_AES_DOUT);
990 set_dout_dlli(&desc[idx],
991 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET),
992 CC_AES_128_BIT_KEY_SIZE, NS_BIT, 0);
993 idx++;
994
995 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
996
997 dma_unmap_single(dev, ctx->key_params.key_dma_addr,
998 ctx->key_params.keylen, DMA_TO_DEVICE);
999 dev_dbg(dev, "Unmapped key-buffer: key_dma_addr=%pad keylen=%u\n",
1000 &ctx->key_params.key_dma_addr, ctx->key_params.keylen);
1001
1002 kfree_sensitive(ctx->key_params.key);
1003
1004 return rc;
1005 }
1006
cc_cmac_setkey(struct crypto_ahash * ahash,const u8 * key,unsigned int keylen)1007 static int cc_cmac_setkey(struct crypto_ahash *ahash,
1008 const u8 *key, unsigned int keylen)
1009 {
1010 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
1011 struct device *dev = drvdata_to_dev(ctx->drvdata);
1012
1013 dev_dbg(dev, "===== setkey (%d) ====\n", keylen);
1014
1015 ctx->is_hmac = true;
1016
1017 switch (keylen) {
1018 case AES_KEYSIZE_128:
1019 case AES_KEYSIZE_192:
1020 case AES_KEYSIZE_256:
1021 break;
1022 default:
1023 return -EINVAL;
1024 }
1025
1026 ctx->key_params.keylen = keylen;
1027
1028 /* STAT_PHASE_1: Copy key to ctx */
1029
1030 dma_sync_single_for_cpu(dev, ctx->opad_tmp_keys_dma_addr,
1031 keylen, DMA_TO_DEVICE);
1032
1033 memcpy(ctx->opad_tmp_keys_buff, key, keylen);
1034 if (keylen == 24) {
1035 memset(ctx->opad_tmp_keys_buff + 24, 0,
1036 CC_AES_KEY_SIZE_MAX - 24);
1037 }
1038
1039 dma_sync_single_for_device(dev, ctx->opad_tmp_keys_dma_addr,
1040 keylen, DMA_TO_DEVICE);
1041
1042 ctx->key_params.keylen = keylen;
1043
1044 return 0;
1045 }
1046
cc_free_ctx(struct cc_hash_ctx * ctx)1047 static void cc_free_ctx(struct cc_hash_ctx *ctx)
1048 {
1049 struct device *dev = drvdata_to_dev(ctx->drvdata);
1050
1051 if (ctx->digest_buff_dma_addr) {
1052 dma_unmap_single(dev, ctx->digest_buff_dma_addr,
1053 sizeof(ctx->digest_buff), DMA_BIDIRECTIONAL);
1054 dev_dbg(dev, "Unmapped digest-buffer: digest_buff_dma_addr=%pad\n",
1055 &ctx->digest_buff_dma_addr);
1056 ctx->digest_buff_dma_addr = 0;
1057 }
1058 if (ctx->opad_tmp_keys_dma_addr) {
1059 dma_unmap_single(dev, ctx->opad_tmp_keys_dma_addr,
1060 sizeof(ctx->opad_tmp_keys_buff),
1061 DMA_BIDIRECTIONAL);
1062 dev_dbg(dev, "Unmapped opad-digest: opad_tmp_keys_dma_addr=%pad\n",
1063 &ctx->opad_tmp_keys_dma_addr);
1064 ctx->opad_tmp_keys_dma_addr = 0;
1065 }
1066
1067 ctx->key_params.keylen = 0;
1068 }
1069
cc_alloc_ctx(struct cc_hash_ctx * ctx)1070 static int cc_alloc_ctx(struct cc_hash_ctx *ctx)
1071 {
1072 struct device *dev = drvdata_to_dev(ctx->drvdata);
1073
1074 ctx->key_params.keylen = 0;
1075
1076 ctx->digest_buff_dma_addr =
1077 dma_map_single(dev, ctx->digest_buff, sizeof(ctx->digest_buff),
1078 DMA_BIDIRECTIONAL);
1079 if (dma_mapping_error(dev, ctx->digest_buff_dma_addr)) {
1080 dev_err(dev, "Mapping digest len %zu B at va=%pK for DMA failed\n",
1081 sizeof(ctx->digest_buff), ctx->digest_buff);
1082 goto fail;
1083 }
1084 dev_dbg(dev, "Mapped digest %zu B at va=%pK to dma=%pad\n",
1085 sizeof(ctx->digest_buff), ctx->digest_buff,
1086 &ctx->digest_buff_dma_addr);
1087
1088 ctx->opad_tmp_keys_dma_addr =
1089 dma_map_single(dev, ctx->opad_tmp_keys_buff,
1090 sizeof(ctx->opad_tmp_keys_buff),
1091 DMA_BIDIRECTIONAL);
1092 if (dma_mapping_error(dev, ctx->opad_tmp_keys_dma_addr)) {
1093 dev_err(dev, "Mapping opad digest %zu B at va=%pK for DMA failed\n",
1094 sizeof(ctx->opad_tmp_keys_buff),
1095 ctx->opad_tmp_keys_buff);
1096 goto fail;
1097 }
1098 dev_dbg(dev, "Mapped opad_tmp_keys %zu B at va=%pK to dma=%pad\n",
1099 sizeof(ctx->opad_tmp_keys_buff), ctx->opad_tmp_keys_buff,
1100 &ctx->opad_tmp_keys_dma_addr);
1101
1102 ctx->is_hmac = false;
1103 return 0;
1104
1105 fail:
1106 cc_free_ctx(ctx);
1107 return -ENOMEM;
1108 }
1109
cc_get_hash_len(struct crypto_tfm * tfm)1110 static int cc_get_hash_len(struct crypto_tfm *tfm)
1111 {
1112 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
1113
1114 if (ctx->hash_mode == DRV_HASH_SM3)
1115 return CC_SM3_HASH_LEN_SIZE;
1116 else
1117 return cc_get_default_hash_len(ctx->drvdata);
1118 }
1119
cc_cra_init(struct crypto_tfm * tfm)1120 static int cc_cra_init(struct crypto_tfm *tfm)
1121 {
1122 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
1123 struct hash_alg_common *hash_alg_common =
1124 container_of(tfm->__crt_alg, struct hash_alg_common, base);
1125 struct ahash_alg *ahash_alg =
1126 container_of(hash_alg_common, struct ahash_alg, halg);
1127 struct cc_hash_alg *cc_alg =
1128 container_of(ahash_alg, struct cc_hash_alg, ahash_alg);
1129
1130 crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),
1131 sizeof(struct ahash_req_ctx));
1132
1133 ctx->hash_mode = cc_alg->hash_mode;
1134 ctx->hw_mode = cc_alg->hw_mode;
1135 ctx->inter_digestsize = cc_alg->inter_digestsize;
1136 ctx->drvdata = cc_alg->drvdata;
1137 ctx->hash_len = cc_get_hash_len(tfm);
1138 return cc_alloc_ctx(ctx);
1139 }
1140
cc_cra_exit(struct crypto_tfm * tfm)1141 static void cc_cra_exit(struct crypto_tfm *tfm)
1142 {
1143 struct cc_hash_ctx *ctx = crypto_tfm_ctx_dma(tfm);
1144 struct device *dev = drvdata_to_dev(ctx->drvdata);
1145
1146 dev_dbg(dev, "cc_cra_exit");
1147 cc_free_ctx(ctx);
1148 }
1149
cc_mac_update(struct ahash_request * req)1150 static int cc_mac_update(struct ahash_request *req)
1151 {
1152 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
1153 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1154 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
1155 struct device *dev = drvdata_to_dev(ctx->drvdata);
1156 unsigned int block_size = crypto_tfm_alg_blocksize(&tfm->base);
1157 struct cc_crypto_req cc_req = {};
1158 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1159 int rc;
1160 u32 idx = 0;
1161 gfp_t flags = cc_gfp_flags(&req->base);
1162
1163 if (req->nbytes == 0) {
1164 /* no real updates required */
1165 return 0;
1166 }
1167
1168 state->xcbc_count++;
1169
1170 rc = cc_map_hash_request_update(ctx->drvdata, state, req->src,
1171 req->nbytes, block_size, flags);
1172 if (rc) {
1173 if (rc == 1) {
1174 dev_dbg(dev, " data size not require HW update %x\n",
1175 req->nbytes);
1176 /* No hardware updates are required */
1177 return 0;
1178 }
1179 dev_err(dev, "map_ahash_request_update() failed\n");
1180 return -ENOMEM;
1181 }
1182
1183 if (cc_map_req(dev, state, ctx)) {
1184 dev_err(dev, "map_ahash_source() failed\n");
1185 return -EINVAL;
1186 }
1187
1188 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC)
1189 cc_setup_xcbc(req, desc, &idx);
1190 else
1191 cc_setup_cmac(req, desc, &idx);
1192
1193 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, true, &idx);
1194
1195 /* store the hash digest result in context */
1196 hw_desc_init(&desc[idx]);
1197 set_cipher_mode(&desc[idx], ctx->hw_mode);
1198 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
1199 ctx->inter_digestsize, NS_BIT, 1);
1200 set_queue_last_ind(ctx->drvdata, &desc[idx]);
1201 set_flow_mode(&desc[idx], S_AES_to_DOUT);
1202 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1203 idx++;
1204
1205 /* Setup request structure */
1206 cc_req.user_cb = cc_update_complete;
1207 cc_req.user_arg = req;
1208
1209 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1210 if (rc != -EINPROGRESS && rc != -EBUSY) {
1211 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1212 cc_unmap_hash_request(dev, state, req->src, true);
1213 cc_unmap_req(dev, state, ctx);
1214 }
1215 return rc;
1216 }
1217
cc_mac_final(struct ahash_request * req)1218 static int cc_mac_final(struct ahash_request *req)
1219 {
1220 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
1221 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1222 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
1223 struct device *dev = drvdata_to_dev(ctx->drvdata);
1224 struct cc_crypto_req cc_req = {};
1225 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1226 int idx = 0;
1227 int rc = 0;
1228 u32 key_size, key_len;
1229 u32 digestsize = crypto_ahash_digestsize(tfm);
1230 gfp_t flags = cc_gfp_flags(&req->base);
1231 u32 rem_cnt = *cc_hash_buf_cnt(state);
1232
1233 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
1234 key_size = CC_AES_128_BIT_KEY_SIZE;
1235 key_len = CC_AES_128_BIT_KEY_SIZE;
1236 } else {
1237 key_size = (ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE :
1238 ctx->key_params.keylen;
1239 key_len = ctx->key_params.keylen;
1240 }
1241
1242 dev_dbg(dev, "===== final xcbc reminder (%d) ====\n", rem_cnt);
1243
1244 if (cc_map_req(dev, state, ctx)) {
1245 dev_err(dev, "map_ahash_source() failed\n");
1246 return -EINVAL;
1247 }
1248
1249 if (cc_map_hash_request_final(ctx->drvdata, state, req->src,
1250 req->nbytes, 0, flags)) {
1251 dev_err(dev, "map_ahash_request_final() failed\n");
1252 cc_unmap_req(dev, state, ctx);
1253 return -ENOMEM;
1254 }
1255
1256 if (cc_map_result(dev, state, digestsize)) {
1257 dev_err(dev, "map_ahash_digest() failed\n");
1258 cc_unmap_hash_request(dev, state, req->src, true);
1259 cc_unmap_req(dev, state, ctx);
1260 return -ENOMEM;
1261 }
1262
1263 /* Setup request structure */
1264 cc_req.user_cb = cc_hash_complete;
1265 cc_req.user_arg = req;
1266
1267 if (state->xcbc_count && rem_cnt == 0) {
1268 /* Load key for ECB decryption */
1269 hw_desc_init(&desc[idx]);
1270 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
1271 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_DECRYPT);
1272 set_din_type(&desc[idx], DMA_DLLI,
1273 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K1_OFFSET),
1274 key_size, NS_BIT);
1275 set_key_size_aes(&desc[idx], key_len);
1276 set_flow_mode(&desc[idx], S_DIN_to_AES);
1277 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1278 idx++;
1279
1280 /* Initiate decryption of block state to previous
1281 * block_state-XOR-M[n]
1282 */
1283 hw_desc_init(&desc[idx]);
1284 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
1285 CC_AES_BLOCK_SIZE, NS_BIT);
1286 set_dout_dlli(&desc[idx], state->digest_buff_dma_addr,
1287 CC_AES_BLOCK_SIZE, NS_BIT, 0);
1288 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1289 idx++;
1290
1291 /* Memory Barrier: wait for axi write to complete */
1292 hw_desc_init(&desc[idx]);
1293 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1294 set_dout_no_dma(&desc[idx], 0, 0, 1);
1295 idx++;
1296 }
1297
1298 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC)
1299 cc_setup_xcbc(req, desc, &idx);
1300 else
1301 cc_setup_cmac(req, desc, &idx);
1302
1303 if (state->xcbc_count == 0) {
1304 hw_desc_init(&desc[idx]);
1305 set_cipher_mode(&desc[idx], ctx->hw_mode);
1306 set_key_size_aes(&desc[idx], key_len);
1307 set_cmac_size0_mode(&desc[idx]);
1308 set_flow_mode(&desc[idx], S_DIN_to_AES);
1309 idx++;
1310 } else if (rem_cnt > 0) {
1311 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);
1312 } else {
1313 hw_desc_init(&desc[idx]);
1314 set_din_const(&desc[idx], 0x00, CC_AES_BLOCK_SIZE);
1315 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1316 idx++;
1317 }
1318
1319 /* Get final MAC result */
1320 hw_desc_init(&desc[idx]);
1321 set_dout_dlli(&desc[idx], state->digest_result_dma_addr,
1322 digestsize, NS_BIT, 1);
1323 set_queue_last_ind(ctx->drvdata, &desc[idx]);
1324 set_flow_mode(&desc[idx], S_AES_to_DOUT);
1325 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1326 set_cipher_mode(&desc[idx], ctx->hw_mode);
1327 idx++;
1328
1329 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1330 if (rc != -EINPROGRESS && rc != -EBUSY) {
1331 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1332 cc_unmap_hash_request(dev, state, req->src, true);
1333 cc_unmap_result(dev, state, digestsize, req->result);
1334 cc_unmap_req(dev, state, ctx);
1335 }
1336 return rc;
1337 }
1338
cc_mac_finup(struct ahash_request * req)1339 static int cc_mac_finup(struct ahash_request *req)
1340 {
1341 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
1342 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1343 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
1344 struct device *dev = drvdata_to_dev(ctx->drvdata);
1345 struct cc_crypto_req cc_req = {};
1346 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1347 int idx = 0;
1348 int rc = 0;
1349 u32 key_len = 0;
1350 u32 digestsize = crypto_ahash_digestsize(tfm);
1351 gfp_t flags = cc_gfp_flags(&req->base);
1352
1353 dev_dbg(dev, "===== finup xcbc(%d) ====\n", req->nbytes);
1354 if (state->xcbc_count > 0 && req->nbytes == 0) {
1355 dev_dbg(dev, "No data to update. Call to fdx_mac_final\n");
1356 return cc_mac_final(req);
1357 }
1358
1359 if (cc_map_req(dev, state, ctx)) {
1360 dev_err(dev, "map_ahash_source() failed\n");
1361 return -EINVAL;
1362 }
1363
1364 if (cc_map_hash_request_final(ctx->drvdata, state, req->src,
1365 req->nbytes, 1, flags)) {
1366 dev_err(dev, "map_ahash_request_final() failed\n");
1367 cc_unmap_req(dev, state, ctx);
1368 return -ENOMEM;
1369 }
1370 if (cc_map_result(dev, state, digestsize)) {
1371 dev_err(dev, "map_ahash_digest() failed\n");
1372 cc_unmap_hash_request(dev, state, req->src, true);
1373 cc_unmap_req(dev, state, ctx);
1374 return -ENOMEM;
1375 }
1376
1377 /* Setup request structure */
1378 cc_req.user_cb = cc_hash_complete;
1379 cc_req.user_arg = req;
1380
1381 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
1382 key_len = CC_AES_128_BIT_KEY_SIZE;
1383 cc_setup_xcbc(req, desc, &idx);
1384 } else {
1385 key_len = ctx->key_params.keylen;
1386 cc_setup_cmac(req, desc, &idx);
1387 }
1388
1389 if (req->nbytes == 0) {
1390 hw_desc_init(&desc[idx]);
1391 set_cipher_mode(&desc[idx], ctx->hw_mode);
1392 set_key_size_aes(&desc[idx], key_len);
1393 set_cmac_size0_mode(&desc[idx]);
1394 set_flow_mode(&desc[idx], S_DIN_to_AES);
1395 idx++;
1396 } else {
1397 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);
1398 }
1399
1400 /* Get final MAC result */
1401 hw_desc_init(&desc[idx]);
1402 set_dout_dlli(&desc[idx], state->digest_result_dma_addr,
1403 digestsize, NS_BIT, 1);
1404 set_queue_last_ind(ctx->drvdata, &desc[idx]);
1405 set_flow_mode(&desc[idx], S_AES_to_DOUT);
1406 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1407 set_cipher_mode(&desc[idx], ctx->hw_mode);
1408 idx++;
1409
1410 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1411 if (rc != -EINPROGRESS && rc != -EBUSY) {
1412 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1413 cc_unmap_hash_request(dev, state, req->src, true);
1414 cc_unmap_result(dev, state, digestsize, req->result);
1415 cc_unmap_req(dev, state, ctx);
1416 }
1417 return rc;
1418 }
1419
cc_mac_digest(struct ahash_request * req)1420 static int cc_mac_digest(struct ahash_request *req)
1421 {
1422 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
1423 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1424 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
1425 struct device *dev = drvdata_to_dev(ctx->drvdata);
1426 u32 digestsize = crypto_ahash_digestsize(tfm);
1427 struct cc_crypto_req cc_req = {};
1428 struct cc_hw_desc desc[CC_MAX_HASH_SEQ_LEN];
1429 u32 key_len;
1430 unsigned int idx = 0;
1431 int rc;
1432 gfp_t flags = cc_gfp_flags(&req->base);
1433
1434 dev_dbg(dev, "===== -digest mac (%d) ====\n", req->nbytes);
1435
1436 cc_init_req(dev, state, ctx);
1437
1438 if (cc_map_req(dev, state, ctx)) {
1439 dev_err(dev, "map_ahash_source() failed\n");
1440 return -ENOMEM;
1441 }
1442 if (cc_map_result(dev, state, digestsize)) {
1443 dev_err(dev, "map_ahash_digest() failed\n");
1444 cc_unmap_req(dev, state, ctx);
1445 return -ENOMEM;
1446 }
1447
1448 if (cc_map_hash_request_final(ctx->drvdata, state, req->src,
1449 req->nbytes, 1, flags)) {
1450 dev_err(dev, "map_ahash_request_final() failed\n");
1451 cc_unmap_req(dev, state, ctx);
1452 return -ENOMEM;
1453 }
1454
1455 /* Setup request structure */
1456 cc_req.user_cb = cc_digest_complete;
1457 cc_req.user_arg = req;
1458
1459 if (ctx->hw_mode == DRV_CIPHER_XCBC_MAC) {
1460 key_len = CC_AES_128_BIT_KEY_SIZE;
1461 cc_setup_xcbc(req, desc, &idx);
1462 } else {
1463 key_len = ctx->key_params.keylen;
1464 cc_setup_cmac(req, desc, &idx);
1465 }
1466
1467 if (req->nbytes == 0) {
1468 hw_desc_init(&desc[idx]);
1469 set_cipher_mode(&desc[idx], ctx->hw_mode);
1470 set_key_size_aes(&desc[idx], key_len);
1471 set_cmac_size0_mode(&desc[idx]);
1472 set_flow_mode(&desc[idx], S_DIN_to_AES);
1473 idx++;
1474 } else {
1475 cc_set_desc(state, ctx, DIN_AES_DOUT, desc, false, &idx);
1476 }
1477
1478 /* Get final MAC result */
1479 hw_desc_init(&desc[idx]);
1480 set_dout_dlli(&desc[idx], state->digest_result_dma_addr,
1481 CC_AES_BLOCK_SIZE, NS_BIT, 1);
1482 set_queue_last_ind(ctx->drvdata, &desc[idx]);
1483 set_flow_mode(&desc[idx], S_AES_to_DOUT);
1484 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1485 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1486 set_cipher_mode(&desc[idx], ctx->hw_mode);
1487 idx++;
1488
1489 rc = cc_send_request(ctx->drvdata, &cc_req, desc, idx, &req->base);
1490 if (rc != -EINPROGRESS && rc != -EBUSY) {
1491 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
1492 cc_unmap_hash_request(dev, state, req->src, true);
1493 cc_unmap_result(dev, state, digestsize, req->result);
1494 cc_unmap_req(dev, state, ctx);
1495 }
1496 return rc;
1497 }
1498
cc_hash_export(struct ahash_request * req,void * out)1499 static int cc_hash_export(struct ahash_request *req, void *out)
1500 {
1501 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1502 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
1503 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
1504 u8 *curr_buff = cc_hash_buf(state);
1505 u32 curr_buff_cnt = *cc_hash_buf_cnt(state);
1506 const u32 tmp = CC_EXPORT_MAGIC;
1507
1508 memcpy(out, &tmp, sizeof(u32));
1509 out += sizeof(u32);
1510
1511 memcpy(out, state->digest_buff, ctx->inter_digestsize);
1512 out += ctx->inter_digestsize;
1513
1514 memcpy(out, state->digest_bytes_len, ctx->hash_len);
1515 out += ctx->hash_len;
1516
1517 memcpy(out, &curr_buff_cnt, sizeof(u32));
1518 out += sizeof(u32);
1519
1520 memcpy(out, curr_buff, curr_buff_cnt);
1521
1522 return 0;
1523 }
1524
cc_hash_import(struct ahash_request * req,const void * in)1525 static int cc_hash_import(struct ahash_request *req, const void *in)
1526 {
1527 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1528 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(ahash);
1529 struct device *dev = drvdata_to_dev(ctx->drvdata);
1530 struct ahash_req_ctx *state = ahash_request_ctx_dma(req);
1531 u32 tmp;
1532
1533 memcpy(&tmp, in, sizeof(u32));
1534 if (tmp != CC_EXPORT_MAGIC)
1535 return -EINVAL;
1536 in += sizeof(u32);
1537
1538 cc_init_req(dev, state, ctx);
1539
1540 memcpy(state->digest_buff, in, ctx->inter_digestsize);
1541 in += ctx->inter_digestsize;
1542
1543 memcpy(state->digest_bytes_len, in, ctx->hash_len);
1544 in += ctx->hash_len;
1545
1546 /* Sanity check the data as much as possible */
1547 memcpy(&tmp, in, sizeof(u32));
1548 if (tmp > CC_MAX_HASH_BLCK_SIZE)
1549 return -EINVAL;
1550 in += sizeof(u32);
1551
1552 state->buf_cnt[0] = tmp;
1553 memcpy(state->buffers[0], in, tmp);
1554
1555 return 0;
1556 }
1557
1558 struct cc_hash_template {
1559 char name[CRYPTO_MAX_ALG_NAME];
1560 char driver_name[CRYPTO_MAX_ALG_NAME];
1561 char mac_name[CRYPTO_MAX_ALG_NAME];
1562 char mac_driver_name[CRYPTO_MAX_ALG_NAME];
1563 unsigned int blocksize;
1564 bool is_mac;
1565 bool synchronize;
1566 struct ahash_alg template_ahash;
1567 int hash_mode;
1568 int hw_mode;
1569 int inter_digestsize;
1570 struct cc_drvdata *drvdata;
1571 u32 min_hw_rev;
1572 enum cc_std_body std_body;
1573 };
1574
1575 #define CC_STATE_SIZE(_x) \
1576 ((_x) + HASH_MAX_LEN_SIZE + CC_MAX_HASH_BLCK_SIZE + (2 * sizeof(u32)))
1577
1578 /* hash descriptors */
1579 static struct cc_hash_template driver_hash[] = {
1580 //Asynchronize hash template
1581 {
1582 .name = "sha1",
1583 .driver_name = "sha1-ccree",
1584 .mac_name = "hmac(sha1)",
1585 .mac_driver_name = "hmac-sha1-ccree",
1586 .blocksize = SHA1_BLOCK_SIZE,
1587 .is_mac = true,
1588 .synchronize = false,
1589 .template_ahash = {
1590 .init = cc_hash_init,
1591 .update = cc_hash_update,
1592 .final = cc_hash_final,
1593 .finup = cc_hash_finup,
1594 .digest = cc_hash_digest,
1595 .export = cc_hash_export,
1596 .import = cc_hash_import,
1597 .setkey = cc_hash_setkey,
1598 .halg = {
1599 .digestsize = SHA1_DIGEST_SIZE,
1600 .statesize = CC_STATE_SIZE(SHA1_DIGEST_SIZE),
1601 },
1602 },
1603 .hash_mode = DRV_HASH_SHA1,
1604 .hw_mode = DRV_HASH_HW_SHA1,
1605 .inter_digestsize = SHA1_DIGEST_SIZE,
1606 .min_hw_rev = CC_HW_REV_630,
1607 .std_body = CC_STD_NIST,
1608 },
1609 {
1610 .name = "sha256",
1611 .driver_name = "sha256-ccree",
1612 .mac_name = "hmac(sha256)",
1613 .mac_driver_name = "hmac-sha256-ccree",
1614 .blocksize = SHA256_BLOCK_SIZE,
1615 .is_mac = true,
1616 .template_ahash = {
1617 .init = cc_hash_init,
1618 .update = cc_hash_update,
1619 .final = cc_hash_final,
1620 .finup = cc_hash_finup,
1621 .digest = cc_hash_digest,
1622 .export = cc_hash_export,
1623 .import = cc_hash_import,
1624 .setkey = cc_hash_setkey,
1625 .halg = {
1626 .digestsize = SHA256_DIGEST_SIZE,
1627 .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE)
1628 },
1629 },
1630 .hash_mode = DRV_HASH_SHA256,
1631 .hw_mode = DRV_HASH_HW_SHA256,
1632 .inter_digestsize = SHA256_DIGEST_SIZE,
1633 .min_hw_rev = CC_HW_REV_630,
1634 .std_body = CC_STD_NIST,
1635 },
1636 {
1637 .name = "sha224",
1638 .driver_name = "sha224-ccree",
1639 .mac_name = "hmac(sha224)",
1640 .mac_driver_name = "hmac-sha224-ccree",
1641 .blocksize = SHA224_BLOCK_SIZE,
1642 .is_mac = true,
1643 .template_ahash = {
1644 .init = cc_hash_init,
1645 .update = cc_hash_update,
1646 .final = cc_hash_final,
1647 .finup = cc_hash_finup,
1648 .digest = cc_hash_digest,
1649 .export = cc_hash_export,
1650 .import = cc_hash_import,
1651 .setkey = cc_hash_setkey,
1652 .halg = {
1653 .digestsize = SHA224_DIGEST_SIZE,
1654 .statesize = CC_STATE_SIZE(SHA256_DIGEST_SIZE),
1655 },
1656 },
1657 .hash_mode = DRV_HASH_SHA224,
1658 .hw_mode = DRV_HASH_HW_SHA256,
1659 .inter_digestsize = SHA256_DIGEST_SIZE,
1660 .min_hw_rev = CC_HW_REV_630,
1661 .std_body = CC_STD_NIST,
1662 },
1663 {
1664 .name = "sha384",
1665 .driver_name = "sha384-ccree",
1666 .mac_name = "hmac(sha384)",
1667 .mac_driver_name = "hmac-sha384-ccree",
1668 .blocksize = SHA384_BLOCK_SIZE,
1669 .is_mac = true,
1670 .template_ahash = {
1671 .init = cc_hash_init,
1672 .update = cc_hash_update,
1673 .final = cc_hash_final,
1674 .finup = cc_hash_finup,
1675 .digest = cc_hash_digest,
1676 .export = cc_hash_export,
1677 .import = cc_hash_import,
1678 .setkey = cc_hash_setkey,
1679 .halg = {
1680 .digestsize = SHA384_DIGEST_SIZE,
1681 .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE),
1682 },
1683 },
1684 .hash_mode = DRV_HASH_SHA384,
1685 .hw_mode = DRV_HASH_HW_SHA512,
1686 .inter_digestsize = SHA512_DIGEST_SIZE,
1687 .min_hw_rev = CC_HW_REV_712,
1688 .std_body = CC_STD_NIST,
1689 },
1690 {
1691 .name = "sha512",
1692 .driver_name = "sha512-ccree",
1693 .mac_name = "hmac(sha512)",
1694 .mac_driver_name = "hmac-sha512-ccree",
1695 .blocksize = SHA512_BLOCK_SIZE,
1696 .is_mac = true,
1697 .template_ahash = {
1698 .init = cc_hash_init,
1699 .update = cc_hash_update,
1700 .final = cc_hash_final,
1701 .finup = cc_hash_finup,
1702 .digest = cc_hash_digest,
1703 .export = cc_hash_export,
1704 .import = cc_hash_import,
1705 .setkey = cc_hash_setkey,
1706 .halg = {
1707 .digestsize = SHA512_DIGEST_SIZE,
1708 .statesize = CC_STATE_SIZE(SHA512_DIGEST_SIZE),
1709 },
1710 },
1711 .hash_mode = DRV_HASH_SHA512,
1712 .hw_mode = DRV_HASH_HW_SHA512,
1713 .inter_digestsize = SHA512_DIGEST_SIZE,
1714 .min_hw_rev = CC_HW_REV_712,
1715 .std_body = CC_STD_NIST,
1716 },
1717 {
1718 .name = "md5",
1719 .driver_name = "md5-ccree",
1720 .mac_name = "hmac(md5)",
1721 .mac_driver_name = "hmac-md5-ccree",
1722 .blocksize = MD5_HMAC_BLOCK_SIZE,
1723 .is_mac = true,
1724 .template_ahash = {
1725 .init = cc_hash_init,
1726 .update = cc_hash_update,
1727 .final = cc_hash_final,
1728 .finup = cc_hash_finup,
1729 .digest = cc_hash_digest,
1730 .export = cc_hash_export,
1731 .import = cc_hash_import,
1732 .setkey = cc_hash_setkey,
1733 .halg = {
1734 .digestsize = MD5_DIGEST_SIZE,
1735 .statesize = CC_STATE_SIZE(MD5_DIGEST_SIZE),
1736 },
1737 },
1738 .hash_mode = DRV_HASH_MD5,
1739 .hw_mode = DRV_HASH_HW_MD5,
1740 .inter_digestsize = MD5_DIGEST_SIZE,
1741 .min_hw_rev = CC_HW_REV_630,
1742 .std_body = CC_STD_NIST,
1743 },
1744 {
1745 .name = "sm3",
1746 .driver_name = "sm3-ccree",
1747 .blocksize = SM3_BLOCK_SIZE,
1748 .is_mac = false,
1749 .template_ahash = {
1750 .init = cc_hash_init,
1751 .update = cc_hash_update,
1752 .final = cc_hash_final,
1753 .finup = cc_hash_finup,
1754 .digest = cc_hash_digest,
1755 .export = cc_hash_export,
1756 .import = cc_hash_import,
1757 .setkey = cc_hash_setkey,
1758 .halg = {
1759 .digestsize = SM3_DIGEST_SIZE,
1760 .statesize = CC_STATE_SIZE(SM3_DIGEST_SIZE),
1761 },
1762 },
1763 .hash_mode = DRV_HASH_SM3,
1764 .hw_mode = DRV_HASH_HW_SM3,
1765 .inter_digestsize = SM3_DIGEST_SIZE,
1766 .min_hw_rev = CC_HW_REV_713,
1767 .std_body = CC_STD_OSCCA,
1768 },
1769 {
1770 .mac_name = "xcbc(aes)",
1771 .mac_driver_name = "xcbc-aes-ccree",
1772 .blocksize = AES_BLOCK_SIZE,
1773 .is_mac = true,
1774 .template_ahash = {
1775 .init = cc_hash_init,
1776 .update = cc_mac_update,
1777 .final = cc_mac_final,
1778 .finup = cc_mac_finup,
1779 .digest = cc_mac_digest,
1780 .setkey = cc_xcbc_setkey,
1781 .export = cc_hash_export,
1782 .import = cc_hash_import,
1783 .halg = {
1784 .digestsize = AES_BLOCK_SIZE,
1785 .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE),
1786 },
1787 },
1788 .hash_mode = DRV_HASH_NULL,
1789 .hw_mode = DRV_CIPHER_XCBC_MAC,
1790 .inter_digestsize = AES_BLOCK_SIZE,
1791 .min_hw_rev = CC_HW_REV_630,
1792 .std_body = CC_STD_NIST,
1793 },
1794 {
1795 .mac_name = "cmac(aes)",
1796 .mac_driver_name = "cmac-aes-ccree",
1797 .blocksize = AES_BLOCK_SIZE,
1798 .is_mac = true,
1799 .template_ahash = {
1800 .init = cc_hash_init,
1801 .update = cc_mac_update,
1802 .final = cc_mac_final,
1803 .finup = cc_mac_finup,
1804 .digest = cc_mac_digest,
1805 .setkey = cc_cmac_setkey,
1806 .export = cc_hash_export,
1807 .import = cc_hash_import,
1808 .halg = {
1809 .digestsize = AES_BLOCK_SIZE,
1810 .statesize = CC_STATE_SIZE(AES_BLOCK_SIZE),
1811 },
1812 },
1813 .hash_mode = DRV_HASH_NULL,
1814 .hw_mode = DRV_CIPHER_CMAC,
1815 .inter_digestsize = AES_BLOCK_SIZE,
1816 .min_hw_rev = CC_HW_REV_630,
1817 .std_body = CC_STD_NIST,
1818 },
1819 };
1820
cc_alloc_hash_alg(struct cc_hash_template * template,struct device * dev,bool keyed)1821 static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template,
1822 struct device *dev, bool keyed)
1823 {
1824 struct cc_hash_alg *t_crypto_alg;
1825 struct crypto_alg *alg;
1826 struct ahash_alg *halg;
1827
1828 t_crypto_alg = devm_kzalloc(dev, sizeof(*t_crypto_alg), GFP_KERNEL);
1829 if (!t_crypto_alg)
1830 return ERR_PTR(-ENOMEM);
1831
1832 t_crypto_alg->ahash_alg = template->template_ahash;
1833 halg = &t_crypto_alg->ahash_alg;
1834 alg = &halg->halg.base;
1835
1836 if (keyed) {
1837 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1838 template->mac_name);
1839 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1840 template->mac_driver_name);
1841 } else {
1842 halg->setkey = NULL;
1843 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1844 template->name);
1845 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1846 template->driver_name);
1847 }
1848 alg->cra_module = THIS_MODULE;
1849 alg->cra_ctxsize = sizeof(struct cc_hash_ctx) + crypto_dma_padding();
1850 alg->cra_priority = CC_CRA_PRIO;
1851 alg->cra_blocksize = template->blocksize;
1852 alg->cra_alignmask = 0;
1853 alg->cra_exit = cc_cra_exit;
1854
1855 alg->cra_init = cc_cra_init;
1856 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
1857
1858 t_crypto_alg->hash_mode = template->hash_mode;
1859 t_crypto_alg->hw_mode = template->hw_mode;
1860 t_crypto_alg->inter_digestsize = template->inter_digestsize;
1861
1862 return t_crypto_alg;
1863 }
1864
cc_init_copy_sram(struct cc_drvdata * drvdata,const u32 * data,unsigned int size,u32 * sram_buff_ofs)1865 static int cc_init_copy_sram(struct cc_drvdata *drvdata, const u32 *data,
1866 unsigned int size, u32 *sram_buff_ofs)
1867 {
1868 struct cc_hw_desc larval_seq[CC_DIGEST_SIZE_MAX / sizeof(u32)];
1869 unsigned int larval_seq_len = 0;
1870 int rc;
1871
1872 cc_set_sram_desc(data, *sram_buff_ofs, size / sizeof(*data),
1873 larval_seq, &larval_seq_len);
1874 rc = send_request_init(drvdata, larval_seq, larval_seq_len);
1875 if (rc)
1876 return rc;
1877
1878 *sram_buff_ofs += size;
1879 return 0;
1880 }
1881
cc_init_hash_sram(struct cc_drvdata * drvdata)1882 int cc_init_hash_sram(struct cc_drvdata *drvdata)
1883 {
1884 struct cc_hash_handle *hash_handle = drvdata->hash_handle;
1885 u32 sram_buff_ofs = hash_handle->digest_len_sram_addr;
1886 bool large_sha_supported = (drvdata->hw_rev >= CC_HW_REV_712);
1887 bool sm3_supported = (drvdata->hw_rev >= CC_HW_REV_713);
1888 int rc = 0;
1889
1890 /* Copy-to-sram digest-len */
1891 rc = cc_init_copy_sram(drvdata, cc_digest_len_init,
1892 sizeof(cc_digest_len_init), &sram_buff_ofs);
1893 if (rc)
1894 goto init_digest_const_err;
1895
1896 if (large_sha_supported) {
1897 /* Copy-to-sram digest-len for sha384/512 */
1898 rc = cc_init_copy_sram(drvdata, cc_digest_len_sha512_init,
1899 sizeof(cc_digest_len_sha512_init),
1900 &sram_buff_ofs);
1901 if (rc)
1902 goto init_digest_const_err;
1903 }
1904
1905 /* The initial digests offset */
1906 hash_handle->larval_digest_sram_addr = sram_buff_ofs;
1907
1908 /* Copy-to-sram initial SHA* digests */
1909 rc = cc_init_copy_sram(drvdata, cc_md5_init, sizeof(cc_md5_init),
1910 &sram_buff_ofs);
1911 if (rc)
1912 goto init_digest_const_err;
1913
1914 rc = cc_init_copy_sram(drvdata, cc_sha1_init, sizeof(cc_sha1_init),
1915 &sram_buff_ofs);
1916 if (rc)
1917 goto init_digest_const_err;
1918
1919 rc = cc_init_copy_sram(drvdata, cc_sha224_init, sizeof(cc_sha224_init),
1920 &sram_buff_ofs);
1921 if (rc)
1922 goto init_digest_const_err;
1923
1924 rc = cc_init_copy_sram(drvdata, cc_sha256_init, sizeof(cc_sha256_init),
1925 &sram_buff_ofs);
1926 if (rc)
1927 goto init_digest_const_err;
1928
1929 if (sm3_supported) {
1930 rc = cc_init_copy_sram(drvdata, cc_sm3_init,
1931 sizeof(cc_sm3_init), &sram_buff_ofs);
1932 if (rc)
1933 goto init_digest_const_err;
1934 }
1935
1936 if (large_sha_supported) {
1937 rc = cc_init_copy_sram(drvdata, cc_sha384_init,
1938 sizeof(cc_sha384_init), &sram_buff_ofs);
1939 if (rc)
1940 goto init_digest_const_err;
1941
1942 rc = cc_init_copy_sram(drvdata, cc_sha512_init,
1943 sizeof(cc_sha512_init), &sram_buff_ofs);
1944 if (rc)
1945 goto init_digest_const_err;
1946 }
1947
1948 init_digest_const_err:
1949 return rc;
1950 }
1951
cc_hash_alloc(struct cc_drvdata * drvdata)1952 int cc_hash_alloc(struct cc_drvdata *drvdata)
1953 {
1954 struct cc_hash_handle *hash_handle;
1955 u32 sram_buff;
1956 u32 sram_size_to_alloc;
1957 struct device *dev = drvdata_to_dev(drvdata);
1958 int rc = 0;
1959 int alg;
1960
1961 hash_handle = devm_kzalloc(dev, sizeof(*hash_handle), GFP_KERNEL);
1962 if (!hash_handle)
1963 return -ENOMEM;
1964
1965 INIT_LIST_HEAD(&hash_handle->hash_list);
1966 drvdata->hash_handle = hash_handle;
1967
1968 sram_size_to_alloc = sizeof(cc_digest_len_init) +
1969 sizeof(cc_md5_init) +
1970 sizeof(cc_sha1_init) +
1971 sizeof(cc_sha224_init) +
1972 sizeof(cc_sha256_init);
1973
1974 if (drvdata->hw_rev >= CC_HW_REV_713)
1975 sram_size_to_alloc += sizeof(cc_sm3_init);
1976
1977 if (drvdata->hw_rev >= CC_HW_REV_712)
1978 sram_size_to_alloc += sizeof(cc_digest_len_sha512_init) +
1979 sizeof(cc_sha384_init) + sizeof(cc_sha512_init);
1980
1981 sram_buff = cc_sram_alloc(drvdata, sram_size_to_alloc);
1982 if (sram_buff == NULL_SRAM_ADDR) {
1983 rc = -ENOMEM;
1984 goto fail;
1985 }
1986
1987 /* The initial digest-len offset */
1988 hash_handle->digest_len_sram_addr = sram_buff;
1989
1990 /*must be set before the alg registration as it is being used there*/
1991 rc = cc_init_hash_sram(drvdata);
1992 if (rc) {
1993 dev_err(dev, "Init digest CONST failed (rc=%d)\n", rc);
1994 goto fail;
1995 }
1996
1997 /* ahash registration */
1998 for (alg = 0; alg < ARRAY_SIZE(driver_hash); alg++) {
1999 struct cc_hash_alg *t_alg;
2000 int hw_mode = driver_hash[alg].hw_mode;
2001
2002 /* Check that the HW revision and variants are suitable */
2003 if ((driver_hash[alg].min_hw_rev > drvdata->hw_rev) ||
2004 !(drvdata->std_bodies & driver_hash[alg].std_body))
2005 continue;
2006
2007 if (driver_hash[alg].is_mac) {
2008 /* register hmac version */
2009 t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, true);
2010 if (IS_ERR(t_alg)) {
2011 rc = PTR_ERR(t_alg);
2012 dev_err(dev, "%s alg allocation failed\n",
2013 driver_hash[alg].driver_name);
2014 goto fail;
2015 }
2016 t_alg->drvdata = drvdata;
2017
2018 rc = crypto_register_ahash(&t_alg->ahash_alg);
2019 if (rc) {
2020 dev_err(dev, "%s alg registration failed\n",
2021 driver_hash[alg].driver_name);
2022 goto fail;
2023 }
2024
2025 list_add_tail(&t_alg->entry, &hash_handle->hash_list);
2026 }
2027 if (hw_mode == DRV_CIPHER_XCBC_MAC ||
2028 hw_mode == DRV_CIPHER_CMAC)
2029 continue;
2030
2031 /* register hash version */
2032 t_alg = cc_alloc_hash_alg(&driver_hash[alg], dev, false);
2033 if (IS_ERR(t_alg)) {
2034 rc = PTR_ERR(t_alg);
2035 dev_err(dev, "%s alg allocation failed\n",
2036 driver_hash[alg].driver_name);
2037 goto fail;
2038 }
2039 t_alg->drvdata = drvdata;
2040
2041 rc = crypto_register_ahash(&t_alg->ahash_alg);
2042 if (rc) {
2043 dev_err(dev, "%s alg registration failed\n",
2044 driver_hash[alg].driver_name);
2045 goto fail;
2046 }
2047
2048 list_add_tail(&t_alg->entry, &hash_handle->hash_list);
2049 }
2050
2051 return 0;
2052
2053 fail:
2054 cc_hash_free(drvdata);
2055 return rc;
2056 }
2057
cc_hash_free(struct cc_drvdata * drvdata)2058 int cc_hash_free(struct cc_drvdata *drvdata)
2059 {
2060 struct cc_hash_alg *t_hash_alg, *hash_n;
2061 struct cc_hash_handle *hash_handle = drvdata->hash_handle;
2062
2063 list_for_each_entry_safe(t_hash_alg, hash_n, &hash_handle->hash_list,
2064 entry) {
2065 crypto_unregister_ahash(&t_hash_alg->ahash_alg);
2066 list_del(&t_hash_alg->entry);
2067 }
2068
2069 return 0;
2070 }
2071
cc_setup_xcbc(struct ahash_request * areq,struct cc_hw_desc desc[],unsigned int * seq_size)2072 static void cc_setup_xcbc(struct ahash_request *areq, struct cc_hw_desc desc[],
2073 unsigned int *seq_size)
2074 {
2075 unsigned int idx = *seq_size;
2076 struct ahash_req_ctx *state = ahash_request_ctx_dma(areq);
2077 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2078 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
2079
2080 /* Setup XCBC MAC K1 */
2081 hw_desc_init(&desc[idx]);
2082 set_din_type(&desc[idx], DMA_DLLI, (ctx->opad_tmp_keys_dma_addr +
2083 XCBC_MAC_K1_OFFSET),
2084 CC_AES_128_BIT_KEY_SIZE, NS_BIT);
2085 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
2086 set_hash_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC, ctx->hash_mode);
2087 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2088 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2089 set_flow_mode(&desc[idx], S_DIN_to_AES);
2090 idx++;
2091
2092 /* Setup XCBC MAC K2 */
2093 hw_desc_init(&desc[idx]);
2094 set_din_type(&desc[idx], DMA_DLLI,
2095 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K2_OFFSET),
2096 CC_AES_128_BIT_KEY_SIZE, NS_BIT);
2097 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
2098 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
2099 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2100 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2101 set_flow_mode(&desc[idx], S_DIN_to_AES);
2102 idx++;
2103
2104 /* Setup XCBC MAC K3 */
2105 hw_desc_init(&desc[idx]);
2106 set_din_type(&desc[idx], DMA_DLLI,
2107 (ctx->opad_tmp_keys_dma_addr + XCBC_MAC_K3_OFFSET),
2108 CC_AES_128_BIT_KEY_SIZE, NS_BIT);
2109 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);
2110 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
2111 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2112 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2113 set_flow_mode(&desc[idx], S_DIN_to_AES);
2114 idx++;
2115
2116 /* Loading MAC state */
2117 hw_desc_init(&desc[idx]);
2118 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
2119 CC_AES_BLOCK_SIZE, NS_BIT);
2120 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
2121 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
2122 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2123 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
2124 set_flow_mode(&desc[idx], S_DIN_to_AES);
2125 idx++;
2126 *seq_size = idx;
2127 }
2128
cc_setup_cmac(struct ahash_request * areq,struct cc_hw_desc desc[],unsigned int * seq_size)2129 static void cc_setup_cmac(struct ahash_request *areq, struct cc_hw_desc desc[],
2130 unsigned int *seq_size)
2131 {
2132 unsigned int idx = *seq_size;
2133 struct ahash_req_ctx *state = ahash_request_ctx_dma(areq);
2134 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2135 struct cc_hash_ctx *ctx = crypto_ahash_ctx_dma(tfm);
2136
2137 /* Setup CMAC Key */
2138 hw_desc_init(&desc[idx]);
2139 set_din_type(&desc[idx], DMA_DLLI, ctx->opad_tmp_keys_dma_addr,
2140 ((ctx->key_params.keylen == 24) ? AES_MAX_KEY_SIZE :
2141 ctx->key_params.keylen), NS_BIT);
2142 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
2143 set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);
2144 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2145 set_key_size_aes(&desc[idx], ctx->key_params.keylen);
2146 set_flow_mode(&desc[idx], S_DIN_to_AES);
2147 idx++;
2148
2149 /* Load MAC state */
2150 hw_desc_init(&desc[idx]);
2151 set_din_type(&desc[idx], DMA_DLLI, state->digest_buff_dma_addr,
2152 CC_AES_BLOCK_SIZE, NS_BIT);
2153 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
2154 set_cipher_mode(&desc[idx], DRV_CIPHER_CMAC);
2155 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
2156 set_key_size_aes(&desc[idx], ctx->key_params.keylen);
2157 set_flow_mode(&desc[idx], S_DIN_to_AES);
2158 idx++;
2159 *seq_size = idx;
2160 }
2161
cc_set_desc(struct ahash_req_ctx * areq_ctx,struct cc_hash_ctx * ctx,unsigned int flow_mode,struct cc_hw_desc desc[],bool is_not_last_data,unsigned int * seq_size)2162 static void cc_set_desc(struct ahash_req_ctx *areq_ctx,
2163 struct cc_hash_ctx *ctx, unsigned int flow_mode,
2164 struct cc_hw_desc desc[], bool is_not_last_data,
2165 unsigned int *seq_size)
2166 {
2167 unsigned int idx = *seq_size;
2168 struct device *dev = drvdata_to_dev(ctx->drvdata);
2169
2170 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_DLLI) {
2171 hw_desc_init(&desc[idx]);
2172 set_din_type(&desc[idx], DMA_DLLI,
2173 sg_dma_address(areq_ctx->curr_sg),
2174 areq_ctx->curr_sg->length, NS_BIT);
2175 set_flow_mode(&desc[idx], flow_mode);
2176 idx++;
2177 } else {
2178 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) {
2179 dev_dbg(dev, " NULL mode\n");
2180 /* nothing to build */
2181 return;
2182 }
2183 /* bypass */
2184 hw_desc_init(&desc[idx]);
2185 set_din_type(&desc[idx], DMA_DLLI,
2186 areq_ctx->mlli_params.mlli_dma_addr,
2187 areq_ctx->mlli_params.mlli_len, NS_BIT);
2188 set_dout_sram(&desc[idx], ctx->drvdata->mlli_sram_addr,
2189 areq_ctx->mlli_params.mlli_len);
2190 set_flow_mode(&desc[idx], BYPASS);
2191 idx++;
2192 /* process */
2193 hw_desc_init(&desc[idx]);
2194 set_din_type(&desc[idx], DMA_MLLI,
2195 ctx->drvdata->mlli_sram_addr,
2196 areq_ctx->mlli_nents, NS_BIT);
2197 set_flow_mode(&desc[idx], flow_mode);
2198 idx++;
2199 }
2200 if (is_not_last_data)
2201 set_din_not_last_indication(&desc[(idx - 1)]);
2202 /* return updated desc sequence size */
2203 *seq_size = idx;
2204 }
2205
cc_larval_digest(struct device * dev,u32 mode)2206 static const void *cc_larval_digest(struct device *dev, u32 mode)
2207 {
2208 switch (mode) {
2209 case DRV_HASH_MD5:
2210 return cc_md5_init;
2211 case DRV_HASH_SHA1:
2212 return cc_sha1_init;
2213 case DRV_HASH_SHA224:
2214 return cc_sha224_init;
2215 case DRV_HASH_SHA256:
2216 return cc_sha256_init;
2217 case DRV_HASH_SHA384:
2218 return cc_sha384_init;
2219 case DRV_HASH_SHA512:
2220 return cc_sha512_init;
2221 case DRV_HASH_SM3:
2222 return cc_sm3_init;
2223 default:
2224 dev_err(dev, "Invalid hash mode (%d)\n", mode);
2225 return cc_md5_init;
2226 }
2227 }
2228
2229 /**
2230 * cc_larval_digest_addr() - Get the address of the initial digest in SRAM
2231 * according to the given hash mode
2232 *
2233 * @drvdata: Associated device driver context
2234 * @mode: The Hash mode. Supported modes: MD5/SHA1/SHA224/SHA256
2235 *
2236 * Return:
2237 * The address of the initial digest in SRAM
2238 */
cc_larval_digest_addr(void * drvdata,u32 mode)2239 u32 cc_larval_digest_addr(void *drvdata, u32 mode)
2240 {
2241 struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata;
2242 struct cc_hash_handle *hash_handle = _drvdata->hash_handle;
2243 struct device *dev = drvdata_to_dev(_drvdata);
2244 bool sm3_supported = (_drvdata->hw_rev >= CC_HW_REV_713);
2245 u32 addr;
2246
2247 switch (mode) {
2248 case DRV_HASH_NULL:
2249 break; /*Ignore*/
2250 case DRV_HASH_MD5:
2251 return (hash_handle->larval_digest_sram_addr);
2252 case DRV_HASH_SHA1:
2253 return (hash_handle->larval_digest_sram_addr +
2254 sizeof(cc_md5_init));
2255 case DRV_HASH_SHA224:
2256 return (hash_handle->larval_digest_sram_addr +
2257 sizeof(cc_md5_init) +
2258 sizeof(cc_sha1_init));
2259 case DRV_HASH_SHA256:
2260 return (hash_handle->larval_digest_sram_addr +
2261 sizeof(cc_md5_init) +
2262 sizeof(cc_sha1_init) +
2263 sizeof(cc_sha224_init));
2264 case DRV_HASH_SM3:
2265 return (hash_handle->larval_digest_sram_addr +
2266 sizeof(cc_md5_init) +
2267 sizeof(cc_sha1_init) +
2268 sizeof(cc_sha224_init) +
2269 sizeof(cc_sha256_init));
2270 case DRV_HASH_SHA384:
2271 addr = (hash_handle->larval_digest_sram_addr +
2272 sizeof(cc_md5_init) +
2273 sizeof(cc_sha1_init) +
2274 sizeof(cc_sha224_init) +
2275 sizeof(cc_sha256_init));
2276 if (sm3_supported)
2277 addr += sizeof(cc_sm3_init);
2278 return addr;
2279 case DRV_HASH_SHA512:
2280 addr = (hash_handle->larval_digest_sram_addr +
2281 sizeof(cc_md5_init) +
2282 sizeof(cc_sha1_init) +
2283 sizeof(cc_sha224_init) +
2284 sizeof(cc_sha256_init) +
2285 sizeof(cc_sha384_init));
2286 if (sm3_supported)
2287 addr += sizeof(cc_sm3_init);
2288 return addr;
2289 default:
2290 dev_err(dev, "Invalid hash mode (%d)\n", mode);
2291 }
2292
2293 /*This is valid wrong value to avoid kernel crash*/
2294 return hash_handle->larval_digest_sram_addr;
2295 }
2296
cc_digest_len_addr(void * drvdata,u32 mode)2297 u32 cc_digest_len_addr(void *drvdata, u32 mode)
2298 {
2299 struct cc_drvdata *_drvdata = (struct cc_drvdata *)drvdata;
2300 struct cc_hash_handle *hash_handle = _drvdata->hash_handle;
2301 u32 digest_len_addr = hash_handle->digest_len_sram_addr;
2302
2303 switch (mode) {
2304 case DRV_HASH_SHA1:
2305 case DRV_HASH_SHA224:
2306 case DRV_HASH_SHA256:
2307 case DRV_HASH_MD5:
2308 return digest_len_addr;
2309 case DRV_HASH_SHA384:
2310 case DRV_HASH_SHA512:
2311 return digest_len_addr + sizeof(cc_digest_len_init);
2312 default:
2313 return digest_len_addr; /*to avoid kernel crash*/
2314 }
2315 }
2316