1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Xilinx ZynqMP AES Driver.
4 * Copyright (C) 2020-2022 Xilinx Inc.
5 * Copyright (C) 2022-2025 Advanced Micro Devices, Inc.
6 */
7
8 #include <crypto/aes.h>
9 #include <crypto/engine.h>
10 #include <crypto/gcm.h>
11 #include <crypto/internal/aead.h>
12 #include <crypto/scatterwalk.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/err.h>
15 #include <linux/firmware/xlnx-zynqmp.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/platform_device.h>
19 #include <linux/string.h>
20
21 #define ZYNQMP_DMA_BIT_MASK 32U
22 #define VERSAL_DMA_BIT_MASK 64U
23 #define XILINX_AES_AUTH_SIZE 16U
24 #define XILINX_AES_BLK_SIZE 1U
25 #define ZYNQMP_AES_MIN_INPUT_BLK_SIZE 4U
26 #define ZYNQMP_AES_WORD_LEN 4U
27
28 #define VERSAL_AES_QWORD_LEN 16U
29 #define ZYNQMP_AES_GCM_TAG_MISMATCH_ERR 0x01
30 #define ZYNQMP_AES_WRONG_KEY_SRC_ERR 0x13
31 #define ZYNQMP_AES_PUF_NOT_PROGRAMMED 0xE300
32 #define XILINX_KEY_MAGIC 0x3EA0
33
34 enum xilinx_aead_op {
35 XILINX_AES_DECRYPT = 0,
36 XILINX_AES_ENCRYPT
37 };
38
39 enum zynqmp_aead_keysrc {
40 ZYNQMP_AES_KUP_KEY = 0,
41 ZYNQMP_AES_DEV_KEY,
42 ZYNQMP_AES_PUF_KEY
43 };
44
45 struct xilinx_aead_dev {
46 struct device *dev;
47 struct crypto_engine *engine;
48 struct xilinx_aead_alg *aead_algs;
49 };
50
51 struct xilinx_aead_alg {
52 struct xilinx_aead_dev *aead_dev;
53 struct aead_engine_alg aead;
54 int (*aes_aead_cipher)(struct aead_request *areq);
55 u8 dma_bit_mask;
56 };
57
58 struct xilinx_hwkey_info {
59 u16 magic;
60 u16 type;
61 } __packed;
62
63 struct zynqmp_aead_hw_req {
64 u64 src;
65 u64 iv;
66 u64 key;
67 u64 dst;
68 u64 size;
69 u64 op;
70 u64 keysrc;
71 };
72
73 struct xilinx_aead_tfm_ctx {
74 struct device *dev;
75 dma_addr_t key_dma_addr;
76 u8 *key;
77 u32 keylen;
78 u32 authsize;
79 u8 keysrc;
80 struct crypto_aead *fbk_cipher;
81 };
82
83 struct xilinx_aead_req_ctx {
84 enum xilinx_aead_op op;
85 };
86
87 static struct xilinx_aead_dev *aead_dev;
88
89 enum versal_aead_keysrc {
90 VERSAL_AES_BBRAM_KEY = 0,
91 VERSAL_AES_BBRAM_RED_KEY,
92 VERSAL_AES_BH_KEY,
93 VERSAL_AES_BH_RED_KEY,
94 VERSAL_AES_EFUSE_KEY,
95 VERSAL_AES_EFUSE_RED_KEY,
96 VERSAL_AES_EFUSE_USER_KEY_0,
97 VERSAL_AES_EFUSE_USER_KEY_1,
98 VERSAL_AES_EFUSE_USER_RED_KEY_0,
99 VERSAL_AES_EFUSE_USER_RED_KEY_1,
100 VERSAL_AES_KUP_KEY,
101 VERSAL_AES_PUF_KEY,
102 VERSAL_AES_USER_KEY_0,
103 VERSAL_AES_USER_KEY_1,
104 VERSAL_AES_USER_KEY_2,
105 VERSAL_AES_USER_KEY_3,
106 VERSAL_AES_USER_KEY_4,
107 VERSAL_AES_USER_KEY_5,
108 VERSAL_AES_USER_KEY_6,
109 VERSAL_AES_USER_KEY_7,
110 VERSAL_AES_EXPANDED_KEYS,
111 VERSAL_AES_ALL_KEYS,
112 };
113
114 enum versal_aead_op {
115 VERSAL_AES_ENCRYPT = 0,
116 VERSAL_AES_DECRYPT
117 };
118
119 enum versal_aes_keysize {
120 HW_AES_KEY_SIZE_128 = 0,
121 HW_AES_KEY_SIZE_256 = 2,
122 };
123
124 struct versal_init_ops {
125 u64 iv;
126 u32 op;
127 u32 keysrc;
128 u32 size;
129 };
130
131 struct versal_in_params {
132 u64 in_data_addr;
133 u32 size;
134 u32 is_last;
135 };
136
zynqmp_aes_aead_cipher(struct aead_request * req)137 static int zynqmp_aes_aead_cipher(struct aead_request *req)
138 {
139 struct crypto_aead *aead = crypto_aead_reqtfm(req);
140 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);
141 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req);
142 dma_addr_t dma_addr_data, dma_addr_hw_req;
143 struct device *dev = tfm_ctx->dev;
144 struct zynqmp_aead_hw_req *hwreq;
145 unsigned int data_size;
146 unsigned int status;
147 int ret;
148 size_t dma_size;
149 void *dmabuf;
150 char *kbuf;
151
152 dma_size = req->cryptlen + XILINX_AES_AUTH_SIZE;
153 kbuf = kmalloc(dma_size, GFP_KERNEL);
154 if (!kbuf)
155 return -ENOMEM;
156
157 dmabuf = kmalloc(sizeof(*hwreq) + GCM_AES_IV_SIZE, GFP_KERNEL);
158 if (!dmabuf) {
159 kfree(kbuf);
160 return -ENOMEM;
161 }
162 hwreq = dmabuf;
163 data_size = req->cryptlen;
164 scatterwalk_map_and_copy(kbuf, req->src, 0, req->cryptlen, 0);
165 memcpy(dmabuf + sizeof(struct zynqmp_aead_hw_req), req->iv, GCM_AES_IV_SIZE);
166 dma_addr_data = dma_map_single(dev, kbuf, dma_size, DMA_BIDIRECTIONAL);
167 if (unlikely(dma_mapping_error(dev, dma_addr_data))) {
168 ret = -ENOMEM;
169 goto freemem;
170 }
171
172 hwreq->src = dma_addr_data;
173 hwreq->dst = dma_addr_data;
174 hwreq->keysrc = tfm_ctx->keysrc;
175 hwreq->op = rq_ctx->op;
176
177 if (hwreq->op == XILINX_AES_ENCRYPT)
178 hwreq->size = data_size;
179 else
180 hwreq->size = data_size - XILINX_AES_AUTH_SIZE;
181
182 if (hwreq->keysrc == ZYNQMP_AES_KUP_KEY)
183 hwreq->key = tfm_ctx->key_dma_addr;
184 else
185 hwreq->key = 0;
186
187 dma_addr_hw_req = dma_map_single(dev, dmabuf, sizeof(struct zynqmp_aead_hw_req) +
188 GCM_AES_IV_SIZE,
189 DMA_TO_DEVICE);
190 if (unlikely(dma_mapping_error(dev, dma_addr_hw_req))) {
191 ret = -ENOMEM;
192 dma_unmap_single(dev, dma_addr_data, dma_size, DMA_BIDIRECTIONAL);
193 goto freemem;
194 }
195 hwreq->iv = dma_addr_hw_req + sizeof(struct zynqmp_aead_hw_req);
196 dma_sync_single_for_device(dev, dma_addr_hw_req, sizeof(struct zynqmp_aead_hw_req) +
197 GCM_AES_IV_SIZE, DMA_TO_DEVICE);
198 ret = zynqmp_pm_aes_engine(dma_addr_hw_req, &status);
199 dma_unmap_single(dev, dma_addr_hw_req, sizeof(struct zynqmp_aead_hw_req) + GCM_AES_IV_SIZE,
200 DMA_TO_DEVICE);
201 dma_unmap_single(dev, dma_addr_data, dma_size, DMA_BIDIRECTIONAL);
202 if (ret) {
203 dev_err(dev, "ERROR: AES PM API failed\n");
204 } else if (status) {
205 switch (status) {
206 case ZYNQMP_AES_GCM_TAG_MISMATCH_ERR:
207 ret = -EBADMSG;
208 break;
209 case ZYNQMP_AES_WRONG_KEY_SRC_ERR:
210 ret = -EINVAL;
211 dev_err(dev, "ERROR: Wrong KeySrc, enable secure mode\n");
212 break;
213 case ZYNQMP_AES_PUF_NOT_PROGRAMMED:
214 ret = -EINVAL;
215 dev_err(dev, "ERROR: PUF is not registered\n");
216 break;
217 default:
218 ret = -EINVAL;
219 break;
220 }
221 } else {
222 if (hwreq->op == XILINX_AES_ENCRYPT)
223 data_size = data_size + crypto_aead_authsize(aead);
224 else
225 data_size = data_size - XILINX_AES_AUTH_SIZE;
226
227 sg_copy_from_buffer(req->dst, sg_nents(req->dst),
228 kbuf, data_size);
229 ret = 0;
230 }
231
232 freemem:
233 memzero_explicit(kbuf, dma_size);
234 kfree(kbuf);
235 memzero_explicit(dmabuf, sizeof(struct zynqmp_aead_hw_req) + GCM_AES_IV_SIZE);
236 kfree(dmabuf);
237
238 return ret;
239 }
240
versal_aes_aead_cipher(struct aead_request * req)241 static int versal_aes_aead_cipher(struct aead_request *req)
242 {
243 struct crypto_aead *aead = crypto_aead_reqtfm(req);
244 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);
245 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req);
246 dma_addr_t dma_addr_data, dma_addr_hw_req, dma_addr_in;
247 u32 total_len = req->assoclen + req->cryptlen;
248 struct device *dev = tfm_ctx->dev;
249 struct versal_init_ops *hwreq;
250 struct versal_in_params *in;
251 u32 gcm_offset, out_len;
252 size_t dmabuf_size;
253 size_t kbuf_size;
254 void *dmabuf;
255 char *kbuf;
256 int ret;
257
258 kbuf_size = total_len + XILINX_AES_AUTH_SIZE;
259 kbuf = kmalloc(kbuf_size, GFP_KERNEL);
260 if (unlikely(!kbuf)) {
261 ret = -ENOMEM;
262 goto err;
263 }
264 dmabuf_size = sizeof(struct versal_init_ops) +
265 sizeof(struct versal_in_params) +
266 GCM_AES_IV_SIZE;
267 dmabuf = kmalloc(dmabuf_size, GFP_KERNEL);
268 if (unlikely(!dmabuf)) {
269 ret = -ENOMEM;
270 goto buf1_free;
271 }
272
273 dma_addr_hw_req = dma_map_single(dev, dmabuf, dmabuf_size, DMA_BIDIRECTIONAL);
274 if (unlikely(dma_mapping_error(dev, dma_addr_hw_req))) {
275 ret = -ENOMEM;
276 goto buf2_free;
277 }
278 scatterwalk_map_and_copy(kbuf, req->src, 0, total_len, 0);
279 dma_addr_data = dma_map_single(dev, kbuf, kbuf_size, DMA_BIDIRECTIONAL);
280 if (unlikely(dma_mapping_error(dev, dma_addr_data))) {
281 dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL);
282 ret = -ENOMEM;
283 goto buf2_free;
284 }
285 hwreq = dmabuf;
286 in = dmabuf + sizeof(struct versal_init_ops);
287 memcpy(dmabuf + sizeof(struct versal_init_ops) +
288 sizeof(struct versal_in_params), req->iv, GCM_AES_IV_SIZE);
289 hwreq->iv = dma_addr_hw_req + sizeof(struct versal_init_ops) +
290 sizeof(struct versal_in_params);
291 hwreq->keysrc = tfm_ctx->keysrc;
292 dma_addr_in = dma_addr_hw_req + sizeof(struct versal_init_ops);
293 if (rq_ctx->op == XILINX_AES_ENCRYPT) {
294 hwreq->op = VERSAL_AES_ENCRYPT;
295 out_len = total_len + crypto_aead_authsize(aead);
296 in->size = req->cryptlen;
297 } else {
298 hwreq->op = VERSAL_AES_DECRYPT;
299 out_len = total_len - XILINX_AES_AUTH_SIZE;
300 in->size = req->cryptlen - XILINX_AES_AUTH_SIZE;
301 }
302
303 if (tfm_ctx->keylen == AES_KEYSIZE_128)
304 hwreq->size = HW_AES_KEY_SIZE_128;
305 else
306 hwreq->size = HW_AES_KEY_SIZE_256;
307
308 /* Request aes key write for volatile user keys */
309 if (hwreq->keysrc >= VERSAL_AES_USER_KEY_0 && hwreq->keysrc <= VERSAL_AES_USER_KEY_7) {
310 ret = versal_pm_aes_key_write(hwreq->size, hwreq->keysrc,
311 tfm_ctx->key_dma_addr);
312 if (ret)
313 goto unmap;
314 }
315
316 in->in_data_addr = dma_addr_data + req->assoclen;
317 in->is_last = 1;
318 gcm_offset = req->assoclen + in->size;
319 dma_sync_single_for_device(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL);
320 ret = versal_pm_aes_op_init(dma_addr_hw_req);
321 if (ret)
322 goto clearkey;
323
324 if (req->assoclen > 0) {
325 /* Currently GMAC is OFF by default */
326 ret = versal_pm_aes_update_aad(dma_addr_data, req->assoclen);
327 if (ret)
328 goto clearkey;
329 }
330 if (rq_ctx->op == XILINX_AES_ENCRYPT) {
331 ret = versal_pm_aes_enc_update(dma_addr_in,
332 dma_addr_data + req->assoclen);
333 if (ret)
334 goto clearkey;
335
336 ret = versal_pm_aes_enc_final(dma_addr_data + gcm_offset);
337 if (ret)
338 goto clearkey;
339 } else {
340 ret = versal_pm_aes_dec_update(dma_addr_in,
341 dma_addr_data + req->assoclen);
342 if (ret)
343 goto clearkey;
344
345 ret = versal_pm_aes_dec_final(dma_addr_data + gcm_offset);
346 if (ret) {
347 ret = -EBADMSG;
348 goto clearkey;
349 }
350 }
351 dma_unmap_single(dev, dma_addr_data, kbuf_size, DMA_BIDIRECTIONAL);
352 dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL);
353 sg_copy_from_buffer(req->dst, sg_nents(req->dst),
354 kbuf, out_len);
355 dma_addr_data = 0;
356 dma_addr_hw_req = 0;
357
358 clearkey:
359 if (hwreq->keysrc >= VERSAL_AES_USER_KEY_0 && hwreq->keysrc <= VERSAL_AES_USER_KEY_7)
360 versal_pm_aes_key_zero(hwreq->keysrc);
361 unmap:
362 if (unlikely(dma_addr_data))
363 dma_unmap_single(dev, dma_addr_data, kbuf_size, DMA_BIDIRECTIONAL);
364 if (unlikely(dma_addr_hw_req))
365 dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL);
366 buf2_free:
367 memzero_explicit(dmabuf, dmabuf_size);
368 kfree(dmabuf);
369 buf1_free:
370 memzero_explicit(kbuf, kbuf_size);
371 kfree(kbuf);
372 err:
373 return ret;
374 }
375
zynqmp_fallback_check(struct xilinx_aead_tfm_ctx * tfm_ctx,struct aead_request * req)376 static int zynqmp_fallback_check(struct xilinx_aead_tfm_ctx *tfm_ctx,
377 struct aead_request *req)
378 {
379 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req);
380
381 if (tfm_ctx->authsize != XILINX_AES_AUTH_SIZE && rq_ctx->op == XILINX_AES_DECRYPT)
382 return 1;
383
384 if (req->assoclen != 0 ||
385 req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE)
386 return 1;
387 if (tfm_ctx->keylen == AES_KEYSIZE_128 ||
388 tfm_ctx->keylen == AES_KEYSIZE_192)
389 return 1;
390
391 if ((req->cryptlen % ZYNQMP_AES_WORD_LEN) != 0)
392 return 1;
393
394 if (rq_ctx->op == XILINX_AES_DECRYPT &&
395 req->cryptlen <= XILINX_AES_AUTH_SIZE)
396 return 1;
397
398 return 0;
399 }
400
versal_fallback_check(struct xilinx_aead_tfm_ctx * tfm_ctx,struct aead_request * req)401 static int versal_fallback_check(struct xilinx_aead_tfm_ctx *tfm_ctx,
402 struct aead_request *req)
403 {
404 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req);
405
406 if (tfm_ctx->authsize != XILINX_AES_AUTH_SIZE && rq_ctx->op == XILINX_AES_DECRYPT)
407 return 1;
408
409 if (tfm_ctx->keylen == AES_KEYSIZE_192)
410 return 1;
411
412 if (req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE ||
413 req->cryptlen % ZYNQMP_AES_WORD_LEN ||
414 req->assoclen % VERSAL_AES_QWORD_LEN)
415 return 1;
416
417 if (rq_ctx->op == XILINX_AES_DECRYPT &&
418 req->cryptlen <= XILINX_AES_AUTH_SIZE)
419 return 1;
420
421 return 0;
422 }
423
xilinx_handle_aes_req(struct crypto_engine * engine,void * req)424 static int xilinx_handle_aes_req(struct crypto_engine *engine, void *req)
425 {
426 struct aead_request *areq =
427 container_of(req, struct aead_request, base);
428 struct crypto_aead *aead = crypto_aead_reqtfm(req);
429 struct aead_alg *alg = crypto_aead_alg(aead);
430 struct xilinx_aead_alg *drv_ctx;
431 int err;
432
433 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base);
434 err = drv_ctx->aes_aead_cipher(areq);
435 local_bh_disable();
436 crypto_finalize_aead_request(engine, areq, err);
437 local_bh_enable();
438
439 return 0;
440 }
441
zynqmp_aes_aead_setkey(struct crypto_aead * aead,const u8 * key,unsigned int keylen)442 static int zynqmp_aes_aead_setkey(struct crypto_aead *aead, const u8 *key,
443 unsigned int keylen)
444 {
445 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
446 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm);
447 int err;
448
449 if (keylen == AES_KEYSIZE_256) {
450 memcpy(tfm_ctx->key, key, keylen);
451 dma_sync_single_for_device(tfm_ctx->dev, tfm_ctx->key_dma_addr,
452 AES_KEYSIZE_256,
453 DMA_TO_DEVICE);
454 }
455
456 tfm_ctx->fbk_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
457 tfm_ctx->fbk_cipher->base.crt_flags |= (aead->base.crt_flags &
458 CRYPTO_TFM_REQ_MASK);
459
460 err = crypto_aead_setkey(tfm_ctx->fbk_cipher, key, keylen);
461 if (err)
462 goto err;
463 tfm_ctx->keylen = keylen;
464 tfm_ctx->keysrc = ZYNQMP_AES_KUP_KEY;
465 err:
466 return err;
467 }
468
zynqmp_paes_aead_setkey(struct crypto_aead * aead,const u8 * key,unsigned int keylen)469 static int zynqmp_paes_aead_setkey(struct crypto_aead *aead, const u8 *key,
470 unsigned int keylen)
471 {
472 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
473 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm);
474 struct xilinx_hwkey_info hwkey;
475 unsigned char keysrc;
476 int err = -EINVAL;
477
478 if (keylen != sizeof(struct xilinx_hwkey_info))
479 return -EINVAL;
480 memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info));
481 if (hwkey.magic != XILINX_KEY_MAGIC)
482 return -EINVAL;
483 keysrc = hwkey.type;
484 if (keysrc == ZYNQMP_AES_DEV_KEY ||
485 keysrc == ZYNQMP_AES_PUF_KEY) {
486 tfm_ctx->keysrc = keysrc;
487 tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info);
488 err = 0;
489 }
490
491 return err;
492 }
493
versal_aes_aead_setkey(struct crypto_aead * aead,const u8 * key,unsigned int keylen)494 static int versal_aes_aead_setkey(struct crypto_aead *aead, const u8 *key,
495 unsigned int keylen)
496 {
497 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
498 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm);
499 struct xilinx_hwkey_info hwkey;
500 unsigned char keysrc;
501 int err;
502
503 tfm_ctx->keysrc = VERSAL_AES_USER_KEY_0;
504 if (keylen == sizeof(struct xilinx_hwkey_info)) {
505 memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info));
506 if (hwkey.magic != XILINX_KEY_MAGIC)
507 return -EINVAL;
508
509 keysrc = hwkey.type;
510 if (keysrc >= VERSAL_AES_USER_KEY_1 &&
511 keysrc <= VERSAL_AES_USER_KEY_7) {
512 tfm_ctx->keysrc = keysrc;
513 tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info);
514 return 0;
515 }
516 return -EINVAL;
517 }
518
519 if (keylen == AES_KEYSIZE_256 || keylen == AES_KEYSIZE_128) {
520 tfm_ctx->keylen = keylen;
521 memcpy(tfm_ctx->key, key, keylen);
522 dma_sync_single_for_device(tfm_ctx->dev, tfm_ctx->key_dma_addr,
523 AES_KEYSIZE_256,
524 DMA_TO_DEVICE);
525 }
526
527 tfm_ctx->fbk_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
528 tfm_ctx->fbk_cipher->base.crt_flags |= (aead->base.crt_flags &
529 CRYPTO_TFM_REQ_MASK);
530 err = crypto_aead_setkey(tfm_ctx->fbk_cipher, key, keylen);
531 if (!err)
532 tfm_ctx->keylen = keylen;
533
534 return err;
535 }
536
versal_paes_aead_setkey(struct crypto_aead * aead,const u8 * key,unsigned int keylen)537 static int versal_paes_aead_setkey(struct crypto_aead *aead, const u8 *key,
538 unsigned int keylen)
539 {
540 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
541 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm);
542 struct xilinx_hwkey_info hwkey;
543 unsigned char keysrc;
544 int err = 0;
545
546 if (keylen != sizeof(struct xilinx_hwkey_info))
547 return -EINVAL;
548
549 memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info));
550 if (hwkey.magic != XILINX_KEY_MAGIC)
551 return -EINVAL;
552
553 keysrc = hwkey.type;
554
555 switch (keysrc) {
556 case VERSAL_AES_EFUSE_USER_KEY_0:
557 case VERSAL_AES_EFUSE_USER_KEY_1:
558 case VERSAL_AES_EFUSE_USER_RED_KEY_0:
559 case VERSAL_AES_EFUSE_USER_RED_KEY_1:
560 case VERSAL_AES_PUF_KEY:
561 tfm_ctx->keysrc = keysrc;
562 tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info);
563 break;
564 default:
565 err = -EINVAL;
566 break;
567 }
568
569 return err;
570 }
571
xilinx_aes_aead_setauthsize(struct crypto_aead * aead,unsigned int authsize)572 static int xilinx_aes_aead_setauthsize(struct crypto_aead *aead,
573 unsigned int authsize)
574 {
575 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
576 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm);
577
578 tfm_ctx->authsize = authsize;
579 return tfm_ctx->fbk_cipher ? crypto_aead_setauthsize(tfm_ctx->fbk_cipher, authsize) : 0;
580 }
581
xilinx_aes_fallback_crypt(struct aead_request * req,bool encrypt)582 static int xilinx_aes_fallback_crypt(struct aead_request *req, bool encrypt)
583 {
584 struct aead_request *subreq = aead_request_ctx(req);
585 struct crypto_aead *aead = crypto_aead_reqtfm(req);
586 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);
587
588 aead_request_set_tfm(subreq, tfm_ctx->fbk_cipher);
589 aead_request_set_callback(subreq, req->base.flags, NULL, NULL);
590 aead_request_set_crypt(subreq, req->src, req->dst,
591 req->cryptlen, req->iv);
592 aead_request_set_ad(subreq, req->assoclen);
593
594 return encrypt ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq);
595 }
596
zynqmp_aes_aead_encrypt(struct aead_request * req)597 static int zynqmp_aes_aead_encrypt(struct aead_request *req)
598 {
599 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req);
600 struct crypto_aead *aead = crypto_aead_reqtfm(req);
601 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);
602 struct aead_alg *alg = crypto_aead_alg(aead);
603 struct xilinx_aead_alg *drv_ctx;
604 int err;
605
606 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base);
607 if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY &&
608 tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info))
609 return -EINVAL;
610
611 rq_ctx->op = XILINX_AES_ENCRYPT;
612 err = zynqmp_fallback_check(tfm_ctx, req);
613 if (err && tfm_ctx->keysrc != ZYNQMP_AES_KUP_KEY)
614 return -EOPNOTSUPP;
615
616 if (err)
617 return xilinx_aes_fallback_crypt(req, true);
618
619 return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req);
620 }
621
versal_aes_aead_encrypt(struct aead_request * req)622 static int versal_aes_aead_encrypt(struct aead_request *req)
623 {
624 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req);
625 struct crypto_aead *aead = crypto_aead_reqtfm(req);
626 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);
627 struct aead_alg *alg = crypto_aead_alg(aead);
628 struct xilinx_aead_alg *drv_ctx;
629 int err;
630
631 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base);
632 rq_ctx->op = XILINX_AES_ENCRYPT;
633 if (tfm_ctx->keysrc >= VERSAL_AES_USER_KEY_0 &&
634 tfm_ctx->keysrc <= VERSAL_AES_USER_KEY_7 &&
635 tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info))
636 return -EINVAL;
637 err = versal_fallback_check(tfm_ctx, req);
638 if (err && (tfm_ctx->keysrc < VERSAL_AES_USER_KEY_0 ||
639 tfm_ctx->keysrc > VERSAL_AES_USER_KEY_7))
640 return -EOPNOTSUPP;
641 if (err)
642 return xilinx_aes_fallback_crypt(req, true);
643
644 return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req);
645 }
646
zynqmp_aes_aead_decrypt(struct aead_request * req)647 static int zynqmp_aes_aead_decrypt(struct aead_request *req)
648 {
649 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req);
650 struct crypto_aead *aead = crypto_aead_reqtfm(req);
651 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);
652 struct aead_alg *alg = crypto_aead_alg(aead);
653 struct xilinx_aead_alg *drv_ctx;
654 int err;
655
656 rq_ctx->op = XILINX_AES_DECRYPT;
657 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base);
658 if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY &&
659 tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info))
660 return -EINVAL;
661 err = zynqmp_fallback_check(tfm_ctx, req);
662 if (err && tfm_ctx->keysrc != ZYNQMP_AES_KUP_KEY)
663 return -EOPNOTSUPP;
664 if (err)
665 return xilinx_aes_fallback_crypt(req, false);
666
667 return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req);
668 }
669
xilinx_paes_aead_init(struct crypto_aead * aead)670 static int xilinx_paes_aead_init(struct crypto_aead *aead)
671 {
672 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
673 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm);
674 struct xilinx_aead_alg *drv_alg;
675 struct aead_alg *alg = crypto_aead_alg(aead);
676
677 drv_alg = container_of(alg, struct xilinx_aead_alg, aead.base);
678 tfm_ctx->dev = drv_alg->aead_dev->dev;
679 tfm_ctx->keylen = 0;
680 tfm_ctx->key = NULL;
681 tfm_ctx->fbk_cipher = NULL;
682 crypto_aead_set_reqsize(aead, sizeof(struct xilinx_aead_req_ctx));
683
684 return 0;
685 }
686
versal_aes_aead_decrypt(struct aead_request * req)687 static int versal_aes_aead_decrypt(struct aead_request *req)
688 {
689 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req);
690 struct crypto_aead *aead = crypto_aead_reqtfm(req);
691 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead);
692 struct aead_alg *alg = crypto_aead_alg(aead);
693 struct xilinx_aead_alg *drv_ctx;
694 int err;
695
696 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base);
697 rq_ctx->op = XILINX_AES_DECRYPT;
698 if (tfm_ctx->keysrc >= VERSAL_AES_USER_KEY_0 &&
699 tfm_ctx->keysrc <= VERSAL_AES_USER_KEY_7 &&
700 tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info))
701 return -EINVAL;
702
703 err = versal_fallback_check(tfm_ctx, req);
704 if (err &&
705 (tfm_ctx->keysrc < VERSAL_AES_USER_KEY_0 ||
706 tfm_ctx->keysrc > VERSAL_AES_USER_KEY_7))
707 return -EOPNOTSUPP;
708 if (err)
709 return xilinx_aes_fallback_crypt(req, false);
710
711 return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req);
712 }
713
xilinx_aes_aead_init(struct crypto_aead * aead)714 static int xilinx_aes_aead_init(struct crypto_aead *aead)
715 {
716 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
717 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm);
718 struct xilinx_aead_alg *drv_ctx;
719 struct aead_alg *alg = crypto_aead_alg(aead);
720
721 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base);
722 tfm_ctx->dev = drv_ctx->aead_dev->dev;
723 tfm_ctx->keylen = 0;
724
725 tfm_ctx->fbk_cipher = crypto_alloc_aead(drv_ctx->aead.base.base.cra_name,
726 0,
727 CRYPTO_ALG_NEED_FALLBACK);
728
729 if (IS_ERR(tfm_ctx->fbk_cipher)) {
730 dev_err(tfm_ctx->dev, "failed to allocate fallback for %s\n",
731 drv_ctx->aead.base.base.cra_name);
732 return PTR_ERR(tfm_ctx->fbk_cipher);
733 }
734 tfm_ctx->key = kmalloc(AES_KEYSIZE_256, GFP_KERNEL);
735 if (!tfm_ctx->key) {
736 crypto_free_aead(tfm_ctx->fbk_cipher);
737 return -ENOMEM;
738 }
739 tfm_ctx->key_dma_addr = dma_map_single(tfm_ctx->dev, tfm_ctx->key,
740 AES_KEYSIZE_256,
741 DMA_TO_DEVICE);
742 if (unlikely(dma_mapping_error(tfm_ctx->dev, tfm_ctx->key_dma_addr))) {
743 kfree(tfm_ctx->key);
744 crypto_free_aead(tfm_ctx->fbk_cipher);
745 tfm_ctx->fbk_cipher = NULL;
746 return -ENOMEM;
747 }
748 crypto_aead_set_reqsize(aead,
749 max(sizeof(struct xilinx_aead_req_ctx),
750 sizeof(struct aead_request) +
751 crypto_aead_reqsize(tfm_ctx->fbk_cipher)));
752 return 0;
753 }
754
xilinx_paes_aead_exit(struct crypto_aead * aead)755 static void xilinx_paes_aead_exit(struct crypto_aead *aead)
756 {
757 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
758 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm);
759
760 memzero_explicit(tfm_ctx, sizeof(struct xilinx_aead_tfm_ctx));
761 }
762
xilinx_aes_aead_exit(struct crypto_aead * aead)763 static void xilinx_aes_aead_exit(struct crypto_aead *aead)
764 {
765 struct crypto_tfm *tfm = crypto_aead_tfm(aead);
766 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm);
767
768 dma_unmap_single(tfm_ctx->dev, tfm_ctx->key_dma_addr, AES_KEYSIZE_256, DMA_TO_DEVICE);
769 kfree(tfm_ctx->key);
770 if (tfm_ctx->fbk_cipher) {
771 crypto_free_aead(tfm_ctx->fbk_cipher);
772 tfm_ctx->fbk_cipher = NULL;
773 }
774 memzero_explicit(tfm_ctx, sizeof(struct xilinx_aead_tfm_ctx));
775 }
776
777 static struct xilinx_aead_alg zynqmp_aes_algs[] = {
778 {
779 .aes_aead_cipher = zynqmp_aes_aead_cipher,
780 .aead.base = {
781 .setkey = zynqmp_aes_aead_setkey,
782 .setauthsize = xilinx_aes_aead_setauthsize,
783 .encrypt = zynqmp_aes_aead_encrypt,
784 .decrypt = zynqmp_aes_aead_decrypt,
785 .init = xilinx_aes_aead_init,
786 .exit = xilinx_aes_aead_exit,
787 .ivsize = GCM_AES_IV_SIZE,
788 .maxauthsize = XILINX_AES_AUTH_SIZE,
789 .base = {
790 .cra_name = "gcm(aes)",
791 .cra_driver_name = "xilinx-zynqmp-aes-gcm",
792 .cra_priority = 200,
793 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
794 CRYPTO_ALG_ASYNC |
795 CRYPTO_ALG_ALLOCATES_MEMORY |
796 CRYPTO_ALG_KERN_DRIVER_ONLY |
797 CRYPTO_ALG_NEED_FALLBACK,
798 .cra_blocksize = XILINX_AES_BLK_SIZE,
799 .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx),
800 .cra_module = THIS_MODULE,
801 }
802 },
803 .aead.op = {
804 .do_one_request = xilinx_handle_aes_req,
805 },
806 .dma_bit_mask = ZYNQMP_DMA_BIT_MASK,
807 },
808 {
809 .aes_aead_cipher = zynqmp_aes_aead_cipher,
810 .aead.base = {
811 .setkey = zynqmp_paes_aead_setkey,
812 .setauthsize = xilinx_aes_aead_setauthsize,
813 .encrypt = zynqmp_aes_aead_encrypt,
814 .decrypt = zynqmp_aes_aead_decrypt,
815 .init = xilinx_paes_aead_init,
816 .exit = xilinx_paes_aead_exit,
817 .ivsize = GCM_AES_IV_SIZE,
818 .maxauthsize = XILINX_AES_AUTH_SIZE,
819 .base = {
820 .cra_name = "gcm(paes)",
821 .cra_driver_name = "xilinx-zynqmp-paes-gcm",
822 .cra_priority = 200,
823 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
824 CRYPTO_ALG_ASYNC |
825 CRYPTO_ALG_ALLOCATES_MEMORY |
826 CRYPTO_ALG_KERN_DRIVER_ONLY,
827 .cra_blocksize = XILINX_AES_BLK_SIZE,
828 .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx),
829 .cra_module = THIS_MODULE,
830 }
831 },
832 .aead.op = {
833 .do_one_request = xilinx_handle_aes_req,
834 },
835 .dma_bit_mask = ZYNQMP_DMA_BIT_MASK,
836 },
837 { /* sentinel */ }
838 };
839
840 static struct xilinx_aead_alg versal_aes_algs[] = {
841 {
842 .aes_aead_cipher = versal_aes_aead_cipher,
843 .aead.base = {
844 .setkey = versal_aes_aead_setkey,
845 .setauthsize = xilinx_aes_aead_setauthsize,
846 .encrypt = versal_aes_aead_encrypt,
847 .decrypt = versal_aes_aead_decrypt,
848 .init = xilinx_aes_aead_init,
849 .exit = xilinx_aes_aead_exit,
850 .ivsize = GCM_AES_IV_SIZE,
851 .maxauthsize = XILINX_AES_AUTH_SIZE,
852 .base = {
853 .cra_name = "gcm(aes)",
854 .cra_driver_name = "versal-aes-gcm",
855 .cra_priority = 300,
856 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
857 CRYPTO_ALG_ASYNC |
858 CRYPTO_ALG_ALLOCATES_MEMORY |
859 CRYPTO_ALG_KERN_DRIVER_ONLY |
860 CRYPTO_ALG_NEED_FALLBACK,
861 .cra_blocksize = XILINX_AES_BLK_SIZE,
862 .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx),
863 .cra_module = THIS_MODULE,
864 }
865 },
866 .aead.op = {
867 .do_one_request = xilinx_handle_aes_req,
868 },
869 .dma_bit_mask = VERSAL_DMA_BIT_MASK,
870 },
871 {
872 .aes_aead_cipher = versal_aes_aead_cipher,
873 .aead.base = {
874 .setkey = versal_paes_aead_setkey,
875 .setauthsize = xilinx_aes_aead_setauthsize,
876 .encrypt = versal_aes_aead_encrypt,
877 .decrypt = versal_aes_aead_decrypt,
878 .init = xilinx_paes_aead_init,
879 .exit = xilinx_paes_aead_exit,
880 .ivsize = GCM_AES_IV_SIZE,
881 .maxauthsize = XILINX_AES_AUTH_SIZE,
882 .base = {
883 .cra_name = "gcm(paes)",
884 .cra_driver_name = "versal-paes-gcm",
885 .cra_priority = 300,
886 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
887 CRYPTO_ALG_ASYNC |
888 CRYPTO_ALG_ALLOCATES_MEMORY |
889 CRYPTO_ALG_KERN_DRIVER_ONLY,
890 .cra_blocksize = XILINX_AES_BLK_SIZE,
891 .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx),
892 .cra_module = THIS_MODULE,
893 }
894 },
895 .aead.op = {
896 .do_one_request = xilinx_handle_aes_req,
897 },
898 .dma_bit_mask = VERSAL_DMA_BIT_MASK,
899 },
900 { /* sentinel */ }
901 };
902
903 static struct xlnx_feature aes_feature_map[] = {
904 {
905 .family = PM_ZYNQMP_FAMILY_CODE,
906 .feature_id = PM_SECURE_AES,
907 .data = zynqmp_aes_algs,
908 },
909 {
910 .family = PM_VERSAL_FAMILY_CODE,
911 .feature_id = XSECURE_API_AES_OP_INIT,
912 .data = versal_aes_algs,
913 },
914 { /* sentinel */ }
915 };
916
xilinx_aes_aead_probe(struct platform_device * pdev)917 static int xilinx_aes_aead_probe(struct platform_device *pdev)
918 {
919 struct xilinx_aead_alg *aead_algs;
920 struct device *dev = &pdev->dev;
921 int err;
922 int i;
923
924 /* Verify the hardware is present */
925 aead_algs = xlnx_get_crypto_dev_data(aes_feature_map);
926 if (IS_ERR(aead_algs)) {
927 dev_err(dev, "AES is not supported on the platform\n");
928 return PTR_ERR(aead_algs);
929 }
930
931 /* ZynqMP AES driver supports only one instance */
932 if (aead_dev)
933 return -ENODEV;
934
935 aead_dev = devm_kzalloc(dev, sizeof(*aead_dev), GFP_KERNEL);
936 if (!aead_dev)
937 return -ENOMEM;
938 aead_dev->dev = dev;
939 aead_dev->aead_algs = aead_algs;
940 platform_set_drvdata(pdev, aead_dev);
941 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aead_algs[0].dma_bit_mask));
942 if (err < 0) {
943 dev_err(dev, "No usable DMA configuration\n");
944 return err;
945 }
946
947 aead_dev->engine = crypto_engine_alloc_init(dev, 1);
948 if (!aead_dev->engine) {
949 dev_err(dev, "Cannot alloc AES engine\n");
950 return -ENOMEM;
951 }
952
953 err = crypto_engine_start(aead_dev->engine);
954 if (err) {
955 dev_err(dev, "Cannot start AES engine\n");
956 goto err_engine_start;
957 }
958
959 for (i = 0; aead_dev->aead_algs[i].dma_bit_mask; i++) {
960 aead_dev->aead_algs[i].aead_dev = aead_dev;
961 err = crypto_engine_register_aead(&aead_dev->aead_algs[i].aead);
962 if (err < 0) {
963 dev_err(dev, "Failed to register AEAD alg %d.\n", i);
964 goto err_alg_register;
965 }
966 }
967
968 return 0;
969
970 err_alg_register:
971 while (i > 0)
972 crypto_engine_unregister_aead(&aead_dev->aead_algs[--i].aead);
973 err_engine_start:
974 crypto_engine_exit(aead_dev->engine);
975
976 return err;
977 }
978
xilinx_aes_aead_remove(struct platform_device * pdev)979 static void xilinx_aes_aead_remove(struct platform_device *pdev)
980 {
981 aead_dev = platform_get_drvdata(pdev);
982 crypto_engine_exit(aead_dev->engine);
983 for (int i = 0; aead_dev->aead_algs[i].dma_bit_mask; i++)
984 crypto_engine_unregister_aead(&aead_dev->aead_algs[i].aead);
985
986 aead_dev = NULL;
987 }
988
989 static struct platform_driver xilinx_aes_driver = {
990 .probe = xilinx_aes_aead_probe,
991 .remove = xilinx_aes_aead_remove,
992 .driver = {
993 .name = "zynqmp-aes",
994 },
995 };
996
997 static struct platform_device *platform_dev;
998
aes_driver_init(void)999 static int __init aes_driver_init(void)
1000 {
1001 int ret;
1002
1003 ret = platform_driver_register(&xilinx_aes_driver);
1004 if (ret)
1005 return ret;
1006
1007 platform_dev = platform_device_register_simple(xilinx_aes_driver.driver.name,
1008 0, NULL, 0);
1009 if (IS_ERR(platform_dev)) {
1010 ret = PTR_ERR(platform_dev);
1011 platform_driver_unregister(&xilinx_aes_driver);
1012 }
1013
1014 return ret;
1015 }
1016
aes_driver_exit(void)1017 static void __exit aes_driver_exit(void)
1018 {
1019 platform_device_unregister(platform_dev);
1020 platform_driver_unregister(&xilinx_aes_driver);
1021 }
1022
1023 module_init(aes_driver_init);
1024 module_exit(aes_driver_exit);
1025 MODULE_DESCRIPTION("zynqmp aes-gcm hardware acceleration support.");
1026 MODULE_LICENSE("GPL");
1027