1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Xilinx ZynqMP AES Driver. 4 * Copyright (C) 2020-2022 Xilinx Inc. 5 * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. 6 */ 7 8 #include <crypto/aes.h> 9 #include <crypto/engine.h> 10 #include <crypto/gcm.h> 11 #include <crypto/internal/aead.h> 12 #include <crypto/scatterwalk.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/err.h> 15 #include <linux/firmware/xlnx-zynqmp.h> 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/mod_devicetable.h> 19 #include <linux/platform_device.h> 20 #include <linux/string.h> 21 22 #define ZYNQMP_DMA_BIT_MASK 32U 23 #define VERSAL_DMA_BIT_MASK 64U 24 #define XILINX_AES_AUTH_SIZE 16U 25 #define XILINX_AES_BLK_SIZE 1U 26 #define ZYNQMP_AES_MIN_INPUT_BLK_SIZE 4U 27 #define ZYNQMP_AES_WORD_LEN 4U 28 29 #define VERSAL_AES_QWORD_LEN 16U 30 #define ZYNQMP_AES_GCM_TAG_MISMATCH_ERR 0x01 31 #define ZYNQMP_AES_WRONG_KEY_SRC_ERR 0x13 32 #define ZYNQMP_AES_PUF_NOT_PROGRAMMED 0xE300 33 #define XILINX_KEY_MAGIC 0x3EA0 34 35 enum xilinx_aead_op { 36 XILINX_AES_DECRYPT = 0, 37 XILINX_AES_ENCRYPT 38 }; 39 40 enum zynqmp_aead_keysrc { 41 ZYNQMP_AES_KUP_KEY = 0, 42 ZYNQMP_AES_DEV_KEY, 43 ZYNQMP_AES_PUF_KEY 44 }; 45 46 struct xilinx_aead_dev { 47 struct device *dev; 48 struct crypto_engine *engine; 49 struct xilinx_aead_alg *aead_algs; 50 }; 51 52 struct xilinx_aead_alg { 53 struct xilinx_aead_dev *aead_dev; 54 struct aead_engine_alg aead; 55 int (*aes_aead_cipher)(struct aead_request *areq); 56 u8 dma_bit_mask; 57 }; 58 59 struct xilinx_hwkey_info { 60 u16 magic; 61 u16 type; 62 } __packed; 63 64 struct zynqmp_aead_hw_req { 65 u64 src; 66 u64 iv; 67 u64 key; 68 u64 dst; 69 u64 size; 70 u64 op; 71 u64 keysrc; 72 }; 73 74 struct xilinx_aead_tfm_ctx { 75 struct device *dev; 76 dma_addr_t key_dma_addr; 77 u8 *key; 78 u32 keylen; 79 u32 authsize; 80 u8 keysrc; 81 struct crypto_aead *fbk_cipher; 82 }; 83 84 struct xilinx_aead_req_ctx { 85 enum xilinx_aead_op op; 86 }; 87 88 static struct xilinx_aead_dev *aead_dev; 89 90 enum versal_aead_keysrc { 91 VERSAL_AES_BBRAM_KEY = 0, 92 VERSAL_AES_BBRAM_RED_KEY, 93 VERSAL_AES_BH_KEY, 94 VERSAL_AES_BH_RED_KEY, 95 VERSAL_AES_EFUSE_KEY, 96 VERSAL_AES_EFUSE_RED_KEY, 97 VERSAL_AES_EFUSE_USER_KEY_0, 98 VERSAL_AES_EFUSE_USER_KEY_1, 99 VERSAL_AES_EFUSE_USER_RED_KEY_0, 100 VERSAL_AES_EFUSE_USER_RED_KEY_1, 101 VERSAL_AES_KUP_KEY, 102 VERSAL_AES_PUF_KEY, 103 VERSAL_AES_USER_KEY_0, 104 VERSAL_AES_USER_KEY_1, 105 VERSAL_AES_USER_KEY_2, 106 VERSAL_AES_USER_KEY_3, 107 VERSAL_AES_USER_KEY_4, 108 VERSAL_AES_USER_KEY_5, 109 VERSAL_AES_USER_KEY_6, 110 VERSAL_AES_USER_KEY_7, 111 VERSAL_AES_EXPANDED_KEYS, 112 VERSAL_AES_ALL_KEYS, 113 }; 114 115 enum versal_aead_op { 116 VERSAL_AES_ENCRYPT = 0, 117 VERSAL_AES_DECRYPT 118 }; 119 120 enum versal_aes_keysize { 121 HW_AES_KEY_SIZE_128 = 0, 122 HW_AES_KEY_SIZE_256 = 2, 123 }; 124 125 struct versal_init_ops { 126 u64 iv; 127 u32 op; 128 u32 keysrc; 129 u32 size; 130 }; 131 132 struct versal_in_params { 133 u64 in_data_addr; 134 u32 size; 135 u32 is_last; 136 }; 137 138 static int zynqmp_aes_aead_cipher(struct aead_request *req) 139 { 140 struct crypto_aead *aead = crypto_aead_reqtfm(req); 141 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 142 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 143 dma_addr_t dma_addr_data, dma_addr_hw_req; 144 struct device *dev = tfm_ctx->dev; 145 struct zynqmp_aead_hw_req *hwreq; 146 unsigned int data_size; 147 unsigned int status; 148 int ret; 149 size_t dma_size; 150 void *dmabuf; 151 char *kbuf; 152 153 dma_size = req->cryptlen + XILINX_AES_AUTH_SIZE; 154 kbuf = kmalloc(dma_size, GFP_KERNEL); 155 if (!kbuf) 156 return -ENOMEM; 157 158 dmabuf = kmalloc(sizeof(*hwreq) + GCM_AES_IV_SIZE, GFP_KERNEL); 159 if (!dmabuf) { 160 kfree(kbuf); 161 return -ENOMEM; 162 } 163 hwreq = dmabuf; 164 data_size = req->cryptlen; 165 scatterwalk_map_and_copy(kbuf, req->src, 0, req->cryptlen, 0); 166 memcpy(dmabuf + sizeof(struct zynqmp_aead_hw_req), req->iv, GCM_AES_IV_SIZE); 167 dma_addr_data = dma_map_single(dev, kbuf, dma_size, DMA_BIDIRECTIONAL); 168 if (unlikely(dma_mapping_error(dev, dma_addr_data))) { 169 ret = -ENOMEM; 170 goto freemem; 171 } 172 173 hwreq->src = dma_addr_data; 174 hwreq->dst = dma_addr_data; 175 hwreq->keysrc = tfm_ctx->keysrc; 176 hwreq->op = rq_ctx->op; 177 178 if (hwreq->op == XILINX_AES_ENCRYPT) 179 hwreq->size = data_size; 180 else 181 hwreq->size = data_size - XILINX_AES_AUTH_SIZE; 182 183 if (hwreq->keysrc == ZYNQMP_AES_KUP_KEY) 184 hwreq->key = tfm_ctx->key_dma_addr; 185 else 186 hwreq->key = 0; 187 188 dma_addr_hw_req = dma_map_single(dev, dmabuf, sizeof(struct zynqmp_aead_hw_req) + 189 GCM_AES_IV_SIZE, 190 DMA_TO_DEVICE); 191 if (unlikely(dma_mapping_error(dev, dma_addr_hw_req))) { 192 ret = -ENOMEM; 193 dma_unmap_single(dev, dma_addr_data, dma_size, DMA_BIDIRECTIONAL); 194 goto freemem; 195 } 196 hwreq->iv = dma_addr_hw_req + sizeof(struct zynqmp_aead_hw_req); 197 dma_sync_single_for_device(dev, dma_addr_hw_req, sizeof(struct zynqmp_aead_hw_req) + 198 GCM_AES_IV_SIZE, DMA_TO_DEVICE); 199 ret = zynqmp_pm_aes_engine(dma_addr_hw_req, &status); 200 dma_unmap_single(dev, dma_addr_hw_req, sizeof(struct zynqmp_aead_hw_req) + GCM_AES_IV_SIZE, 201 DMA_TO_DEVICE); 202 dma_unmap_single(dev, dma_addr_data, dma_size, DMA_BIDIRECTIONAL); 203 if (ret) { 204 dev_err(dev, "ERROR: AES PM API failed\n"); 205 } else if (status) { 206 switch (status) { 207 case ZYNQMP_AES_GCM_TAG_MISMATCH_ERR: 208 ret = -EBADMSG; 209 break; 210 case ZYNQMP_AES_WRONG_KEY_SRC_ERR: 211 ret = -EINVAL; 212 dev_err(dev, "ERROR: Wrong KeySrc, enable secure mode\n"); 213 break; 214 case ZYNQMP_AES_PUF_NOT_PROGRAMMED: 215 ret = -EINVAL; 216 dev_err(dev, "ERROR: PUF is not registered\n"); 217 break; 218 default: 219 ret = -EINVAL; 220 break; 221 } 222 } else { 223 if (hwreq->op == XILINX_AES_ENCRYPT) 224 data_size = data_size + crypto_aead_authsize(aead); 225 else 226 data_size = data_size - XILINX_AES_AUTH_SIZE; 227 228 sg_copy_from_buffer(req->dst, sg_nents(req->dst), 229 kbuf, data_size); 230 ret = 0; 231 } 232 233 freemem: 234 memzero_explicit(kbuf, dma_size); 235 kfree(kbuf); 236 memzero_explicit(dmabuf, sizeof(struct zynqmp_aead_hw_req) + GCM_AES_IV_SIZE); 237 kfree(dmabuf); 238 239 return ret; 240 } 241 242 static int versal_aes_aead_cipher(struct aead_request *req) 243 { 244 struct crypto_aead *aead = crypto_aead_reqtfm(req); 245 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 246 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 247 dma_addr_t dma_addr_data, dma_addr_hw_req, dma_addr_in; 248 u32 total_len = req->assoclen + req->cryptlen; 249 struct device *dev = tfm_ctx->dev; 250 struct versal_init_ops *hwreq; 251 struct versal_in_params *in; 252 u32 gcm_offset, out_len; 253 size_t dmabuf_size; 254 size_t kbuf_size; 255 void *dmabuf; 256 char *kbuf; 257 int ret; 258 259 kbuf_size = total_len + XILINX_AES_AUTH_SIZE; 260 kbuf = kmalloc(kbuf_size, GFP_KERNEL); 261 if (unlikely(!kbuf)) { 262 ret = -ENOMEM; 263 goto err; 264 } 265 dmabuf_size = sizeof(struct versal_init_ops) + 266 sizeof(struct versal_in_params) + 267 GCM_AES_IV_SIZE; 268 dmabuf = kmalloc(dmabuf_size, GFP_KERNEL); 269 if (unlikely(!dmabuf)) { 270 ret = -ENOMEM; 271 goto buf1_free; 272 } 273 274 dma_addr_hw_req = dma_map_single(dev, dmabuf, dmabuf_size, DMA_BIDIRECTIONAL); 275 if (unlikely(dma_mapping_error(dev, dma_addr_hw_req))) { 276 ret = -ENOMEM; 277 goto buf2_free; 278 } 279 scatterwalk_map_and_copy(kbuf, req->src, 0, total_len, 0); 280 dma_addr_data = dma_map_single(dev, kbuf, kbuf_size, DMA_BIDIRECTIONAL); 281 if (unlikely(dma_mapping_error(dev, dma_addr_data))) { 282 dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); 283 ret = -ENOMEM; 284 goto buf2_free; 285 } 286 hwreq = dmabuf; 287 in = dmabuf + sizeof(struct versal_init_ops); 288 memcpy(dmabuf + sizeof(struct versal_init_ops) + 289 sizeof(struct versal_in_params), req->iv, GCM_AES_IV_SIZE); 290 hwreq->iv = dma_addr_hw_req + sizeof(struct versal_init_ops) + 291 sizeof(struct versal_in_params); 292 hwreq->keysrc = tfm_ctx->keysrc; 293 dma_addr_in = dma_addr_hw_req + sizeof(struct versal_init_ops); 294 if (rq_ctx->op == XILINX_AES_ENCRYPT) { 295 hwreq->op = VERSAL_AES_ENCRYPT; 296 out_len = total_len + crypto_aead_authsize(aead); 297 in->size = req->cryptlen; 298 } else { 299 hwreq->op = VERSAL_AES_DECRYPT; 300 out_len = total_len - XILINX_AES_AUTH_SIZE; 301 in->size = req->cryptlen - XILINX_AES_AUTH_SIZE; 302 } 303 304 if (tfm_ctx->keylen == AES_KEYSIZE_128) 305 hwreq->size = HW_AES_KEY_SIZE_128; 306 else 307 hwreq->size = HW_AES_KEY_SIZE_256; 308 309 /* Request aes key write for volatile user keys */ 310 if (hwreq->keysrc >= VERSAL_AES_USER_KEY_0 && hwreq->keysrc <= VERSAL_AES_USER_KEY_7) { 311 ret = versal_pm_aes_key_write(hwreq->size, hwreq->keysrc, 312 tfm_ctx->key_dma_addr); 313 if (ret) 314 goto unmap; 315 } 316 317 in->in_data_addr = dma_addr_data + req->assoclen; 318 in->is_last = 1; 319 gcm_offset = req->assoclen + in->size; 320 dma_sync_single_for_device(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); 321 ret = versal_pm_aes_op_init(dma_addr_hw_req); 322 if (ret) 323 goto clearkey; 324 325 if (req->assoclen > 0) { 326 /* Currently GMAC is OFF by default */ 327 ret = versal_pm_aes_update_aad(dma_addr_data, req->assoclen); 328 if (ret) 329 goto clearkey; 330 } 331 if (rq_ctx->op == XILINX_AES_ENCRYPT) { 332 ret = versal_pm_aes_enc_update(dma_addr_in, 333 dma_addr_data + req->assoclen); 334 if (ret) 335 goto clearkey; 336 337 ret = versal_pm_aes_enc_final(dma_addr_data + gcm_offset); 338 if (ret) 339 goto clearkey; 340 } else { 341 ret = versal_pm_aes_dec_update(dma_addr_in, 342 dma_addr_data + req->assoclen); 343 if (ret) 344 goto clearkey; 345 346 ret = versal_pm_aes_dec_final(dma_addr_data + gcm_offset); 347 if (ret) { 348 ret = -EBADMSG; 349 goto clearkey; 350 } 351 } 352 dma_unmap_single(dev, dma_addr_data, kbuf_size, DMA_BIDIRECTIONAL); 353 dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); 354 sg_copy_from_buffer(req->dst, sg_nents(req->dst), 355 kbuf, out_len); 356 dma_addr_data = 0; 357 dma_addr_hw_req = 0; 358 359 clearkey: 360 if (hwreq->keysrc >= VERSAL_AES_USER_KEY_0 && hwreq->keysrc <= VERSAL_AES_USER_KEY_7) 361 versal_pm_aes_key_zero(hwreq->keysrc); 362 unmap: 363 if (unlikely(dma_addr_data)) 364 dma_unmap_single(dev, dma_addr_data, kbuf_size, DMA_BIDIRECTIONAL); 365 if (unlikely(dma_addr_hw_req)) 366 dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); 367 buf2_free: 368 memzero_explicit(dmabuf, dmabuf_size); 369 kfree(dmabuf); 370 buf1_free: 371 memzero_explicit(kbuf, kbuf_size); 372 kfree(kbuf); 373 err: 374 return ret; 375 } 376 377 static int zynqmp_fallback_check(struct xilinx_aead_tfm_ctx *tfm_ctx, 378 struct aead_request *req) 379 { 380 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 381 382 if (tfm_ctx->authsize != XILINX_AES_AUTH_SIZE && rq_ctx->op == XILINX_AES_DECRYPT) 383 return 1; 384 385 if (req->assoclen != 0 || 386 req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE) 387 return 1; 388 if (tfm_ctx->keylen == AES_KEYSIZE_128 || 389 tfm_ctx->keylen == AES_KEYSIZE_192) 390 return 1; 391 392 if ((req->cryptlen % ZYNQMP_AES_WORD_LEN) != 0) 393 return 1; 394 395 if (rq_ctx->op == XILINX_AES_DECRYPT && 396 req->cryptlen <= XILINX_AES_AUTH_SIZE) 397 return 1; 398 399 return 0; 400 } 401 402 static int versal_fallback_check(struct xilinx_aead_tfm_ctx *tfm_ctx, 403 struct aead_request *req) 404 { 405 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 406 407 if (tfm_ctx->authsize != XILINX_AES_AUTH_SIZE && rq_ctx->op == XILINX_AES_DECRYPT) 408 return 1; 409 410 if (tfm_ctx->keylen == AES_KEYSIZE_192) 411 return 1; 412 413 if (req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE || 414 req->cryptlen % ZYNQMP_AES_WORD_LEN || 415 req->assoclen % VERSAL_AES_QWORD_LEN) 416 return 1; 417 418 if (rq_ctx->op == XILINX_AES_DECRYPT && 419 req->cryptlen <= XILINX_AES_AUTH_SIZE) 420 return 1; 421 422 return 0; 423 } 424 425 static int xilinx_handle_aes_req(struct crypto_engine *engine, void *req) 426 { 427 struct aead_request *areq = 428 container_of(req, struct aead_request, base); 429 struct crypto_aead *aead = crypto_aead_reqtfm(req); 430 struct aead_alg *alg = crypto_aead_alg(aead); 431 struct xilinx_aead_alg *drv_ctx; 432 int err; 433 434 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 435 err = drv_ctx->aes_aead_cipher(areq); 436 local_bh_disable(); 437 crypto_finalize_aead_request(engine, areq, err); 438 local_bh_enable(); 439 440 return 0; 441 } 442 443 static int zynqmp_aes_aead_setkey(struct crypto_aead *aead, const u8 *key, 444 unsigned int keylen) 445 { 446 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 447 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 448 int err; 449 450 if (keylen == AES_KEYSIZE_256) { 451 memcpy(tfm_ctx->key, key, keylen); 452 dma_sync_single_for_device(tfm_ctx->dev, tfm_ctx->key_dma_addr, 453 AES_KEYSIZE_256, 454 DMA_TO_DEVICE); 455 } 456 457 tfm_ctx->fbk_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; 458 tfm_ctx->fbk_cipher->base.crt_flags |= (aead->base.crt_flags & 459 CRYPTO_TFM_REQ_MASK); 460 461 err = crypto_aead_setkey(tfm_ctx->fbk_cipher, key, keylen); 462 if (err) 463 goto err; 464 tfm_ctx->keylen = keylen; 465 tfm_ctx->keysrc = ZYNQMP_AES_KUP_KEY; 466 err: 467 return err; 468 } 469 470 static int zynqmp_paes_aead_setkey(struct crypto_aead *aead, const u8 *key, 471 unsigned int keylen) 472 { 473 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 474 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 475 struct xilinx_hwkey_info hwkey; 476 unsigned char keysrc; 477 int err = -EINVAL; 478 479 if (keylen != sizeof(struct xilinx_hwkey_info)) 480 return -EINVAL; 481 memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info)); 482 if (hwkey.magic != XILINX_KEY_MAGIC) 483 return -EINVAL; 484 keysrc = hwkey.type; 485 if (keysrc == ZYNQMP_AES_DEV_KEY || 486 keysrc == ZYNQMP_AES_PUF_KEY) { 487 tfm_ctx->keysrc = keysrc; 488 tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info); 489 err = 0; 490 } 491 492 return err; 493 } 494 495 static int versal_aes_aead_setkey(struct crypto_aead *aead, const u8 *key, 496 unsigned int keylen) 497 { 498 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 499 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 500 struct xilinx_hwkey_info hwkey; 501 unsigned char keysrc; 502 int err; 503 504 tfm_ctx->keysrc = VERSAL_AES_USER_KEY_0; 505 if (keylen == sizeof(struct xilinx_hwkey_info)) { 506 memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info)); 507 if (hwkey.magic != XILINX_KEY_MAGIC) 508 return -EINVAL; 509 510 keysrc = hwkey.type; 511 if (keysrc >= VERSAL_AES_USER_KEY_1 && 512 keysrc <= VERSAL_AES_USER_KEY_7) { 513 tfm_ctx->keysrc = keysrc; 514 tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info); 515 return 0; 516 } 517 return -EINVAL; 518 } 519 520 if (keylen == AES_KEYSIZE_256 || keylen == AES_KEYSIZE_128) { 521 tfm_ctx->keylen = keylen; 522 memcpy(tfm_ctx->key, key, keylen); 523 dma_sync_single_for_device(tfm_ctx->dev, tfm_ctx->key_dma_addr, 524 AES_KEYSIZE_256, 525 DMA_TO_DEVICE); 526 } 527 528 tfm_ctx->fbk_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; 529 tfm_ctx->fbk_cipher->base.crt_flags |= (aead->base.crt_flags & 530 CRYPTO_TFM_REQ_MASK); 531 err = crypto_aead_setkey(tfm_ctx->fbk_cipher, key, keylen); 532 if (!err) 533 tfm_ctx->keylen = keylen; 534 535 return err; 536 } 537 538 static int versal_paes_aead_setkey(struct crypto_aead *aead, const u8 *key, 539 unsigned int keylen) 540 { 541 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 542 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 543 struct xilinx_hwkey_info hwkey; 544 unsigned char keysrc; 545 int err = 0; 546 547 if (keylen != sizeof(struct xilinx_hwkey_info)) 548 return -EINVAL; 549 550 memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info)); 551 if (hwkey.magic != XILINX_KEY_MAGIC) 552 return -EINVAL; 553 554 keysrc = hwkey.type; 555 556 switch (keysrc) { 557 case VERSAL_AES_EFUSE_USER_KEY_0: 558 case VERSAL_AES_EFUSE_USER_KEY_1: 559 case VERSAL_AES_EFUSE_USER_RED_KEY_0: 560 case VERSAL_AES_EFUSE_USER_RED_KEY_1: 561 case VERSAL_AES_PUF_KEY: 562 tfm_ctx->keysrc = keysrc; 563 tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info); 564 break; 565 default: 566 err = -EINVAL; 567 break; 568 } 569 570 return err; 571 } 572 573 static int xilinx_aes_aead_setauthsize(struct crypto_aead *aead, 574 unsigned int authsize) 575 { 576 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 577 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 578 579 tfm_ctx->authsize = authsize; 580 return tfm_ctx->fbk_cipher ? crypto_aead_setauthsize(tfm_ctx->fbk_cipher, authsize) : 0; 581 } 582 583 static int xilinx_aes_fallback_crypt(struct aead_request *req, bool encrypt) 584 { 585 struct aead_request *subreq = aead_request_ctx(req); 586 struct crypto_aead *aead = crypto_aead_reqtfm(req); 587 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 588 589 aead_request_set_tfm(subreq, tfm_ctx->fbk_cipher); 590 aead_request_set_callback(subreq, req->base.flags, NULL, NULL); 591 aead_request_set_crypt(subreq, req->src, req->dst, 592 req->cryptlen, req->iv); 593 aead_request_set_ad(subreq, req->assoclen); 594 595 return encrypt ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); 596 } 597 598 static int zynqmp_aes_aead_encrypt(struct aead_request *req) 599 { 600 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 601 struct crypto_aead *aead = crypto_aead_reqtfm(req); 602 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 603 struct aead_alg *alg = crypto_aead_alg(aead); 604 struct xilinx_aead_alg *drv_ctx; 605 int err; 606 607 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 608 if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY && 609 tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) 610 return -EINVAL; 611 612 rq_ctx->op = XILINX_AES_ENCRYPT; 613 err = zynqmp_fallback_check(tfm_ctx, req); 614 if (err && tfm_ctx->keysrc != ZYNQMP_AES_KUP_KEY) 615 return -EOPNOTSUPP; 616 617 if (err) 618 return xilinx_aes_fallback_crypt(req, true); 619 620 return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); 621 } 622 623 static int versal_aes_aead_encrypt(struct aead_request *req) 624 { 625 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 626 struct crypto_aead *aead = crypto_aead_reqtfm(req); 627 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 628 struct aead_alg *alg = crypto_aead_alg(aead); 629 struct xilinx_aead_alg *drv_ctx; 630 int err; 631 632 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 633 rq_ctx->op = XILINX_AES_ENCRYPT; 634 if (tfm_ctx->keysrc >= VERSAL_AES_USER_KEY_0 && 635 tfm_ctx->keysrc <= VERSAL_AES_USER_KEY_7 && 636 tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) 637 return -EINVAL; 638 err = versal_fallback_check(tfm_ctx, req); 639 if (err && (tfm_ctx->keysrc < VERSAL_AES_USER_KEY_0 || 640 tfm_ctx->keysrc > VERSAL_AES_USER_KEY_7)) 641 return -EOPNOTSUPP; 642 if (err) 643 return xilinx_aes_fallback_crypt(req, true); 644 645 return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); 646 } 647 648 static int zynqmp_aes_aead_decrypt(struct aead_request *req) 649 { 650 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 651 struct crypto_aead *aead = crypto_aead_reqtfm(req); 652 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 653 struct aead_alg *alg = crypto_aead_alg(aead); 654 struct xilinx_aead_alg *drv_ctx; 655 int err; 656 657 rq_ctx->op = XILINX_AES_DECRYPT; 658 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 659 if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY && 660 tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) 661 return -EINVAL; 662 err = zynqmp_fallback_check(tfm_ctx, req); 663 if (err && tfm_ctx->keysrc != ZYNQMP_AES_KUP_KEY) 664 return -EOPNOTSUPP; 665 if (err) 666 return xilinx_aes_fallback_crypt(req, false); 667 668 return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); 669 } 670 671 static int xilinx_paes_aead_init(struct crypto_aead *aead) 672 { 673 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 674 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 675 struct xilinx_aead_alg *drv_alg; 676 struct aead_alg *alg = crypto_aead_alg(aead); 677 678 drv_alg = container_of(alg, struct xilinx_aead_alg, aead.base); 679 tfm_ctx->dev = drv_alg->aead_dev->dev; 680 tfm_ctx->keylen = 0; 681 tfm_ctx->key = NULL; 682 tfm_ctx->fbk_cipher = NULL; 683 crypto_aead_set_reqsize(aead, sizeof(struct xilinx_aead_req_ctx)); 684 685 return 0; 686 } 687 688 static int versal_aes_aead_decrypt(struct aead_request *req) 689 { 690 struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 691 struct crypto_aead *aead = crypto_aead_reqtfm(req); 692 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 693 struct aead_alg *alg = crypto_aead_alg(aead); 694 struct xilinx_aead_alg *drv_ctx; 695 int err; 696 697 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 698 rq_ctx->op = XILINX_AES_DECRYPT; 699 if (tfm_ctx->keysrc >= VERSAL_AES_USER_KEY_0 && 700 tfm_ctx->keysrc <= VERSAL_AES_USER_KEY_7 && 701 tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) 702 return -EINVAL; 703 704 err = versal_fallback_check(tfm_ctx, req); 705 if (err && 706 (tfm_ctx->keysrc < VERSAL_AES_USER_KEY_0 || 707 tfm_ctx->keysrc > VERSAL_AES_USER_KEY_7)) 708 return -EOPNOTSUPP; 709 if (err) 710 return xilinx_aes_fallback_crypt(req, false); 711 712 return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); 713 } 714 715 static int xilinx_aes_aead_init(struct crypto_aead *aead) 716 { 717 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 718 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 719 struct xilinx_aead_alg *drv_ctx; 720 struct aead_alg *alg = crypto_aead_alg(aead); 721 722 drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 723 tfm_ctx->dev = drv_ctx->aead_dev->dev; 724 tfm_ctx->keylen = 0; 725 726 tfm_ctx->fbk_cipher = crypto_alloc_aead(drv_ctx->aead.base.base.cra_name, 727 0, 728 CRYPTO_ALG_NEED_FALLBACK); 729 730 if (IS_ERR(tfm_ctx->fbk_cipher)) { 731 dev_err(tfm_ctx->dev, "failed to allocate fallback for %s\n", 732 drv_ctx->aead.base.base.cra_name); 733 return PTR_ERR(tfm_ctx->fbk_cipher); 734 } 735 tfm_ctx->key = kmalloc(AES_KEYSIZE_256, GFP_KERNEL); 736 if (!tfm_ctx->key) { 737 crypto_free_aead(tfm_ctx->fbk_cipher); 738 return -ENOMEM; 739 } 740 tfm_ctx->key_dma_addr = dma_map_single(tfm_ctx->dev, tfm_ctx->key, 741 AES_KEYSIZE_256, 742 DMA_TO_DEVICE); 743 if (unlikely(dma_mapping_error(tfm_ctx->dev, tfm_ctx->key_dma_addr))) { 744 kfree(tfm_ctx->key); 745 crypto_free_aead(tfm_ctx->fbk_cipher); 746 tfm_ctx->fbk_cipher = NULL; 747 return -ENOMEM; 748 } 749 crypto_aead_set_reqsize(aead, 750 max(sizeof(struct xilinx_aead_req_ctx), 751 sizeof(struct aead_request) + 752 crypto_aead_reqsize(tfm_ctx->fbk_cipher))); 753 return 0; 754 } 755 756 static void xilinx_paes_aead_exit(struct crypto_aead *aead) 757 { 758 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 759 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 760 761 memzero_explicit(tfm_ctx, sizeof(struct xilinx_aead_tfm_ctx)); 762 } 763 764 static void xilinx_aes_aead_exit(struct crypto_aead *aead) 765 { 766 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 767 struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 768 769 dma_unmap_single(tfm_ctx->dev, tfm_ctx->key_dma_addr, AES_KEYSIZE_256, DMA_TO_DEVICE); 770 kfree(tfm_ctx->key); 771 if (tfm_ctx->fbk_cipher) { 772 crypto_free_aead(tfm_ctx->fbk_cipher); 773 tfm_ctx->fbk_cipher = NULL; 774 } 775 memzero_explicit(tfm_ctx, sizeof(struct xilinx_aead_tfm_ctx)); 776 } 777 778 static struct xilinx_aead_alg zynqmp_aes_algs[] = { 779 { 780 .aes_aead_cipher = zynqmp_aes_aead_cipher, 781 .aead.base = { 782 .setkey = zynqmp_aes_aead_setkey, 783 .setauthsize = xilinx_aes_aead_setauthsize, 784 .encrypt = zynqmp_aes_aead_encrypt, 785 .decrypt = zynqmp_aes_aead_decrypt, 786 .init = xilinx_aes_aead_init, 787 .exit = xilinx_aes_aead_exit, 788 .ivsize = GCM_AES_IV_SIZE, 789 .maxauthsize = XILINX_AES_AUTH_SIZE, 790 .base = { 791 .cra_name = "gcm(aes)", 792 .cra_driver_name = "xilinx-zynqmp-aes-gcm", 793 .cra_priority = 200, 794 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 795 CRYPTO_ALG_ASYNC | 796 CRYPTO_ALG_ALLOCATES_MEMORY | 797 CRYPTO_ALG_KERN_DRIVER_ONLY | 798 CRYPTO_ALG_NEED_FALLBACK, 799 .cra_blocksize = XILINX_AES_BLK_SIZE, 800 .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), 801 .cra_module = THIS_MODULE, 802 } 803 }, 804 .aead.op = { 805 .do_one_request = xilinx_handle_aes_req, 806 }, 807 .dma_bit_mask = ZYNQMP_DMA_BIT_MASK, 808 }, 809 { 810 .aes_aead_cipher = zynqmp_aes_aead_cipher, 811 .aead.base = { 812 .setkey = zynqmp_paes_aead_setkey, 813 .setauthsize = xilinx_aes_aead_setauthsize, 814 .encrypt = zynqmp_aes_aead_encrypt, 815 .decrypt = zynqmp_aes_aead_decrypt, 816 .init = xilinx_paes_aead_init, 817 .exit = xilinx_paes_aead_exit, 818 .ivsize = GCM_AES_IV_SIZE, 819 .maxauthsize = XILINX_AES_AUTH_SIZE, 820 .base = { 821 .cra_name = "gcm(paes)", 822 .cra_driver_name = "xilinx-zynqmp-paes-gcm", 823 .cra_priority = 200, 824 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 825 CRYPTO_ALG_ASYNC | 826 CRYPTO_ALG_ALLOCATES_MEMORY | 827 CRYPTO_ALG_KERN_DRIVER_ONLY, 828 .cra_blocksize = XILINX_AES_BLK_SIZE, 829 .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), 830 .cra_module = THIS_MODULE, 831 } 832 }, 833 .aead.op = { 834 .do_one_request = xilinx_handle_aes_req, 835 }, 836 .dma_bit_mask = ZYNQMP_DMA_BIT_MASK, 837 }, 838 { /* sentinel */ } 839 }; 840 841 static struct xilinx_aead_alg versal_aes_algs[] = { 842 { 843 .aes_aead_cipher = versal_aes_aead_cipher, 844 .aead.base = { 845 .setkey = versal_aes_aead_setkey, 846 .setauthsize = xilinx_aes_aead_setauthsize, 847 .encrypt = versal_aes_aead_encrypt, 848 .decrypt = versal_aes_aead_decrypt, 849 .init = xilinx_aes_aead_init, 850 .exit = xilinx_aes_aead_exit, 851 .ivsize = GCM_AES_IV_SIZE, 852 .maxauthsize = XILINX_AES_AUTH_SIZE, 853 .base = { 854 .cra_name = "gcm(aes)", 855 .cra_driver_name = "versal-aes-gcm", 856 .cra_priority = 300, 857 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 858 CRYPTO_ALG_ASYNC | 859 CRYPTO_ALG_ALLOCATES_MEMORY | 860 CRYPTO_ALG_KERN_DRIVER_ONLY | 861 CRYPTO_ALG_NEED_FALLBACK, 862 .cra_blocksize = XILINX_AES_BLK_SIZE, 863 .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), 864 .cra_module = THIS_MODULE, 865 } 866 }, 867 .aead.op = { 868 .do_one_request = xilinx_handle_aes_req, 869 }, 870 .dma_bit_mask = VERSAL_DMA_BIT_MASK, 871 }, 872 { 873 .aes_aead_cipher = versal_aes_aead_cipher, 874 .aead.base = { 875 .setkey = versal_paes_aead_setkey, 876 .setauthsize = xilinx_aes_aead_setauthsize, 877 .encrypt = versal_aes_aead_encrypt, 878 .decrypt = versal_aes_aead_decrypt, 879 .init = xilinx_paes_aead_init, 880 .exit = xilinx_paes_aead_exit, 881 .ivsize = GCM_AES_IV_SIZE, 882 .maxauthsize = XILINX_AES_AUTH_SIZE, 883 .base = { 884 .cra_name = "gcm(paes)", 885 .cra_driver_name = "versal-paes-gcm", 886 .cra_priority = 300, 887 .cra_flags = CRYPTO_ALG_TYPE_AEAD | 888 CRYPTO_ALG_ASYNC | 889 CRYPTO_ALG_ALLOCATES_MEMORY | 890 CRYPTO_ALG_KERN_DRIVER_ONLY, 891 .cra_blocksize = XILINX_AES_BLK_SIZE, 892 .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), 893 .cra_module = THIS_MODULE, 894 } 895 }, 896 .aead.op = { 897 .do_one_request = xilinx_handle_aes_req, 898 }, 899 .dma_bit_mask = VERSAL_DMA_BIT_MASK, 900 }, 901 { /* sentinel */ } 902 }; 903 904 static struct xlnx_feature aes_feature_map[] = { 905 { 906 .family = PM_ZYNQMP_FAMILY_CODE, 907 .feature_id = PM_SECURE_AES, 908 .data = zynqmp_aes_algs, 909 }, 910 { 911 .family = PM_VERSAL_FAMILY_CODE, 912 .feature_id = XSECURE_API_AES_OP_INIT, 913 .data = versal_aes_algs, 914 }, 915 { /* sentinel */ } 916 }; 917 918 static int xilinx_aes_aead_probe(struct platform_device *pdev) 919 { 920 struct xilinx_aead_alg *aead_algs; 921 struct device *dev = &pdev->dev; 922 int err; 923 int i; 924 925 /* Verify the hardware is present */ 926 aead_algs = xlnx_get_crypto_dev_data(aes_feature_map); 927 if (IS_ERR(aead_algs)) { 928 dev_err(dev, "AES is not supported on the platform\n"); 929 return PTR_ERR(aead_algs); 930 } 931 932 /* ZynqMP AES driver supports only one instance */ 933 if (aead_dev) 934 return -ENODEV; 935 936 aead_dev = devm_kzalloc(dev, sizeof(*aead_dev), GFP_KERNEL); 937 if (!aead_dev) 938 return -ENOMEM; 939 aead_dev->dev = dev; 940 aead_dev->aead_algs = aead_algs; 941 platform_set_drvdata(pdev, aead_dev); 942 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aead_algs[0].dma_bit_mask)); 943 if (err < 0) { 944 dev_err(dev, "No usable DMA configuration\n"); 945 return err; 946 } 947 948 aead_dev->engine = crypto_engine_alloc_init(dev, 1); 949 if (!aead_dev->engine) { 950 dev_err(dev, "Cannot alloc AES engine\n"); 951 return -ENOMEM; 952 } 953 954 err = crypto_engine_start(aead_dev->engine); 955 if (err) { 956 dev_err(dev, "Cannot start AES engine\n"); 957 goto err_engine_start; 958 } 959 960 for (i = 0; aead_dev->aead_algs[i].dma_bit_mask; i++) { 961 aead_dev->aead_algs[i].aead_dev = aead_dev; 962 err = crypto_engine_register_aead(&aead_dev->aead_algs[i].aead); 963 if (err < 0) { 964 dev_err(dev, "Failed to register AEAD alg %d.\n", i); 965 goto err_alg_register; 966 } 967 } 968 969 return 0; 970 971 err_alg_register: 972 while (i > 0) 973 crypto_engine_unregister_aead(&aead_dev->aead_algs[--i].aead); 974 err_engine_start: 975 crypto_engine_exit(aead_dev->engine); 976 977 return err; 978 } 979 980 static void xilinx_aes_aead_remove(struct platform_device *pdev) 981 { 982 aead_dev = platform_get_drvdata(pdev); 983 crypto_engine_exit(aead_dev->engine); 984 for (int i = 0; aead_dev->aead_algs[i].dma_bit_mask; i++) 985 crypto_engine_unregister_aead(&aead_dev->aead_algs[i].aead); 986 987 aead_dev = NULL; 988 } 989 990 static struct platform_driver xilinx_aes_driver = { 991 .probe = xilinx_aes_aead_probe, 992 .remove = xilinx_aes_aead_remove, 993 .driver = { 994 .name = "zynqmp-aes", 995 }, 996 }; 997 998 static struct platform_device *platform_dev; 999 1000 static int __init aes_driver_init(void) 1001 { 1002 int ret; 1003 1004 ret = platform_driver_register(&xilinx_aes_driver); 1005 if (ret) 1006 return ret; 1007 1008 platform_dev = platform_device_register_simple(xilinx_aes_driver.driver.name, 1009 0, NULL, 0); 1010 if (IS_ERR(platform_dev)) { 1011 ret = PTR_ERR(platform_dev); 1012 platform_driver_unregister(&xilinx_aes_driver); 1013 } 1014 1015 return ret; 1016 } 1017 1018 static void __exit aes_driver_exit(void) 1019 { 1020 platform_device_unregister(platform_dev); 1021 platform_driver_unregister(&xilinx_aes_driver); 1022 } 1023 1024 module_init(aes_driver_init); 1025 module_exit(aes_driver_exit); 1026 MODULE_DESCRIPTION("zynqmp aes-gcm hardware acceleration support."); 1027 MODULE_LICENSE("GPL"); 1028