1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 - 2021 4 * 5 * Richard van Schagen <vschagen@icloud.com> 6 * Christian Marangi <ansuelsmth@gmail.com 7 */ 8 9 #include <linux/atomic.h> 10 #include <linux/clk.h> 11 #include <linux/delay.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/interrupt.h> 14 #include <linux/module.h> 15 #include <linux/of.h> 16 #include <linux/platform_device.h> 17 #include <linux/spinlock.h> 18 #include <crypto/aes.h> 19 #include <crypto/ctr.h> 20 21 #include "eip93-main.h" 22 #include "eip93-regs.h" 23 #include "eip93-common.h" 24 #include "eip93-cipher.h" 25 #include "eip93-aes.h" 26 #include "eip93-des.h" 27 #include "eip93-aead.h" 28 #include "eip93-hash.h" 29 30 static struct eip93_alg_template *eip93_algs[] = { 31 &eip93_alg_ecb_des, 32 &eip93_alg_cbc_des, 33 &eip93_alg_ecb_des3_ede, 34 &eip93_alg_cbc_des3_ede, 35 &eip93_alg_ecb_aes, 36 &eip93_alg_cbc_aes, 37 &eip93_alg_ctr_aes, 38 &eip93_alg_rfc3686_aes, 39 &eip93_alg_authenc_hmac_md5_cbc_des, 40 &eip93_alg_authenc_hmac_sha1_cbc_des, 41 &eip93_alg_authenc_hmac_sha224_cbc_des, 42 &eip93_alg_authenc_hmac_sha256_cbc_des, 43 &eip93_alg_authenc_hmac_md5_cbc_des3_ede, 44 &eip93_alg_authenc_hmac_sha1_cbc_des3_ede, 45 &eip93_alg_authenc_hmac_sha224_cbc_des3_ede, 46 &eip93_alg_authenc_hmac_sha256_cbc_des3_ede, 47 &eip93_alg_authenc_hmac_md5_cbc_aes, 48 &eip93_alg_authenc_hmac_sha1_cbc_aes, 49 &eip93_alg_authenc_hmac_sha224_cbc_aes, 50 &eip93_alg_authenc_hmac_sha256_cbc_aes, 51 &eip93_alg_authenc_hmac_md5_rfc3686_aes, 52 &eip93_alg_authenc_hmac_sha1_rfc3686_aes, 53 &eip93_alg_authenc_hmac_sha224_rfc3686_aes, 54 &eip93_alg_authenc_hmac_sha256_rfc3686_aes, 55 &eip93_alg_md5, 56 &eip93_alg_sha1, 57 &eip93_alg_sha224, 58 &eip93_alg_sha256, 59 &eip93_alg_hmac_md5, 60 &eip93_alg_hmac_sha1, 61 &eip93_alg_hmac_sha224, 62 &eip93_alg_hmac_sha256, 63 }; 64 65 inline void eip93_irq_disable(struct eip93_device *eip93, u32 mask) 66 { 67 __raw_writel(mask, eip93->base + EIP93_REG_MASK_DISABLE); 68 } 69 70 inline void eip93_irq_enable(struct eip93_device *eip93, u32 mask) 71 { 72 __raw_writel(mask, eip93->base + EIP93_REG_MASK_ENABLE); 73 } 74 75 inline void eip93_irq_clear(struct eip93_device *eip93, u32 mask) 76 { 77 __raw_writel(mask, eip93->base + EIP93_REG_INT_CLR); 78 } 79 80 static int eip93_algo_is_supported(u32 alg_flags, u32 supported_algo_flags) 81 { 82 if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && 83 !(supported_algo_flags & EIP93_PE_OPTION_TDES)) 84 return 0; 85 86 if (IS_AES(alg_flags) && 87 !(supported_algo_flags & EIP93_PE_OPTION_AES)) 88 return 0; 89 90 if (IS_HASH_MD5(alg_flags) && 91 !(supported_algo_flags & EIP93_PE_OPTION_MD5)) 92 return 0; 93 94 if (IS_HASH_SHA1(alg_flags) && 95 !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) 96 return 0; 97 98 if (IS_HASH_SHA224(alg_flags) && 99 !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) 100 return 0; 101 102 if (IS_HASH_SHA256(alg_flags) && 103 !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) 104 return 0; 105 106 return 1; 107 } 108 109 static void eip93_unregister_algs(u32 supported_algo_flags, unsigned int i) 110 { 111 unsigned int j; 112 113 for (j = 0; j < i; j++) { 114 if (!eip93_algo_is_supported(eip93_algs[j]->flags, 115 supported_algo_flags)) 116 continue; 117 118 switch (eip93_algs[j]->type) { 119 case EIP93_ALG_TYPE_SKCIPHER: 120 crypto_unregister_skcipher(&eip93_algs[j]->alg.skcipher); 121 break; 122 case EIP93_ALG_TYPE_AEAD: 123 crypto_unregister_aead(&eip93_algs[j]->alg.aead); 124 break; 125 case EIP93_ALG_TYPE_HASH: 126 crypto_unregister_ahash(&eip93_algs[j]->alg.ahash); 127 break; 128 } 129 } 130 } 131 132 static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_flags) 133 { 134 unsigned int i; 135 int ret = 0; 136 137 for (i = 0; i < ARRAY_SIZE(eip93_algs); i++) { 138 u32 alg_flags = eip93_algs[i]->flags; 139 140 eip93_algs[i]->eip93 = eip93; 141 142 if (!eip93_algo_is_supported(alg_flags, supported_algo_flags)) 143 continue; 144 145 if (IS_AES(alg_flags) && !IS_HMAC(alg_flags)) { 146 if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) 147 eip93_algs[i]->alg.skcipher.max_keysize = 148 AES_KEYSIZE_128; 149 150 if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) 151 eip93_algs[i]->alg.skcipher.max_keysize = 152 AES_KEYSIZE_192; 153 154 if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) 155 eip93_algs[i]->alg.skcipher.max_keysize = 156 AES_KEYSIZE_256; 157 158 if (IS_RFC3686(alg_flags)) 159 eip93_algs[i]->alg.skcipher.max_keysize += 160 CTR_RFC3686_NONCE_SIZE; 161 } 162 163 switch (eip93_algs[i]->type) { 164 case EIP93_ALG_TYPE_SKCIPHER: 165 ret = crypto_register_skcipher(&eip93_algs[i]->alg.skcipher); 166 break; 167 case EIP93_ALG_TYPE_AEAD: 168 ret = crypto_register_aead(&eip93_algs[i]->alg.aead); 169 break; 170 case EIP93_ALG_TYPE_HASH: 171 ret = crypto_register_ahash(&eip93_algs[i]->alg.ahash); 172 break; 173 } 174 if (ret) 175 goto fail; 176 } 177 178 return 0; 179 180 fail: 181 eip93_unregister_algs(supported_algo_flags, i); 182 183 return ret; 184 } 185 186 static void eip93_handle_result_descriptor(struct eip93_device *eip93) 187 { 188 struct crypto_async_request *async; 189 struct eip93_descriptor *rdesc; 190 u16 desc_flags, crypto_idr; 191 bool last_entry; 192 int handled, left, err; 193 u32 pe_ctrl_stat; 194 u32 pe_length; 195 196 get_more: 197 handled = 0; 198 199 left = readl(eip93->base + EIP93_REG_PE_RD_COUNT) & EIP93_PE_RD_COUNT; 200 201 if (!left) { 202 eip93_irq_clear(eip93, EIP93_INT_RDR_THRESH); 203 eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH); 204 return; 205 } 206 207 last_entry = false; 208 209 while (left) { 210 scoped_guard(spinlock_irqsave, &eip93->ring->read_lock) 211 rdesc = eip93_get_descriptor(eip93); 212 if (IS_ERR(rdesc)) { 213 dev_err(eip93->dev, "Ndesc: %d nreq: %d\n", 214 handled, left); 215 err = -EIO; 216 break; 217 } 218 /* make sure DMA is finished writing */ 219 do { 220 pe_ctrl_stat = READ_ONCE(rdesc->pe_ctrl_stat_word); 221 pe_length = READ_ONCE(rdesc->pe_length_word); 222 } while (FIELD_GET(EIP93_PE_CTRL_PE_READY_DES_TRING_OWN, pe_ctrl_stat) != 223 EIP93_PE_CTRL_PE_READY || 224 FIELD_GET(EIP93_PE_LENGTH_HOST_PE_READY, pe_length) != 225 EIP93_PE_LENGTH_PE_READY); 226 227 err = rdesc->pe_ctrl_stat_word & (EIP93_PE_CTRL_PE_EXT_ERR_CODE | 228 EIP93_PE_CTRL_PE_EXT_ERR | 229 EIP93_PE_CTRL_PE_SEQNUM_ERR | 230 EIP93_PE_CTRL_PE_PAD_ERR | 231 EIP93_PE_CTRL_PE_AUTH_ERR); 232 233 desc_flags = FIELD_GET(EIP93_PE_USER_ID_DESC_FLAGS, rdesc->user_id); 234 crypto_idr = FIELD_GET(EIP93_PE_USER_ID_CRYPTO_IDR, rdesc->user_id); 235 236 writel(1, eip93->base + EIP93_REG_PE_RD_COUNT); 237 eip93_irq_clear(eip93, EIP93_INT_RDR_THRESH); 238 239 handled++; 240 left--; 241 242 if (desc_flags & EIP93_DESC_LAST) { 243 last_entry = true; 244 break; 245 } 246 } 247 248 if (!last_entry) 249 goto get_more; 250 251 /* Get crypto async ref only for last descriptor */ 252 scoped_guard(spinlock_bh, &eip93->ring->idr_lock) { 253 async = idr_find(&eip93->ring->crypto_async_idr, crypto_idr); 254 idr_remove(&eip93->ring->crypto_async_idr, crypto_idr); 255 } 256 257 /* Parse error in ctrl stat word */ 258 err = eip93_parse_ctrl_stat_err(eip93, err); 259 260 if (desc_flags & EIP93_DESC_SKCIPHER) 261 eip93_skcipher_handle_result(async, err); 262 263 if (desc_flags & EIP93_DESC_AEAD) 264 eip93_aead_handle_result(async, err); 265 266 if (desc_flags & EIP93_DESC_HASH) 267 eip93_hash_handle_result(async, err); 268 269 goto get_more; 270 } 271 272 static void eip93_done_task(unsigned long data) 273 { 274 struct eip93_device *eip93 = (struct eip93_device *)data; 275 276 eip93_handle_result_descriptor(eip93); 277 } 278 279 static irqreturn_t eip93_irq_handler(int irq, void *data) 280 { 281 struct eip93_device *eip93 = data; 282 u32 irq_status; 283 284 irq_status = readl(eip93->base + EIP93_REG_INT_MASK_STAT); 285 if (FIELD_GET(EIP93_INT_RDR_THRESH, irq_status)) { 286 eip93_irq_disable(eip93, EIP93_INT_RDR_THRESH); 287 tasklet_schedule(&eip93->ring->done_task); 288 return IRQ_HANDLED; 289 } 290 291 /* Ignore errors in AUTO mode, handled by the RDR */ 292 eip93_irq_clear(eip93, irq_status); 293 if (irq_status) 294 eip93_irq_disable(eip93, irq_status); 295 296 return IRQ_NONE; 297 } 298 299 static void eip93_initialize(struct eip93_device *eip93, u32 supported_algo_flags) 300 { 301 u32 val; 302 303 /* Reset PE and rings */ 304 val = EIP93_PE_CONFIG_RST_PE | EIP93_PE_CONFIG_RST_RING; 305 val |= EIP93_PE_TARGET_AUTO_RING_MODE; 306 /* For Auto more, update the CDR ring owner after processing */ 307 val |= EIP93_PE_CONFIG_EN_CDR_UPDATE; 308 writel(val, eip93->base + EIP93_REG_PE_CONFIG); 309 310 /* Wait for PE and ring to reset */ 311 usleep_range(10, 20); 312 313 /* Release PE and ring reset */ 314 val = readl(eip93->base + EIP93_REG_PE_CONFIG); 315 val &= ~(EIP93_PE_CONFIG_RST_PE | EIP93_PE_CONFIG_RST_RING); 316 writel(val, eip93->base + EIP93_REG_PE_CONFIG); 317 318 /* Config Clocks */ 319 val = EIP93_PE_CLOCK_EN_PE_CLK; 320 if (supported_algo_flags & EIP93_PE_OPTION_TDES) 321 val |= EIP93_PE_CLOCK_EN_DES_CLK; 322 if (supported_algo_flags & EIP93_PE_OPTION_AES) 323 val |= EIP93_PE_CLOCK_EN_AES_CLK; 324 if (supported_algo_flags & 325 (EIP93_PE_OPTION_MD5 | EIP93_PE_OPTION_SHA_1 | EIP93_PE_OPTION_SHA_224 | 326 EIP93_PE_OPTION_SHA_256)) 327 val |= EIP93_PE_CLOCK_EN_HASH_CLK; 328 writel(val, eip93->base + EIP93_REG_PE_CLOCK_CTRL); 329 330 /* Config DMA thresholds */ 331 val = FIELD_PREP(EIP93_PE_OUTBUF_THRESH, 128) | 332 FIELD_PREP(EIP93_PE_INBUF_THRESH, 128); 333 writel(val, eip93->base + EIP93_REG_PE_BUF_THRESH); 334 335 /* Clear/ack all interrupts before disable all */ 336 eip93_irq_clear(eip93, EIP93_INT_ALL); 337 eip93_irq_disable(eip93, EIP93_INT_ALL); 338 339 /* Setup CRD threshold to trigger interrupt */ 340 val = FIELD_PREP(EIPR93_PE_CDR_THRESH, EIP93_RING_NUM - EIP93_RING_BUSY); 341 /* 342 * Configure RDR interrupt to be triggered if RD counter is not 0 343 * for more than 2^(N+10) system clocks. 344 */ 345 val |= FIELD_PREP(EIPR93_PE_RD_TIMEOUT, 5) | EIPR93_PE_TIMEROUT_EN; 346 writel(val, eip93->base + EIP93_REG_PE_RING_THRESH); 347 } 348 349 static void eip93_desc_free(struct eip93_device *eip93) 350 { 351 writel(0, eip93->base + EIP93_REG_PE_RING_CONFIG); 352 writel(0, eip93->base + EIP93_REG_PE_CDR_BASE); 353 writel(0, eip93->base + EIP93_REG_PE_RDR_BASE); 354 } 355 356 static int eip93_set_ring(struct eip93_device *eip93, struct eip93_desc_ring *ring) 357 { 358 ring->offset = sizeof(struct eip93_descriptor); 359 ring->base = dmam_alloc_coherent(eip93->dev, 360 sizeof(struct eip93_descriptor) * EIP93_RING_NUM, 361 &ring->base_dma, GFP_KERNEL); 362 if (!ring->base) 363 return -ENOMEM; 364 365 ring->write = ring->base; 366 ring->base_end = ring->base + sizeof(struct eip93_descriptor) * (EIP93_RING_NUM - 1); 367 ring->read = ring->base; 368 369 return 0; 370 } 371 372 static int eip93_desc_init(struct eip93_device *eip93) 373 { 374 struct eip93_desc_ring *cdr = &eip93->ring->cdr; 375 struct eip93_desc_ring *rdr = &eip93->ring->rdr; 376 int ret; 377 u32 val; 378 379 ret = eip93_set_ring(eip93, cdr); 380 if (ret) 381 return ret; 382 383 ret = eip93_set_ring(eip93, rdr); 384 if (ret) 385 return ret; 386 387 writel((u32 __force)cdr->base_dma, eip93->base + EIP93_REG_PE_CDR_BASE); 388 writel((u32 __force)rdr->base_dma, eip93->base + EIP93_REG_PE_RDR_BASE); 389 390 val = FIELD_PREP(EIP93_PE_RING_SIZE, EIP93_RING_NUM - 1); 391 writel(val, eip93->base + EIP93_REG_PE_RING_CONFIG); 392 393 return 0; 394 } 395 396 static void eip93_cleanup(struct eip93_device *eip93) 397 { 398 tasklet_kill(&eip93->ring->done_task); 399 400 /* Clear/ack all interrupts before disable all */ 401 eip93_irq_clear(eip93, EIP93_INT_ALL); 402 eip93_irq_disable(eip93, EIP93_INT_ALL); 403 404 writel(0, eip93->base + EIP93_REG_PE_CLOCK_CTRL); 405 406 eip93_desc_free(eip93); 407 408 idr_destroy(&eip93->ring->crypto_async_idr); 409 } 410 411 static int eip93_crypto_probe(struct platform_device *pdev) 412 { 413 struct device *dev = &pdev->dev; 414 struct eip93_device *eip93; 415 u32 ver, algo_flags; 416 int ret; 417 418 eip93 = devm_kzalloc(dev, sizeof(*eip93), GFP_KERNEL); 419 if (!eip93) 420 return -ENOMEM; 421 422 eip93->dev = dev; 423 platform_set_drvdata(pdev, eip93); 424 425 eip93->base = devm_platform_ioremap_resource(pdev, 0); 426 if (IS_ERR(eip93->base)) 427 return PTR_ERR(eip93->base); 428 429 eip93->irq = platform_get_irq(pdev, 0); 430 if (eip93->irq < 0) 431 return eip93->irq; 432 433 ret = devm_request_threaded_irq(eip93->dev, eip93->irq, eip93_irq_handler, 434 NULL, IRQF_ONESHOT, 435 dev_name(eip93->dev), eip93); 436 437 eip93->ring = devm_kcalloc(eip93->dev, 1, sizeof(*eip93->ring), GFP_KERNEL); 438 if (!eip93->ring) 439 return -ENOMEM; 440 441 ret = eip93_desc_init(eip93); 442 443 if (ret) 444 return ret; 445 446 tasklet_init(&eip93->ring->done_task, eip93_done_task, (unsigned long)eip93); 447 448 spin_lock_init(&eip93->ring->read_lock); 449 spin_lock_init(&eip93->ring->write_lock); 450 451 spin_lock_init(&eip93->ring->idr_lock); 452 idr_init(&eip93->ring->crypto_async_idr); 453 454 algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); 455 456 eip93_initialize(eip93, algo_flags); 457 458 /* Init finished, enable RDR interrupt */ 459 eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH); 460 461 ret = eip93_register_algs(eip93, algo_flags); 462 if (ret) { 463 eip93_cleanup(eip93); 464 return ret; 465 } 466 467 ver = readl(eip93->base + EIP93_REG_PE_REVISION); 468 /* EIP_EIP_NO:MAJOR_HW_REV:MINOR_HW_REV:HW_PATCH,PE(ALGO_FLAGS) */ 469 dev_info(eip93->dev, "EIP%lu:%lx:%lx:%lx,PE(0x%x:0x%x)\n", 470 FIELD_GET(EIP93_PE_REVISION_EIP_NO, ver), 471 FIELD_GET(EIP93_PE_REVISION_MAJ_HW_REV, ver), 472 FIELD_GET(EIP93_PE_REVISION_MIN_HW_REV, ver), 473 FIELD_GET(EIP93_PE_REVISION_HW_PATCH, ver), 474 algo_flags, 475 readl(eip93->base + EIP93_REG_PE_OPTION_0)); 476 477 return 0; 478 } 479 480 static void eip93_crypto_remove(struct platform_device *pdev) 481 { 482 struct eip93_device *eip93 = platform_get_drvdata(pdev); 483 u32 algo_flags; 484 485 algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); 486 487 eip93_unregister_algs(algo_flags, ARRAY_SIZE(eip93_algs)); 488 eip93_cleanup(eip93); 489 } 490 491 static const struct of_device_id eip93_crypto_of_match[] = { 492 { .compatible = "inside-secure,safexcel-eip93i", }, 493 { .compatible = "inside-secure,safexcel-eip93ie", }, 494 { .compatible = "inside-secure,safexcel-eip93is", }, 495 { .compatible = "inside-secure,safexcel-eip93ies", }, 496 /* IW not supported currently, missing AES-XCB-MAC/AES-CCM */ 497 /* { .compatible = "inside-secure,safexcel-eip93iw", }, */ 498 {} 499 }; 500 MODULE_DEVICE_TABLE(of, eip93_crypto_of_match); 501 502 static struct platform_driver eip93_crypto_driver = { 503 .probe = eip93_crypto_probe, 504 .remove = eip93_crypto_remove, 505 .driver = { 506 .name = "inside-secure-eip93", 507 .of_match_table = eip93_crypto_of_match, 508 }, 509 }; 510 module_platform_driver(eip93_crypto_driver); 511 512 MODULE_AUTHOR("Richard van Schagen <vschagen@cs.com>"); 513 MODULE_AUTHOR("Christian Marangi <ansuelsmth@gmail.com>"); 514 MODULE_DESCRIPTION("Mediatek EIP-93 crypto engine driver"); 515 MODULE_LICENSE("GPL"); 516