1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 - 2021 4 * 5 * Richard van Schagen <vschagen@icloud.com> 6 * Christian Marangi <ansuelsmth@gmail.com 7 */ 8 9 #include <linux/atomic.h> 10 #include <linux/clk.h> 11 #include <linux/delay.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/interrupt.h> 14 #include <linux/module.h> 15 #include <linux/of.h> 16 #include <linux/platform_device.h> 17 #include <linux/spinlock.h> 18 #include <crypto/aes.h> 19 #include <crypto/ctr.h> 20 21 #include "eip93-main.h" 22 #include "eip93-regs.h" 23 #include "eip93-common.h" 24 #include "eip93-cipher.h" 25 #include "eip93-aes.h" 26 #include "eip93-des.h" 27 #include "eip93-aead.h" 28 #include "eip93-hash.h" 29 30 static struct eip93_alg_template *eip93_algs[] = { 31 &eip93_alg_ecb_des, 32 &eip93_alg_cbc_des, 33 &eip93_alg_ecb_des3_ede, 34 &eip93_alg_cbc_des3_ede, 35 &eip93_alg_ecb_aes, 36 &eip93_alg_cbc_aes, 37 &eip93_alg_ctr_aes, 38 &eip93_alg_rfc3686_aes, 39 &eip93_alg_authenc_hmac_md5_cbc_des, 40 &eip93_alg_authenc_hmac_sha1_cbc_des, 41 &eip93_alg_authenc_hmac_sha224_cbc_des, 42 &eip93_alg_authenc_hmac_sha256_cbc_des, 43 &eip93_alg_authenc_hmac_md5_cbc_des3_ede, 44 &eip93_alg_authenc_hmac_sha1_cbc_des3_ede, 45 &eip93_alg_authenc_hmac_sha224_cbc_des3_ede, 46 &eip93_alg_authenc_hmac_sha256_cbc_des3_ede, 47 &eip93_alg_authenc_hmac_md5_cbc_aes, 48 &eip93_alg_authenc_hmac_sha1_cbc_aes, 49 &eip93_alg_authenc_hmac_sha224_cbc_aes, 50 &eip93_alg_authenc_hmac_sha256_cbc_aes, 51 &eip93_alg_authenc_hmac_md5_rfc3686_aes, 52 &eip93_alg_authenc_hmac_sha1_rfc3686_aes, 53 &eip93_alg_authenc_hmac_sha224_rfc3686_aes, 54 &eip93_alg_authenc_hmac_sha256_rfc3686_aes, 55 &eip93_alg_md5, 56 &eip93_alg_sha1, 57 &eip93_alg_sha224, 58 &eip93_alg_sha256, 59 &eip93_alg_hmac_md5, 60 &eip93_alg_hmac_sha1, 61 &eip93_alg_hmac_sha224, 62 &eip93_alg_hmac_sha256, 63 }; 64 65 inline void eip93_irq_disable(struct eip93_device *eip93, u32 mask) 66 { 67 __raw_writel(mask, eip93->base + EIP93_REG_MASK_DISABLE); 68 } 69 70 inline void eip93_irq_enable(struct eip93_device *eip93, u32 mask) 71 { 72 __raw_writel(mask, eip93->base + EIP93_REG_MASK_ENABLE); 73 } 74 75 inline void eip93_irq_clear(struct eip93_device *eip93, u32 mask) 76 { 77 __raw_writel(mask, eip93->base + EIP93_REG_INT_CLR); 78 } 79 80 static void eip93_unregister_algs(unsigned int i) 81 { 82 unsigned int j; 83 84 for (j = 0; j < i; j++) { 85 switch (eip93_algs[j]->type) { 86 case EIP93_ALG_TYPE_SKCIPHER: 87 crypto_unregister_skcipher(&eip93_algs[j]->alg.skcipher); 88 break; 89 case EIP93_ALG_TYPE_AEAD: 90 crypto_unregister_aead(&eip93_algs[j]->alg.aead); 91 break; 92 case EIP93_ALG_TYPE_HASH: 93 crypto_unregister_ahash(&eip93_algs[i]->alg.ahash); 94 break; 95 } 96 } 97 } 98 99 static int eip93_register_algs(struct eip93_device *eip93, u32 supported_algo_flags) 100 { 101 unsigned int i; 102 int ret = 0; 103 104 for (i = 0; i < ARRAY_SIZE(eip93_algs); i++) { 105 u32 alg_flags = eip93_algs[i]->flags; 106 107 eip93_algs[i]->eip93 = eip93; 108 109 if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && 110 !(supported_algo_flags & EIP93_PE_OPTION_TDES)) 111 continue; 112 113 if (IS_AES(alg_flags)) { 114 if (!(supported_algo_flags & EIP93_PE_OPTION_AES)) 115 continue; 116 117 if (!IS_HMAC(alg_flags)) { 118 if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) 119 eip93_algs[i]->alg.skcipher.max_keysize = 120 AES_KEYSIZE_128; 121 122 if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) 123 eip93_algs[i]->alg.skcipher.max_keysize = 124 AES_KEYSIZE_192; 125 126 if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) 127 eip93_algs[i]->alg.skcipher.max_keysize = 128 AES_KEYSIZE_256; 129 130 if (IS_RFC3686(alg_flags)) 131 eip93_algs[i]->alg.skcipher.max_keysize += 132 CTR_RFC3686_NONCE_SIZE; 133 } 134 } 135 136 if (IS_HASH_MD5(alg_flags) && 137 !(supported_algo_flags & EIP93_PE_OPTION_MD5)) 138 continue; 139 140 if (IS_HASH_SHA1(alg_flags) && 141 !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) 142 continue; 143 144 if (IS_HASH_SHA224(alg_flags) && 145 !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) 146 continue; 147 148 if (IS_HASH_SHA256(alg_flags) && 149 !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) 150 continue; 151 152 switch (eip93_algs[i]->type) { 153 case EIP93_ALG_TYPE_SKCIPHER: 154 ret = crypto_register_skcipher(&eip93_algs[i]->alg.skcipher); 155 break; 156 case EIP93_ALG_TYPE_AEAD: 157 ret = crypto_register_aead(&eip93_algs[i]->alg.aead); 158 break; 159 case EIP93_ALG_TYPE_HASH: 160 ret = crypto_register_ahash(&eip93_algs[i]->alg.ahash); 161 break; 162 } 163 if (ret) 164 goto fail; 165 } 166 167 return 0; 168 169 fail: 170 eip93_unregister_algs(i); 171 172 return ret; 173 } 174 175 static void eip93_handle_result_descriptor(struct eip93_device *eip93) 176 { 177 struct crypto_async_request *async; 178 struct eip93_descriptor *rdesc; 179 u16 desc_flags, crypto_idr; 180 bool last_entry; 181 int handled, left, err; 182 u32 pe_ctrl_stat; 183 u32 pe_length; 184 185 get_more: 186 handled = 0; 187 188 left = readl(eip93->base + EIP93_REG_PE_RD_COUNT) & EIP93_PE_RD_COUNT; 189 190 if (!left) { 191 eip93_irq_clear(eip93, EIP93_INT_RDR_THRESH); 192 eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH); 193 return; 194 } 195 196 last_entry = false; 197 198 while (left) { 199 scoped_guard(spinlock_irqsave, &eip93->ring->read_lock) 200 rdesc = eip93_get_descriptor(eip93); 201 if (IS_ERR(rdesc)) { 202 dev_err(eip93->dev, "Ndesc: %d nreq: %d\n", 203 handled, left); 204 err = -EIO; 205 break; 206 } 207 /* make sure DMA is finished writing */ 208 do { 209 pe_ctrl_stat = READ_ONCE(rdesc->pe_ctrl_stat_word); 210 pe_length = READ_ONCE(rdesc->pe_length_word); 211 } while (FIELD_GET(EIP93_PE_CTRL_PE_READY_DES_TRING_OWN, pe_ctrl_stat) != 212 EIP93_PE_CTRL_PE_READY || 213 FIELD_GET(EIP93_PE_LENGTH_HOST_PE_READY, pe_length) != 214 EIP93_PE_LENGTH_PE_READY); 215 216 err = rdesc->pe_ctrl_stat_word & (EIP93_PE_CTRL_PE_EXT_ERR_CODE | 217 EIP93_PE_CTRL_PE_EXT_ERR | 218 EIP93_PE_CTRL_PE_SEQNUM_ERR | 219 EIP93_PE_CTRL_PE_PAD_ERR | 220 EIP93_PE_CTRL_PE_AUTH_ERR); 221 222 desc_flags = FIELD_GET(EIP93_PE_USER_ID_DESC_FLAGS, rdesc->user_id); 223 crypto_idr = FIELD_GET(EIP93_PE_USER_ID_CRYPTO_IDR, rdesc->user_id); 224 225 writel(1, eip93->base + EIP93_REG_PE_RD_COUNT); 226 eip93_irq_clear(eip93, EIP93_INT_RDR_THRESH); 227 228 handled++; 229 left--; 230 231 if (desc_flags & EIP93_DESC_LAST) { 232 last_entry = true; 233 break; 234 } 235 } 236 237 if (!last_entry) 238 goto get_more; 239 240 /* Get crypto async ref only for last descriptor */ 241 scoped_guard(spinlock_bh, &eip93->ring->idr_lock) { 242 async = idr_find(&eip93->ring->crypto_async_idr, crypto_idr); 243 idr_remove(&eip93->ring->crypto_async_idr, crypto_idr); 244 } 245 246 /* Parse error in ctrl stat word */ 247 err = eip93_parse_ctrl_stat_err(eip93, err); 248 249 if (desc_flags & EIP93_DESC_SKCIPHER) 250 eip93_skcipher_handle_result(async, err); 251 252 if (desc_flags & EIP93_DESC_AEAD) 253 eip93_aead_handle_result(async, err); 254 255 if (desc_flags & EIP93_DESC_HASH) 256 eip93_hash_handle_result(async, err); 257 258 goto get_more; 259 } 260 261 static void eip93_done_task(unsigned long data) 262 { 263 struct eip93_device *eip93 = (struct eip93_device *)data; 264 265 eip93_handle_result_descriptor(eip93); 266 } 267 268 static irqreturn_t eip93_irq_handler(int irq, void *data) 269 { 270 struct eip93_device *eip93 = data; 271 u32 irq_status; 272 273 irq_status = readl(eip93->base + EIP93_REG_INT_MASK_STAT); 274 if (FIELD_GET(EIP93_INT_RDR_THRESH, irq_status)) { 275 eip93_irq_disable(eip93, EIP93_INT_RDR_THRESH); 276 tasklet_schedule(&eip93->ring->done_task); 277 return IRQ_HANDLED; 278 } 279 280 /* Ignore errors in AUTO mode, handled by the RDR */ 281 eip93_irq_clear(eip93, irq_status); 282 if (irq_status) 283 eip93_irq_disable(eip93, irq_status); 284 285 return IRQ_NONE; 286 } 287 288 static void eip93_initialize(struct eip93_device *eip93, u32 supported_algo_flags) 289 { 290 u32 val; 291 292 /* Reset PE and rings */ 293 val = EIP93_PE_CONFIG_RST_PE | EIP93_PE_CONFIG_RST_RING; 294 val |= EIP93_PE_TARGET_AUTO_RING_MODE; 295 /* For Auto more, update the CDR ring owner after processing */ 296 val |= EIP93_PE_CONFIG_EN_CDR_UPDATE; 297 writel(val, eip93->base + EIP93_REG_PE_CONFIG); 298 299 /* Wait for PE and ring to reset */ 300 usleep_range(10, 20); 301 302 /* Release PE and ring reset */ 303 val = readl(eip93->base + EIP93_REG_PE_CONFIG); 304 val &= ~(EIP93_PE_CONFIG_RST_PE | EIP93_PE_CONFIG_RST_RING); 305 writel(val, eip93->base + EIP93_REG_PE_CONFIG); 306 307 /* Config Clocks */ 308 val = EIP93_PE_CLOCK_EN_PE_CLK; 309 if (supported_algo_flags & EIP93_PE_OPTION_TDES) 310 val |= EIP93_PE_CLOCK_EN_DES_CLK; 311 if (supported_algo_flags & EIP93_PE_OPTION_AES) 312 val |= EIP93_PE_CLOCK_EN_AES_CLK; 313 if (supported_algo_flags & 314 (EIP93_PE_OPTION_MD5 | EIP93_PE_OPTION_SHA_1 | EIP93_PE_OPTION_SHA_224 | 315 EIP93_PE_OPTION_SHA_256)) 316 val |= EIP93_PE_CLOCK_EN_HASH_CLK; 317 writel(val, eip93->base + EIP93_REG_PE_CLOCK_CTRL); 318 319 /* Config DMA thresholds */ 320 val = FIELD_PREP(EIP93_PE_OUTBUF_THRESH, 128) | 321 FIELD_PREP(EIP93_PE_INBUF_THRESH, 128); 322 writel(val, eip93->base + EIP93_REG_PE_BUF_THRESH); 323 324 /* Clear/ack all interrupts before disable all */ 325 eip93_irq_clear(eip93, EIP93_INT_ALL); 326 eip93_irq_disable(eip93, EIP93_INT_ALL); 327 328 /* Setup CRD threshold to trigger interrupt */ 329 val = FIELD_PREP(EIPR93_PE_CDR_THRESH, EIP93_RING_NUM - EIP93_RING_BUSY); 330 /* 331 * Configure RDR interrupt to be triggered if RD counter is not 0 332 * for more than 2^(N+10) system clocks. 333 */ 334 val |= FIELD_PREP(EIPR93_PE_RD_TIMEOUT, 5) | EIPR93_PE_TIMEROUT_EN; 335 writel(val, eip93->base + EIP93_REG_PE_RING_THRESH); 336 } 337 338 static void eip93_desc_free(struct eip93_device *eip93) 339 { 340 writel(0, eip93->base + EIP93_REG_PE_RING_CONFIG); 341 writel(0, eip93->base + EIP93_REG_PE_CDR_BASE); 342 writel(0, eip93->base + EIP93_REG_PE_RDR_BASE); 343 } 344 345 static int eip93_set_ring(struct eip93_device *eip93, struct eip93_desc_ring *ring) 346 { 347 ring->offset = sizeof(struct eip93_descriptor); 348 ring->base = dmam_alloc_coherent(eip93->dev, 349 sizeof(struct eip93_descriptor) * EIP93_RING_NUM, 350 &ring->base_dma, GFP_KERNEL); 351 if (!ring->base) 352 return -ENOMEM; 353 354 ring->write = ring->base; 355 ring->base_end = ring->base + sizeof(struct eip93_descriptor) * (EIP93_RING_NUM - 1); 356 ring->read = ring->base; 357 358 return 0; 359 } 360 361 static int eip93_desc_init(struct eip93_device *eip93) 362 { 363 struct eip93_desc_ring *cdr = &eip93->ring->cdr; 364 struct eip93_desc_ring *rdr = &eip93->ring->rdr; 365 int ret; 366 u32 val; 367 368 ret = eip93_set_ring(eip93, cdr); 369 if (ret) 370 return ret; 371 372 ret = eip93_set_ring(eip93, rdr); 373 if (ret) 374 return ret; 375 376 writel((u32 __force)cdr->base_dma, eip93->base + EIP93_REG_PE_CDR_BASE); 377 writel((u32 __force)rdr->base_dma, eip93->base + EIP93_REG_PE_RDR_BASE); 378 379 val = FIELD_PREP(EIP93_PE_RING_SIZE, EIP93_RING_NUM - 1); 380 writel(val, eip93->base + EIP93_REG_PE_RING_CONFIG); 381 382 return 0; 383 } 384 385 static void eip93_cleanup(struct eip93_device *eip93) 386 { 387 tasklet_kill(&eip93->ring->done_task); 388 389 /* Clear/ack all interrupts before disable all */ 390 eip93_irq_clear(eip93, EIP93_INT_ALL); 391 eip93_irq_disable(eip93, EIP93_INT_ALL); 392 393 writel(0, eip93->base + EIP93_REG_PE_CLOCK_CTRL); 394 395 eip93_desc_free(eip93); 396 397 idr_destroy(&eip93->ring->crypto_async_idr); 398 } 399 400 static int eip93_crypto_probe(struct platform_device *pdev) 401 { 402 struct device *dev = &pdev->dev; 403 struct eip93_device *eip93; 404 u32 ver, algo_flags; 405 int ret; 406 407 eip93 = devm_kzalloc(dev, sizeof(*eip93), GFP_KERNEL); 408 if (!eip93) 409 return -ENOMEM; 410 411 eip93->dev = dev; 412 platform_set_drvdata(pdev, eip93); 413 414 eip93->base = devm_platform_ioremap_resource(pdev, 0); 415 if (IS_ERR(eip93->base)) 416 return PTR_ERR(eip93->base); 417 418 eip93->irq = platform_get_irq(pdev, 0); 419 if (eip93->irq < 0) 420 return eip93->irq; 421 422 ret = devm_request_threaded_irq(eip93->dev, eip93->irq, eip93_irq_handler, 423 NULL, IRQF_ONESHOT, 424 dev_name(eip93->dev), eip93); 425 426 eip93->ring = devm_kcalloc(eip93->dev, 1, sizeof(*eip93->ring), GFP_KERNEL); 427 if (!eip93->ring) 428 return -ENOMEM; 429 430 ret = eip93_desc_init(eip93); 431 432 if (ret) 433 return ret; 434 435 tasklet_init(&eip93->ring->done_task, eip93_done_task, (unsigned long)eip93); 436 437 spin_lock_init(&eip93->ring->read_lock); 438 spin_lock_init(&eip93->ring->write_lock); 439 440 spin_lock_init(&eip93->ring->idr_lock); 441 idr_init(&eip93->ring->crypto_async_idr); 442 443 algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); 444 445 eip93_initialize(eip93, algo_flags); 446 447 /* Init finished, enable RDR interrupt */ 448 eip93_irq_enable(eip93, EIP93_INT_RDR_THRESH); 449 450 ret = eip93_register_algs(eip93, algo_flags); 451 if (ret) { 452 eip93_cleanup(eip93); 453 return ret; 454 } 455 456 ver = readl(eip93->base + EIP93_REG_PE_REVISION); 457 /* EIP_EIP_NO:MAJOR_HW_REV:MINOR_HW_REV:HW_PATCH,PE(ALGO_FLAGS) */ 458 dev_info(eip93->dev, "EIP%lu:%lx:%lx:%lx,PE(0x%x:0x%x)\n", 459 FIELD_GET(EIP93_PE_REVISION_EIP_NO, ver), 460 FIELD_GET(EIP93_PE_REVISION_MAJ_HW_REV, ver), 461 FIELD_GET(EIP93_PE_REVISION_MIN_HW_REV, ver), 462 FIELD_GET(EIP93_PE_REVISION_HW_PATCH, ver), 463 algo_flags, 464 readl(eip93->base + EIP93_REG_PE_OPTION_0)); 465 466 return 0; 467 } 468 469 static void eip93_crypto_remove(struct platform_device *pdev) 470 { 471 struct eip93_device *eip93 = platform_get_drvdata(pdev); 472 473 eip93_unregister_algs(ARRAY_SIZE(eip93_algs)); 474 eip93_cleanup(eip93); 475 } 476 477 static const struct of_device_id eip93_crypto_of_match[] = { 478 { .compatible = "inside-secure,safexcel-eip93i", }, 479 { .compatible = "inside-secure,safexcel-eip93ie", }, 480 { .compatible = "inside-secure,safexcel-eip93is", }, 481 { .compatible = "inside-secure,safexcel-eip93ies", }, 482 /* IW not supported currently, missing AES-XCB-MAC/AES-CCM */ 483 /* { .compatible = "inside-secure,safexcel-eip93iw", }, */ 484 {} 485 }; 486 MODULE_DEVICE_TABLE(of, eip93_crypto_of_match); 487 488 static struct platform_driver eip93_crypto_driver = { 489 .probe = eip93_crypto_probe, 490 .remove = eip93_crypto_remove, 491 .driver = { 492 .name = "inside-secure-eip93", 493 .of_match_table = eip93_crypto_of_match, 494 }, 495 }; 496 module_platform_driver(eip93_crypto_driver); 497 498 MODULE_AUTHOR("Richard van Schagen <vschagen@cs.com>"); 499 MODULE_AUTHOR("Christian Marangi <ansuelsmth@gmail.com>"); 500 MODULE_DESCRIPTION("Mediatek EIP-93 crypto engine driver"); 501 MODULE_LICENSE("GPL"); 502