1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2015-2021, 2023 Linaro Limited 4 * Copyright (c) 2016, EPAM Systems 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/arm-smccc.h> 10 #include <linux/cpuhotplug.h> 11 #include <linux/errno.h> 12 #include <linux/firmware.h> 13 #include <linux/interrupt.h> 14 #include <linux/io.h> 15 #include <linux/irqdomain.h> 16 #include <linux/kernel.h> 17 #include <linux/mm.h> 18 #include <linux/module.h> 19 #include <linux/of.h> 20 #include <linux/of_irq.h> 21 #include <linux/of_platform.h> 22 #include <linux/platform_device.h> 23 #include <linux/rpmb.h> 24 #include <linux/sched.h> 25 #include <linux/slab.h> 26 #include <linux/string.h> 27 #include <linux/tee_core.h> 28 #include <linux/types.h> 29 #include <linux/workqueue.h> 30 #include "optee_private.h" 31 #include "optee_smc.h" 32 #include "optee_rpc_cmd.h" 33 #include <linux/kmemleak.h> 34 #define CREATE_TRACE_POINTS 35 #include "optee_trace.h" 36 37 /* 38 * This file implement the SMC ABI used when communicating with secure world 39 * OP-TEE OS via raw SMCs. 40 * This file is divided into the following sections: 41 * 1. Convert between struct tee_param and struct optee_msg_param 42 * 2. Low level support functions to register shared memory in secure world 43 * 3. Dynamic shared memory pool based on alloc_pages() 44 * 4. Do a normal scheduled call into secure world 45 * 5. Asynchronous notification 46 * 6. Driver initialization. 47 */ 48 49 /* 50 * A typical OP-TEE private shm allocation is 224 bytes (argument struct 51 * with 6 parameters, needed for open session). So with an alignment of 512 52 * we'll waste a bit more than 50%. However, it's only expected that we'll 53 * have a handful of these structs allocated at a time. Most memory will 54 * be allocated aligned to the page size, So all in all this should scale 55 * up and down quite well. 56 */ 57 #define OPTEE_MIN_STATIC_POOL_ALIGN 9 /* 512 bytes aligned */ 58 59 /* SMC ABI considers at most a single TEE firmware */ 60 static unsigned int pcpu_irq_num; 61 62 static int optee_cpuhp_enable_pcpu_irq(unsigned int cpu) 63 { 64 enable_percpu_irq(pcpu_irq_num, IRQ_TYPE_NONE); 65 66 return 0; 67 } 68 69 static int optee_cpuhp_disable_pcpu_irq(unsigned int cpu) 70 { 71 disable_percpu_irq(pcpu_irq_num); 72 73 return 0; 74 } 75 76 /* 77 * 1. Convert between struct tee_param and struct optee_msg_param 78 * 79 * optee_from_msg_param() and optee_to_msg_param() are the main 80 * functions. 81 */ 82 83 static int from_msg_param_tmp_mem(struct tee_param *p, u32 attr, 84 const struct optee_msg_param *mp) 85 { 86 struct tee_shm *shm; 87 phys_addr_t pa; 88 int rc; 89 90 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT + 91 attr - OPTEE_MSG_ATTR_TYPE_TMEM_INPUT; 92 p->u.memref.size = mp->u.tmem.size; 93 shm = (struct tee_shm *)(unsigned long)mp->u.tmem.shm_ref; 94 if (!shm) { 95 p->u.memref.shm_offs = 0; 96 p->u.memref.shm = NULL; 97 return 0; 98 } 99 100 rc = tee_shm_get_pa(shm, 0, &pa); 101 if (rc) 102 return rc; 103 104 p->u.memref.shm_offs = mp->u.tmem.buf_ptr - pa; 105 p->u.memref.shm = shm; 106 107 return 0; 108 } 109 110 static void from_msg_param_reg_mem(struct tee_param *p, u32 attr, 111 const struct optee_msg_param *mp) 112 { 113 struct tee_shm *shm; 114 115 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT + 116 attr - OPTEE_MSG_ATTR_TYPE_RMEM_INPUT; 117 p->u.memref.size = mp->u.rmem.size; 118 shm = (struct tee_shm *)(unsigned long)mp->u.rmem.shm_ref; 119 120 if (shm) { 121 p->u.memref.shm_offs = mp->u.rmem.offs; 122 p->u.memref.shm = shm; 123 } else { 124 p->u.memref.shm_offs = 0; 125 p->u.memref.shm = NULL; 126 } 127 } 128 129 /** 130 * optee_from_msg_param() - convert from OPTEE_MSG parameters to 131 * struct tee_param 132 * @optee: main service struct 133 * @params: subsystem internal parameter representation 134 * @num_params: number of elements in the parameter arrays 135 * @msg_params: OPTEE_MSG parameters 136 * Returns 0 on success or <0 on failure 137 */ 138 static int optee_from_msg_param(struct optee *optee, struct tee_param *params, 139 size_t num_params, 140 const struct optee_msg_param *msg_params) 141 { 142 int rc; 143 size_t n; 144 145 for (n = 0; n < num_params; n++) { 146 struct tee_param *p = params + n; 147 const struct optee_msg_param *mp = msg_params + n; 148 u32 attr = mp->attr & OPTEE_MSG_ATTR_TYPE_MASK; 149 150 switch (attr) { 151 case OPTEE_MSG_ATTR_TYPE_NONE: 152 p->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE; 153 memset(&p->u, 0, sizeof(p->u)); 154 break; 155 case OPTEE_MSG_ATTR_TYPE_VALUE_INPUT: 156 case OPTEE_MSG_ATTR_TYPE_VALUE_OUTPUT: 157 case OPTEE_MSG_ATTR_TYPE_VALUE_INOUT: 158 optee_from_msg_param_value(p, attr, mp); 159 break; 160 case OPTEE_MSG_ATTR_TYPE_TMEM_INPUT: 161 case OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT: 162 case OPTEE_MSG_ATTR_TYPE_TMEM_INOUT: 163 rc = from_msg_param_tmp_mem(p, attr, mp); 164 if (rc) 165 return rc; 166 break; 167 case OPTEE_MSG_ATTR_TYPE_RMEM_INPUT: 168 case OPTEE_MSG_ATTR_TYPE_RMEM_OUTPUT: 169 case OPTEE_MSG_ATTR_TYPE_RMEM_INOUT: 170 from_msg_param_reg_mem(p, attr, mp); 171 break; 172 173 default: 174 return -EINVAL; 175 } 176 } 177 return 0; 178 } 179 180 static int to_msg_param_tmp_mem(struct optee_msg_param *mp, 181 const struct tee_param *p) 182 { 183 int rc; 184 phys_addr_t pa; 185 186 mp->attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT + p->attr - 187 TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT; 188 189 mp->u.tmem.shm_ref = (unsigned long)p->u.memref.shm; 190 mp->u.tmem.size = p->u.memref.size; 191 192 if (!p->u.memref.shm) { 193 mp->u.tmem.buf_ptr = 0; 194 return 0; 195 } 196 197 rc = tee_shm_get_pa(p->u.memref.shm, p->u.memref.shm_offs, &pa); 198 if (rc) 199 return rc; 200 201 mp->u.tmem.buf_ptr = pa; 202 mp->attr |= OPTEE_MSG_ATTR_CACHE_PREDEFINED << 203 OPTEE_MSG_ATTR_CACHE_SHIFT; 204 205 return 0; 206 } 207 208 static int to_msg_param_reg_mem(struct optee_msg_param *mp, 209 const struct tee_param *p) 210 { 211 mp->attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT + p->attr - 212 TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT; 213 214 mp->u.rmem.shm_ref = (unsigned long)p->u.memref.shm; 215 mp->u.rmem.size = p->u.memref.size; 216 mp->u.rmem.offs = p->u.memref.shm_offs; 217 return 0; 218 } 219 220 /** 221 * optee_to_msg_param() - convert from struct tee_params to OPTEE_MSG parameters 222 * @optee: main service struct 223 * @msg_params: OPTEE_MSG parameters 224 * @num_params: number of elements in the parameter arrays 225 * @params: subsystem itnernal parameter representation 226 * Returns 0 on success or <0 on failure 227 */ 228 static int optee_to_msg_param(struct optee *optee, 229 struct optee_msg_param *msg_params, 230 size_t num_params, const struct tee_param *params) 231 { 232 int rc; 233 size_t n; 234 235 for (n = 0; n < num_params; n++) { 236 const struct tee_param *p = params + n; 237 struct optee_msg_param *mp = msg_params + n; 238 239 switch (p->attr) { 240 case TEE_IOCTL_PARAM_ATTR_TYPE_NONE: 241 mp->attr = TEE_IOCTL_PARAM_ATTR_TYPE_NONE; 242 memset(&mp->u, 0, sizeof(mp->u)); 243 break; 244 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INPUT: 245 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_OUTPUT: 246 case TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT: 247 optee_to_msg_param_value(mp, p); 248 break; 249 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INPUT: 250 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_OUTPUT: 251 case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: 252 if (tee_shm_is_dynamic(p->u.memref.shm)) 253 rc = to_msg_param_reg_mem(mp, p); 254 else 255 rc = to_msg_param_tmp_mem(mp, p); 256 if (rc) 257 return rc; 258 break; 259 default: 260 return -EINVAL; 261 } 262 } 263 return 0; 264 } 265 266 /* 267 * 2. Low level support functions to register shared memory in secure world 268 * 269 * Functions to enable/disable shared memory caching in secure world, that 270 * is, lazy freeing of previously allocated shared memory. Freeing is 271 * performed when a request has been compled. 272 * 273 * Functions to register and unregister shared memory both for normal 274 * clients and for tee-supplicant. 275 */ 276 277 /** 278 * optee_enable_shm_cache() - Enables caching of some shared memory allocation 279 * in OP-TEE 280 * @optee: main service struct 281 */ 282 static void optee_enable_shm_cache(struct optee *optee) 283 { 284 struct optee_call_waiter w; 285 286 /* We need to retry until secure world isn't busy. */ 287 optee_cq_wait_init(&optee->call_queue, &w, false); 288 while (true) { 289 struct arm_smccc_res res; 290 291 optee->smc.invoke_fn(OPTEE_SMC_ENABLE_SHM_CACHE, 292 0, 0, 0, 0, 0, 0, 0, &res); 293 if (res.a0 == OPTEE_SMC_RETURN_OK) 294 break; 295 optee_cq_wait_for_completion(&optee->call_queue, &w); 296 } 297 optee_cq_wait_final(&optee->call_queue, &w); 298 } 299 300 /** 301 * __optee_disable_shm_cache() - Disables caching of some shared memory 302 * allocation in OP-TEE 303 * @optee: main service struct 304 * @is_mapped: true if the cached shared memory addresses were mapped by this 305 * kernel, are safe to dereference, and should be freed 306 */ 307 static void __optee_disable_shm_cache(struct optee *optee, bool is_mapped) 308 { 309 struct optee_call_waiter w; 310 311 /* We need to retry until secure world isn't busy. */ 312 optee_cq_wait_init(&optee->call_queue, &w, false); 313 while (true) { 314 union { 315 struct arm_smccc_res smccc; 316 struct optee_smc_disable_shm_cache_result result; 317 } res; 318 319 optee->smc.invoke_fn(OPTEE_SMC_DISABLE_SHM_CACHE, 320 0, 0, 0, 0, 0, 0, 0, &res.smccc); 321 if (res.result.status == OPTEE_SMC_RETURN_ENOTAVAIL) 322 break; /* All shm's freed */ 323 if (res.result.status == OPTEE_SMC_RETURN_OK) { 324 struct tee_shm *shm; 325 326 /* 327 * Shared memory references that were not mapped by 328 * this kernel must be ignored to prevent a crash. 329 */ 330 if (!is_mapped) 331 continue; 332 333 shm = reg_pair_to_ptr(res.result.shm_upper32, 334 res.result.shm_lower32); 335 tee_shm_free(shm); 336 } else { 337 optee_cq_wait_for_completion(&optee->call_queue, &w); 338 } 339 } 340 optee_cq_wait_final(&optee->call_queue, &w); 341 } 342 343 /** 344 * optee_disable_shm_cache() - Disables caching of mapped shared memory 345 * allocations in OP-TEE 346 * @optee: main service struct 347 */ 348 static void optee_disable_shm_cache(struct optee *optee) 349 { 350 return __optee_disable_shm_cache(optee, true); 351 } 352 353 /** 354 * optee_disable_unmapped_shm_cache() - Disables caching of shared memory 355 * allocations in OP-TEE which are not 356 * currently mapped 357 * @optee: main service struct 358 */ 359 static void optee_disable_unmapped_shm_cache(struct optee *optee) 360 { 361 return __optee_disable_shm_cache(optee, false); 362 } 363 364 #define PAGELIST_ENTRIES_PER_PAGE \ 365 ((OPTEE_MSG_NONCONTIG_PAGE_SIZE / sizeof(u64)) - 1) 366 367 /* 368 * The final entry in each pagelist page is a pointer to the next 369 * pagelist page. 370 */ 371 static size_t get_pages_list_size(size_t num_entries) 372 { 373 int pages = DIV_ROUND_UP(num_entries, PAGELIST_ENTRIES_PER_PAGE); 374 375 return pages * OPTEE_MSG_NONCONTIG_PAGE_SIZE; 376 } 377 378 static u64 *optee_allocate_pages_list(size_t num_entries) 379 { 380 return alloc_pages_exact(get_pages_list_size(num_entries), GFP_KERNEL); 381 } 382 383 static void optee_free_pages_list(void *list, size_t num_entries) 384 { 385 free_pages_exact(list, get_pages_list_size(num_entries)); 386 } 387 388 /** 389 * optee_fill_pages_list() - write list of user pages to given shared 390 * buffer. 391 * 392 * @dst: page-aligned buffer where list of pages will be stored 393 * @pages: array of pages that represents shared buffer 394 * @num_pages: number of entries in @pages 395 * @page_offset: offset of user buffer from page start 396 * 397 * @dst should be big enough to hold list of user page addresses and 398 * links to the next pages of buffer 399 */ 400 static void optee_fill_pages_list(u64 *dst, struct page **pages, int num_pages, 401 size_t page_offset) 402 { 403 int n = 0; 404 phys_addr_t optee_page; 405 /* 406 * Refer to OPTEE_MSG_ATTR_NONCONTIG description in optee_msg.h 407 * for details. 408 */ 409 struct { 410 u64 pages_list[PAGELIST_ENTRIES_PER_PAGE]; 411 u64 next_page_data; 412 } *pages_data; 413 414 /* 415 * Currently OP-TEE uses 4k page size and it does not looks 416 * like this will change in the future. On other hand, there are 417 * no know ARM architectures with page size < 4k. 418 * Thus the next built assert looks redundant. But the following 419 * code heavily relies on this assumption, so it is better be 420 * safe than sorry. 421 */ 422 BUILD_BUG_ON(PAGE_SIZE < OPTEE_MSG_NONCONTIG_PAGE_SIZE); 423 424 pages_data = (void *)dst; 425 /* 426 * If linux page is bigger than 4k, and user buffer offset is 427 * larger than 4k/8k/12k/etc this will skip first 4k pages, 428 * because they bear no value data for OP-TEE. 429 */ 430 optee_page = page_to_phys(*pages) + 431 round_down(page_offset, OPTEE_MSG_NONCONTIG_PAGE_SIZE); 432 433 while (true) { 434 pages_data->pages_list[n++] = optee_page; 435 436 if (n == PAGELIST_ENTRIES_PER_PAGE) { 437 pages_data->next_page_data = 438 virt_to_phys(pages_data + 1); 439 pages_data++; 440 n = 0; 441 } 442 443 optee_page += OPTEE_MSG_NONCONTIG_PAGE_SIZE; 444 if (!(optee_page & ~PAGE_MASK)) { 445 if (!--num_pages) 446 break; 447 pages++; 448 optee_page = page_to_phys(*pages); 449 } 450 } 451 } 452 453 static int optee_shm_register(struct tee_context *ctx, struct tee_shm *shm, 454 struct page **pages, size_t num_pages, 455 unsigned long start) 456 { 457 struct optee *optee = tee_get_drvdata(ctx->teedev); 458 struct optee_msg_arg *msg_arg; 459 struct tee_shm *shm_arg; 460 u64 *pages_list; 461 size_t sz; 462 int rc; 463 464 if (!num_pages) 465 return -EINVAL; 466 467 rc = optee_check_mem_type(start, num_pages); 468 if (rc) 469 return rc; 470 471 pages_list = optee_allocate_pages_list(num_pages); 472 if (!pages_list) 473 return -ENOMEM; 474 475 /* 476 * We're about to register shared memory we can't register shared 477 * memory for this request or there's a catch-22. 478 * 479 * So in this we'll have to do the good old temporary private 480 * allocation instead of using optee_get_msg_arg(). 481 */ 482 sz = optee_msg_arg_size(optee->rpc_param_count); 483 shm_arg = tee_shm_alloc_priv_buf(ctx, sz); 484 if (IS_ERR(shm_arg)) { 485 rc = PTR_ERR(shm_arg); 486 goto out; 487 } 488 msg_arg = tee_shm_get_va(shm_arg, 0); 489 if (IS_ERR(msg_arg)) { 490 rc = PTR_ERR(msg_arg); 491 goto out; 492 } 493 494 optee_fill_pages_list(pages_list, pages, num_pages, 495 tee_shm_get_page_offset(shm)); 496 497 memset(msg_arg, 0, OPTEE_MSG_GET_ARG_SIZE(1)); 498 msg_arg->num_params = 1; 499 msg_arg->cmd = OPTEE_MSG_CMD_REGISTER_SHM; 500 msg_arg->params->attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT | 501 OPTEE_MSG_ATTR_NONCONTIG; 502 msg_arg->params->u.tmem.shm_ref = (unsigned long)shm; 503 msg_arg->params->u.tmem.size = tee_shm_get_size(shm); 504 /* 505 * In the least bits of msg_arg->params->u.tmem.buf_ptr we 506 * store buffer offset from 4k page, as described in OP-TEE ABI. 507 */ 508 msg_arg->params->u.tmem.buf_ptr = virt_to_phys(pages_list) | 509 (tee_shm_get_page_offset(shm) & (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1)); 510 511 if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) || 512 msg_arg->ret != TEEC_SUCCESS) 513 rc = -EINVAL; 514 515 tee_shm_free(shm_arg); 516 out: 517 optee_free_pages_list(pages_list, num_pages); 518 return rc; 519 } 520 521 static int optee_shm_unregister(struct tee_context *ctx, struct tee_shm *shm) 522 { 523 struct optee *optee = tee_get_drvdata(ctx->teedev); 524 struct optee_msg_arg *msg_arg; 525 struct tee_shm *shm_arg; 526 int rc = 0; 527 size_t sz; 528 529 /* 530 * We're about to unregister shared memory and we may not be able 531 * register shared memory for this request in case we're called 532 * from optee_shm_arg_cache_uninit(). 533 * 534 * So in order to keep things simple in this function just as in 535 * optee_shm_register() we'll use temporary private allocation 536 * instead of using optee_get_msg_arg(). 537 */ 538 sz = optee_msg_arg_size(optee->rpc_param_count); 539 shm_arg = tee_shm_alloc_priv_buf(ctx, sz); 540 if (IS_ERR(shm_arg)) 541 return PTR_ERR(shm_arg); 542 msg_arg = tee_shm_get_va(shm_arg, 0); 543 if (IS_ERR(msg_arg)) { 544 rc = PTR_ERR(msg_arg); 545 goto out; 546 } 547 548 memset(msg_arg, 0, sz); 549 msg_arg->num_params = 1; 550 msg_arg->cmd = OPTEE_MSG_CMD_UNREGISTER_SHM; 551 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT; 552 msg_arg->params[0].u.rmem.shm_ref = (unsigned long)shm; 553 554 if (optee->ops->do_call_with_arg(ctx, shm_arg, 0, false) || 555 msg_arg->ret != TEEC_SUCCESS) 556 rc = -EINVAL; 557 out: 558 tee_shm_free(shm_arg); 559 return rc; 560 } 561 562 static int optee_shm_register_supp(struct tee_context *ctx, struct tee_shm *shm, 563 struct page **pages, size_t num_pages, 564 unsigned long start) 565 { 566 /* 567 * We don't want to register supplicant memory in OP-TEE. 568 * Instead information about it will be passed in RPC code. 569 */ 570 return optee_check_mem_type(start, num_pages); 571 } 572 573 static int optee_shm_unregister_supp(struct tee_context *ctx, 574 struct tee_shm *shm) 575 { 576 return 0; 577 } 578 579 /* 580 * 3. Dynamic shared memory pool based on alloc_pages() 581 * 582 * Implements an OP-TEE specific shared memory pool which is used 583 * when dynamic shared memory is supported by secure world. 584 * 585 * The main function is optee_shm_pool_alloc_pages(). 586 */ 587 588 static int pool_op_alloc(struct tee_shm_pool *pool, 589 struct tee_shm *shm, size_t size, size_t align) 590 { 591 /* 592 * Shared memory private to the OP-TEE driver doesn't need 593 * to be registered with OP-TEE. 594 */ 595 if (shm->flags & TEE_SHM_PRIV) 596 return tee_dyn_shm_alloc_helper(shm, size, align, NULL); 597 598 return tee_dyn_shm_alloc_helper(shm, size, align, optee_shm_register); 599 } 600 601 static void pool_op_free(struct tee_shm_pool *pool, 602 struct tee_shm *shm) 603 { 604 if (!(shm->flags & TEE_SHM_PRIV)) 605 tee_dyn_shm_free_helper(shm, optee_shm_unregister); 606 else 607 tee_dyn_shm_free_helper(shm, NULL); 608 } 609 610 static void pool_op_destroy_pool(struct tee_shm_pool *pool) 611 { 612 kfree(pool); 613 } 614 615 static const struct tee_shm_pool_ops pool_ops = { 616 .alloc = pool_op_alloc, 617 .free = pool_op_free, 618 .destroy_pool = pool_op_destroy_pool, 619 }; 620 621 /** 622 * optee_shm_pool_alloc_pages() - create page-based allocator pool 623 * 624 * This pool is used when OP-TEE supports dymanic SHM. In this case 625 * command buffers and such are allocated from kernel's own memory. 626 */ 627 static struct tee_shm_pool *optee_shm_pool_alloc_pages(void) 628 { 629 struct tee_shm_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL); 630 631 if (!pool) 632 return ERR_PTR(-ENOMEM); 633 634 pool->ops = &pool_ops; 635 636 return pool; 637 } 638 639 /* 640 * 4. Do a normal scheduled call into secure world 641 * 642 * The function optee_smc_do_call_with_arg() performs a normal scheduled 643 * call into secure world. During this call may normal world request help 644 * from normal world using RPCs, Remote Procedure Calls. This includes 645 * delivery of non-secure interrupts to for instance allow rescheduling of 646 * the current task. 647 */ 648 649 static void handle_rpc_func_cmd_shm_free(struct tee_context *ctx, 650 struct optee_msg_arg *arg) 651 { 652 struct tee_shm *shm; 653 654 arg->ret_origin = TEEC_ORIGIN_COMMS; 655 656 if (arg->num_params != 1 || 657 arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) { 658 arg->ret = TEEC_ERROR_BAD_PARAMETERS; 659 return; 660 } 661 662 shm = (struct tee_shm *)(unsigned long)arg->params[0].u.value.b; 663 switch (arg->params[0].u.value.a) { 664 case OPTEE_RPC_SHM_TYPE_APPL: 665 optee_rpc_cmd_free_suppl(ctx, shm); 666 break; 667 case OPTEE_RPC_SHM_TYPE_KERNEL: 668 tee_shm_free(shm); 669 break; 670 default: 671 arg->ret = TEEC_ERROR_BAD_PARAMETERS; 672 } 673 arg->ret = TEEC_SUCCESS; 674 } 675 676 static void handle_rpc_func_cmd_shm_alloc(struct tee_context *ctx, 677 struct optee *optee, 678 struct optee_msg_arg *arg, 679 struct optee_call_ctx *call_ctx) 680 { 681 struct tee_shm *shm; 682 size_t sz; 683 size_t n; 684 struct page **pages; 685 size_t page_count; 686 687 arg->ret_origin = TEEC_ORIGIN_COMMS; 688 689 if (!arg->num_params || 690 arg->params[0].attr != OPTEE_MSG_ATTR_TYPE_VALUE_INPUT) { 691 arg->ret = TEEC_ERROR_BAD_PARAMETERS; 692 return; 693 } 694 695 for (n = 1; n < arg->num_params; n++) { 696 if (arg->params[n].attr != OPTEE_MSG_ATTR_TYPE_NONE) { 697 arg->ret = TEEC_ERROR_BAD_PARAMETERS; 698 return; 699 } 700 } 701 702 sz = arg->params[0].u.value.b; 703 switch (arg->params[0].u.value.a) { 704 case OPTEE_RPC_SHM_TYPE_APPL: 705 shm = optee_rpc_cmd_alloc_suppl(ctx, sz); 706 break; 707 case OPTEE_RPC_SHM_TYPE_KERNEL: 708 shm = tee_shm_alloc_priv_buf(optee->ctx, sz); 709 break; 710 default: 711 arg->ret = TEEC_ERROR_BAD_PARAMETERS; 712 return; 713 } 714 715 if (IS_ERR(shm)) { 716 arg->ret = TEEC_ERROR_OUT_OF_MEMORY; 717 return; 718 } 719 720 /* 721 * If there are pages it's dynamically allocated shared memory (not 722 * from the reserved shared memory pool) and needs to be 723 * registered. 724 */ 725 pages = tee_shm_get_pages(shm, &page_count); 726 if (pages) { 727 u64 *pages_list; 728 729 pages_list = optee_allocate_pages_list(page_count); 730 if (!pages_list) { 731 arg->ret = TEEC_ERROR_OUT_OF_MEMORY; 732 goto bad; 733 } 734 735 call_ctx->pages_list = pages_list; 736 call_ctx->num_entries = page_count; 737 738 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT | 739 OPTEE_MSG_ATTR_NONCONTIG; 740 /* 741 * In the least bits of u.tmem.buf_ptr we store buffer offset 742 * from 4k page, as described in OP-TEE ABI. 743 */ 744 arg->params[0].u.tmem.buf_ptr = virt_to_phys(pages_list) | 745 (tee_shm_get_page_offset(shm) & 746 (OPTEE_MSG_NONCONTIG_PAGE_SIZE - 1)); 747 748 optee_fill_pages_list(pages_list, pages, page_count, 749 tee_shm_get_page_offset(shm)); 750 } else { 751 phys_addr_t pa; 752 753 if (tee_shm_get_pa(shm, 0, &pa)) { 754 arg->ret = TEEC_ERROR_BAD_PARAMETERS; 755 goto bad; 756 } 757 758 arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_TMEM_OUTPUT; 759 arg->params[0].u.tmem.buf_ptr = pa; 760 } 761 arg->params[0].u.tmem.size = tee_shm_get_size(shm); 762 arg->params[0].u.tmem.shm_ref = (unsigned long)shm; 763 764 arg->ret = TEEC_SUCCESS; 765 return; 766 bad: 767 tee_shm_free(shm); 768 } 769 770 static void free_pages_list(struct optee_call_ctx *call_ctx) 771 { 772 if (call_ctx->pages_list) { 773 optee_free_pages_list(call_ctx->pages_list, 774 call_ctx->num_entries); 775 call_ctx->pages_list = NULL; 776 call_ctx->num_entries = 0; 777 } 778 } 779 780 static void optee_rpc_finalize_call(struct optee_call_ctx *call_ctx) 781 { 782 free_pages_list(call_ctx); 783 } 784 785 static void handle_rpc_func_cmd(struct tee_context *ctx, struct optee *optee, 786 struct optee_msg_arg *arg, 787 struct optee_call_ctx *call_ctx) 788 { 789 790 switch (arg->cmd) { 791 case OPTEE_RPC_CMD_SHM_ALLOC: 792 free_pages_list(call_ctx); 793 handle_rpc_func_cmd_shm_alloc(ctx, optee, arg, call_ctx); 794 break; 795 case OPTEE_RPC_CMD_SHM_FREE: 796 handle_rpc_func_cmd_shm_free(ctx, arg); 797 break; 798 default: 799 optee_rpc_cmd(ctx, optee, arg); 800 } 801 } 802 803 /** 804 * optee_handle_rpc() - handle RPC from secure world 805 * @ctx: context doing the RPC 806 * @rpc_arg: pointer to RPC arguments if any, or NULL if none 807 * @param: value of registers for the RPC 808 * @call_ctx: call context. Preserved during one OP-TEE invocation 809 * 810 * Result of RPC is written back into @param. 811 */ 812 static void optee_handle_rpc(struct tee_context *ctx, 813 struct optee_msg_arg *rpc_arg, 814 struct optee_rpc_param *param, 815 struct optee_call_ctx *call_ctx) 816 { 817 struct tee_device *teedev = ctx->teedev; 818 struct optee *optee = tee_get_drvdata(teedev); 819 struct optee_msg_arg *arg; 820 struct tee_shm *shm; 821 phys_addr_t pa; 822 823 switch (OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)) { 824 case OPTEE_SMC_RPC_FUNC_ALLOC: 825 shm = tee_shm_alloc_priv_buf(optee->ctx, param->a1); 826 if (!IS_ERR(shm) && !tee_shm_get_pa(shm, 0, &pa)) { 827 reg_pair_from_64(¶m->a1, ¶m->a2, pa); 828 reg_pair_from_64(¶m->a4, ¶m->a5, 829 (unsigned long)shm); 830 } else { 831 param->a1 = 0; 832 param->a2 = 0; 833 param->a4 = 0; 834 param->a5 = 0; 835 } 836 kmemleak_not_leak(shm); 837 break; 838 case OPTEE_SMC_RPC_FUNC_FREE: 839 shm = reg_pair_to_ptr(param->a1, param->a2); 840 tee_shm_free(shm); 841 break; 842 case OPTEE_SMC_RPC_FUNC_FOREIGN_INTR: 843 /* 844 * A foreign interrupt was raised while secure world was 845 * executing, since they are handled in Linux a dummy RPC is 846 * performed to let Linux take the interrupt through the normal 847 * vector. 848 */ 849 break; 850 case OPTEE_SMC_RPC_FUNC_CMD: 851 if (rpc_arg) { 852 arg = rpc_arg; 853 } else { 854 shm = reg_pair_to_ptr(param->a1, param->a2); 855 arg = tee_shm_get_va(shm, 0); 856 if (IS_ERR(arg)) { 857 pr_err("%s: tee_shm_get_va %p failed\n", 858 __func__, shm); 859 break; 860 } 861 } 862 863 handle_rpc_func_cmd(ctx, optee, arg, call_ctx); 864 break; 865 default: 866 pr_warn("Unknown RPC func 0x%x\n", 867 (u32)OPTEE_SMC_RETURN_GET_RPC_FUNC(param->a0)); 868 break; 869 } 870 871 param->a0 = OPTEE_SMC_CALL_RETURN_FROM_RPC; 872 } 873 874 /** 875 * optee_smc_do_call_with_arg() - Do an SMC to OP-TEE in secure world 876 * @ctx: calling context 877 * @shm: shared memory holding the message to pass to secure world 878 * @offs: offset of the message in @shm 879 * @system_thread: true if caller requests TEE system thread support 880 * 881 * Does and SMC to OP-TEE in secure world and handles eventual resulting 882 * Remote Procedure Calls (RPC) from OP-TEE. 883 * 884 * Returns return code from secure world, 0 is OK 885 */ 886 static int optee_smc_do_call_with_arg(struct tee_context *ctx, 887 struct tee_shm *shm, u_int offs, 888 bool system_thread) 889 { 890 struct optee *optee = tee_get_drvdata(ctx->teedev); 891 struct optee_call_waiter w; 892 struct optee_rpc_param param = { }; 893 struct optee_call_ctx call_ctx = { }; 894 struct optee_msg_arg *rpc_arg = NULL; 895 int rc; 896 897 if (optee->rpc_param_count) { 898 struct optee_msg_arg *arg; 899 unsigned int rpc_arg_offs; 900 901 arg = tee_shm_get_va(shm, offs); 902 if (IS_ERR(arg)) 903 return PTR_ERR(arg); 904 905 rpc_arg_offs = OPTEE_MSG_GET_ARG_SIZE(arg->num_params); 906 rpc_arg = tee_shm_get_va(shm, offs + rpc_arg_offs); 907 if (IS_ERR(rpc_arg)) 908 return PTR_ERR(rpc_arg); 909 } 910 911 if (rpc_arg && tee_shm_is_dynamic(shm)) { 912 param.a0 = OPTEE_SMC_CALL_WITH_REGD_ARG; 913 reg_pair_from_64(¶m.a1, ¶m.a2, (u_long)shm); 914 param.a3 = offs; 915 } else { 916 phys_addr_t parg; 917 918 rc = tee_shm_get_pa(shm, offs, &parg); 919 if (rc) 920 return rc; 921 922 if (rpc_arg) 923 param.a0 = OPTEE_SMC_CALL_WITH_RPC_ARG; 924 else 925 param.a0 = OPTEE_SMC_CALL_WITH_ARG; 926 reg_pair_from_64(¶m.a1, ¶m.a2, parg); 927 } 928 /* Initialize waiter */ 929 optee_cq_wait_init(&optee->call_queue, &w, system_thread); 930 while (true) { 931 struct arm_smccc_res res; 932 933 trace_optee_invoke_fn_begin(¶m); 934 optee->smc.invoke_fn(param.a0, param.a1, param.a2, param.a3, 935 param.a4, param.a5, param.a6, param.a7, 936 &res); 937 trace_optee_invoke_fn_end(¶m, &res); 938 939 if (res.a0 == OPTEE_SMC_RETURN_ETHREAD_LIMIT) { 940 /* 941 * Out of threads in secure world, wait for a thread 942 * become available. 943 */ 944 optee_cq_wait_for_completion(&optee->call_queue, &w); 945 } else if (OPTEE_SMC_RETURN_IS_RPC(res.a0)) { 946 cond_resched(); 947 param.a0 = res.a0; 948 param.a1 = res.a1; 949 param.a2 = res.a2; 950 param.a3 = res.a3; 951 optee_handle_rpc(ctx, rpc_arg, ¶m, &call_ctx); 952 } else { 953 rc = res.a0; 954 break; 955 } 956 } 957 958 optee_rpc_finalize_call(&call_ctx); 959 /* 960 * We're done with our thread in secure world, if there's any 961 * thread waiters wake up one. 962 */ 963 optee_cq_wait_final(&optee->call_queue, &w); 964 965 return rc; 966 } 967 968 static int optee_smc_lend_protmem(struct optee *optee, struct tee_shm *protmem, 969 u32 *mem_attrs, unsigned int ma_count, 970 u32 use_case) 971 { 972 struct optee_shm_arg_entry *entry; 973 struct optee_msg_arg *msg_arg; 974 struct tee_shm *shm; 975 u_int offs; 976 int rc; 977 978 msg_arg = optee_get_msg_arg(optee->ctx, 2, &entry, &shm, &offs); 979 if (IS_ERR(msg_arg)) 980 return PTR_ERR(msg_arg); 981 982 msg_arg->cmd = OPTEE_MSG_CMD_LEND_PROTMEM; 983 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_VALUE_INPUT; 984 msg_arg->params[0].u.value.a = use_case; 985 msg_arg->params[1].attr = OPTEE_MSG_ATTR_TYPE_TMEM_INPUT; 986 msg_arg->params[1].u.tmem.buf_ptr = protmem->paddr; 987 msg_arg->params[1].u.tmem.size = protmem->size; 988 msg_arg->params[1].u.tmem.shm_ref = (u_long)protmem; 989 990 rc = optee->ops->do_call_with_arg(optee->ctx, shm, offs, false); 991 if (rc) 992 goto out; 993 if (msg_arg->ret != TEEC_SUCCESS) { 994 rc = -EINVAL; 995 goto out; 996 } 997 protmem->sec_world_id = (u_long)protmem; 998 999 out: 1000 optee_free_msg_arg(optee->ctx, entry, offs); 1001 return rc; 1002 } 1003 1004 static int optee_smc_reclaim_protmem(struct optee *optee, 1005 struct tee_shm *protmem) 1006 { 1007 struct optee_shm_arg_entry *entry; 1008 struct optee_msg_arg *msg_arg; 1009 struct tee_shm *shm; 1010 u_int offs; 1011 int rc; 1012 1013 msg_arg = optee_get_msg_arg(optee->ctx, 1, &entry, &shm, &offs); 1014 if (IS_ERR(msg_arg)) 1015 return PTR_ERR(msg_arg); 1016 1017 msg_arg->cmd = OPTEE_MSG_CMD_RECLAIM_PROTMEM; 1018 msg_arg->params[0].attr = OPTEE_MSG_ATTR_TYPE_RMEM_INPUT; 1019 msg_arg->params[0].u.rmem.shm_ref = (u_long)protmem; 1020 1021 rc = optee->ops->do_call_with_arg(optee->ctx, shm, offs, false); 1022 if (rc) 1023 goto out; 1024 if (msg_arg->ret != TEEC_SUCCESS) 1025 rc = -EINVAL; 1026 1027 out: 1028 optee_free_msg_arg(optee->ctx, entry, offs); 1029 return rc; 1030 } 1031 1032 /* 1033 * 5. Asynchronous notification 1034 */ 1035 1036 static u32 get_async_notif_value(optee_invoke_fn *invoke_fn, bool *value_valid, 1037 bool *value_pending) 1038 { 1039 struct arm_smccc_res res; 1040 1041 invoke_fn(OPTEE_SMC_GET_ASYNC_NOTIF_VALUE, 0, 0, 0, 0, 0, 0, 0, &res); 1042 1043 if (res.a0) { 1044 *value_valid = false; 1045 return 0; 1046 } 1047 *value_valid = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_VALID); 1048 *value_pending = (res.a2 & OPTEE_SMC_ASYNC_NOTIF_VALUE_PENDING); 1049 return res.a1; 1050 } 1051 1052 static irqreturn_t irq_handler(struct optee *optee) 1053 { 1054 bool do_bottom_half = false; 1055 bool value_valid; 1056 bool value_pending; 1057 u32 value; 1058 1059 do { 1060 value = get_async_notif_value(optee->smc.invoke_fn, 1061 &value_valid, &value_pending); 1062 if (!value_valid) 1063 break; 1064 1065 if (value == OPTEE_SMC_ASYNC_NOTIF_VALUE_DO_BOTTOM_HALF) 1066 do_bottom_half = true; 1067 else 1068 optee_notif_send(optee, value); 1069 } while (value_pending); 1070 1071 if (do_bottom_half) 1072 return IRQ_WAKE_THREAD; 1073 return IRQ_HANDLED; 1074 } 1075 1076 static irqreturn_t notif_irq_handler(int irq, void *dev_id) 1077 { 1078 struct optee *optee = dev_id; 1079 1080 return irq_handler(optee); 1081 } 1082 1083 static irqreturn_t notif_irq_thread_fn(int irq, void *dev_id) 1084 { 1085 struct optee *optee = dev_id; 1086 1087 optee_do_bottom_half(optee->ctx); 1088 1089 return IRQ_HANDLED; 1090 } 1091 1092 static int init_irq(struct optee *optee, u_int irq) 1093 { 1094 int rc; 1095 1096 rc = request_threaded_irq(irq, notif_irq_handler, 1097 notif_irq_thread_fn, 1098 0, "optee_notification", optee); 1099 if (rc) 1100 return rc; 1101 1102 optee->smc.notif_irq = irq; 1103 1104 return 0; 1105 } 1106 1107 static irqreturn_t notif_pcpu_irq_handler(int irq, void *dev_id) 1108 { 1109 struct optee_pcpu *pcpu = dev_id; 1110 struct optee *optee = pcpu->optee; 1111 1112 if (irq_handler(optee) == IRQ_WAKE_THREAD) 1113 queue_work(optee->smc.notif_pcpu_wq, 1114 &optee->smc.notif_pcpu_work); 1115 1116 return IRQ_HANDLED; 1117 } 1118 1119 static void notif_pcpu_irq_work_fn(struct work_struct *work) 1120 { 1121 struct optee_smc *optee_smc = container_of(work, struct optee_smc, 1122 notif_pcpu_work); 1123 struct optee *optee = container_of(optee_smc, struct optee, smc); 1124 1125 optee_do_bottom_half(optee->ctx); 1126 } 1127 1128 static int init_pcpu_irq(struct optee *optee, u_int irq) 1129 { 1130 struct optee_pcpu __percpu *optee_pcpu; 1131 int cpu, rc; 1132 1133 optee_pcpu = alloc_percpu(struct optee_pcpu); 1134 if (!optee_pcpu) 1135 return -ENOMEM; 1136 1137 for_each_present_cpu(cpu) 1138 per_cpu_ptr(optee_pcpu, cpu)->optee = optee; 1139 1140 rc = request_percpu_irq(irq, notif_pcpu_irq_handler, 1141 "optee_pcpu_notification", optee_pcpu); 1142 if (rc) 1143 goto err_free_pcpu; 1144 1145 INIT_WORK(&optee->smc.notif_pcpu_work, notif_pcpu_irq_work_fn); 1146 optee->smc.notif_pcpu_wq = create_workqueue("optee_pcpu_notification"); 1147 if (!optee->smc.notif_pcpu_wq) { 1148 rc = -EINVAL; 1149 goto err_free_pcpu_irq; 1150 } 1151 1152 optee->smc.optee_pcpu = optee_pcpu; 1153 optee->smc.notif_irq = irq; 1154 1155 pcpu_irq_num = irq; 1156 rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee/pcpu-notif:starting", 1157 optee_cpuhp_enable_pcpu_irq, 1158 optee_cpuhp_disable_pcpu_irq); 1159 if (!rc) 1160 rc = -EINVAL; 1161 if (rc < 0) 1162 goto err_free_pcpu_irq; 1163 1164 optee->smc.notif_cpuhp_state = rc; 1165 1166 return 0; 1167 1168 err_free_pcpu_irq: 1169 free_percpu_irq(irq, optee_pcpu); 1170 err_free_pcpu: 1171 free_percpu(optee_pcpu); 1172 1173 return rc; 1174 } 1175 1176 static int optee_smc_notif_init_irq(struct optee *optee, u_int irq) 1177 { 1178 if (irq_is_percpu_devid(irq)) 1179 return init_pcpu_irq(optee, irq); 1180 else 1181 return init_irq(optee, irq); 1182 } 1183 1184 static void uninit_pcpu_irq(struct optee *optee) 1185 { 1186 cpuhp_remove_state(optee->smc.notif_cpuhp_state); 1187 1188 destroy_workqueue(optee->smc.notif_pcpu_wq); 1189 1190 free_percpu_irq(optee->smc.notif_irq, optee->smc.optee_pcpu); 1191 free_percpu(optee->smc.optee_pcpu); 1192 } 1193 1194 static void optee_smc_notif_uninit_irq(struct optee *optee) 1195 { 1196 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) { 1197 optee_stop_async_notif(optee->ctx); 1198 if (optee->smc.notif_irq) { 1199 if (irq_is_percpu_devid(optee->smc.notif_irq)) 1200 uninit_pcpu_irq(optee); 1201 else 1202 free_irq(optee->smc.notif_irq, optee); 1203 1204 irq_dispose_mapping(optee->smc.notif_irq); 1205 } 1206 } 1207 } 1208 1209 /* 1210 * 6. Driver initialization 1211 * 1212 * During driver initialization is secure world probed to find out which 1213 * features it supports so the driver can be initialized with a matching 1214 * configuration. This involves for instance support for dynamic shared 1215 * memory instead of a static memory carvout. 1216 */ 1217 1218 static void optee_get_version(struct tee_device *teedev, 1219 struct tee_ioctl_version_data *vers) 1220 { 1221 struct tee_ioctl_version_data v = { 1222 .impl_id = TEE_IMPL_ID_OPTEE, 1223 .impl_caps = TEE_OPTEE_CAP_TZ, 1224 .gen_caps = TEE_GEN_CAP_GP, 1225 }; 1226 struct optee *optee = tee_get_drvdata(teedev); 1227 1228 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) 1229 v.gen_caps |= TEE_GEN_CAP_REG_MEM; 1230 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL) 1231 v.gen_caps |= TEE_GEN_CAP_MEMREF_NULL; 1232 *vers = v; 1233 } 1234 1235 static int optee_smc_open(struct tee_context *ctx) 1236 { 1237 struct optee *optee = tee_get_drvdata(ctx->teedev); 1238 u32 sec_caps = optee->smc.sec_caps; 1239 1240 return optee_open(ctx, sec_caps & OPTEE_SMC_SEC_CAP_MEMREF_NULL); 1241 } 1242 1243 static const struct tee_driver_ops optee_clnt_ops = { 1244 .get_version = optee_get_version, 1245 .open = optee_smc_open, 1246 .release = optee_release, 1247 .open_session = optee_open_session, 1248 .close_session = optee_close_session, 1249 .system_session = optee_system_session, 1250 .invoke_func = optee_invoke_func, 1251 .cancel_req = optee_cancel_req, 1252 .shm_register = optee_shm_register, 1253 .shm_unregister = optee_shm_unregister, 1254 }; 1255 1256 static const struct tee_desc optee_clnt_desc = { 1257 .name = DRIVER_NAME "-clnt", 1258 .ops = &optee_clnt_ops, 1259 .owner = THIS_MODULE, 1260 }; 1261 1262 static const struct tee_driver_ops optee_supp_ops = { 1263 .get_version = optee_get_version, 1264 .open = optee_smc_open, 1265 .release = optee_release_supp, 1266 .supp_recv = optee_supp_recv, 1267 .supp_send = optee_supp_send, 1268 .shm_register = optee_shm_register_supp, 1269 .shm_unregister = optee_shm_unregister_supp, 1270 }; 1271 1272 static const struct tee_desc optee_supp_desc = { 1273 .name = DRIVER_NAME "-supp", 1274 .ops = &optee_supp_ops, 1275 .owner = THIS_MODULE, 1276 .flags = TEE_DESC_PRIVILEGED, 1277 }; 1278 1279 static const struct optee_ops optee_ops = { 1280 .do_call_with_arg = optee_smc_do_call_with_arg, 1281 .to_msg_param = optee_to_msg_param, 1282 .from_msg_param = optee_from_msg_param, 1283 .lend_protmem = optee_smc_lend_protmem, 1284 .reclaim_protmem = optee_smc_reclaim_protmem, 1285 }; 1286 1287 static int enable_async_notif(optee_invoke_fn *invoke_fn) 1288 { 1289 struct arm_smccc_res res; 1290 1291 invoke_fn(OPTEE_SMC_ENABLE_ASYNC_NOTIF, 0, 0, 0, 0, 0, 0, 0, &res); 1292 1293 if (res.a0) 1294 return -EINVAL; 1295 return 0; 1296 } 1297 1298 static bool optee_msg_api_uid_is_optee_api(optee_invoke_fn *invoke_fn) 1299 { 1300 struct arm_smccc_res res; 1301 1302 invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res); 1303 1304 if (res.a0 == OPTEE_MSG_UID_0 && res.a1 == OPTEE_MSG_UID_1 && 1305 res.a2 == OPTEE_MSG_UID_2 && res.a3 == OPTEE_MSG_UID_3) 1306 return true; 1307 return false; 1308 } 1309 1310 #ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE 1311 static bool optee_msg_api_uid_is_optee_image_load(optee_invoke_fn *invoke_fn) 1312 { 1313 struct arm_smccc_res res; 1314 1315 invoke_fn(OPTEE_SMC_CALLS_UID, 0, 0, 0, 0, 0, 0, 0, &res); 1316 1317 if (res.a0 == OPTEE_MSG_IMAGE_LOAD_UID_0 && 1318 res.a1 == OPTEE_MSG_IMAGE_LOAD_UID_1 && 1319 res.a2 == OPTEE_MSG_IMAGE_LOAD_UID_2 && 1320 res.a3 == OPTEE_MSG_IMAGE_LOAD_UID_3) 1321 return true; 1322 return false; 1323 } 1324 #endif 1325 1326 static void optee_msg_get_os_revision(optee_invoke_fn *invoke_fn) 1327 { 1328 union { 1329 struct arm_smccc_res smccc; 1330 struct optee_smc_call_get_os_revision_result result; 1331 } res = { 1332 .result = { 1333 .build_id = 0 1334 } 1335 }; 1336 1337 invoke_fn(OPTEE_SMC_CALL_GET_OS_REVISION, 0, 0, 0, 0, 0, 0, 0, 1338 &res.smccc); 1339 1340 if (res.result.build_id) 1341 pr_info("revision %lu.%lu (%0*lx)", res.result.major, 1342 res.result.minor, (int)sizeof(res.result.build_id) * 2, 1343 res.result.build_id); 1344 else 1345 pr_info("revision %lu.%lu", res.result.major, res.result.minor); 1346 } 1347 1348 static bool optee_msg_api_revision_is_compatible(optee_invoke_fn *invoke_fn) 1349 { 1350 union { 1351 struct arm_smccc_res smccc; 1352 struct optee_smc_calls_revision_result result; 1353 } res; 1354 1355 invoke_fn(OPTEE_SMC_CALLS_REVISION, 0, 0, 0, 0, 0, 0, 0, &res.smccc); 1356 1357 if (res.result.major == OPTEE_MSG_REVISION_MAJOR && 1358 (int)res.result.minor >= OPTEE_MSG_REVISION_MINOR) 1359 return true; 1360 return false; 1361 } 1362 1363 static bool optee_msg_exchange_capabilities(optee_invoke_fn *invoke_fn, 1364 u32 *sec_caps, u32 *max_notif_value, 1365 unsigned int *rpc_param_count) 1366 { 1367 union { 1368 struct arm_smccc_res smccc; 1369 struct optee_smc_exchange_capabilities_result result; 1370 } res; 1371 u32 a1 = 0; 1372 1373 /* 1374 * TODO This isn't enough to tell if it's UP system (from kernel 1375 * point of view) or not, is_smp() returns the information 1376 * needed, but can't be called directly from here. 1377 */ 1378 if (!IS_ENABLED(CONFIG_SMP) || nr_cpu_ids == 1) 1379 a1 |= OPTEE_SMC_NSEC_CAP_UNIPROCESSOR; 1380 1381 invoke_fn(OPTEE_SMC_EXCHANGE_CAPABILITIES, a1, 0, 0, 0, 0, 0, 0, 1382 &res.smccc); 1383 1384 if (res.result.status != OPTEE_SMC_RETURN_OK) 1385 return false; 1386 1387 *sec_caps = res.result.capabilities; 1388 if (*sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) 1389 *max_notif_value = res.result.max_notif_value; 1390 else 1391 *max_notif_value = OPTEE_DEFAULT_MAX_NOTIF_VALUE; 1392 if (*sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG) 1393 *rpc_param_count = (u8)res.result.data; 1394 else 1395 *rpc_param_count = 0; 1396 1397 return true; 1398 } 1399 1400 static unsigned int optee_msg_get_thread_count(optee_invoke_fn *invoke_fn) 1401 { 1402 struct arm_smccc_res res; 1403 1404 invoke_fn(OPTEE_SMC_GET_THREAD_COUNT, 0, 0, 0, 0, 0, 0, 0, &res); 1405 if (res.a0) 1406 return 0; 1407 return res.a1; 1408 } 1409 1410 static struct tee_shm_pool * 1411 optee_config_shm_memremap(optee_invoke_fn *invoke_fn, void **memremaped_shm) 1412 { 1413 union { 1414 struct arm_smccc_res smccc; 1415 struct optee_smc_get_shm_config_result result; 1416 } res; 1417 unsigned long vaddr; 1418 phys_addr_t paddr; 1419 size_t size; 1420 phys_addr_t begin; 1421 phys_addr_t end; 1422 void *va; 1423 void *rc; 1424 1425 invoke_fn(OPTEE_SMC_GET_SHM_CONFIG, 0, 0, 0, 0, 0, 0, 0, &res.smccc); 1426 if (res.result.status != OPTEE_SMC_RETURN_OK) { 1427 pr_err("static shm service not available\n"); 1428 return ERR_PTR(-ENOENT); 1429 } 1430 1431 if (res.result.settings != OPTEE_SMC_SHM_CACHED) { 1432 pr_err("only normal cached shared memory supported\n"); 1433 return ERR_PTR(-EINVAL); 1434 } 1435 1436 begin = roundup(res.result.start, PAGE_SIZE); 1437 end = rounddown(res.result.start + res.result.size, PAGE_SIZE); 1438 paddr = begin; 1439 size = end - begin; 1440 1441 va = memremap(paddr, size, MEMREMAP_WB); 1442 if (!va) { 1443 pr_err("shared memory ioremap failed\n"); 1444 return ERR_PTR(-EINVAL); 1445 } 1446 vaddr = (unsigned long)va; 1447 1448 rc = tee_shm_pool_alloc_res_mem(vaddr, paddr, size, 1449 OPTEE_MIN_STATIC_POOL_ALIGN); 1450 if (IS_ERR(rc)) 1451 memunmap(va); 1452 else 1453 *memremaped_shm = va; 1454 1455 return rc; 1456 } 1457 1458 /* Simple wrapper functions to be able to use a function pointer */ 1459 static void optee_smccc_smc(unsigned long a0, unsigned long a1, 1460 unsigned long a2, unsigned long a3, 1461 unsigned long a4, unsigned long a5, 1462 unsigned long a6, unsigned long a7, 1463 struct arm_smccc_res *res) 1464 { 1465 arm_smccc_smc(a0, a1, a2, a3, a4, a5, a6, a7, res); 1466 } 1467 1468 static void optee_smccc_hvc(unsigned long a0, unsigned long a1, 1469 unsigned long a2, unsigned long a3, 1470 unsigned long a4, unsigned long a5, 1471 unsigned long a6, unsigned long a7, 1472 struct arm_smccc_res *res) 1473 { 1474 arm_smccc_hvc(a0, a1, a2, a3, a4, a5, a6, a7, res); 1475 } 1476 1477 static optee_invoke_fn *get_invoke_func(struct device *dev) 1478 { 1479 const char *method; 1480 1481 pr_info("probing for conduit method.\n"); 1482 1483 if (device_property_read_string(dev, "method", &method)) { 1484 pr_warn("missing \"method\" property\n"); 1485 return ERR_PTR(-ENXIO); 1486 } 1487 1488 if (!strcmp("hvc", method)) 1489 return optee_smccc_hvc; 1490 else if (!strcmp("smc", method)) 1491 return optee_smccc_smc; 1492 1493 pr_warn("invalid \"method\" property: %s\n", method); 1494 return ERR_PTR(-EINVAL); 1495 } 1496 1497 /* optee_remove - Device Removal Routine 1498 * @pdev: platform device information struct 1499 * 1500 * optee_remove is called by platform subsystem to alert the driver 1501 * that it should release the device 1502 */ 1503 static void optee_smc_remove(struct platform_device *pdev) 1504 { 1505 struct optee *optee = platform_get_drvdata(pdev); 1506 1507 /* 1508 * Ask OP-TEE to free all cached shared memory objects to decrease 1509 * reference counters and also avoid wild pointers in secure world 1510 * into the old shared memory range. 1511 */ 1512 if (!optee->rpc_param_count) 1513 optee_disable_shm_cache(optee); 1514 1515 optee_smc_notif_uninit_irq(optee); 1516 1517 optee_remove_common(optee); 1518 1519 if (optee->smc.memremaped_shm) 1520 memunmap(optee->smc.memremaped_shm); 1521 1522 kfree(optee); 1523 } 1524 1525 /* optee_shutdown - Device Removal Routine 1526 * @pdev: platform device information struct 1527 * 1528 * platform_shutdown is called by the platform subsystem to alert 1529 * the driver that a shutdown, reboot, or kexec is happening and 1530 * device must be disabled. 1531 */ 1532 static void optee_shutdown(struct platform_device *pdev) 1533 { 1534 struct optee *optee = platform_get_drvdata(pdev); 1535 1536 if (!optee->rpc_param_count) 1537 optee_disable_shm_cache(optee); 1538 } 1539 1540 #ifdef CONFIG_OPTEE_INSECURE_LOAD_IMAGE 1541 1542 #define OPTEE_FW_IMAGE "optee/tee.bin" 1543 1544 static optee_invoke_fn *cpuhp_invoke_fn; 1545 1546 static int optee_cpuhp_probe(unsigned int cpu) 1547 { 1548 /* 1549 * Invoking a call on a CPU will cause OP-TEE to perform the required 1550 * setup for that CPU. Just invoke the call to get the UID since that 1551 * has no side effects. 1552 */ 1553 if (optee_msg_api_uid_is_optee_api(cpuhp_invoke_fn)) 1554 return 0; 1555 else 1556 return -EINVAL; 1557 } 1558 1559 static int optee_load_fw(struct platform_device *pdev, 1560 optee_invoke_fn *invoke_fn) 1561 { 1562 const struct firmware *fw = NULL; 1563 struct arm_smccc_res res; 1564 phys_addr_t data_pa; 1565 u8 *data_buf = NULL; 1566 u64 data_size; 1567 u32 data_pa_high, data_pa_low; 1568 u32 data_size_high, data_size_low; 1569 int rc; 1570 int hp_state; 1571 1572 if (!optee_msg_api_uid_is_optee_image_load(invoke_fn)) 1573 return 0; 1574 1575 rc = request_firmware(&fw, OPTEE_FW_IMAGE, &pdev->dev); 1576 if (rc) { 1577 /* 1578 * The firmware in the rootfs will not be accessible until we 1579 * are in the SYSTEM_RUNNING state, so return EPROBE_DEFER until 1580 * that point. 1581 */ 1582 if (system_state < SYSTEM_RUNNING) 1583 return -EPROBE_DEFER; 1584 goto fw_err; 1585 } 1586 1587 data_size = fw->size; 1588 /* 1589 * This uses the GFP_DMA flag to ensure we are allocated memory in the 1590 * 32-bit space since TF-A cannot map memory beyond the 32-bit boundary. 1591 */ 1592 data_buf = kmemdup(fw->data, fw->size, GFP_KERNEL | GFP_DMA); 1593 if (!data_buf) { 1594 rc = -ENOMEM; 1595 goto fw_err; 1596 } 1597 data_pa = virt_to_phys(data_buf); 1598 reg_pair_from_64(&data_pa_high, &data_pa_low, data_pa); 1599 reg_pair_from_64(&data_size_high, &data_size_low, data_size); 1600 goto fw_load; 1601 1602 fw_err: 1603 pr_warn("image loading failed\n"); 1604 data_pa_high = 0; 1605 data_pa_low = 0; 1606 data_size_high = 0; 1607 data_size_low = 0; 1608 1609 fw_load: 1610 /* 1611 * Always invoke the SMC, even if loading the image fails, to indicate 1612 * to EL3 that we have passed the point where it should allow invoking 1613 * this SMC. 1614 */ 1615 pr_warn("OP-TEE image loaded from kernel, this can be insecure"); 1616 invoke_fn(OPTEE_SMC_CALL_LOAD_IMAGE, data_size_high, data_size_low, 1617 data_pa_high, data_pa_low, 0, 0, 0, &res); 1618 if (!rc) 1619 rc = res.a0; 1620 release_firmware(fw); 1621 kfree(data_buf); 1622 1623 if (!rc) { 1624 /* 1625 * We need to initialize OP-TEE on all other running cores as 1626 * well. Any cores that aren't running yet will get initialized 1627 * when they are brought up by the power management functions in 1628 * TF-A which are registered by the OP-TEE SPD. Due to that we 1629 * can un-register the callback right after registering it. 1630 */ 1631 cpuhp_invoke_fn = invoke_fn; 1632 hp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "optee:probe", 1633 optee_cpuhp_probe, NULL); 1634 if (hp_state < 0) { 1635 pr_warn("Failed with CPU hotplug setup for OP-TEE"); 1636 return -EINVAL; 1637 } 1638 cpuhp_remove_state(hp_state); 1639 cpuhp_invoke_fn = NULL; 1640 } 1641 1642 return rc; 1643 } 1644 #else 1645 static inline int optee_load_fw(struct platform_device *pdev, 1646 optee_invoke_fn *invoke_fn) 1647 { 1648 return 0; 1649 } 1650 #endif 1651 1652 static struct tee_protmem_pool *static_protmem_pool_init(struct optee *optee) 1653 { 1654 #if IS_ENABLED(CONFIG_OPTEE_STATIC_PROTMEM_POOL) 1655 union { 1656 struct arm_smccc_res smccc; 1657 struct optee_smc_get_protmem_config_result result; 1658 } res; 1659 struct tee_protmem_pool *pool; 1660 void *p; 1661 int rc; 1662 1663 optee->smc.invoke_fn(OPTEE_SMC_GET_PROTMEM_CONFIG, 0, 0, 0, 0, 1664 0, 0, 0, &res.smccc); 1665 if (res.result.status != OPTEE_SMC_RETURN_OK) 1666 return ERR_PTR(-EINVAL); 1667 1668 rc = optee_set_dma_mask(optee, res.result.pa_width); 1669 if (rc) 1670 return ERR_PTR(rc); 1671 1672 /* 1673 * Map the memory as uncached to make sure the kernel can work with 1674 * __pfn_to_page() and friends since that's needed when passing the 1675 * protected DMA-buf to a device. The memory should otherwise not 1676 * be touched by the kernel since it's likely to cause an external 1677 * abort due to the protection status. 1678 */ 1679 p = devm_memremap(&optee->teedev->dev, res.result.start, 1680 res.result.size, MEMREMAP_WC); 1681 if (IS_ERR(p)) 1682 return p; 1683 1684 pool = tee_protmem_static_pool_alloc(res.result.start, res.result.size); 1685 if (IS_ERR(pool)) 1686 devm_memunmap(&optee->teedev->dev, p); 1687 1688 return pool; 1689 #else 1690 return ERR_PTR(-EINVAL); 1691 #endif 1692 } 1693 1694 static int optee_protmem_pool_init(struct optee *optee) 1695 { 1696 bool protm = optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_PROTMEM; 1697 bool dyn_protm = optee->smc.sec_caps & 1698 OPTEE_SMC_SEC_CAP_DYNAMIC_PROTMEM; 1699 enum tee_dma_heap_id heap_id = TEE_DMA_HEAP_SECURE_VIDEO_PLAY; 1700 struct tee_protmem_pool *pool = ERR_PTR(-EINVAL); 1701 int rc = -EINVAL; 1702 1703 if (!protm && !dyn_protm) 1704 return 0; 1705 1706 if (protm) 1707 pool = static_protmem_pool_init(optee); 1708 if (dyn_protm && IS_ERR(pool)) 1709 pool = optee_protmem_alloc_dyn_pool(optee, heap_id); 1710 if (IS_ERR(pool)) 1711 return PTR_ERR(pool); 1712 1713 rc = tee_device_register_dma_heap(optee->teedev, heap_id, pool); 1714 if (rc) 1715 pool->ops->destroy_pool(pool); 1716 1717 return rc; 1718 } 1719 1720 static int optee_probe(struct platform_device *pdev) 1721 { 1722 optee_invoke_fn *invoke_fn; 1723 struct tee_shm_pool *pool = ERR_PTR(-EINVAL); 1724 struct optee *optee = NULL; 1725 void *memremaped_shm = NULL; 1726 unsigned int rpc_param_count; 1727 unsigned int thread_count; 1728 struct tee_device *teedev; 1729 struct tee_context *ctx; 1730 u32 max_notif_value; 1731 u32 arg_cache_flags; 1732 u32 sec_caps; 1733 int rc; 1734 1735 invoke_fn = get_invoke_func(&pdev->dev); 1736 if (IS_ERR(invoke_fn)) 1737 return PTR_ERR(invoke_fn); 1738 1739 rc = optee_load_fw(pdev, invoke_fn); 1740 if (rc) 1741 return rc; 1742 1743 if (!optee_msg_api_uid_is_optee_api(invoke_fn)) { 1744 pr_warn("api uid mismatch\n"); 1745 return -EINVAL; 1746 } 1747 1748 optee_msg_get_os_revision(invoke_fn); 1749 1750 if (!optee_msg_api_revision_is_compatible(invoke_fn)) { 1751 pr_warn("api revision mismatch\n"); 1752 return -EINVAL; 1753 } 1754 1755 thread_count = optee_msg_get_thread_count(invoke_fn); 1756 if (!optee_msg_exchange_capabilities(invoke_fn, &sec_caps, 1757 &max_notif_value, 1758 &rpc_param_count)) { 1759 pr_warn("capabilities mismatch\n"); 1760 return -EINVAL; 1761 } 1762 1763 /* 1764 * Try to use dynamic shared memory if possible 1765 */ 1766 if (sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) { 1767 /* 1768 * If we have OPTEE_SMC_SEC_CAP_RPC_ARG we can ask 1769 * optee_get_msg_arg() to pre-register (by having 1770 * OPTEE_SHM_ARG_ALLOC_PRIV cleared) the page used to pass 1771 * an argument struct. 1772 * 1773 * With the page is pre-registered we can use a non-zero 1774 * offset for argument struct, this is indicated with 1775 * OPTEE_SHM_ARG_SHARED. 1776 * 1777 * This means that optee_smc_do_call_with_arg() will use 1778 * OPTEE_SMC_CALL_WITH_REGD_ARG for pre-registered pages. 1779 */ 1780 if (sec_caps & OPTEE_SMC_SEC_CAP_RPC_ARG) 1781 arg_cache_flags = OPTEE_SHM_ARG_SHARED; 1782 else 1783 arg_cache_flags = OPTEE_SHM_ARG_ALLOC_PRIV; 1784 1785 pool = optee_shm_pool_alloc_pages(); 1786 } 1787 1788 /* 1789 * If dynamic shared memory is not available or failed - try static one 1790 */ 1791 if (IS_ERR(pool) && (sec_caps & OPTEE_SMC_SEC_CAP_HAVE_RESERVED_SHM)) { 1792 /* 1793 * The static memory pool can use non-zero page offsets so 1794 * let optee_get_msg_arg() know that with OPTEE_SHM_ARG_SHARED. 1795 * 1796 * optee_get_msg_arg() should not pre-register the 1797 * allocated page used to pass an argument struct, this is 1798 * indicated with OPTEE_SHM_ARG_ALLOC_PRIV. 1799 * 1800 * This means that optee_smc_do_call_with_arg() will use 1801 * OPTEE_SMC_CALL_WITH_ARG if rpc_param_count is 0, else 1802 * OPTEE_SMC_CALL_WITH_RPC_ARG. 1803 */ 1804 arg_cache_flags = OPTEE_SHM_ARG_SHARED | 1805 OPTEE_SHM_ARG_ALLOC_PRIV; 1806 pool = optee_config_shm_memremap(invoke_fn, &memremaped_shm); 1807 } 1808 1809 if (IS_ERR(pool)) 1810 return PTR_ERR(pool); 1811 1812 optee = kzalloc(sizeof(*optee), GFP_KERNEL); 1813 if (!optee) { 1814 rc = -ENOMEM; 1815 goto err_free_shm_pool; 1816 } 1817 1818 optee->ops = &optee_ops; 1819 optee->smc.invoke_fn = invoke_fn; 1820 optee->smc.sec_caps = sec_caps; 1821 optee->rpc_param_count = rpc_param_count; 1822 1823 if (IS_REACHABLE(CONFIG_RPMB) && 1824 (sec_caps & OPTEE_SMC_SEC_CAP_RPMB_PROBE)) 1825 optee->in_kernel_rpmb_routing = true; 1826 1827 teedev = tee_device_alloc(&optee_clnt_desc, NULL, pool, optee); 1828 if (IS_ERR(teedev)) { 1829 rc = PTR_ERR(teedev); 1830 goto err_free_optee; 1831 } 1832 optee->teedev = teedev; 1833 1834 teedev = tee_device_alloc(&optee_supp_desc, NULL, pool, optee); 1835 if (IS_ERR(teedev)) { 1836 rc = PTR_ERR(teedev); 1837 goto err_unreg_teedev; 1838 } 1839 optee->supp_teedev = teedev; 1840 1841 optee_set_dev_group(optee); 1842 1843 rc = tee_device_register(optee->teedev); 1844 if (rc) 1845 goto err_unreg_supp_teedev; 1846 1847 rc = tee_device_register(optee->supp_teedev); 1848 if (rc) 1849 goto err_unreg_supp_teedev; 1850 1851 optee_cq_init(&optee->call_queue, thread_count); 1852 optee_supp_init(&optee->supp); 1853 optee->smc.memremaped_shm = memremaped_shm; 1854 optee->pool = pool; 1855 optee_shm_arg_cache_init(optee, arg_cache_flags); 1856 mutex_init(&optee->rpmb_dev_mutex); 1857 1858 platform_set_drvdata(pdev, optee); 1859 ctx = teedev_open(optee->teedev); 1860 if (IS_ERR(ctx)) { 1861 rc = PTR_ERR(ctx); 1862 goto err_supp_uninit; 1863 } 1864 optee->ctx = ctx; 1865 rc = optee_notif_init(optee, max_notif_value); 1866 if (rc) 1867 goto err_close_ctx; 1868 1869 if (sec_caps & OPTEE_SMC_SEC_CAP_ASYNC_NOTIF) { 1870 unsigned int irq; 1871 1872 rc = platform_get_irq(pdev, 0); 1873 if (rc < 0) { 1874 pr_err("platform_get_irq: ret %d\n", rc); 1875 goto err_notif_uninit; 1876 } 1877 irq = rc; 1878 1879 rc = optee_smc_notif_init_irq(optee, irq); 1880 if (rc) { 1881 irq_dispose_mapping(irq); 1882 goto err_notif_uninit; 1883 } 1884 enable_async_notif(optee->smc.invoke_fn); 1885 pr_info("Asynchronous notifications enabled\n"); 1886 } 1887 1888 if (optee_protmem_pool_init(optee)) 1889 pr_info("Protected memory service not available\n"); 1890 1891 /* 1892 * Ensure that there are no pre-existing shm objects before enabling 1893 * the shm cache so that there's no chance of receiving an invalid 1894 * address during shutdown. This could occur, for example, if we're 1895 * kexec booting from an older kernel that did not properly cleanup the 1896 * shm cache. 1897 */ 1898 optee_disable_unmapped_shm_cache(optee); 1899 1900 /* 1901 * Only enable the shm cache in case we're not able to pass the RPC 1902 * arg struct right after the normal arg struct. 1903 */ 1904 if (!optee->rpc_param_count) 1905 optee_enable_shm_cache(optee); 1906 1907 if (optee->smc.sec_caps & OPTEE_SMC_SEC_CAP_DYNAMIC_SHM) 1908 pr_info("dynamic shared memory is enabled\n"); 1909 1910 rc = optee_enumerate_devices(PTA_CMD_GET_DEVICES); 1911 if (rc) 1912 goto err_disable_shm_cache; 1913 1914 INIT_WORK(&optee->rpmb_scan_bus_work, optee_bus_scan_rpmb); 1915 optee->rpmb_intf.notifier_call = optee_rpmb_intf_rdev; 1916 blocking_notifier_chain_register(&optee_rpmb_intf_added, 1917 &optee->rpmb_intf); 1918 pr_info("initialized driver\n"); 1919 return 0; 1920 1921 err_disable_shm_cache: 1922 if (!optee->rpc_param_count) 1923 optee_disable_shm_cache(optee); 1924 optee_smc_notif_uninit_irq(optee); 1925 optee_unregister_devices(); 1926 err_notif_uninit: 1927 optee_notif_uninit(optee); 1928 err_close_ctx: 1929 teedev_close_context(ctx); 1930 err_supp_uninit: 1931 rpmb_dev_put(optee->rpmb_dev); 1932 mutex_destroy(&optee->rpmb_dev_mutex); 1933 optee_shm_arg_cache_uninit(optee); 1934 optee_supp_uninit(&optee->supp); 1935 mutex_destroy(&optee->call_queue.mutex); 1936 err_unreg_supp_teedev: 1937 tee_device_unregister(optee->supp_teedev); 1938 err_unreg_teedev: 1939 tee_device_unregister(optee->teedev); 1940 err_free_optee: 1941 kfree(optee); 1942 err_free_shm_pool: 1943 tee_shm_pool_free(pool); 1944 if (memremaped_shm) 1945 memunmap(memremaped_shm); 1946 return rc; 1947 } 1948 1949 static const struct of_device_id optee_dt_match[] = { 1950 { .compatible = "linaro,optee-tz" }, 1951 {}, 1952 }; 1953 MODULE_DEVICE_TABLE(of, optee_dt_match); 1954 1955 static struct platform_driver optee_driver = { 1956 .probe = optee_probe, 1957 .remove = optee_smc_remove, 1958 .shutdown = optee_shutdown, 1959 .driver = { 1960 .name = "optee", 1961 .of_match_table = optee_dt_match, 1962 }, 1963 }; 1964 1965 int optee_smc_abi_register(void) 1966 { 1967 return platform_driver_register(&optee_driver); 1968 } 1969 1970 void optee_smc_abi_unregister(void) 1971 { 1972 platform_driver_unregister(&optee_driver); 1973 } 1974