1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * FF-A v1.0 proxy to filter out invalid memory-sharing SMC calls issued by 4 * the host. FF-A is a slightly more palatable abbreviation of "Arm Firmware 5 * Framework for Arm A-profile", which is specified by Arm in document 6 * number DEN0077. 7 * 8 * Copyright (C) 2022 - Google LLC 9 * Author: Andrew Walbran <qwandor@google.com> 10 * 11 * This driver hooks into the SMC trapping logic for the host and intercepts 12 * all calls falling within the FF-A range. Each call is either: 13 * 14 * - Forwarded on unmodified to the SPMD at EL3 15 * - Rejected as "unsupported" 16 * - Accompanied by a host stage-2 page-table check/update and reissued 17 * 18 * Consequently, any attempts by the host to make guest memory pages 19 * accessible to the secure world using FF-A will be detected either here 20 * (in the case that the memory is already owned by the guest) or during 21 * donation to the guest (in the case that the memory was previously shared 22 * with the secure world). 23 * 24 * To allow the rolling-back of page-table updates and FF-A calls in the 25 * event of failure, operations involving the RXTX buffers are locked for 26 * the duration and are therefore serialised. 27 */ 28 29 #include <linux/arm-smccc.h> 30 #include <linux/arm_ffa.h> 31 #include <asm/kvm_pkvm.h> 32 33 #include <nvhe/ffa.h> 34 #include <nvhe/mem_protect.h> 35 #include <nvhe/memory.h> 36 #include <nvhe/trap_handler.h> 37 #include <nvhe/spinlock.h> 38 39 /* 40 * "ID value 0 must be returned at the Non-secure physical FF-A instance" 41 * We share this ID with the host. 42 */ 43 #define HOST_FFA_ID 0 44 45 /* 46 * A buffer to hold the maximum descriptor size we can see from the host, 47 * which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP 48 * when resolving the handle on the reclaim path. 49 */ 50 struct kvm_ffa_descriptor_buffer { 51 void *buf; 52 size_t len; 53 }; 54 55 static struct kvm_ffa_descriptor_buffer ffa_desc_buf; 56 57 struct kvm_ffa_buffers { 58 hyp_spinlock_t lock; 59 void *tx; 60 void *rx; 61 }; 62 63 /* 64 * Note that we don't currently lock these buffers explicitly, instead 65 * relying on the locking of the host FFA buffers as we only have one 66 * client. 67 */ 68 static struct kvm_ffa_buffers hyp_buffers; 69 static struct kvm_ffa_buffers host_buffers; 70 static u32 hyp_ffa_version; 71 static bool has_version_negotiated; 72 static hyp_spinlock_t version_lock; 73 74 static void ffa_to_smccc_error(struct arm_smccc_1_2_regs *res, u64 ffa_errno) 75 { 76 *res = (struct arm_smccc_1_2_regs) { 77 .a0 = FFA_ERROR, 78 .a2 = ffa_errno, 79 }; 80 } 81 82 static void ffa_to_smccc_res_prop(struct arm_smccc_1_2_regs *res, int ret, u64 prop) 83 { 84 if (ret == FFA_RET_SUCCESS) { 85 *res = (struct arm_smccc_1_2_regs) { .a0 = FFA_SUCCESS, 86 .a2 = prop }; 87 } else { 88 ffa_to_smccc_error(res, ret); 89 } 90 } 91 92 static void ffa_to_smccc_res(struct arm_smccc_1_2_regs *res, int ret) 93 { 94 ffa_to_smccc_res_prop(res, ret, 0); 95 } 96 97 static void ffa_set_retval(struct kvm_cpu_context *ctxt, 98 struct arm_smccc_1_2_regs *res) 99 { 100 cpu_reg(ctxt, 0) = res->a0; 101 cpu_reg(ctxt, 1) = res->a1; 102 cpu_reg(ctxt, 2) = res->a2; 103 cpu_reg(ctxt, 3) = res->a3; 104 cpu_reg(ctxt, 4) = res->a4; 105 cpu_reg(ctxt, 5) = res->a5; 106 cpu_reg(ctxt, 6) = res->a6; 107 cpu_reg(ctxt, 7) = res->a7; 108 109 /* 110 * DEN0028C 2.6: SMC32/HVC32 call from aarch64 must preserve x8-x30. 111 * 112 * In FF-A 1.2, we cannot rely on the function ID sent by the caller to 113 * detect 32-bit calls because the CPU cycle management interfaces (e.g. 114 * FFA_MSG_WAIT, FFA_RUN) are 32-bit only but can have 64-bit responses. 115 * 116 * FFA-1.3 introduces 64-bit variants of the CPU cycle management 117 * interfaces. Moreover, FF-A 1.3 clarifies that SMC32 direct requests 118 * complete with SMC32 direct reponses which *should* allow us use the 119 * function ID sent by the caller to determine whether to return x8-x17. 120 * 121 * Note that we also cannot rely on function IDs in the response. 122 * 123 * Given the above, assume SMC64 and send back x0-x17 unconditionally 124 * as the passthrough code (__kvm_hyp_host_forward_smc) does the same. 125 */ 126 cpu_reg(ctxt, 8) = res->a8; 127 cpu_reg(ctxt, 9) = res->a9; 128 cpu_reg(ctxt, 10) = res->a10; 129 cpu_reg(ctxt, 11) = res->a11; 130 cpu_reg(ctxt, 12) = res->a12; 131 cpu_reg(ctxt, 13) = res->a13; 132 cpu_reg(ctxt, 14) = res->a14; 133 cpu_reg(ctxt, 15) = res->a15; 134 cpu_reg(ctxt, 16) = res->a16; 135 cpu_reg(ctxt, 17) = res->a17; 136 } 137 138 static bool is_ffa_call(u64 func_id) 139 { 140 return ARM_SMCCC_IS_FAST_CALL(func_id) && 141 ARM_SMCCC_OWNER_NUM(func_id) == ARM_SMCCC_OWNER_STANDARD && 142 ARM_SMCCC_FUNC_NUM(func_id) >= FFA_MIN_FUNC_NUM && 143 ARM_SMCCC_FUNC_NUM(func_id) <= FFA_MAX_FUNC_NUM; 144 } 145 146 static int ffa_map_hyp_buffers(u64 ffa_page_count) 147 { 148 struct arm_smccc_1_2_regs res; 149 150 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 151 .a0 = FFA_FN64_RXTX_MAP, 152 .a1 = hyp_virt_to_phys(hyp_buffers.tx), 153 .a2 = hyp_virt_to_phys(hyp_buffers.rx), 154 .a3 = ffa_page_count, 155 }, &res); 156 157 return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2; 158 } 159 160 static int ffa_unmap_hyp_buffers(void) 161 { 162 struct arm_smccc_1_2_regs res; 163 164 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 165 .a0 = FFA_RXTX_UNMAP, 166 .a1 = HOST_FFA_ID, 167 }, &res); 168 169 return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2; 170 } 171 172 static void ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, u32 handle_lo, 173 u32 handle_hi, u32 fraglen, u32 endpoint_id) 174 { 175 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 176 .a0 = FFA_MEM_FRAG_TX, 177 .a1 = handle_lo, 178 .a2 = handle_hi, 179 .a3 = fraglen, 180 .a4 = endpoint_id, 181 }, res); 182 } 183 184 static void ffa_mem_frag_rx(struct arm_smccc_1_2_regs *res, u32 handle_lo, 185 u32 handle_hi, u32 fragoff) 186 { 187 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 188 .a0 = FFA_MEM_FRAG_RX, 189 .a1 = handle_lo, 190 .a2 = handle_hi, 191 .a3 = fragoff, 192 .a4 = HOST_FFA_ID, 193 }, res); 194 } 195 196 static void ffa_mem_xfer(struct arm_smccc_1_2_regs *res, u64 func_id, u32 len, 197 u32 fraglen) 198 { 199 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 200 .a0 = func_id, 201 .a1 = len, 202 .a2 = fraglen, 203 }, res); 204 } 205 206 static void ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, u32 handle_lo, 207 u32 handle_hi, u32 flags) 208 { 209 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 210 .a0 = FFA_MEM_RECLAIM, 211 .a1 = handle_lo, 212 .a2 = handle_hi, 213 .a3 = flags, 214 }, res); 215 } 216 217 static void ffa_retrieve_req(struct arm_smccc_1_2_regs *res, u32 len) 218 { 219 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 220 .a0 = FFA_FN64_MEM_RETRIEVE_REQ, 221 .a1 = len, 222 .a2 = len, 223 }, res); 224 } 225 226 static void ffa_rx_release(struct arm_smccc_1_2_regs *res) 227 { 228 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 229 .a0 = FFA_RX_RELEASE, 230 }, res); 231 } 232 233 static void do_ffa_rxtx_map(struct arm_smccc_1_2_regs *res, 234 struct kvm_cpu_context *ctxt) 235 { 236 DECLARE_REG(phys_addr_t, tx, ctxt, 1); 237 DECLARE_REG(phys_addr_t, rx, ctxt, 2); 238 DECLARE_REG(u32, npages, ctxt, 3); 239 int ret = 0; 240 void *rx_virt, *tx_virt; 241 242 if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE) { 243 ret = FFA_RET_INVALID_PARAMETERS; 244 goto out; 245 } 246 247 if (!PAGE_ALIGNED(tx) || !PAGE_ALIGNED(rx)) { 248 ret = FFA_RET_INVALID_PARAMETERS; 249 goto out; 250 } 251 252 hyp_spin_lock(&host_buffers.lock); 253 if (host_buffers.tx) { 254 ret = FFA_RET_DENIED; 255 goto out_unlock; 256 } 257 258 /* 259 * Map our hypervisor buffers into the SPMD before mapping and 260 * pinning the host buffers in our own address space. 261 */ 262 ret = ffa_map_hyp_buffers(npages); 263 if (ret) 264 goto out_unlock; 265 266 ret = __pkvm_host_share_hyp(hyp_phys_to_pfn(tx)); 267 if (ret) { 268 ret = FFA_RET_INVALID_PARAMETERS; 269 goto err_unmap; 270 } 271 272 ret = __pkvm_host_share_hyp(hyp_phys_to_pfn(rx)); 273 if (ret) { 274 ret = FFA_RET_INVALID_PARAMETERS; 275 goto err_unshare_tx; 276 } 277 278 tx_virt = hyp_phys_to_virt(tx); 279 ret = hyp_pin_shared_mem(tx_virt, tx_virt + 1); 280 if (ret) { 281 ret = FFA_RET_INVALID_PARAMETERS; 282 goto err_unshare_rx; 283 } 284 285 rx_virt = hyp_phys_to_virt(rx); 286 ret = hyp_pin_shared_mem(rx_virt, rx_virt + 1); 287 if (ret) { 288 ret = FFA_RET_INVALID_PARAMETERS; 289 goto err_unpin_tx; 290 } 291 292 host_buffers.tx = tx_virt; 293 host_buffers.rx = rx_virt; 294 295 out_unlock: 296 hyp_spin_unlock(&host_buffers.lock); 297 out: 298 ffa_to_smccc_res(res, ret); 299 return; 300 301 err_unpin_tx: 302 hyp_unpin_shared_mem(tx_virt, tx_virt + 1); 303 err_unshare_rx: 304 __pkvm_host_unshare_hyp(hyp_phys_to_pfn(rx)); 305 err_unshare_tx: 306 __pkvm_host_unshare_hyp(hyp_phys_to_pfn(tx)); 307 err_unmap: 308 ffa_unmap_hyp_buffers(); 309 goto out_unlock; 310 } 311 312 static void do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs *res, 313 struct kvm_cpu_context *ctxt) 314 { 315 DECLARE_REG(u32, id, ctxt, 1); 316 int ret = 0; 317 318 if (id != HOST_FFA_ID) { 319 ret = FFA_RET_INVALID_PARAMETERS; 320 goto out; 321 } 322 323 hyp_spin_lock(&host_buffers.lock); 324 if (!host_buffers.tx) { 325 ret = FFA_RET_INVALID_PARAMETERS; 326 goto out_unlock; 327 } 328 329 hyp_unpin_shared_mem(host_buffers.tx, host_buffers.tx + 1); 330 WARN_ON(__pkvm_host_unshare_hyp(hyp_virt_to_pfn(host_buffers.tx))); 331 host_buffers.tx = NULL; 332 333 hyp_unpin_shared_mem(host_buffers.rx, host_buffers.rx + 1); 334 WARN_ON(__pkvm_host_unshare_hyp(hyp_virt_to_pfn(host_buffers.rx))); 335 host_buffers.rx = NULL; 336 337 ffa_unmap_hyp_buffers(); 338 339 out_unlock: 340 hyp_spin_unlock(&host_buffers.lock); 341 out: 342 ffa_to_smccc_res(res, ret); 343 } 344 345 static u32 __ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges, 346 u32 nranges) 347 { 348 u32 i; 349 350 for (i = 0; i < nranges; ++i) { 351 struct ffa_mem_region_addr_range *range = &ranges[i]; 352 u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE; 353 u64 pfn = hyp_phys_to_pfn(range->address); 354 355 if (!PAGE_ALIGNED(sz)) 356 break; 357 358 if (__pkvm_host_share_ffa(pfn, sz / PAGE_SIZE)) 359 break; 360 } 361 362 return i; 363 } 364 365 static u32 __ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges, 366 u32 nranges) 367 { 368 u32 i; 369 370 for (i = 0; i < nranges; ++i) { 371 struct ffa_mem_region_addr_range *range = &ranges[i]; 372 u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE; 373 u64 pfn = hyp_phys_to_pfn(range->address); 374 375 if (!PAGE_ALIGNED(sz)) 376 break; 377 378 if (__pkvm_host_unshare_ffa(pfn, sz / PAGE_SIZE)) 379 break; 380 } 381 382 return i; 383 } 384 385 static int ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges, 386 u32 nranges) 387 { 388 u32 nshared = __ffa_host_share_ranges(ranges, nranges); 389 int ret = 0; 390 391 if (nshared != nranges) { 392 WARN_ON(__ffa_host_unshare_ranges(ranges, nshared) != nshared); 393 ret = FFA_RET_DENIED; 394 } 395 396 return ret; 397 } 398 399 static int ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges, 400 u32 nranges) 401 { 402 u32 nunshared = __ffa_host_unshare_ranges(ranges, nranges); 403 int ret = 0; 404 405 if (nunshared != nranges) { 406 WARN_ON(__ffa_host_share_ranges(ranges, nunshared) != nunshared); 407 ret = FFA_RET_DENIED; 408 } 409 410 return ret; 411 } 412 413 static void do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, 414 struct kvm_cpu_context *ctxt) 415 { 416 DECLARE_REG(u32, handle_lo, ctxt, 1); 417 DECLARE_REG(u32, handle_hi, ctxt, 2); 418 DECLARE_REG(u32, fraglen, ctxt, 3); 419 DECLARE_REG(u32, endpoint_id, ctxt, 4); 420 struct ffa_mem_region_addr_range *buf; 421 int ret = FFA_RET_INVALID_PARAMETERS; 422 u32 nr_ranges; 423 424 if (fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) 425 goto out; 426 427 if (fraglen % sizeof(*buf)) 428 goto out; 429 430 hyp_spin_lock(&host_buffers.lock); 431 if (!host_buffers.tx) 432 goto out_unlock; 433 434 buf = hyp_buffers.tx; 435 memcpy(buf, host_buffers.tx, fraglen); 436 nr_ranges = fraglen / sizeof(*buf); 437 438 ret = ffa_host_share_ranges(buf, nr_ranges); 439 if (ret) { 440 /* 441 * We're effectively aborting the transaction, so we need 442 * to restore the global state back to what it was prior to 443 * transmission of the first fragment. 444 */ 445 ffa_mem_reclaim(res, handle_lo, handle_hi, 0); 446 WARN_ON(res->a0 != FFA_SUCCESS); 447 goto out_unlock; 448 } 449 450 ffa_mem_frag_tx(res, handle_lo, handle_hi, fraglen, endpoint_id); 451 if (res->a0 != FFA_SUCCESS && res->a0 != FFA_MEM_FRAG_RX) 452 WARN_ON(ffa_host_unshare_ranges(buf, nr_ranges)); 453 454 out_unlock: 455 hyp_spin_unlock(&host_buffers.lock); 456 out: 457 if (ret) 458 ffa_to_smccc_res(res, ret); 459 460 /* 461 * If for any reason this did not succeed, we're in trouble as we have 462 * now lost the content of the previous fragments and we can't rollback 463 * the host stage-2 changes. The pages previously marked as shared will 464 * remain stuck in that state forever, hence preventing the host from 465 * sharing/donating them again and may possibly lead to subsequent 466 * failures, but this will not compromise confidentiality. 467 */ 468 return; 469 } 470 471 static void __do_ffa_mem_xfer(const u64 func_id, 472 struct arm_smccc_1_2_regs *res, 473 struct kvm_cpu_context *ctxt) 474 { 475 DECLARE_REG(u32, len, ctxt, 1); 476 DECLARE_REG(u32, fraglen, ctxt, 2); 477 DECLARE_REG(u64, addr_mbz, ctxt, 3); 478 DECLARE_REG(u32, npages_mbz, ctxt, 4); 479 struct ffa_mem_region_attributes *ep_mem_access; 480 struct ffa_composite_mem_region *reg; 481 struct ffa_mem_region *buf; 482 u32 offset, nr_ranges, checked_offset; 483 int ret = 0; 484 485 if (addr_mbz || npages_mbz || fraglen > len || 486 fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { 487 ret = FFA_RET_INVALID_PARAMETERS; 488 goto out; 489 } 490 491 if (fraglen < sizeof(struct ffa_mem_region) + 492 sizeof(struct ffa_mem_region_attributes)) { 493 ret = FFA_RET_INVALID_PARAMETERS; 494 goto out; 495 } 496 497 hyp_spin_lock(&host_buffers.lock); 498 if (!host_buffers.tx) { 499 ret = FFA_RET_INVALID_PARAMETERS; 500 goto out_unlock; 501 } 502 503 if (len > ffa_desc_buf.len) { 504 ret = FFA_RET_NO_MEMORY; 505 goto out_unlock; 506 } 507 508 buf = hyp_buffers.tx; 509 memcpy(buf, host_buffers.tx, fraglen); 510 511 ep_mem_access = (void *)buf + 512 ffa_mem_desc_offset(buf, 0, hyp_ffa_version); 513 offset = ep_mem_access->composite_off; 514 if (!offset || buf->ep_count != 1 || buf->sender_id != HOST_FFA_ID) { 515 ret = FFA_RET_INVALID_PARAMETERS; 516 goto out_unlock; 517 } 518 519 if (check_add_overflow(offset, sizeof(struct ffa_composite_mem_region), &checked_offset)) { 520 ret = FFA_RET_INVALID_PARAMETERS; 521 goto out_unlock; 522 } 523 524 if (fraglen < checked_offset) { 525 ret = FFA_RET_INVALID_PARAMETERS; 526 goto out_unlock; 527 } 528 529 reg = (void *)buf + offset; 530 nr_ranges = ((void *)buf + fraglen) - (void *)reg->constituents; 531 if (nr_ranges % sizeof(reg->constituents[0])) { 532 ret = FFA_RET_INVALID_PARAMETERS; 533 goto out_unlock; 534 } 535 536 nr_ranges /= sizeof(reg->constituents[0]); 537 ret = ffa_host_share_ranges(reg->constituents, nr_ranges); 538 if (ret) 539 goto out_unlock; 540 541 ffa_mem_xfer(res, func_id, len, fraglen); 542 if (fraglen != len) { 543 if (res->a0 != FFA_MEM_FRAG_RX) 544 goto err_unshare; 545 546 if (res->a3 != fraglen) 547 goto err_unshare; 548 } else if (res->a0 != FFA_SUCCESS) { 549 goto err_unshare; 550 } 551 552 out_unlock: 553 hyp_spin_unlock(&host_buffers.lock); 554 out: 555 if (ret) 556 ffa_to_smccc_res(res, ret); 557 return; 558 559 err_unshare: 560 WARN_ON(ffa_host_unshare_ranges(reg->constituents, nr_ranges)); 561 goto out_unlock; 562 } 563 564 #define do_ffa_mem_xfer(fid, res, ctxt) \ 565 do { \ 566 BUILD_BUG_ON((fid) != FFA_FN64_MEM_SHARE && \ 567 (fid) != FFA_FN64_MEM_LEND); \ 568 __do_ffa_mem_xfer((fid), (res), (ctxt)); \ 569 } while (0); 570 571 static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, 572 struct kvm_cpu_context *ctxt) 573 { 574 DECLARE_REG(u32, handle_lo, ctxt, 1); 575 DECLARE_REG(u32, handle_hi, ctxt, 2); 576 DECLARE_REG(u32, flags, ctxt, 3); 577 struct ffa_mem_region_attributes *ep_mem_access; 578 struct ffa_composite_mem_region *reg; 579 u32 offset, len, fraglen, fragoff; 580 struct ffa_mem_region *buf; 581 int ret = 0; 582 u64 handle; 583 584 handle = PACK_HANDLE(handle_lo, handle_hi); 585 586 hyp_spin_lock(&host_buffers.lock); 587 588 buf = hyp_buffers.tx; 589 *buf = (struct ffa_mem_region) { 590 .sender_id = HOST_FFA_ID, 591 .handle = handle, 592 }; 593 594 ffa_retrieve_req(res, sizeof(*buf)); 595 buf = hyp_buffers.rx; 596 if (res->a0 != FFA_MEM_RETRIEVE_RESP) 597 goto out_unlock; 598 599 len = res->a1; 600 fraglen = res->a2; 601 602 ep_mem_access = (void *)buf + 603 ffa_mem_desc_offset(buf, 0, hyp_ffa_version); 604 offset = ep_mem_access->composite_off; 605 /* 606 * We can trust the SPMD to get this right, but let's at least 607 * check that we end up with something that doesn't look _completely_ 608 * bogus. 609 */ 610 if (WARN_ON(offset > len || 611 fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)) { 612 ret = FFA_RET_ABORTED; 613 ffa_rx_release(res); 614 goto out_unlock; 615 } 616 617 if (len > ffa_desc_buf.len) { 618 ret = FFA_RET_NO_MEMORY; 619 ffa_rx_release(res); 620 goto out_unlock; 621 } 622 623 buf = ffa_desc_buf.buf; 624 memcpy(buf, hyp_buffers.rx, fraglen); 625 ffa_rx_release(res); 626 627 for (fragoff = fraglen; fragoff < len; fragoff += fraglen) { 628 ffa_mem_frag_rx(res, handle_lo, handle_hi, fragoff); 629 if (res->a0 != FFA_MEM_FRAG_TX) { 630 ret = FFA_RET_INVALID_PARAMETERS; 631 goto out_unlock; 632 } 633 634 fraglen = res->a3; 635 memcpy((void *)buf + fragoff, hyp_buffers.rx, fraglen); 636 ffa_rx_release(res); 637 } 638 639 ffa_mem_reclaim(res, handle_lo, handle_hi, flags); 640 if (res->a0 != FFA_SUCCESS) 641 goto out_unlock; 642 643 reg = (void *)buf + offset; 644 /* If the SPMD was happy, then we should be too. */ 645 WARN_ON(ffa_host_unshare_ranges(reg->constituents, 646 reg->addr_range_cnt)); 647 out_unlock: 648 hyp_spin_unlock(&host_buffers.lock); 649 650 if (ret) 651 ffa_to_smccc_res(res, ret); 652 } 653 654 /* 655 * Is a given FFA function supported, either by forwarding on directly 656 * or by handling at EL2? 657 */ 658 static bool ffa_call_supported(u64 func_id) 659 { 660 switch (func_id) { 661 /* Unsupported memory management calls */ 662 case FFA_FN64_MEM_RETRIEVE_REQ: 663 case FFA_MEM_RETRIEVE_RESP: 664 case FFA_MEM_RELINQUISH: 665 case FFA_MEM_OP_PAUSE: 666 case FFA_MEM_OP_RESUME: 667 case FFA_MEM_FRAG_RX: 668 case FFA_FN64_MEM_DONATE: 669 /* Indirect message passing via RX/TX buffers */ 670 case FFA_MSG_SEND: 671 case FFA_MSG_POLL: 672 case FFA_MSG_WAIT: 673 /* 32-bit variants of 64-bit calls */ 674 case FFA_MSG_SEND_DIRECT_RESP: 675 case FFA_RXTX_MAP: 676 case FFA_MEM_DONATE: 677 case FFA_MEM_RETRIEVE_REQ: 678 /* Optional notification interfaces added in FF-A 1.1 */ 679 case FFA_NOTIFICATION_BITMAP_CREATE: 680 case FFA_NOTIFICATION_BITMAP_DESTROY: 681 case FFA_NOTIFICATION_BIND: 682 case FFA_NOTIFICATION_UNBIND: 683 case FFA_NOTIFICATION_SET: 684 case FFA_NOTIFICATION_GET: 685 case FFA_NOTIFICATION_INFO_GET: 686 /* Optional interfaces added in FF-A 1.2 */ 687 case FFA_MSG_SEND_DIRECT_REQ2: /* Optional per 7.5.1 */ 688 case FFA_MSG_SEND_DIRECT_RESP2: /* Optional per 7.5.1 */ 689 case FFA_CONSOLE_LOG: /* Optional per 13.1: not in Table 13.1 */ 690 case FFA_PARTITION_INFO_GET_REGS: /* Optional for virtual instances per 13.1 */ 691 return false; 692 } 693 694 return true; 695 } 696 697 static bool do_ffa_features(struct arm_smccc_1_2_regs *res, 698 struct kvm_cpu_context *ctxt) 699 { 700 DECLARE_REG(u32, id, ctxt, 1); 701 u64 prop = 0; 702 int ret = 0; 703 704 if (!ffa_call_supported(id)) { 705 ret = FFA_RET_NOT_SUPPORTED; 706 goto out_handled; 707 } 708 709 switch (id) { 710 case FFA_MEM_SHARE: 711 case FFA_FN64_MEM_SHARE: 712 case FFA_MEM_LEND: 713 case FFA_FN64_MEM_LEND: 714 ret = FFA_RET_SUCCESS; 715 prop = 0; /* No support for dynamic buffers */ 716 goto out_handled; 717 default: 718 return false; 719 } 720 721 out_handled: 722 ffa_to_smccc_res_prop(res, ret, prop); 723 return true; 724 } 725 726 static int hyp_ffa_post_init(void) 727 { 728 size_t min_rxtx_sz; 729 struct arm_smccc_1_2_regs res; 730 731 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){ 732 .a0 = FFA_ID_GET, 733 }, &res); 734 if (res.a0 != FFA_SUCCESS) 735 return -EOPNOTSUPP; 736 737 if (res.a2 != HOST_FFA_ID) 738 return -EINVAL; 739 740 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){ 741 .a0 = FFA_FEATURES, 742 .a1 = FFA_FN64_RXTX_MAP, 743 }, &res); 744 if (res.a0 != FFA_SUCCESS) 745 return -EOPNOTSUPP; 746 747 switch (res.a2 & FFA_FEAT_RXTX_MIN_SZ_MASK) { 748 case FFA_FEAT_RXTX_MIN_SZ_4K: 749 min_rxtx_sz = SZ_4K; 750 break; 751 case FFA_FEAT_RXTX_MIN_SZ_16K: 752 min_rxtx_sz = SZ_16K; 753 break; 754 case FFA_FEAT_RXTX_MIN_SZ_64K: 755 min_rxtx_sz = SZ_64K; 756 break; 757 default: 758 return -EINVAL; 759 } 760 761 if (min_rxtx_sz > PAGE_SIZE) 762 return -EOPNOTSUPP; 763 764 return 0; 765 } 766 767 static void do_ffa_version(struct arm_smccc_1_2_regs *res, 768 struct kvm_cpu_context *ctxt) 769 { 770 DECLARE_REG(u32, ffa_req_version, ctxt, 1); 771 772 if (FFA_MAJOR_VERSION(ffa_req_version) != 1) { 773 res->a0 = FFA_RET_NOT_SUPPORTED; 774 return; 775 } 776 777 hyp_spin_lock(&version_lock); 778 if (has_version_negotiated) { 779 if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version)) 780 res->a0 = FFA_RET_NOT_SUPPORTED; 781 else 782 res->a0 = hyp_ffa_version; 783 goto unlock; 784 } 785 786 /* 787 * If the client driver tries to downgrade the version, we need to ask 788 * first if TEE supports it. 789 */ 790 if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version)) { 791 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 792 .a0 = FFA_VERSION, 793 .a1 = ffa_req_version, 794 }, res); 795 if (res->a0 == FFA_RET_NOT_SUPPORTED) 796 goto unlock; 797 798 hyp_ffa_version = ffa_req_version; 799 } 800 801 if (hyp_ffa_post_init()) { 802 res->a0 = FFA_RET_NOT_SUPPORTED; 803 } else { 804 smp_store_release(&has_version_negotiated, true); 805 res->a0 = hyp_ffa_version; 806 } 807 unlock: 808 hyp_spin_unlock(&version_lock); 809 } 810 811 static void do_ffa_part_get(struct arm_smccc_1_2_regs *res, 812 struct kvm_cpu_context *ctxt) 813 { 814 DECLARE_REG(u32, uuid0, ctxt, 1); 815 DECLARE_REG(u32, uuid1, ctxt, 2); 816 DECLARE_REG(u32, uuid2, ctxt, 3); 817 DECLARE_REG(u32, uuid3, ctxt, 4); 818 DECLARE_REG(u32, flags, ctxt, 5); 819 u32 count, partition_sz, copy_sz; 820 821 hyp_spin_lock(&host_buffers.lock); 822 if (!host_buffers.rx) { 823 ffa_to_smccc_res(res, FFA_RET_BUSY); 824 goto out_unlock; 825 } 826 827 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 828 .a0 = FFA_PARTITION_INFO_GET, 829 .a1 = uuid0, 830 .a2 = uuid1, 831 .a3 = uuid2, 832 .a4 = uuid3, 833 .a5 = flags, 834 }, res); 835 836 if (res->a0 != FFA_SUCCESS) 837 goto out_unlock; 838 839 count = res->a2; 840 if (!count) 841 goto out_unlock; 842 843 if (hyp_ffa_version > FFA_VERSION_1_0) { 844 /* Get the number of partitions deployed in the system */ 845 if (flags & 0x1) 846 goto out_unlock; 847 848 partition_sz = res->a3; 849 } else { 850 /* FFA_VERSION_1_0 lacks the size in the response */ 851 partition_sz = FFA_1_0_PARTITON_INFO_SZ; 852 } 853 854 copy_sz = partition_sz * count; 855 if (copy_sz > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) { 856 ffa_to_smccc_res(res, FFA_RET_ABORTED); 857 goto out_unlock; 858 } 859 860 memcpy(host_buffers.rx, hyp_buffers.rx, copy_sz); 861 out_unlock: 862 hyp_spin_unlock(&host_buffers.lock); 863 } 864 865 bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id) 866 { 867 struct arm_smccc_1_2_regs res; 868 869 /* 870 * There's no way we can tell what a non-standard SMC call might 871 * be up to. Ideally, we would terminate these here and return 872 * an error to the host, but sadly devices make use of custom 873 * firmware calls for things like power management, debugging, 874 * RNG access and crash reporting. 875 * 876 * Given that the architecture requires us to trust EL3 anyway, 877 * we forward unrecognised calls on under the assumption that 878 * the firmware doesn't expose a mechanism to access arbitrary 879 * non-secure memory. Short of a per-device table of SMCs, this 880 * is the best we can do. 881 */ 882 if (!is_ffa_call(func_id)) 883 return false; 884 885 if (func_id != FFA_VERSION && 886 !smp_load_acquire(&has_version_negotiated)) { 887 ffa_to_smccc_error(&res, FFA_RET_INVALID_PARAMETERS); 888 goto out_handled; 889 } 890 891 switch (func_id) { 892 case FFA_FEATURES: 893 if (!do_ffa_features(&res, host_ctxt)) 894 return false; 895 goto out_handled; 896 /* Memory management */ 897 case FFA_FN64_RXTX_MAP: 898 do_ffa_rxtx_map(&res, host_ctxt); 899 goto out_handled; 900 case FFA_RXTX_UNMAP: 901 do_ffa_rxtx_unmap(&res, host_ctxt); 902 goto out_handled; 903 case FFA_MEM_SHARE: 904 case FFA_FN64_MEM_SHARE: 905 do_ffa_mem_xfer(FFA_FN64_MEM_SHARE, &res, host_ctxt); 906 goto out_handled; 907 case FFA_MEM_RECLAIM: 908 do_ffa_mem_reclaim(&res, host_ctxt); 909 goto out_handled; 910 case FFA_MEM_LEND: 911 case FFA_FN64_MEM_LEND: 912 do_ffa_mem_xfer(FFA_FN64_MEM_LEND, &res, host_ctxt); 913 goto out_handled; 914 case FFA_MEM_FRAG_TX: 915 do_ffa_mem_frag_tx(&res, host_ctxt); 916 goto out_handled; 917 case FFA_VERSION: 918 do_ffa_version(&res, host_ctxt); 919 goto out_handled; 920 case FFA_PARTITION_INFO_GET: 921 do_ffa_part_get(&res, host_ctxt); 922 goto out_handled; 923 } 924 925 if (ffa_call_supported(func_id)) 926 return false; /* Pass through */ 927 928 ffa_to_smccc_error(&res, FFA_RET_NOT_SUPPORTED); 929 out_handled: 930 ffa_set_retval(host_ctxt, &res); 931 return true; 932 } 933 934 int hyp_ffa_init(void *pages) 935 { 936 struct arm_smccc_1_2_regs res; 937 void *tx, *rx; 938 939 if (kvm_host_psci_config.smccc_version < ARM_SMCCC_VERSION_1_2) 940 return 0; 941 942 arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) { 943 .a0 = FFA_VERSION, 944 .a1 = FFA_VERSION_1_2, 945 }, &res); 946 if (res.a0 == FFA_RET_NOT_SUPPORTED) 947 return 0; 948 949 /* 950 * Firmware returns the maximum supported version of the FF-A 951 * implementation. Check that the returned version is 952 * backwards-compatible with the hyp according to the rules in DEN0077A 953 * v1.1 REL0 13.2.1. 954 * 955 * Of course, things are never simple when dealing with firmware. v1.1 956 * broke ABI with v1.0 on several structures, which is itself 957 * incompatible with the aforementioned versioning scheme. The 958 * expectation is that v1.x implementations that do not support the v1.0 959 * ABI return NOT_SUPPORTED rather than a version number, according to 960 * DEN0077A v1.1 REL0 18.6.4. 961 */ 962 if (FFA_MAJOR_VERSION(res.a0) != 1) 963 return -EOPNOTSUPP; 964 965 if (FFA_MINOR_VERSION(res.a0) < FFA_MINOR_VERSION(FFA_VERSION_1_2)) 966 hyp_ffa_version = res.a0; 967 else 968 hyp_ffa_version = FFA_VERSION_1_2; 969 970 tx = pages; 971 pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE; 972 rx = pages; 973 pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE; 974 975 ffa_desc_buf = (struct kvm_ffa_descriptor_buffer) { 976 .buf = pages, 977 .len = PAGE_SIZE * 978 (hyp_ffa_proxy_pages() - (2 * KVM_FFA_MBOX_NR_PAGES)), 979 }; 980 981 hyp_buffers = (struct kvm_ffa_buffers) { 982 .lock = __HYP_SPIN_LOCK_UNLOCKED, 983 .tx = tx, 984 .rx = rx, 985 }; 986 987 host_buffers = (struct kvm_ffa_buffers) { 988 .lock = __HYP_SPIN_LOCK_UNLOCKED, 989 }; 990 991 version_lock = __HYP_SPIN_LOCK_UNLOCKED; 992 return 0; 993 } 994