1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * FF-A v1.0 proxy to filter out invalid memory-sharing SMC calls issued by
4 * the host. FF-A is a slightly more palatable abbreviation of "Arm Firmware
5 * Framework for Arm A-profile", which is specified by Arm in document
6 * number DEN0077.
7 *
8 * Copyright (C) 2022 - Google LLC
9 * Author: Andrew Walbran <qwandor@google.com>
10 *
11 * This driver hooks into the SMC trapping logic for the host and intercepts
12 * all calls falling within the FF-A range. Each call is either:
13 *
14 * - Forwarded on unmodified to the SPMD at EL3
15 * - Rejected as "unsupported"
16 * - Accompanied by a host stage-2 page-table check/update and reissued
17 *
18 * Consequently, any attempts by the host to make guest memory pages
19 * accessible to the secure world using FF-A will be detected either here
20 * (in the case that the memory is already owned by the guest) or during
21 * donation to the guest (in the case that the memory was previously shared
22 * with the secure world).
23 *
24 * To allow the rolling-back of page-table updates and FF-A calls in the
25 * event of failure, operations involving the RXTX buffers are locked for
26 * the duration and are therefore serialised.
27 */
28
29 #include <linux/arm_ffa.h>
30 #include <asm/kvm_pkvm.h>
31
32 #include <nvhe/arm-smccc.h>
33 #include <nvhe/ffa.h>
34 #include <nvhe/mem_protect.h>
35 #include <nvhe/memory.h>
36 #include <nvhe/trap_handler.h>
37 #include <nvhe/spinlock.h>
38
39 /*
40 * "ID value 0 must be returned at the Non-secure physical FF-A instance"
41 * We share this ID with the host.
42 */
43 #define HOST_FFA_ID 0
44
45 /*
46 * A buffer to hold the maximum descriptor size we can see from the host,
47 * which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP
48 * when resolving the handle on the reclaim path.
49 */
50 struct kvm_ffa_descriptor_buffer {
51 void *buf;
52 size_t len;
53 };
54
55 static struct kvm_ffa_descriptor_buffer ffa_desc_buf;
56
57 struct kvm_ffa_buffers {
58 hyp_spinlock_t lock;
59 void *tx;
60 void *rx;
61 };
62
63 /*
64 * Note that we don't currently lock these buffers explicitly, instead
65 * relying on the locking of the host FFA buffers as we only have one
66 * client.
67 */
68 static struct kvm_ffa_buffers hyp_buffers;
69 static struct kvm_ffa_buffers host_buffers;
70 static u32 hyp_ffa_version;
71 static bool has_version_negotiated;
72 static hyp_spinlock_t version_lock;
73
ffa_to_smccc_error(struct arm_smccc_1_2_regs * res,u64 ffa_errno)74 static void ffa_to_smccc_error(struct arm_smccc_1_2_regs *res, u64 ffa_errno)
75 {
76 *res = (struct arm_smccc_1_2_regs) {
77 .a0 = FFA_ERROR,
78 .a2 = ffa_errno,
79 };
80 }
81
ffa_to_smccc_res_prop(struct arm_smccc_1_2_regs * res,int ret,u64 prop)82 static void ffa_to_smccc_res_prop(struct arm_smccc_1_2_regs *res, int ret, u64 prop)
83 {
84 if (ret == FFA_RET_SUCCESS) {
85 *res = (struct arm_smccc_1_2_regs) { .a0 = FFA_SUCCESS,
86 .a2 = prop };
87 } else {
88 ffa_to_smccc_error(res, ret);
89 }
90 }
91
ffa_to_smccc_res(struct arm_smccc_1_2_regs * res,int ret)92 static void ffa_to_smccc_res(struct arm_smccc_1_2_regs *res, int ret)
93 {
94 ffa_to_smccc_res_prop(res, ret, 0);
95 }
96
ffa_set_retval(struct kvm_cpu_context * ctxt,struct arm_smccc_1_2_regs * res)97 static void ffa_set_retval(struct kvm_cpu_context *ctxt,
98 struct arm_smccc_1_2_regs *res)
99 {
100 cpu_reg(ctxt, 0) = res->a0;
101 cpu_reg(ctxt, 1) = res->a1;
102 cpu_reg(ctxt, 2) = res->a2;
103 cpu_reg(ctxt, 3) = res->a3;
104 cpu_reg(ctxt, 4) = res->a4;
105 cpu_reg(ctxt, 5) = res->a5;
106 cpu_reg(ctxt, 6) = res->a6;
107 cpu_reg(ctxt, 7) = res->a7;
108
109 /*
110 * DEN0028C 2.6: SMC32/HVC32 call from aarch64 must preserve x8-x30.
111 *
112 * In FF-A 1.2, we cannot rely on the function ID sent by the caller to
113 * detect 32-bit calls because the CPU cycle management interfaces (e.g.
114 * FFA_MSG_WAIT, FFA_RUN) are 32-bit only but can have 64-bit responses.
115 *
116 * FFA-1.3 introduces 64-bit variants of the CPU cycle management
117 * interfaces. Moreover, FF-A 1.3 clarifies that SMC32 direct requests
118 * complete with SMC32 direct responses which *should* allow us use the
119 * function ID sent by the caller to determine whether to return x8-x17.
120 *
121 * Note that we also cannot rely on function IDs in the response.
122 *
123 * Given the above, assume SMC64 and send back x0-x17 unconditionally
124 * as the passthrough code (__kvm_hyp_host_forward_smc) does the same.
125 */
126 cpu_reg(ctxt, 8) = res->a8;
127 cpu_reg(ctxt, 9) = res->a9;
128 cpu_reg(ctxt, 10) = res->a10;
129 cpu_reg(ctxt, 11) = res->a11;
130 cpu_reg(ctxt, 12) = res->a12;
131 cpu_reg(ctxt, 13) = res->a13;
132 cpu_reg(ctxt, 14) = res->a14;
133 cpu_reg(ctxt, 15) = res->a15;
134 cpu_reg(ctxt, 16) = res->a16;
135 cpu_reg(ctxt, 17) = res->a17;
136 }
137
is_ffa_call(u64 func_id)138 static bool is_ffa_call(u64 func_id)
139 {
140 return ARM_SMCCC_IS_FAST_CALL(func_id) &&
141 ARM_SMCCC_OWNER_NUM(func_id) == ARM_SMCCC_OWNER_STANDARD &&
142 ARM_SMCCC_FUNC_NUM(func_id) >= FFA_MIN_FUNC_NUM &&
143 ARM_SMCCC_FUNC_NUM(func_id) <= FFA_MAX_FUNC_NUM;
144 }
145
ffa_map_hyp_buffers(u64 ffa_page_count)146 static int ffa_map_hyp_buffers(u64 ffa_page_count)
147 {
148 struct arm_smccc_1_2_regs res;
149
150 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
151 .a0 = FFA_FN64_RXTX_MAP,
152 .a1 = hyp_virt_to_phys(hyp_buffers.tx),
153 .a2 = hyp_virt_to_phys(hyp_buffers.rx),
154 .a3 = ffa_page_count,
155 }, &res);
156
157 return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2;
158 }
159
ffa_unmap_hyp_buffers(void)160 static int ffa_unmap_hyp_buffers(void)
161 {
162 struct arm_smccc_1_2_regs res;
163
164 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
165 .a0 = FFA_RXTX_UNMAP,
166 .a1 = HOST_FFA_ID,
167 }, &res);
168
169 return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2;
170 }
171
ffa_mem_frag_tx(struct arm_smccc_1_2_regs * res,u32 handle_lo,u32 handle_hi,u32 fraglen,u32 endpoint_id)172 static void ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, u32 handle_lo,
173 u32 handle_hi, u32 fraglen, u32 endpoint_id)
174 {
175 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
176 .a0 = FFA_MEM_FRAG_TX,
177 .a1 = handle_lo,
178 .a2 = handle_hi,
179 .a3 = fraglen,
180 .a4 = endpoint_id,
181 }, res);
182 }
183
ffa_mem_frag_rx(struct arm_smccc_1_2_regs * res,u32 handle_lo,u32 handle_hi,u32 fragoff)184 static void ffa_mem_frag_rx(struct arm_smccc_1_2_regs *res, u32 handle_lo,
185 u32 handle_hi, u32 fragoff)
186 {
187 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
188 .a0 = FFA_MEM_FRAG_RX,
189 .a1 = handle_lo,
190 .a2 = handle_hi,
191 .a3 = fragoff,
192 .a4 = HOST_FFA_ID,
193 }, res);
194 }
195
ffa_mem_xfer(struct arm_smccc_1_2_regs * res,u64 func_id,u32 len,u32 fraglen)196 static void ffa_mem_xfer(struct arm_smccc_1_2_regs *res, u64 func_id, u32 len,
197 u32 fraglen)
198 {
199 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
200 .a0 = func_id,
201 .a1 = len,
202 .a2 = fraglen,
203 }, res);
204 }
205
ffa_mem_reclaim(struct arm_smccc_1_2_regs * res,u32 handle_lo,u32 handle_hi,u32 flags)206 static void ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, u32 handle_lo,
207 u32 handle_hi, u32 flags)
208 {
209 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
210 .a0 = FFA_MEM_RECLAIM,
211 .a1 = handle_lo,
212 .a2 = handle_hi,
213 .a3 = flags,
214 }, res);
215 }
216
ffa_retrieve_req(struct arm_smccc_1_2_regs * res,u32 len)217 static void ffa_retrieve_req(struct arm_smccc_1_2_regs *res, u32 len)
218 {
219 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
220 .a0 = FFA_FN64_MEM_RETRIEVE_REQ,
221 .a1 = len,
222 .a2 = len,
223 }, res);
224 }
225
ffa_rx_release(struct arm_smccc_1_2_regs * res)226 static void ffa_rx_release(struct arm_smccc_1_2_regs *res)
227 {
228 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
229 .a0 = FFA_RX_RELEASE,
230 }, res);
231 }
232
do_ffa_rxtx_map(struct arm_smccc_1_2_regs * res,struct kvm_cpu_context * ctxt)233 static void do_ffa_rxtx_map(struct arm_smccc_1_2_regs *res,
234 struct kvm_cpu_context *ctxt)
235 {
236 DECLARE_REG(phys_addr_t, tx, ctxt, 1);
237 DECLARE_REG(phys_addr_t, rx, ctxt, 2);
238 DECLARE_REG(u32, npages, ctxt, 3);
239 int ret = 0;
240 void *rx_virt, *tx_virt;
241
242 if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE) {
243 ret = FFA_RET_INVALID_PARAMETERS;
244 goto out;
245 }
246
247 if (!PAGE_ALIGNED(tx) || !PAGE_ALIGNED(rx)) {
248 ret = FFA_RET_INVALID_PARAMETERS;
249 goto out;
250 }
251
252 hyp_spin_lock(&host_buffers.lock);
253 if (host_buffers.tx) {
254 ret = FFA_RET_DENIED;
255 goto out_unlock;
256 }
257
258 /*
259 * Map our hypervisor buffers into the SPMD before mapping and
260 * pinning the host buffers in our own address space.
261 */
262 ret = ffa_map_hyp_buffers(npages);
263 if (ret)
264 goto out_unlock;
265
266 ret = __pkvm_host_share_hyp(hyp_phys_to_pfn(tx));
267 if (ret) {
268 ret = FFA_RET_INVALID_PARAMETERS;
269 goto err_unmap;
270 }
271
272 ret = __pkvm_host_share_hyp(hyp_phys_to_pfn(rx));
273 if (ret) {
274 ret = FFA_RET_INVALID_PARAMETERS;
275 goto err_unshare_tx;
276 }
277
278 tx_virt = hyp_phys_to_virt(tx);
279 ret = hyp_pin_shared_mem(tx_virt, tx_virt + 1);
280 if (ret) {
281 ret = FFA_RET_INVALID_PARAMETERS;
282 goto err_unshare_rx;
283 }
284
285 rx_virt = hyp_phys_to_virt(rx);
286 ret = hyp_pin_shared_mem(rx_virt, rx_virt + 1);
287 if (ret) {
288 ret = FFA_RET_INVALID_PARAMETERS;
289 goto err_unpin_tx;
290 }
291
292 host_buffers.tx = tx_virt;
293 host_buffers.rx = rx_virt;
294
295 out_unlock:
296 hyp_spin_unlock(&host_buffers.lock);
297 out:
298 ffa_to_smccc_res(res, ret);
299 return;
300
301 err_unpin_tx:
302 hyp_unpin_shared_mem(tx_virt, tx_virt + 1);
303 err_unshare_rx:
304 __pkvm_host_unshare_hyp(hyp_phys_to_pfn(rx));
305 err_unshare_tx:
306 __pkvm_host_unshare_hyp(hyp_phys_to_pfn(tx));
307 err_unmap:
308 ffa_unmap_hyp_buffers();
309 goto out_unlock;
310 }
311
do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs * res,struct kvm_cpu_context * ctxt)312 static void do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs *res,
313 struct kvm_cpu_context *ctxt)
314 {
315 DECLARE_REG(u32, id, ctxt, 1);
316 int ret = 0;
317
318 if (id != HOST_FFA_ID) {
319 ret = FFA_RET_INVALID_PARAMETERS;
320 goto out;
321 }
322
323 hyp_spin_lock(&host_buffers.lock);
324 if (!host_buffers.tx) {
325 ret = FFA_RET_INVALID_PARAMETERS;
326 goto out_unlock;
327 }
328
329 hyp_unpin_shared_mem(host_buffers.tx, host_buffers.tx + 1);
330 WARN_ON(__pkvm_host_unshare_hyp(hyp_virt_to_pfn(host_buffers.tx)));
331 host_buffers.tx = NULL;
332
333 hyp_unpin_shared_mem(host_buffers.rx, host_buffers.rx + 1);
334 WARN_ON(__pkvm_host_unshare_hyp(hyp_virt_to_pfn(host_buffers.rx)));
335 host_buffers.rx = NULL;
336
337 ffa_unmap_hyp_buffers();
338
339 out_unlock:
340 hyp_spin_unlock(&host_buffers.lock);
341 out:
342 ffa_to_smccc_res(res, ret);
343 }
344
__ffa_host_share_ranges(struct ffa_mem_region_addr_range * ranges,u32 nranges)345 static u32 __ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges,
346 u32 nranges)
347 {
348 u32 i;
349
350 for (i = 0; i < nranges; ++i) {
351 struct ffa_mem_region_addr_range *range = &ranges[i];
352 u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE;
353 u64 pfn = hyp_phys_to_pfn(range->address);
354
355 if (!PAGE_ALIGNED(sz | range->address))
356 break;
357
358 if (__pkvm_host_share_ffa(pfn, sz / PAGE_SIZE))
359 break;
360 }
361
362 return i;
363 }
364
__ffa_host_unshare_ranges(struct ffa_mem_region_addr_range * ranges,u32 nranges)365 static u32 __ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges,
366 u32 nranges)
367 {
368 u32 i;
369
370 for (i = 0; i < nranges; ++i) {
371 struct ffa_mem_region_addr_range *range = &ranges[i];
372 u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE;
373 u64 pfn = hyp_phys_to_pfn(range->address);
374
375 if (!PAGE_ALIGNED(sz | range->address))
376 break;
377
378 if (__pkvm_host_unshare_ffa(pfn, sz / PAGE_SIZE))
379 break;
380 }
381
382 return i;
383 }
384
ffa_host_share_ranges(struct ffa_mem_region_addr_range * ranges,u32 nranges)385 static int ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges,
386 u32 nranges)
387 {
388 u32 nshared = __ffa_host_share_ranges(ranges, nranges);
389 int ret = 0;
390
391 if (nshared != nranges) {
392 WARN_ON(__ffa_host_unshare_ranges(ranges, nshared) != nshared);
393 ret = FFA_RET_DENIED;
394 }
395
396 return ret;
397 }
398
ffa_host_unshare_ranges(struct ffa_mem_region_addr_range * ranges,u32 nranges)399 static int ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges,
400 u32 nranges)
401 {
402 u32 nunshared = __ffa_host_unshare_ranges(ranges, nranges);
403 int ret = 0;
404
405 if (nunshared != nranges) {
406 WARN_ON(__ffa_host_share_ranges(ranges, nunshared) != nunshared);
407 ret = FFA_RET_DENIED;
408 }
409
410 return ret;
411 }
412
do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs * res,struct kvm_cpu_context * ctxt)413 static void do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res,
414 struct kvm_cpu_context *ctxt)
415 {
416 DECLARE_REG(u32, handle_lo, ctxt, 1);
417 DECLARE_REG(u32, handle_hi, ctxt, 2);
418 DECLARE_REG(u32, fraglen, ctxt, 3);
419 DECLARE_REG(u32, endpoint_id, ctxt, 4);
420 struct ffa_mem_region_addr_range *buf;
421 int ret = FFA_RET_INVALID_PARAMETERS;
422 u32 nr_ranges;
423
424 if (fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)
425 goto out;
426
427 if (fraglen % sizeof(*buf))
428 goto out;
429
430 hyp_spin_lock(&host_buffers.lock);
431 if (!host_buffers.tx)
432 goto out_unlock;
433
434 buf = hyp_buffers.tx;
435 memcpy(buf, host_buffers.tx, fraglen);
436 nr_ranges = fraglen / sizeof(*buf);
437
438 ret = ffa_host_share_ranges(buf, nr_ranges);
439 if (ret) {
440 /*
441 * We're effectively aborting the transaction, so we need
442 * to restore the global state back to what it was prior to
443 * transmission of the first fragment.
444 */
445 ffa_mem_reclaim(res, handle_lo, handle_hi, 0);
446 WARN_ON(res->a0 != FFA_SUCCESS);
447 goto out_unlock;
448 }
449
450 ffa_mem_frag_tx(res, handle_lo, handle_hi, fraglen, endpoint_id);
451 if (res->a0 != FFA_SUCCESS && res->a0 != FFA_MEM_FRAG_RX)
452 WARN_ON(ffa_host_unshare_ranges(buf, nr_ranges));
453
454 out_unlock:
455 hyp_spin_unlock(&host_buffers.lock);
456 out:
457 if (ret)
458 ffa_to_smccc_res(res, ret);
459
460 /*
461 * If for any reason this did not succeed, we're in trouble as we have
462 * now lost the content of the previous fragments and we can't rollback
463 * the host stage-2 changes. The pages previously marked as shared will
464 * remain stuck in that state forever, hence preventing the host from
465 * sharing/donating them again and may possibly lead to subsequent
466 * failures, but this will not compromise confidentiality.
467 */
468 return;
469 }
470
__do_ffa_mem_xfer(const u64 func_id,struct arm_smccc_1_2_regs * res,struct kvm_cpu_context * ctxt)471 static void __do_ffa_mem_xfer(const u64 func_id,
472 struct arm_smccc_1_2_regs *res,
473 struct kvm_cpu_context *ctxt)
474 {
475 DECLARE_REG(u32, len, ctxt, 1);
476 DECLARE_REG(u32, fraglen, ctxt, 2);
477 DECLARE_REG(u64, addr_mbz, ctxt, 3);
478 DECLARE_REG(u32, npages_mbz, ctxt, 4);
479 u32 offset, nr_ranges, checked_offset, em_mem_access_off;
480 struct ffa_mem_region_attributes *ep_mem_access;
481 struct ffa_composite_mem_region *reg;
482 struct ffa_mem_region *buf;
483 int ret = 0;
484 size_t mem_region_len = FFA_MEM_REGION_SZ(hyp_ffa_version);
485
486 if (addr_mbz || npages_mbz || fraglen > len ||
487 fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
488 ret = FFA_RET_INVALID_PARAMETERS;
489 goto out;
490 }
491
492 if (fraglen < mem_region_len + ffa_emad_size_get(hyp_ffa_version)) {
493 ret = FFA_RET_INVALID_PARAMETERS;
494 goto out;
495 }
496
497 hyp_spin_lock(&host_buffers.lock);
498 if (!host_buffers.tx) {
499 ret = FFA_RET_INVALID_PARAMETERS;
500 goto out_unlock;
501 }
502
503 if (len > ffa_desc_buf.len) {
504 ret = FFA_RET_NO_MEMORY;
505 goto out_unlock;
506 }
507
508 buf = hyp_buffers.tx;
509 memcpy(buf, host_buffers.tx, fraglen);
510
511 em_mem_access_off = ffa_mem_desc_offset(buf, 0, hyp_ffa_version);
512 if ((u64)em_mem_access_off + ffa_emad_size_get(hyp_ffa_version) > fraglen) {
513 ret = FFA_RET_INVALID_PARAMETERS;
514 goto out_unlock;
515 }
516
517 ep_mem_access = (void *)buf + em_mem_access_off;
518 offset = ep_mem_access->composite_off;
519 if (!offset || buf->ep_count != 1 || buf->sender_id != HOST_FFA_ID) {
520 ret = FFA_RET_INVALID_PARAMETERS;
521 goto out_unlock;
522 }
523
524 if (check_add_overflow(offset, sizeof(struct ffa_composite_mem_region), &checked_offset)) {
525 ret = FFA_RET_INVALID_PARAMETERS;
526 goto out_unlock;
527 }
528
529 if (fraglen < checked_offset) {
530 ret = FFA_RET_INVALID_PARAMETERS;
531 goto out_unlock;
532 }
533
534 reg = (void *)buf + offset;
535 nr_ranges = ((void *)buf + fraglen) - (void *)reg->constituents;
536 if (nr_ranges % sizeof(reg->constituents[0])) {
537 ret = FFA_RET_INVALID_PARAMETERS;
538 goto out_unlock;
539 }
540
541 nr_ranges /= sizeof(reg->constituents[0]);
542 ret = ffa_host_share_ranges(reg->constituents, nr_ranges);
543 if (ret)
544 goto out_unlock;
545
546 ffa_mem_xfer(res, func_id, len, fraglen);
547 if (fraglen != len) {
548 if (res->a0 != FFA_MEM_FRAG_RX)
549 goto err_unshare;
550
551 if (res->a3 != fraglen)
552 goto err_unshare;
553 } else if (res->a0 != FFA_SUCCESS) {
554 goto err_unshare;
555 }
556
557 out_unlock:
558 hyp_spin_unlock(&host_buffers.lock);
559 out:
560 if (ret)
561 ffa_to_smccc_res(res, ret);
562 return;
563
564 err_unshare:
565 WARN_ON(ffa_host_unshare_ranges(reg->constituents, nr_ranges));
566 goto out_unlock;
567 }
568
569 #define do_ffa_mem_xfer(fid, res, ctxt) \
570 do { \
571 BUILD_BUG_ON((fid) != FFA_FN64_MEM_SHARE && \
572 (fid) != FFA_FN64_MEM_LEND); \
573 __do_ffa_mem_xfer((fid), (res), (ctxt)); \
574 } while (0);
575
do_ffa_mem_reclaim(struct arm_smccc_1_2_regs * res,struct kvm_cpu_context * ctxt)576 static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res,
577 struct kvm_cpu_context *ctxt)
578 {
579 DECLARE_REG(u32, handle_lo, ctxt, 1);
580 DECLARE_REG(u32, handle_hi, ctxt, 2);
581 DECLARE_REG(u32, flags, ctxt, 3);
582 u32 offset, len, fraglen, fragoff, em_mem_access_off;
583 struct ffa_mem_region_attributes *ep_mem_access;
584 struct ffa_composite_mem_region *reg;
585 struct ffa_mem_region *buf;
586 int ret = 0;
587 u64 handle;
588
589 handle = PACK_HANDLE(handle_lo, handle_hi);
590
591 hyp_spin_lock(&host_buffers.lock);
592
593 buf = hyp_buffers.tx;
594 *buf = (struct ffa_mem_region) {
595 .sender_id = HOST_FFA_ID,
596 .handle = handle,
597 };
598
599 ffa_retrieve_req(res, sizeof(*buf));
600 buf = hyp_buffers.rx;
601 if (res->a0 != FFA_MEM_RETRIEVE_RESP)
602 goto out_unlock;
603
604 len = res->a1;
605 fraglen = res->a2;
606
607 em_mem_access_off = ffa_mem_desc_offset(buf, 0, hyp_ffa_version);
608 if ((u64)em_mem_access_off + ffa_emad_size_get(hyp_ffa_version) > fraglen) {
609 ret = FFA_RET_INVALID_PARAMETERS;
610 ffa_rx_release(res);
611 goto out_unlock;
612 }
613
614 ep_mem_access = (void *)buf + em_mem_access_off;
615 offset = ep_mem_access->composite_off;
616 /*
617 * We can trust the SPMD to get this right, but let's at least
618 * check that we end up with something that doesn't look _completely_
619 * bogus.
620 */
621 if (offset + CONSTITUENTS_OFFSET(0) > len ||
622 fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
623 ret = FFA_RET_ABORTED;
624 ffa_rx_release(res);
625 goto out_unlock;
626 }
627
628 if (len > ffa_desc_buf.len) {
629 ret = FFA_RET_NO_MEMORY;
630 ffa_rx_release(res);
631 goto out_unlock;
632 }
633
634 buf = ffa_desc_buf.buf;
635 memcpy(buf, hyp_buffers.rx, fraglen);
636 ffa_rx_release(res);
637
638 for (fragoff = fraglen; fragoff < len; fragoff += fraglen) {
639 ffa_mem_frag_rx(res, handle_lo, handle_hi, fragoff);
640 if (res->a0 != FFA_MEM_FRAG_TX) {
641 ret = FFA_RET_INVALID_PARAMETERS;
642 goto out_unlock;
643 }
644
645 fraglen = res->a3;
646 memcpy((void *)buf + fragoff, hyp_buffers.rx, fraglen);
647 ffa_rx_release(res);
648 }
649
650 reg = (void *)buf + offset;
651 if (offset + CONSTITUENTS_OFFSET(reg->addr_range_cnt) > len) {
652 ret = FFA_RET_ABORTED;
653 goto out_unlock;
654 }
655
656 ffa_mem_reclaim(res, handle_lo, handle_hi, flags);
657 if (res->a0 != FFA_SUCCESS)
658 goto out_unlock;
659
660 /* If the SPMD was happy, then we should be too. */
661 WARN_ON(ffa_host_unshare_ranges(reg->constituents,
662 reg->addr_range_cnt));
663 out_unlock:
664 hyp_spin_unlock(&host_buffers.lock);
665
666 if (ret)
667 ffa_to_smccc_res(res, ret);
668 }
669
670 /*
671 * Is a given FFA function supported, either by forwarding on directly
672 * or by handling at EL2?
673 */
ffa_call_supported(u64 func_id)674 static bool ffa_call_supported(u64 func_id)
675 {
676 switch (func_id) {
677 /* Unsupported memory management calls */
678 case FFA_FN64_MEM_RETRIEVE_REQ:
679 case FFA_MEM_RETRIEVE_RESP:
680 case FFA_MEM_RELINQUISH:
681 case FFA_MEM_OP_PAUSE:
682 case FFA_MEM_OP_RESUME:
683 case FFA_MEM_FRAG_RX:
684 case FFA_FN64_MEM_DONATE:
685 /* Indirect message passing via RX/TX buffers */
686 case FFA_MSG_SEND:
687 case FFA_MSG_POLL:
688 case FFA_MSG_WAIT:
689 /* 32-bit variants of 64-bit calls */
690 case FFA_MSG_SEND_DIRECT_RESP:
691 case FFA_RXTX_MAP:
692 case FFA_MEM_DONATE:
693 case FFA_MEM_RETRIEVE_REQ:
694 /* Optional notification interfaces added in FF-A 1.1 */
695 case FFA_NOTIFICATION_BITMAP_CREATE:
696 case FFA_NOTIFICATION_BITMAP_DESTROY:
697 case FFA_NOTIFICATION_BIND:
698 case FFA_NOTIFICATION_UNBIND:
699 case FFA_NOTIFICATION_SET:
700 case FFA_NOTIFICATION_GET:
701 case FFA_NOTIFICATION_INFO_GET:
702 /* Optional interfaces added in FF-A 1.2 */
703 case FFA_MSG_SEND_DIRECT_REQ2: /* Optional per 7.5.1 */
704 case FFA_MSG_SEND_DIRECT_RESP2: /* Optional per 7.5.1 */
705 case FFA_CONSOLE_LOG: /* Optional per 13.1: not in Table 13.1 */
706 case FFA_PARTITION_INFO_GET_REGS: /* Optional for virtual instances per 13.1 */
707 return false;
708 }
709
710 return true;
711 }
712
do_ffa_features(struct arm_smccc_1_2_regs * res,struct kvm_cpu_context * ctxt)713 static bool do_ffa_features(struct arm_smccc_1_2_regs *res,
714 struct kvm_cpu_context *ctxt)
715 {
716 DECLARE_REG(u32, id, ctxt, 1);
717 u64 prop = 0;
718 int ret = 0;
719
720 if (!ffa_call_supported(id)) {
721 ret = FFA_RET_NOT_SUPPORTED;
722 goto out_handled;
723 }
724
725 switch (id) {
726 case FFA_MEM_SHARE:
727 case FFA_FN64_MEM_SHARE:
728 case FFA_MEM_LEND:
729 case FFA_FN64_MEM_LEND:
730 ret = FFA_RET_SUCCESS;
731 prop = 0; /* No support for dynamic buffers */
732 goto out_handled;
733 default:
734 return false;
735 }
736
737 out_handled:
738 ffa_to_smccc_res_prop(res, ret, prop);
739 return true;
740 }
741
hyp_ffa_post_init(void)742 static int hyp_ffa_post_init(void)
743 {
744 size_t min_rxtx_sz;
745 struct arm_smccc_1_2_regs res;
746
747 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){
748 .a0 = FFA_ID_GET,
749 }, &res);
750 if (res.a0 != FFA_SUCCESS)
751 return -EOPNOTSUPP;
752
753 if (res.a2 != HOST_FFA_ID)
754 return -EINVAL;
755
756 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){
757 .a0 = FFA_FEATURES,
758 .a1 = FFA_FN64_RXTX_MAP,
759 }, &res);
760 if (res.a0 != FFA_SUCCESS)
761 return -EOPNOTSUPP;
762
763 switch (res.a2 & FFA_FEAT_RXTX_MIN_SZ_MASK) {
764 case FFA_FEAT_RXTX_MIN_SZ_4K:
765 min_rxtx_sz = SZ_4K;
766 break;
767 case FFA_FEAT_RXTX_MIN_SZ_16K:
768 min_rxtx_sz = SZ_16K;
769 break;
770 case FFA_FEAT_RXTX_MIN_SZ_64K:
771 min_rxtx_sz = SZ_64K;
772 break;
773 default:
774 return -EINVAL;
775 }
776
777 if (min_rxtx_sz > PAGE_SIZE)
778 return -EOPNOTSUPP;
779
780 return 0;
781 }
782
do_ffa_version(struct arm_smccc_1_2_regs * res,struct kvm_cpu_context * ctxt)783 static void do_ffa_version(struct arm_smccc_1_2_regs *res,
784 struct kvm_cpu_context *ctxt)
785 {
786 DECLARE_REG(u32, ffa_req_version, ctxt, 1);
787
788 if (FFA_MAJOR_VERSION(ffa_req_version) != 1) {
789 res->a0 = FFA_RET_NOT_SUPPORTED;
790 return;
791 }
792
793 hyp_spin_lock(&version_lock);
794 if (has_version_negotiated) {
795 if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version))
796 res->a0 = FFA_RET_NOT_SUPPORTED;
797 else
798 res->a0 = hyp_ffa_version;
799 goto unlock;
800 }
801
802 /*
803 * If the client driver tries to downgrade the version, we need to ask
804 * first if TEE supports it.
805 */
806 if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version)) {
807 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
808 .a0 = FFA_VERSION,
809 .a1 = ffa_req_version,
810 }, res);
811 if ((s32)res->a0 == FFA_RET_NOT_SUPPORTED)
812 goto unlock;
813
814 hyp_ffa_version = ffa_req_version;
815 }
816
817 if (hyp_ffa_post_init()) {
818 res->a0 = FFA_RET_NOT_SUPPORTED;
819 } else {
820 smp_store_release(&has_version_negotiated, true);
821 res->a0 = hyp_ffa_version;
822 }
823 unlock:
824 hyp_spin_unlock(&version_lock);
825 }
826
do_ffa_part_get(struct arm_smccc_1_2_regs * res,struct kvm_cpu_context * ctxt)827 static void do_ffa_part_get(struct arm_smccc_1_2_regs *res,
828 struct kvm_cpu_context *ctxt)
829 {
830 DECLARE_REG(u32, uuid0, ctxt, 1);
831 DECLARE_REG(u32, uuid1, ctxt, 2);
832 DECLARE_REG(u32, uuid2, ctxt, 3);
833 DECLARE_REG(u32, uuid3, ctxt, 4);
834 DECLARE_REG(u32, flags, ctxt, 5);
835 u32 count, partition_sz, copy_sz;
836
837 hyp_spin_lock(&host_buffers.lock);
838 if (!host_buffers.rx) {
839 ffa_to_smccc_res(res, FFA_RET_BUSY);
840 goto out_unlock;
841 }
842
843 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
844 .a0 = FFA_PARTITION_INFO_GET,
845 .a1 = uuid0,
846 .a2 = uuid1,
847 .a3 = uuid2,
848 .a4 = uuid3,
849 .a5 = flags,
850 }, res);
851
852 if (res->a0 != FFA_SUCCESS)
853 goto out_unlock;
854
855 count = res->a2;
856 if (!count)
857 goto out_unlock;
858
859 if (hyp_ffa_version > FFA_VERSION_1_0) {
860 /* Get the number of partitions deployed in the system */
861 if (flags & 0x1)
862 goto out_unlock;
863
864 partition_sz = res->a3;
865 } else {
866 /* FFA_VERSION_1_0 lacks the size in the response */
867 partition_sz = FFA_1_0_PARTITON_INFO_SZ;
868 }
869
870 copy_sz = partition_sz * count;
871 if (copy_sz > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
872 ffa_to_smccc_res(res, FFA_RET_ABORTED);
873 goto out_unlock;
874 }
875
876 memcpy(host_buffers.rx, hyp_buffers.rx, copy_sz);
877 out_unlock:
878 hyp_spin_unlock(&host_buffers.lock);
879 }
880
kvm_host_ffa_handler(struct kvm_cpu_context * host_ctxt,u32 func_id)881 bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
882 {
883 struct arm_smccc_1_2_regs res = {0};
884
885 /*
886 * There's no way we can tell what a non-standard SMC call might
887 * be up to. Ideally, we would terminate these here and return
888 * an error to the host, but sadly devices make use of custom
889 * firmware calls for things like power management, debugging,
890 * RNG access and crash reporting.
891 *
892 * Given that the architecture requires us to trust EL3 anyway,
893 * we forward unrecognised calls on under the assumption that
894 * the firmware doesn't expose a mechanism to access arbitrary
895 * non-secure memory. Short of a per-device table of SMCs, this
896 * is the best we can do.
897 */
898 if (!is_ffa_call(func_id))
899 return false;
900
901 if (func_id != FFA_VERSION &&
902 !smp_load_acquire(&has_version_negotiated)) {
903 ffa_to_smccc_error(&res, FFA_RET_INVALID_PARAMETERS);
904 goto out_handled;
905 }
906
907 switch (func_id) {
908 case FFA_FEATURES:
909 if (!do_ffa_features(&res, host_ctxt))
910 return false;
911 goto out_handled;
912 /* Memory management */
913 case FFA_FN64_RXTX_MAP:
914 do_ffa_rxtx_map(&res, host_ctxt);
915 goto out_handled;
916 case FFA_RXTX_UNMAP:
917 do_ffa_rxtx_unmap(&res, host_ctxt);
918 goto out_handled;
919 case FFA_MEM_SHARE:
920 case FFA_FN64_MEM_SHARE:
921 do_ffa_mem_xfer(FFA_FN64_MEM_SHARE, &res, host_ctxt);
922 goto out_handled;
923 case FFA_MEM_RECLAIM:
924 do_ffa_mem_reclaim(&res, host_ctxt);
925 goto out_handled;
926 case FFA_MEM_LEND:
927 case FFA_FN64_MEM_LEND:
928 do_ffa_mem_xfer(FFA_FN64_MEM_LEND, &res, host_ctxt);
929 goto out_handled;
930 case FFA_MEM_FRAG_TX:
931 do_ffa_mem_frag_tx(&res, host_ctxt);
932 goto out_handled;
933 case FFA_VERSION:
934 do_ffa_version(&res, host_ctxt);
935 goto out_handled;
936 case FFA_PARTITION_INFO_GET:
937 do_ffa_part_get(&res, host_ctxt);
938 goto out_handled;
939 }
940
941 if (ffa_call_supported(func_id))
942 return false; /* Pass through */
943
944 ffa_to_smccc_error(&res, FFA_RET_NOT_SUPPORTED);
945 out_handled:
946 ffa_set_retval(host_ctxt, &res);
947 return true;
948 }
949
hyp_ffa_init(void * pages)950 int hyp_ffa_init(void *pages)
951 {
952 struct arm_smccc_1_2_regs res;
953 void *tx, *rx;
954
955 if (kvm_host_psci_config.smccc_version < ARM_SMCCC_VERSION_1_2)
956 return 0;
957
958 hyp_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
959 .a0 = FFA_VERSION,
960 .a1 = FFA_VERSION_1_2,
961 }, &res);
962 if ((s32)res.a0 == FFA_RET_NOT_SUPPORTED)
963 return 0;
964
965 /*
966 * Firmware returns the maximum supported version of the FF-A
967 * implementation. Check that the returned version is
968 * backwards-compatible with the hyp according to the rules in DEN0077A
969 * v1.1 REL0 13.2.1.
970 *
971 * Of course, things are never simple when dealing with firmware. v1.1
972 * broke ABI with v1.0 on several structures, which is itself
973 * incompatible with the aforementioned versioning scheme. The
974 * expectation is that v1.x implementations that do not support the v1.0
975 * ABI return NOT_SUPPORTED rather than a version number, according to
976 * DEN0077A v1.1 REL0 18.6.4.
977 */
978 if (FFA_MAJOR_VERSION(res.a0) != 1)
979 return -EOPNOTSUPP;
980
981 if (FFA_MINOR_VERSION(res.a0) < FFA_MINOR_VERSION(FFA_VERSION_1_2))
982 hyp_ffa_version = res.a0;
983 else
984 hyp_ffa_version = FFA_VERSION_1_2;
985
986 tx = pages;
987 pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE;
988 rx = pages;
989 pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE;
990
991 ffa_desc_buf = (struct kvm_ffa_descriptor_buffer) {
992 .buf = pages,
993 .len = PAGE_SIZE *
994 (hyp_ffa_proxy_pages() - (2 * KVM_FFA_MBOX_NR_PAGES)),
995 };
996
997 hyp_buffers = (struct kvm_ffa_buffers) {
998 .lock = __HYP_SPIN_LOCK_UNLOCKED,
999 .tx = tx,
1000 .rx = rx,
1001 };
1002
1003 host_buffers = (struct kvm_ffa_buffers) {
1004 .lock = __HYP_SPIN_LOCK_UNLOCKED,
1005 };
1006
1007 version_lock = __HYP_SPIN_LOCK_UNLOCKED;
1008 return 0;
1009 }
1010