1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2023-2026 ARM Ltd.
4 */
5
6 #include <linux/arm-rmi-cmds.h>
7 #include <linux/cpufeature.h>
8 #include <linux/memblock.h>
9 #include <linux/memory.h>
10 #include <linux/mmzone.h>
11 #include <linux/slab.h>
12
13 #include <asm/memory.h>
14 #include <asm/pgtable-hwdef.h>
15
16 /* RMM v2.0 defines RmiFeatureRegister0 to RmiFeatureRegister4. */
17 static unsigned long rmi_feat_reg_cache[5] __ro_after_init;
18 static bool arm64_rmm_active;
19 static bool arm64_rmi_is_available;
20
21 /* is_rmm_active: Returns if the RMM is ACTIVE */
is_rmm_active(void)22 bool is_rmm_active(void)
23 {
24 return READ_ONCE(arm64_rmm_active);
25 }
26 EXPORT_SYMBOL_GPL(is_rmm_active);
27
28 /*
29 * is_rmi_available: Returns if the RMM is ACTIVE and is configured with static
30 * carveout for metadata nad L1GPTs.
31 */
is_rmi_available(void)32 bool is_rmi_available(void)
33 {
34 return READ_ONCE(arm64_rmi_is_available);
35 }
36 EXPORT_SYMBOL_GPL(is_rmi_available);
37
38 /*
39 * rmi_granule_range_delegate() - Delegate granules
40 * @base: PA of the first granule of the range
41 * @top: PA of the first granule after the range
42 * @out_top: PA of the first granule not delegated
43 *
44 * Delegate a range of granule for use by the realm world. If the entire range
45 * was delegated then @out_top == @top, otherwise the function should be called
46 * again with @base == @out_top.
47 *
48 * Return: 0 on success, positive RMI result code or negative Linux error code
49 */
rmi_granule_range_delegate(unsigned long base,unsigned long top,unsigned long * out_top)50 static long rmi_granule_range_delegate(unsigned long base,
51 unsigned long top,
52 unsigned long *out_top)
53 {
54 struct arm_smccc_1_2_regs regs = {
55 SMC_RMI_GRANULE_RANGE_DELEGATE, base, top
56 };
57 long ret = rmi_sro_execute(®s);
58
59 if (ret == RMI_SUCCESS && out_top)
60 *out_top = regs.a1;
61
62 return ret;
63 }
64
65 /*
66 * rmi_delegate_range: Delegate a physically contiguous range.
67 * We iterate over the range until we hit an error. So we may
68 * return an error, but with a partially delegated range. The
69 * caller must always look at the @out_phys to figure out, how
70 * much progress was made.
71 *
72 * @phys: Base of the physical address range
73 * @size: Size of the physical address range
74 * @out_phys: Top of the range that was completed. This is always
75 * valid, irrespective of the result.
76 *
77 * Returns RMI_SUCCESS on successful completion. Otherwise, returns
78 * the Linux error number or the RMI status code as described
79 * by the RMM spec for RMI_GRANULE_DELEGATE_RANGE or RMI_BLOCKED.
80 */
rmi_delegate_range(phys_addr_t phys,unsigned long size,phys_addr_t * out_phys)81 int rmi_delegate_range(phys_addr_t phys,
82 unsigned long size,
83 phys_addr_t *out_phys)
84 {
85 long ret = 0;
86 unsigned long top = phys + size;
87 unsigned long out_top;
88
89 while (phys < top) {
90 ret = rmi_granule_range_delegate(phys, top, &out_top);
91
92 if (ret != RMI_SUCCESS)
93 break;
94 /*
95 * Buggy RMM ? Let the caller handle the failure.
96 * We can't know how far the RMM delegated in this
97 * iteration, so we return the best known good limit.
98 * RMM can deal with granules already in "undelegated"
99 * in a given range. So, it is fine for the caller to
100 * try the range we return.
101 */
102 if (WARN_ON(out_top <= phys)) {
103 ret = -ENXIO;
104 break;
105 }
106 phys = out_top;
107 }
108
109 if (out_phys)
110 *out_phys = phys;
111
112 return ret;
113 }
114 EXPORT_SYMBOL_GPL(rmi_delegate_range);
115
116 /*
117 * rmi_granule_range_undelegate() - Undelegate a range of granules
118 * @base: Base PA of the target range
119 * @top: Top PA of the target range
120 * @out_top: Returns the top PA of range whose state is undelegated
121 *
122 * Undelegate a range of granules to allow use by the normal world. Will fail
123 * if the granules are in use by RMM. RMM can ignore granules that are already
124 * undelegated and thus is safe to be called on a range with a mix of delegated
125 * and undelegated granules.
126 *
127 * Return: 0 on success, positive RMI result code or negative Linux error code
128 */
rmi_granule_range_undelegate(unsigned long base,unsigned long top,unsigned long * out_top)129 static long rmi_granule_range_undelegate(unsigned long base,
130 unsigned long top,
131 unsigned long *out_top)
132 {
133 struct arm_smccc_1_2_regs regs = {
134 SMC_RMI_GRANULE_RANGE_UNDELEGATE, base, top
135 };
136 long ret = rmi_sro_execute(®s);
137
138 if (ret == RMI_SUCCESS && out_top)
139 *out_top = regs.a1;
140
141 return ret;
142 }
143
rmi_undelegate_range(phys_addr_t phys,unsigned long size)144 int rmi_undelegate_range(phys_addr_t phys,
145 unsigned long size)
146 {
147 long ret = 0;
148 unsigned long top = phys + size;
149 unsigned long out_top;
150
151 while (phys < top) {
152 ret = rmi_granule_range_undelegate(phys, top, &out_top);
153
154 if (ret != RMI_SUCCESS)
155 break;
156 /* Buggy RMM ? Let the caller leak the pages */
157 if (WARN_ON(out_top <= phys))
158 return -ENXIO;
159 phys = out_top;
160 }
161
162 return ret;
163 }
164 EXPORT_SYMBOL_GPL(rmi_undelegate_range);
165
166 /*
167 * rmi_free_delegated_page: Undelegate and free a page that has been previously
168 * delegated to the Realm world. If we are unable to undelegate it, the page is
169 * leaked.
170 * NOTE: Do not use this helper if the page could be concurrently operated by
171 * another thread, as it may get leaked if the undelegation fails due to RMI_BLOCKED
172 */
rmi_free_delegated_page(phys_addr_t phys)173 int rmi_free_delegated_page(phys_addr_t phys)
174 {
175 if (WARN_ON_ONCE(rmi_undelegate_page(phys))) {
176 /* Undelegate failed: leak the page */
177 return -EBUSY;
178 }
179
180 free_page((unsigned long)phys_to_virt(phys));
181
182 return 0;
183 }
184 EXPORT_SYMBOL_GPL(rmi_free_delegated_page);
185
186 /*
187 * Convert the RmiAddrBlockSize to actual size. This is used in RmiDonateReq
188 * and RmiAddrRangeDesc*.
189 */
rmi_addr_block_size_to_bytes(unsigned long block_size_fld)190 static unsigned long rmi_addr_block_size_to_bytes(unsigned long block_size_fld)
191 {
192 return BIT(ARM64_HW_PGTABLE_LEVEL_SHIFT(3 - block_size_fld));
193 }
194
195 /*
196 * free_addr_range: Free memory described by the address range entry, which may
197 * be partially consumed by RMM.
198 *
199 * @entry: RMI_ADDR_RANGE descriptor
200 * @consumed_size: Page aligned size consumed by the RMM from the address range.
201 *
202 * If the state of the address is DELEGATED, undelegate it back, before freeing.
203 * Leaks the memory if we cannot undelegate the range.
204 */
free_addr_range(unsigned long entry,unsigned long consumed_size)205 static void free_addr_range(unsigned long entry, unsigned long consumed_size)
206 {
207 unsigned long phys = RMI_ADDR_RANGE_ADDR(entry);
208 unsigned long block_size_fld = RMI_ADDR_RANGE_BLOCK_SIZE(entry);
209 unsigned long count = RMI_ADDR_RANGE_COUNT(entry);
210 unsigned long state = RMI_ADDR_RANGE_STATE(entry);
211 unsigned long size = rmi_addr_block_size_to_bytes(block_size_fld) * count;
212
213 WARN_ON(!PAGE_ALIGNED(phys) || !PAGE_ALIGNED(consumed_size));
214
215 /* We shouldn't see this in reclaim path, leak it for now */
216 if (WARN_ON((state != RMI_OP_MEM_DELEGATED) &&
217 (state != RMI_OP_MEM_UNDELEGATED)))
218 return;
219
220 /* Adjust the address and size for partially consumed entry */
221 phys += consumed_size;
222 size -= consumed_size;
223 /*
224 * Undelegate the pages back if required. If we can't
225 * change them back, leak the pages.
226 */
227 if (state == RMI_OP_MEM_DELEGATED &&
228 WARN_ON(rmi_undelegate_range(phys, size)))
229 return;
230 free_pages_exact(phys_to_virt(phys), size);
231 }
232
rmi_op_continue(unsigned long sro_handle,unsigned long flags,struct arm_smccc_1_2_regs * out_regs)233 static void rmi_op_continue(unsigned long sro_handle, unsigned long flags,
234 struct arm_smccc_1_2_regs *out_regs)
235 {
236 *out_regs = (struct arm_smccc_1_2_regs) {
237 SMC_RMI_OP_CONTINUE, sro_handle, flags
238 };
239
240 rmi_smccc_invoke(out_regs);
241 }
242
rmi_op_cancel(unsigned long sro_handle,struct arm_smccc_1_2_regs * out_regs)243 static void rmi_op_cancel(unsigned long sro_handle,
244 struct arm_smccc_1_2_regs *out_regs)
245 {
246 *out_regs = (struct arm_smccc_1_2_regs) {
247 SMC_RMI_OP_CANCEL, sro_handle
248 };
249
250 rmi_smccc_invoke(out_regs);
251 }
252
rmi_op_mem_donate(unsigned long sro_handle,unsigned long list_addr,unsigned long list_count,unsigned long flags,struct arm_smccc_1_2_regs * out_regs)253 static void rmi_op_mem_donate(unsigned long sro_handle, unsigned long list_addr,
254 unsigned long list_count, unsigned long flags,
255 struct arm_smccc_1_2_regs *out_regs)
256 {
257 *out_regs = (struct arm_smccc_1_2_regs) {
258 SMC_RMI_OP_MEM_DONATE, sro_handle, list_addr, list_count, flags
259 };
260
261 /*
262 * The output donated count (a1) is always valid, irrespective
263 * of the return result. i.e., 0 if there was an error
264 */
265 rmi_smccc_invoke(out_regs);
266 }
267
rmi_op_mem_reclaim(unsigned long sro_handle,unsigned long list_addr,unsigned long list_count,struct arm_smccc_1_2_regs * out_regs)268 static void rmi_op_mem_reclaim(unsigned long sro_handle,
269 unsigned long list_addr,
270 unsigned long list_count,
271 struct arm_smccc_1_2_regs *out_regs)
272 {
273 *out_regs = (struct arm_smccc_1_2_regs) {
274 SMC_RMI_OP_MEM_RECLAIM, sro_handle, list_addr, list_count
275 };
276
277 rmi_smccc_invoke(out_regs);
278 }
279
rmi_sro_ensure_capacity(struct rmi_sro_state * sro,unsigned long count)280 static int rmi_sro_ensure_capacity(struct rmi_sro_state *sro,
281 unsigned long count)
282 {
283 if (WARN_ON_ONCE(sro->addr_count > RMI_MAX_ADDR_LIST))
284 return -EOVERFLOW;
285
286 if (count > RMI_MAX_ADDR_LIST - sro->addr_count)
287 return -ENOSPC;
288
289 return 0;
290 }
291
rmi_sro_donate_contig(struct rmi_sro_state * sro,unsigned long sro_handle,unsigned long donatereq,struct arm_smccc_1_2_regs * out_regs,gfp_t gfp)292 static int rmi_sro_donate_contig(struct rmi_sro_state *sro,
293 unsigned long sro_handle,
294 unsigned long donatereq,
295 struct arm_smccc_1_2_regs *out_regs,
296 gfp_t gfp)
297 {
298 unsigned long block_size_fld = RMI_DONATE_BLOCK_SIZE(donatereq);
299 unsigned long block_size = rmi_addr_block_size_to_bytes(block_size_fld);
300 unsigned long count = RMI_DONATE_COUNT(donatereq);
301 unsigned long state = RMI_DONATE_STATE(donatereq);
302 unsigned long size = block_size * count;
303 unsigned long addr_range;
304 unsigned long donated_size;
305 int ret;
306 void *virt;
307 phys_addr_t phys;
308
309 /*
310 * The RMM specification requires contiguous allocations are always a
311 * power of 2
312 */
313 if (WARN_ON_ONCE(!is_power_of_2(size)))
314 return -EINVAL;
315 /*
316 * RMM clamps the Maximum value of RmiOpMemDonateReq:count to prevent
317 * overflow in the RMI_ADDR_RANGE_COUNT field.
318 */
319 if (WARN_ON_ONCE(count > FIELD_MAX(RMI_ADDR_RANGE_COUNT_MASK)))
320 return -EINVAL;
321
322 /*
323 * We can't allocate blocks more than what the buddy allocator can
324 * satisfy. TODO: Add support for larger allocations.
325 */
326 if (get_order(size) > MAX_PAGE_ORDER)
327 return -ENOMEM;
328
329 /* Reuse the cached address range if we have one */
330 for (int i = 0; i < sro->addr_count; i++) {
331 unsigned long entry = sro->addr_list[i];
332
333 if (RMI_ADDR_RANGE_BLOCK_SIZE(entry) == block_size_fld &&
334 RMI_ADDR_RANGE_COUNT(entry) == count &&
335 RMI_ADDR_RANGE_STATE(entry) == state &&
336 IS_ALIGNED(RMI_ADDR_RANGE_ADDR(entry), size)) {
337 sro->addr_count--;
338 swap(sro->addr_list[sro->addr_count],
339 sro->addr_list[i]);
340
341 goto mem_donate;
342 }
343 }
344
345 ret = rmi_sro_ensure_capacity(sro, 1);
346 /* In the odd case we have filled up the cache, free up the entire list */
347 if (ret)
348 rmi_sro_free(sro);
349
350 virt = alloc_pages_exact(size, gfp);
351 if (!virt)
352 return -ENOMEM;
353 phys = virt_to_phys(virt);
354
355 if (state == RMI_OP_MEM_DELEGATED) {
356 phys_addr_t delegated_phys;
357
358 if (rmi_delegate_range(phys, size, &delegated_phys)) {
359 if (!rmi_undelegate_range(phys, delegated_phys - phys))
360 free_pages_exact(virt, size);
361 return -ENXIO;
362 }
363 }
364
365 addr_range = phys & RMI_ADDR_RANGE_ADDR_MASK;
366 FIELD_MODIFY(RMI_ADDR_RANGE_BLOCK_SIZE_MASK, &addr_range, block_size_fld);
367 FIELD_MODIFY(RMI_ADDR_RANGE_COUNT_MASK, &addr_range, count);
368 FIELD_MODIFY(RMI_ADDR_RANGE_STATE_MASK, &addr_range, state);
369
370 sro->addr_list[sro->addr_count] = addr_range;
371
372 mem_donate:
373 rmi_op_mem_donate(sro_handle,
374 virt_to_phys(&sro->addr_list[sro->addr_count]), 1,
375 0, out_regs);
376 donated_size = out_regs->a1 << PAGE_SHIFT;
377
378 if (WARN_ON(out_regs->a1 > (size >> PAGE_SHIFT)))
379 donated_size = size;
380
381 /* All granules consumed by the RMM */
382 if (donated_size == size)
383 return 0;
384 /*
385 * No granules were consumed by the RMM, cache them. This is a valid
386 * case, where the RMM was donated memory by another competing thread.
387 */
388 if (donated_size == 0) {
389 sro->addr_count++;
390 return 0;
391 }
392
393 /* The granules were partially consumed, reclaim the unused ones. */
394 free_addr_range(sro->addr_list[sro->addr_count], donated_size);
395
396 return 0;
397 }
398
rmi_sro_donate_noncontig(struct rmi_sro_state * sro,unsigned long sro_handle,unsigned long donatereq,struct arm_smccc_1_2_regs * out_regs,gfp_t gfp)399 static int rmi_sro_donate_noncontig(struct rmi_sro_state *sro,
400 unsigned long sro_handle,
401 unsigned long donatereq,
402 struct arm_smccc_1_2_regs *out_regs,
403 gfp_t gfp)
404 {
405 unsigned long block_size_fld = RMI_DONATE_BLOCK_SIZE(donatereq);
406 unsigned long block_size = rmi_addr_block_size_to_bytes(block_size_fld);
407 unsigned long count = RMI_DONATE_COUNT(donatereq);
408 unsigned long state = RMI_DONATE_STATE(donatereq);
409 unsigned long found = 0;
410 unsigned long donated_granules;
411 unsigned long granules_per_block = block_size >> PAGE_SHIFT;
412 unsigned long consumed_blocks;
413 int addr_list_start = sro->addr_count;
414 int ret, i, src;
415
416 /*
417 * We can't allocate blocks more than what the buddy allocator can
418 * satisfy. TODO: Add support for larger allocations.
419 */
420 if (get_order(block_size) > MAX_PAGE_ORDER)
421 return -ENOMEM;
422 /*
423 * Maximum value of RmiOpMemDonateReq:count is 2^(PAGE_SHIFT-2) - 1.
424 * But we further clamp it down by the number of entries we can do
425 * in one go. The RMM can request the remaining in the next iteration.
426 */
427 if (count > RMI_MAX_ADDR_LIST)
428 count = RMI_MAX_ADDR_LIST;
429
430 /* Gather the suitable entries to the end of the list */
431 i = 0;
432 while (i < addr_list_start && found < count) {
433 unsigned long entry = sro->addr_list[i];
434
435 if (RMI_ADDR_RANGE_BLOCK_SIZE(entry) == block_size_fld &&
436 RMI_ADDR_RANGE_COUNT(entry) == 1 &&
437 RMI_ADDR_RANGE_STATE(entry) == state) {
438 addr_list_start--;
439 swap(sro->addr_list[addr_list_start],
440 sro->addr_list[i]);
441 found++;
442 /* Continue from the swapped in entry */
443 continue;
444 }
445 /* Skip past the entry */
446 i++;
447 }
448
449 ret = rmi_sro_ensure_capacity(sro, count - found);
450 if (ret) {
451 /* If we have found some entries, donate them and try again */
452 if (found)
453 goto mem_donate;
454 /* Otherwise free up the list and start again */
455 rmi_sro_free(sro);
456 /* Reset the addr_list_start to match sro->addr_count */
457 addr_list_start = 0;
458 }
459
460 for (; found < count; found++) {
461 unsigned long addr_range;
462 void *virt = alloc_pages_exact(block_size, gfp);
463 phys_addr_t phys;
464
465 if (!virt)
466 return -ENOMEM;
467
468 phys = virt_to_phys(virt);
469
470 if (state == RMI_OP_MEM_DELEGATED) {
471 phys_addr_t delegated_phys;
472
473 if (rmi_delegate_range(phys, block_size, &delegated_phys)) {
474 if (!rmi_undelegate_range(phys, delegated_phys - phys))
475 free_pages_exact(virt, block_size);
476 return -ENXIO;
477 }
478 }
479
480 addr_range = phys & RMI_ADDR_RANGE_ADDR_MASK;
481 FIELD_MODIFY(RMI_ADDR_RANGE_BLOCK_SIZE_MASK, &addr_range, block_size_fld);
482 FIELD_MODIFY(RMI_ADDR_RANGE_COUNT_MASK, &addr_range, 1);
483 FIELD_MODIFY(RMI_ADDR_RANGE_STATE_MASK, &addr_range, state);
484
485 sro->addr_list[sro->addr_count++] = addr_range;
486 }
487
488 mem_donate:
489 rmi_op_mem_donate(sro_handle,
490 virt_to_phys(&sro->addr_list[addr_list_start]),
491 found, 0, out_regs);
492
493 donated_granules = out_regs->a1;
494 /*
495 * The RMM shouldn't report more granules than we provided, but clamp
496 * just in case.
497 */
498 if (WARN_ON_ONCE(donated_granules > found * granules_per_block))
499 donated_granules = found * granules_per_block;
500
501 /*
502 * The RMM reports the consumed memory in terms of granules, but we
503 * track in the address lists in block-sized ranges. So divide to get
504 * the number of (complete) consumed blocks.
505 */
506 consumed_blocks = donated_granules / granules_per_block;
507 if (donated_granules % granules_per_block) {
508 /*
509 * A block has been partially consumed, the start is owned by
510 * the RMM, the tail is owned by the host
511 */
512 unsigned long entry =
513 sro->addr_list[addr_list_start + consumed_blocks];
514 unsigned long donated_size =
515 (donated_granules % granules_per_block) << PAGE_SHIFT;
516
517 free_addr_range(entry, donated_size);
518 /*
519 * This block is now fully 'consumed' (either held by the RMM or
520 * freed)
521 */
522 consumed_blocks++;
523 }
524
525 /*
526 * Keep just the blocks the RMM didn't use in addr_list
527 * RMM claimed consumed_blocks entries from addr_list_start.
528 * Move the entries left out at the end i.e.,
529 * [ addr_list_start + consumed_blocks, addr_list_start + found)
530 * to the rest of the valid entries and adjust the addr_count to
531 * reflect the available entries.
532 */
533 for (i = 0, src = addr_list_start + consumed_blocks;
534 i < found - consumed_blocks; i++)
535 sro->addr_list[addr_list_start + i] = sro->addr_list[src + i];
536
537 sro->addr_count -= consumed_blocks;
538
539 return 0;
540 }
541
rmi_sro_donate(struct rmi_sro_state * sro,unsigned long sro_handle,unsigned long donatereq,struct arm_smccc_1_2_regs * regs,gfp_t gfp)542 static int rmi_sro_donate(struct rmi_sro_state *sro,
543 unsigned long sro_handle,
544 unsigned long donatereq,
545 struct arm_smccc_1_2_regs *regs,
546 gfp_t gfp)
547 {
548 unsigned long state = RMI_DONATE_STATE(donatereq);
549
550 if (WARN_ON_ONCE(!RMI_DONATE_COUNT(donatereq)))
551 return -EINVAL;
552
553 /*
554 * We do not support RMI_OP_MEM_CONDITIONAL yet. This is only required
555 * for use in RMI_GRANULE_TRACKING_SET, which we don't support yet.
556 */
557 if (WARN_ON_ONCE((state != RMI_OP_MEM_DELEGATED) &&
558 (state != RMI_OP_MEM_UNDELEGATED)))
559 return -EINVAL;
560
561 if (RMI_DONATE_CONTIG(donatereq) == RMI_OP_MEM_CONTIG)
562 return rmi_sro_donate_contig(sro, sro_handle, donatereq, regs, gfp);
563 else
564 return rmi_sro_donate_noncontig(sro, sro_handle, donatereq, regs, gfp);
565 }
566
rmi_sro_reclaim(struct rmi_sro_state * sro,unsigned long sro_handle,struct arm_smccc_1_2_regs * out_regs)567 static int rmi_sro_reclaim(struct rmi_sro_state *sro,
568 unsigned long sro_handle,
569 struct arm_smccc_1_2_regs *out_regs)
570 {
571 unsigned long capacity;
572
573 /*
574 * We don't do a partial free of the entries. So for now free the
575 * entire address list as we prepare to reclaim more from the RMM.
576 */
577 if (rmi_sro_ensure_capacity(sro, 1))
578 rmi_sro_free(sro);
579
580 capacity = RMI_MAX_ADDR_LIST - sro->addr_count;
581
582 rmi_op_mem_reclaim(sro_handle,
583 virt_to_phys(&sro->addr_list[sro->addr_count]),
584 capacity, out_regs);
585
586 /*
587 * RMI_OP_MEM_RECLAIM always return RMI_INCOMPLETE, except when the
588 * input parameters were invalid.
589 */
590 if (WARN_ON_ONCE(RMI_RESULT_STATUS(out_regs->a0) != RMI_INCOMPLETE))
591 return -EINVAL;
592 if (WARN_ON_ONCE(out_regs->a1 > capacity))
593 out_regs->a1 = capacity;
594
595 sro->addr_count += out_regs->a1;
596
597 return 0;
598 }
599
rmi_sro_free(struct rmi_sro_state * sro)600 void rmi_sro_free(struct rmi_sro_state *sro)
601 {
602 /* Handle the worse */
603 if (WARN_ON(sro->addr_count < 0))
604 return;
605
606 if (WARN_ON(sro->addr_count > RMI_MAX_ADDR_LIST))
607 sro->addr_count = RMI_MAX_ADDR_LIST;
608
609 for (int i = 0; i < sro->addr_count; i++)
610 free_addr_range(sro->addr_list[i], 0);
611
612 sro->addr_count = 0;
613 }
614 EXPORT_SYMBOL_GPL(rmi_sro_free);
615
rmi_sro_memxfer_execute(struct rmi_sro_state * sro,gfp_t gfp)616 long rmi_sro_memxfer_execute(struct rmi_sro_state *sro, gfp_t gfp)
617 {
618 struct arm_smccc_1_2_regs *regs = &sro->regs;
619 bool cancelled = false;
620 unsigned long sro_handle;
621
622 rmi_smccc_invoke(regs);
623
624 sro_handle = regs->a1;
625 while (RMI_RESULT_STATUS(regs->a0) == RMI_INCOMPLETE) {
626 bool can_cancel = RMI_RESULT_CAN_CANCEL(regs->a0) == RMI_OP_CAN_CANCEL;
627 int ret = 0;
628
629 switch (RMI_RESULT_MEMREQ(regs->a0)) {
630 case RMI_OP_MEM_REQ_NONE:
631 rmi_op_continue(sro_handle, RMI_CONTINUE_KEEP_GOING,
632 regs);
633 break;
634 case RMI_OP_MEM_REQ_DONATE:
635 ret = rmi_sro_donate(sro, sro_handle, regs->a2, regs,
636 gfp);
637 break;
638 case RMI_OP_MEM_REQ_RECLAIM:
639 ret = rmi_sro_reclaim(sro, sro_handle, regs);
640 break;
641 default:
642 WARN_ON_ONCE(1);
643 ret = -ENXIO;
644 }
645
646 if (ret) {
647 /*
648 * All memory donating SROs must be cancellable. So a
649 * failure in memory allocation shouldn't be an issue.
650 * However, if we encounter a random failure (e.g.,
651 * buggy RMM), don't loop forever, just give up.
652 */
653 if (WARN_ON_ONCE(!can_cancel))
654 return ret;
655 /*
656 * If we have already cancelled, and came back here due
657 * to an error in MEMREQ, then there is no point
658 * in going in loops.
659 */
660 if (WARN_ON_ONCE(cancelled))
661 break;
662 rmi_op_cancel(sro_handle, regs);
663 cancelled = true;
664
665 if (WARN_ON_ONCE(RMI_RESULT_STATUS(regs->a0) != RMI_INCOMPLETE))
666 return ret;
667 }
668 }
669
670 if (cancelled)
671 return -ECANCELED;
672
673 return regs->a0;
674 }
675 EXPORT_SYMBOL_GPL(rmi_sro_memxfer_execute);
676
677 /*
678 * rmi_sro_execute: Execute an RMI command that is stateful but not memory
679 * transferring. Takes regs, filled with the FIDs and the arguments in place.
680 *
681 * Returns :
682 * -ECANCELED - If the operation had to be aborted and SRO was cancellable.
683 * Otherwise, returns the result of the RMI command.
684 */
rmi_sro_execute(struct arm_smccc_1_2_regs * regs)685 long rmi_sro_execute(struct arm_smccc_1_2_regs *regs)
686 {
687 bool cancelled = false;
688 unsigned long sro_handle;
689
690 rmi_smccc_invoke(regs);
691
692 sro_handle = regs->a1;
693 while (RMI_RESULT_STATUS(regs->a0) == RMI_INCOMPLETE) {
694 bool can_cancel = RMI_RESULT_CAN_CANCEL(regs->a0) == RMI_OP_CAN_CANCEL;
695
696 switch (RMI_RESULT_MEMREQ(regs->a0)) {
697 case RMI_OP_MEM_REQ_NONE:
698 rmi_op_continue(sro_handle, RMI_CONTINUE_KEEP_GOING,
699 regs);
700 break;
701 default:
702 WARN_ON_ONCE(1);
703 if (!can_cancel)
704 return regs->a0;
705 /*
706 * We can't get here normally, but handle this anyway
707 * for a buggy RMM implementation.
708 */
709 if (cancelled)
710 return -ECANCELED;
711 rmi_op_cancel(sro_handle, regs);
712 cancelled = true;
713 }
714 }
715
716 if (cancelled)
717 return -ECANCELED;
718
719 return regs->a0;
720 }
721 EXPORT_SYMBOL_GPL(rmi_sro_execute);
722
rmi_check_version(void)723 static int rmi_check_version(void)
724 {
725 unsigned short version_major, version_minor;
726 unsigned long host_version = RMI_ABI_VERSION(RMI_ABI_MAJOR_VERSION,
727 RMI_ABI_MINOR_VERSION);
728 unsigned long aa64pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
729 struct arm_smccc_1_2_regs res = {
730 SMC_RMI_VERSION, host_version,
731 };
732
733 /* If RME isn't supported, then RMI can't be */
734 if (cpuid_feature_extract_unsigned_field(aa64pfr0, ID_AA64PFR0_EL1_RME_SHIFT) == 0)
735 return -ENXIO;
736
737 if (arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_SMC)
738 return -ENXIO;
739
740 rmi_smccc_invoke(&res);
741 if (res.a0 == SMCCC_RET_NOT_SUPPORTED)
742 return -ENXIO;
743
744 version_major = RMI_ABI_VERSION_GET_MAJOR(res.a1);
745 version_minor = RMI_ABI_VERSION_GET_MINOR(res.a1);
746
747 if (res.a0 != RMI_SUCCESS) {
748 unsigned short high_version_major, high_version_minor;
749
750 high_version_major = RMI_ABI_VERSION_GET_MAJOR(res.a2);
751 high_version_minor = RMI_ABI_VERSION_GET_MINOR(res.a2);
752
753 pr_err("Unsupported RMI ABI (v%d.%d - v%d.%d) we want v%d.%d\n",
754 version_major, version_minor,
755 high_version_major, high_version_minor,
756 RMI_ABI_MAJOR_VERSION,
757 RMI_ABI_MINOR_VERSION);
758 return -ENXIO;
759 }
760
761 pr_info("RMI ABI version %d.%d\n", version_major, version_minor);
762
763 return 0;
764 }
765
rmi_read_features(void)766 static int rmi_read_features(void)
767 {
768 /*
769 * Since we've negotiated a compatible version these feature registers
770 * should always be available
771 */
772 for (int i = 0; i < ARRAY_SIZE(rmi_feat_reg_cache); i++) {
773 struct arm_smccc_1_2_regs args = {
774 SMC_RMI_FEATURES, i,
775 };
776
777 rmi_smccc_invoke(&args);
778 if (WARN_ON(args.a0 != RMI_SUCCESS))
779 return -EINVAL;
780
781 rmi_feat_reg_cache[i] = args.a1;
782 }
783
784 return 0;
785 }
786
rmi_feat_reg(unsigned int index)787 unsigned long rmi_feat_reg(unsigned int index)
788 {
789 if (WARN_ON(index >= ARRAY_SIZE(rmi_feat_reg_cache)))
790 return 0;
791
792 return rmi_feat_reg_cache[index];
793 }
794 EXPORT_SYMBOL_GPL(rmi_feat_reg);
795
rmi_rmm_config_set(unsigned long cfg_ptr)796 static int rmi_rmm_config_set(unsigned long cfg_ptr)
797 {
798 struct arm_smccc_1_2_regs regs = {
799 SMC_RMI_RMM_CONFIG_SET, cfg_ptr,
800 };
801
802 rmi_smccc_invoke(®s);
803
804 return regs.a0;
805 }
806
rmi_configure(void)807 static int rmi_configure(void)
808 {
809 unsigned long granule_feature;
810 unsigned long granule_size;
811 int ret;
812
813 switch (PAGE_SIZE) {
814 case SZ_4K:
815 granule_size = RMI_GRANULE_SIZE_4KB;
816 granule_feature = RMI_FEATURE_REGISTER_1_RMI_GRAN_SZ_4KB;
817 break;
818 case SZ_16K:
819 granule_size = RMI_GRANULE_SIZE_16KB;
820 granule_feature = RMI_FEATURE_REGISTER_1_RMI_GRAN_SZ_16KB;
821 break;
822 case SZ_64K:
823 granule_size = RMI_GRANULE_SIZE_64KB;
824 granule_feature = RMI_FEATURE_REGISTER_1_RMI_GRAN_SZ_64KB;
825 break;
826 default:
827 BUILD_BUG();
828 }
829
830 if (!(rmi_feat_reg(1) & granule_feature)) {
831 pr_err("RMM does not support %luKB granules\n",
832 PAGE_SIZE >> 10);
833 return -ENXIO;
834 }
835
836 struct rmm_config *config __free(free_page) =
837 (struct rmm_config *)get_zeroed_page(GFP_KERNEL);
838
839 if (!config) {
840 pr_err("Unable to allocate memory for RMM config\n");
841 return -ENOMEM;
842 }
843
844 config->rmi_granule_size = granule_size;
845
846 /*
847 * For now we set the tracking_region_size to 0 which is the only option
848 * for 4KB PAGE_SIZE (1GB for 4KB PAGE_SIZE, 32MB/512MB for 16KB/64KB).
849 * TODO: Support other tracking sizes via Kconfig option for other
850 * PAGE_SIZES
851 */
852 config->tracking_region_size = 0;
853
854 ret = rmi_rmm_config_set(virt_to_phys(config));
855 if (ret) {
856 pr_err("RMM config set failed (%d)\n", ret);
857 return -EINVAL;
858 }
859
860 return 0;
861 }
862
863 /**
864 * rmi_granule_tracking_get() - Get configuration of a Granule tracking region
865 * @start: Base PA of the tracking region
866 * @end: End of the PA region
867 * @out_category: Memory category
868 * @out_state: Tracking region state
869 * @out_top: Top of the memory region
870 *
871 * Return: RMI return code
872 */
rmi_granule_tracking_get(unsigned long start,unsigned long end,unsigned long * out_category,unsigned long * out_state,unsigned long * out_top)873 static int rmi_granule_tracking_get(unsigned long start,
874 unsigned long end,
875 unsigned long *out_category,
876 unsigned long *out_state,
877 unsigned long *out_top)
878 {
879 struct arm_smccc_1_2_regs regs = {
880 SMC_RMI_GRANULE_TRACKING_GET, start, end,
881 };
882
883 rmi_smccc_invoke(®s);
884
885 if (regs.a0 != RMI_SUCCESS)
886 return regs.a0;
887
888 if (out_category)
889 *out_category = regs.a1;
890 if (out_state)
891 *out_state = regs.a2;
892 if (out_top)
893 *out_top = regs.a3;
894
895 return RMI_SUCCESS;
896 }
897
898 /*
899 * Make sure the area is tracked by RMM at FINE granularity.
900 * We do not support changing the tracking yet.
901 */
rmi_verify_memory_tracking(phys_addr_t start,phys_addr_t end)902 static int rmi_verify_memory_tracking(phys_addr_t start, phys_addr_t end)
903 {
904 while (start < end) {
905 unsigned long ret, category, state, next;
906
907 ret = rmi_granule_tracking_get(start, end, &category, &state, &next);
908 if (ret != RMI_SUCCESS) {
909 pr_err("RMI_GRANULE_TRACKING_GET failed for %llx-%llx: (%lu)\n",
910 start, end, ret);
911 return -ENXIO;
912 }
913
914 if (WARN_ON(next <= start))
915 return -ENXIO;
916
917 if (state != RMI_TRACKING_FINE ||
918 category != RMI_MEM_CATEGORY_CONVENTIONAL) {
919 /* TODO: Set granule tracking in this case */
920 pr_err("Granule tracking for region isn't fine/conventional: %llx-%lx\n",
921 start, next);
922 return -ENODEV;
923 }
924 start = next;
925 }
926
927 return 0;
928 }
929
930 /*
931 * rmi_gpt_info - Query the GPT info for the given PAR.
932 * @start: Base of the physical address region
933 * @end: Top of the physical address region
934 * @out_top: Top of the physical address region for which the GPT @out_gpt_par_state
935 * is valid
936 * @out_gpt_par_state: State of the GPT covered by [start, out_top)
937 */
rmi_gpt_info(unsigned long start,unsigned long end,unsigned long * out_top,unsigned long * out_gpt_par_state)938 static long rmi_gpt_info(unsigned long start, unsigned long end,
939 unsigned long *out_top,
940 unsigned long *out_gpt_par_state)
941 {
942 struct arm_smccc_1_2_regs regs = {
943 SMC_RMI_GPT_INFO, start, end,
944 };
945
946 rmi_smccc_invoke(®s);
947 if (regs.a0 != RMI_SUCCESS)
948 return regs.a0;
949
950 if (out_top)
951 *out_top = regs.a1;
952 if (out_gpt_par_state)
953 *out_gpt_par_state = regs.a2;
954
955 return RMI_SUCCESS;
956 }
957
958 /*
959 * We do not support creating L1 GPTs yet. So, make sure that all the regions
960 * are managed by the firmware.
961 */
rmi_verify_gpt_firmware_managed(phys_addr_t start,phys_addr_t end)962 static int rmi_verify_gpt_firmware_managed(phys_addr_t start, phys_addr_t end)
963 {
964 unsigned long l0gpt_sz;
965 unsigned long next, par_state;
966
967 l0gpt_sz = 1UL << (30 + FIELD_GET(RMI_FEATURE_REGISTER_1_L0GPTSZ,
968 rmi_feat_reg(1)));
969 start = ALIGN_DOWN(start, l0gpt_sz);
970 end = ALIGN(end, l0gpt_sz);
971
972 while (start < end) {
973 long ret = rmi_gpt_info(start, end, &next, &par_state);
974
975 if (ret != RMI_SUCCESS) {
976 pr_err("RMI_GPT_INFO failed for %llx-%llx: (%ld)\n",
977 start, end, ret);
978 return -ENXIO;
979 }
980
981 if (WARN_ON(next <= start))
982 return -ENXIO;
983
984 if (par_state != RMI_GPT_PAR_PLAT) {
985 pr_err("GPT for the region is not managed by firmware %llx-%lx\n",
986 start, next);
987 return -ENODEV;
988 }
989 start = next;
990 }
991
992 return 0;
993 }
994
rmi_prepare_memory(phys_addr_t start,phys_addr_t end)995 static int rmi_prepare_memory(phys_addr_t start, phys_addr_t end)
996 {
997 int ret;
998
999 if (start >= end)
1000 return -EINVAL;
1001
1002 ret = rmi_verify_memory_tracking(start, end);
1003 if (ret)
1004 return ret;
1005
1006 return rmi_verify_gpt_firmware_managed(start, end);
1007 }
1008
rmi_init_metadata(void)1009 static int rmi_init_metadata(void)
1010 {
1011 phys_addr_t start, end;
1012 struct memblock_region *r;
1013
1014 for_each_mem_region(r) {
1015 int ret;
1016
1017 /* Firmware-reserved NOMAP regions are not usable system RAM */
1018 if (memblock_is_nomap(r))
1019 continue;
1020
1021 start = PAGE_ALIGN(r->base);
1022 /*
1023 * We always deal with PAGE_SIZE and if in the odd case this
1024 * region boundary is not PAGE aligned, we stick to the page
1025 * that we can use from the region.
1026 */
1027 end = PAGE_ALIGN_DOWN(r->base + r->size);
1028 /* Too small ? */
1029 if (start >= end)
1030 continue;
1031
1032 ret = rmi_prepare_memory(start, end);
1033 if (ret)
1034 return ret;
1035 }
1036
1037 return 0;
1038 }
1039
rmi_memory_notifier(struct notifier_block * nb,unsigned long action,void * data)1040 static int rmi_memory_notifier(struct notifier_block *nb,
1041 unsigned long action, void *data)
1042 {
1043 struct memory_notify *arg = data;
1044 phys_addr_t start, end;
1045 int ret;
1046
1047 if (action != MEM_GOING_ONLINE)
1048 return NOTIFY_DONE;
1049
1050 start = PFN_PHYS(arg->start_pfn);
1051 end = PFN_PHYS(arg->start_pfn + arg->nr_pages);
1052
1053 /*
1054 * If the hotplugged memory region is in ZONE_MOVABLE, neither the
1055 * guest-memfd nor the kernel allocations can come from there. So we are
1056 * fine to skip this.
1057 * For boot time memory, we don't allow this discount, as the
1058 * decision to move boot memory pages to ZONE_MOVABLE is purely
1059 * a kernel choice and the firmware must have decided what it covers.
1060 */
1061 if (is_zone_movable_page(pfn_to_page(arg->start_pfn)))
1062 return NOTIFY_DONE;
1063
1064 ret = rmi_prepare_memory(start, end);
1065
1066 return notifier_from_errno(ret);
1067 }
1068
1069 static struct notifier_block rmi_memory_nb = {
1070 .notifier_call = rmi_memory_notifier,
1071 };
1072
rmi_init_memory(void)1073 static int rmi_init_memory(void)
1074 {
1075 int ret;
1076
1077 ret = rmi_init_metadata();
1078 if (ret)
1079 return ret;
1080
1081 return register_memory_notifier(&rmi_memory_nb);
1082 }
1083
arm64_init_rmi(void)1084 static int __init arm64_init_rmi(void)
1085 {
1086 int ret;
1087
1088 /* If we can't agree on the RMI ABI version, don't proceed further */
1089 ret = rmi_check_version();
1090 if (ret)
1091 return ret;
1092
1093 ret = rmi_read_features();
1094 if (ret)
1095 return ret;
1096
1097 ret = rmi_configure();
1098 if (ret)
1099 return ret;
1100
1101 /* Activate the RMM */
1102 struct rmi_sro_state *sro __free(kfree) = kmalloc_obj(*sro);
1103 if (!sro)
1104 return -ENOMEM;
1105
1106 ret = rmi_sro_memxfer_cmd(sro, GFP_KERNEL, SMC_RMI_RMM_ACTIVATE);
1107 if (ret) {
1108 pr_err("RMM activate failed (%d)\n", ret);
1109 ret = ret < 0 ? ret : -ENXIO;
1110 return ret;
1111 }
1112
1113 WRITE_ONCE(arm64_rmm_active, true);
1114
1115 ret = rmi_init_memory();
1116 if (ret) {
1117 /* Deactivate the RMM */
1118 if (!WARN_ON(rmi_sro_memxfer_cmd(sro, GFP_KERNEL, SMC_RMI_RMM_DEACTIVATE)))
1119 WRITE_ONCE(arm64_rmm_active, false);
1120 return ret;
1121 }
1122
1123 /* Mark the RMM is usable */
1124 WRITE_ONCE(arm64_rmi_is_available, true);
1125 pr_info("RMI configured\n");
1126
1127 return 0;
1128 }
1129
1130 /*
1131 * Note arm64_init_rmi() must be called before kvm_init_rmi() otherwise KVM
1132 * will not support realm guests. subsys_initcall() is called before
1133 * module_init() (used for KVM) so this is OK.
1134 */
1135 subsys_initcall(arm64_init_rmi);
1136