1// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 2// See https://llvm.org/LICENSE.txt for license information. 3// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 4 5// This patch implements the support routines for the SME ABI, 6// described here: 7// https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#sme-support-routines 8 9#include "../assembly.h" 10 11 12#if !defined(__APPLE__) 13#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0) 14#define TPIDR2_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0) 15#else 16// MachO requires @page/@pageoff directives because the global is defined 17// in a different file. Otherwise this file may fail to build. 18#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@page 19#define TPIDR2_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@pageoff 20#endif 21 22.arch armv9-a+sme 23 24// Utility function which calls a system's abort() routine. Because the function 25// is streaming-compatible it should disable streaming-SVE mode before calling 26// abort(). Note that there is no need to preserve any state before the call, 27// because the function does not return. 28DEFINE_COMPILERRT_PRIVATE_FUNCTION(do_abort) 29.cfi_startproc 30 .variant_pcs SYMBOL_NAME(do_abort) 31 stp x29, x30, [sp, #-32]! 32 cntd x0 33 // Store VG to a stack location that we describe with .cfi_offset 34 str x0, [sp, #16] 35 .cfi_def_cfa_offset 32 36 .cfi_offset w30, -24 37 .cfi_offset w29, -32 38 .cfi_offset 46, -16 39 bl __arm_sme_state 40 tbz x0, #0, 2f 411: 42 smstop sm 432: 44 // We can't make this into a tail-call because the unwinder would 45 // need to restore the value of VG. 46 bl SYMBOL_NAME(abort) 47.cfi_endproc 48END_COMPILERRT_FUNCTION(do_abort) 49 50// __arm_sme_state fills the result registers based on a local 51// that is set as part of the compiler-rt startup code. 52// __aarch64_has_sme_and_tpidr2_el0 53DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_sme_state) 54 .variant_pcs __arm_sme_state 55 mov x0, xzr 56 mov x1, xzr 57 58 adrp x16, TPIDR2_SYMBOL 59 ldrb w16, [x16, TPIDR2_SYMBOL_OFFSET] 60 cbz w16, 1f 610: 62 orr x0, x0, #0xC000000000000000 63 mrs x16, SVCR 64 bfxil x0, x16, #0, #2 65 mrs x1, TPIDR2_EL0 661: 67 ret 68END_COMPILERRT_OUTLINE_FUNCTION(__arm_sme_state) 69 70DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_restore) 71 .variant_pcs __arm_tpidr2_restore 72 // If TPIDR2_EL0 is nonnull, the subroutine aborts in some platform-specific 73 // manner. 74 mrs x14, TPIDR2_EL0 75 cbnz x14, 2f 76 77 // If any of the reserved bytes in the first 16 bytes of BLK are nonzero, 78 // the subroutine [..] aborts in some platform-defined manner. 79 ldrh w14, [x0, #10] 80 cbnz w14, 2f 81 ldr w14, [x0, #12] 82 cbnz w14, 2f 83 84 // If BLK.za_save_buffer is NULL, the subroutine does nothing. 85 ldr x16, [x0] 86 cbz x16, 1f 87 88 // If BLK.num_za_save_slices is zero, the subroutine does nothing. 89 ldrh w14, [x0, #8] 90 cbz x14, 1f 91 92 mov x15, xzr 930: 94 ldr za[w15,0], [x16] 95 addsvl x16, x16, #1 96 add x15, x15, #1 97 cmp x14, x15 98 b.ne 0b 991: 100 ret 1012: 102 b SYMBOL_NAME(do_abort) 103END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_restore) 104 105DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_save) 106 .variant_pcs __arm_tpidr2_restore 107 // If the current thread does not have access to TPIDR2_EL0, the subroutine 108 // does nothing. 109 adrp x14, TPIDR2_SYMBOL 110 ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET] 111 cbz w14, 1f 112 113 // If TPIDR2_EL0 is null, the subroutine does nothing. 114 mrs x16, TPIDR2_EL0 115 cbz x16, 1f 116 117 // If any of the reserved bytes in the first 16 bytes of the TPIDR2 block are 118 // nonzero, the subroutine [..] aborts in some platform-defined manner. 119 ldrh w14, [x16, #10] 120 cbnz w14, 2f 121 ldr w14, [x16, #12] 122 cbnz w14, 2f 123 124 // If num_za_save_slices is zero, the subroutine does nothing. 125 ldrh w14, [x16, #8] 126 cbz x14, 1f 127 128 // If za_save_buffer is NULL, the subroutine does nothing. 129 ldr x16, [x16] 130 cbz x16, 1f 131 132 mov x15, xzr 1330: 134 str za[w15,0], [x16] 135 addsvl x16, x16, #1 136 add x15, x15, #1 137 cmp x14, x15 138 b.ne 0b 1391: 140 ret 1412: 142 b SYMBOL_NAME(do_abort) 143END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_save) 144 145DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_za_disable) 146 .variant_pcs __arm_tpidr2_restore 147 // If the current thread does not have access to SME, the subroutine does 148 // nothing. 149 adrp x14, TPIDR2_SYMBOL 150 ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET] 151 cbz w14, 0f 152 153 // Otherwise, the subroutine behaves as if it did the following: 154 // * Call __arm_tpidr2_save. 155 stp x29, x30, [sp, #-16]! 156 .cfi_def_cfa_offset 16 157 mov x29, sp 158 .cfi_def_cfa w29, 16 159 .cfi_offset w30, -8 160 .cfi_offset w29, -16 161 bl __arm_tpidr2_save 162 163 // * Set TPIDR2_EL0 to null. 164 msr TPIDR2_EL0, xzr 165 166 // * Set PSTATE.ZA to 0. 167 smstop za 168 169 .cfi_def_cfa wsp, 16 170 ldp x29, x30, [sp], #16 171 .cfi_def_cfa_offset 0 172 .cfi_restore w30 173 .cfi_restore w29 1740: 175 ret 176END_COMPILERRT_OUTLINE_FUNCTION(__arm_za_disable) 177