1// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 2// See https://llvm.org/LICENSE.txt for license information. 3// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 4 5// This patch implements the support routines for the SME ABI, 6// described here: 7// https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#sme-support-routines 8 9#include "../assembly.h" 10 11 12#if !defined(__APPLE__) 13#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0) 14#define TPIDR2_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0) 15#define CPU_FEATS_SYMBOL SYMBOL_NAME(__aarch64_cpu_features) 16#define CPU_FEATS_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_cpu_features) 17#else 18// MachO requires @page/@pageoff directives because the global is defined 19// in a different file. Otherwise this file may fail to build. 20#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@page 21#define TPIDR2_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@pageoff 22#define CPU_FEATS_SYMBOL SYMBOL_NAME(__aarch64_cpu_features)@page 23#define CPU_FEATS_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_cpu_features)@pageoff 24#endif 25 26.arch armv9-a+sme 27 28// Utility function which calls a system's abort() routine. Because the function 29// is streaming-compatible it should disable streaming-SVE mode before calling 30// abort(). Note that there is no need to preserve any state before the call, 31// because the function does not return. 32DEFINE_COMPILERRT_PRIVATE_FUNCTION(do_abort) 33 .cfi_startproc 34 .variant_pcs SYMBOL_NAME(do_abort) 35 BTI_C 36 stp x29, x30, [sp, #-32]! 37 cntd x0 38 // Store VG to a stack location that we describe with .cfi_offset 39 str x0, [sp, #16] 40 .cfi_def_cfa_offset 32 41 .cfi_offset w30, -24 42 .cfi_offset w29, -32 43 .cfi_offset 46, -16 44 bl __arm_sme_state 45 tbz x0, #0, 2f 461: 47 smstop sm 482: 49 // We can't make this into a tail-call because the unwinder would 50 // need to restore the value of VG. 51 bl SYMBOL_NAME(abort) 52 .cfi_endproc 53END_COMPILERRT_FUNCTION(do_abort) 54 55// __arm_sme_state fills the result registers based on a local 56// that is set as part of the compiler-rt startup code. 57// __aarch64_has_sme_and_tpidr2_el0 58DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_sme_state) 59 .variant_pcs __arm_sme_state 60 BTI_C 61 mov x0, xzr 62 mov x1, xzr 63 64 adrp x16, TPIDR2_SYMBOL 65 ldrb w16, [x16, TPIDR2_SYMBOL_OFFSET] 66 cbz w16, 1f 670: 68 orr x0, x0, #0xC000000000000000 69 mrs x16, SVCR 70 bfxil x0, x16, #0, #2 71 mrs x1, TPIDR2_EL0 721: 73 ret 74END_COMPILERRT_OUTLINE_FUNCTION(__arm_sme_state) 75 76DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_restore) 77 .variant_pcs __arm_tpidr2_restore 78 BTI_C 79 // If TPIDR2_EL0 is nonnull, the subroutine aborts in some platform-specific 80 // manner. 81 mrs x14, TPIDR2_EL0 82 cbnz x14, 2f 83 84 // If any of the reserved bytes in the first 16 bytes of BLK are nonzero, 85 // the subroutine [..] aborts in some platform-defined manner. 86 ldrh w14, [x0, #10] 87 cbnz w14, 2f 88 ldr w14, [x0, #12] 89 cbnz w14, 2f 90 91 // If BLK.za_save_buffer is NULL, the subroutine does nothing. 92 ldr x16, [x0] 93 cbz x16, 1f 94 95 // If BLK.num_za_save_slices is zero, the subroutine does nothing. 96 ldrh w14, [x0, #8] 97 cbz x14, 1f 98 99 mov x15, xzr 1000: 101 ldr za[w15,0], [x16] 102 addsvl x16, x16, #1 103 add x15, x15, #1 104 cmp x14, x15 105 b.ne 0b 1061: 107 ret 1082: 109 b SYMBOL_NAME(do_abort) 110END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_restore) 111 112DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_save) 113 .variant_pcs __arm_tpidr2_restore 114 BTI_C 115 // If the current thread does not have access to TPIDR2_EL0, the subroutine 116 // does nothing. 117 adrp x14, TPIDR2_SYMBOL 118 ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET] 119 cbz w14, 1f 120 121 // If TPIDR2_EL0 is null, the subroutine does nothing. 122 mrs x16, TPIDR2_EL0 123 cbz x16, 1f 124 125 // If any of the reserved bytes in the first 16 bytes of the TPIDR2 block are 126 // nonzero, the subroutine [..] aborts in some platform-defined manner. 127 ldrh w14, [x16, #10] 128 cbnz w14, 2f 129 ldr w14, [x16, #12] 130 cbnz w14, 2f 131 132 // If num_za_save_slices is zero, the subroutine does nothing. 133 ldrh w14, [x16, #8] 134 cbz x14, 1f 135 136 // If za_save_buffer is NULL, the subroutine does nothing. 137 ldr x16, [x16] 138 cbz x16, 1f 139 140 mov x15, xzr 1410: 142 str za[w15,0], [x16] 143 addsvl x16, x16, #1 144 add x15, x15, #1 145 cmp x14, x15 146 b.ne 0b 1471: 148 ret 1492: 150 b SYMBOL_NAME(do_abort) 151END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_save) 152 153DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_za_disable) 154 .variant_pcs __arm_tpidr2_restore 155 BTI_C 156 // If the current thread does not have access to SME, the subroutine does 157 // nothing. 158 adrp x14, TPIDR2_SYMBOL 159 ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET] 160 cbz w14, 0f 161 162 // Otherwise, the subroutine behaves as if it did the following: 163 // * Call __arm_tpidr2_save. 164 stp x29, x30, [sp, #-16]! 165 .cfi_def_cfa_offset 16 166 mov x29, sp 167 .cfi_def_cfa w29, 16 168 .cfi_offset w30, -8 169 .cfi_offset w29, -16 170 bl __arm_tpidr2_save 171 172 // * Set TPIDR2_EL0 to null. 173 msr TPIDR2_EL0, xzr 174 175 // * Set PSTATE.ZA to 0. 176 smstop za 177 178 .cfi_def_cfa wsp, 16 179 ldp x29, x30, [sp], #16 180 .cfi_def_cfa_offset 0 181 .cfi_restore w30 182 .cfi_restore w29 1830: 184 ret 185END_COMPILERRT_OUTLINE_FUNCTION(__arm_za_disable) 186 187DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_get_current_vg) 188 .variant_pcs __arm_get_current_vg 189 BTI_C 190 191 stp x29, x30, [sp, #-16]! 192 .cfi_def_cfa_offset 16 193 mov x29, sp 194 .cfi_def_cfa w29, 16 195 .cfi_offset w30, -8 196 .cfi_offset w29, -16 197 adrp x17, CPU_FEATS_SYMBOL 198 ldr w17, [x17, CPU_FEATS_SYMBOL_OFFSET] 199 tbnz w17, #30, 0f 200 adrp x16, TPIDR2_SYMBOL 201 ldrb w16, [x16, TPIDR2_SYMBOL_OFFSET] 202 cbz w16, 1f 2030: 204 mov x18, x1 205 bl __arm_sme_state 206 mov x1, x18 207 and x17, x17, #0x40000000 208 bfxil x17, x0, #0, #1 209 cbz x17, 1f 210 cntd x0 211 .cfi_def_cfa wsp, 16 212 ldp x29, x30, [sp], #16 213 .cfi_def_cfa_offset 0 214 .cfi_restore w30 215 .cfi_restore w29 216 ret 2171: 218 mov x0, xzr 219 .cfi_def_cfa wsp, 16 220 ldp x29, x30, [sp], #16 221 .cfi_def_cfa_offset 0 222 .cfi_restore w30 223 .cfi_restore w29 224 ret 225END_COMPILERRT_OUTLINE_FUNCTION(__arm_get_current_vg) 226 227NO_EXEC_STACK_DIRECTIVE 228 229// GNU property note for BTI and PAC 230GNU_PROPERTY_BTI_PAC 231