1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2022 - Google LLC 4 * Author: Ard Biesheuvel <ardb@google.com> 5 */ 6 7 #include <linux/errno.h> 8 #include <linux/init.h> 9 #include <linux/linkage.h> 10 #include <linux/types.h> 11 12 #include <asm/scs.h> 13 14 #include "pi.h" 15 16 bool dynamic_scs_is_enabled; 17 18 // 19 // This minimal DWARF CFI parser is partially based on the code in 20 // arch/arc/kernel/unwind.c, and on the document below: 21 // https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html 22 // 23 24 #define DW_CFA_nop 0x00 25 #define DW_CFA_set_loc 0x01 26 #define DW_CFA_advance_loc1 0x02 27 #define DW_CFA_advance_loc2 0x03 28 #define DW_CFA_advance_loc4 0x04 29 #define DW_CFA_offset_extended 0x05 30 #define DW_CFA_restore_extended 0x06 31 #define DW_CFA_undefined 0x07 32 #define DW_CFA_same_value 0x08 33 #define DW_CFA_register 0x09 34 #define DW_CFA_remember_state 0x0a 35 #define DW_CFA_restore_state 0x0b 36 #define DW_CFA_def_cfa 0x0c 37 #define DW_CFA_def_cfa_register 0x0d 38 #define DW_CFA_def_cfa_offset 0x0e 39 #define DW_CFA_def_cfa_expression 0x0f 40 #define DW_CFA_expression 0x10 41 #define DW_CFA_offset_extended_sf 0x11 42 #define DW_CFA_def_cfa_sf 0x12 43 #define DW_CFA_def_cfa_offset_sf 0x13 44 #define DW_CFA_val_offset 0x14 45 #define DW_CFA_val_offset_sf 0x15 46 #define DW_CFA_val_expression 0x16 47 #define DW_CFA_lo_user 0x1c 48 #define DW_CFA_negate_ra_state 0x2d 49 #define DW_CFA_GNU_args_size 0x2e 50 #define DW_CFA_GNU_negative_offset_extended 0x2f 51 #define DW_CFA_hi_user 0x3f 52 53 #define DW_EH_PE_sdata4 0x0b 54 #define DW_EH_PE_sdata8 0x0c 55 #define DW_EH_PE_pcrel 0x10 56 57 enum { 58 PACIASP = 0xd503233f, 59 AUTIASP = 0xd50323bf, 60 SCS_PUSH = 0xf800865e, 61 SCS_POP = 0xf85f8e5e, 62 }; 63 64 static void __always_inline scs_patch_loc(u64 loc) 65 { 66 u32 insn = le32_to_cpup((void *)loc); 67 68 switch (insn) { 69 case PACIASP: 70 *(u32 *)loc = cpu_to_le32(SCS_PUSH); 71 break; 72 case AUTIASP: 73 *(u32 *)loc = cpu_to_le32(SCS_POP); 74 break; 75 default: 76 /* 77 * While the DW_CFA_negate_ra_state directive is guaranteed to 78 * appear right after a PACIASP/AUTIASP instruction, it may 79 * also appear after a DW_CFA_restore_state directive that 80 * restores a state that is only partially accurate, and is 81 * followed by DW_CFA_negate_ra_state directive to toggle the 82 * PAC bit again. So we permit other instructions here, and ignore 83 * them. 84 */ 85 return; 86 } 87 if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_CLEAN_CACHE)) 88 asm("dc civac, %0" :: "r"(loc)); 89 else 90 asm(ALTERNATIVE("dc cvau, %0", "nop", ARM64_HAS_CACHE_IDC) 91 :: "r"(loc)); 92 } 93 94 /* 95 * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes 96 * except the last one have bit #7 set. 97 */ 98 static int __always_inline skip_xleb128(const u8 **opcode, int size) 99 { 100 u8 c; 101 102 do { 103 c = *(*opcode)++; 104 size--; 105 } while (c & BIT(7)); 106 107 return size; 108 } 109 110 struct eh_frame { 111 /* 112 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list. 113 */ 114 u32 size; 115 116 /* 117 * The first frame is a Common Information Entry (CIE) frame, followed 118 * by one or more Frame Description Entry (FDE) frames. In the former 119 * case, this field is 0, otherwise it is the negated offset relative 120 * to the associated CIE frame. 121 */ 122 u32 cie_id_or_pointer; 123 124 union { 125 struct { // CIE 126 u8 version; 127 u8 augmentation_string[3]; 128 u8 code_alignment_factor; 129 u8 data_alignment_factor; 130 u8 return_address_register; 131 u8 augmentation_data_size; 132 u8 fde_pointer_format; 133 }; 134 135 struct { // FDE 136 s32 initial_loc; 137 s32 range; 138 u8 opcodes[]; 139 }; 140 141 struct { // FDE 142 s64 initial_loc64; 143 s64 range64; 144 u8 opcodes64[]; 145 }; 146 }; 147 }; 148 149 static int scs_handle_fde_frame(const struct eh_frame *frame, 150 int code_alignment_factor, 151 bool use_sdata8, 152 bool dry_run) 153 { 154 int size = frame->size - offsetof(struct eh_frame, opcodes) + 4; 155 u64 loc = (u64)offset_to_ptr(&frame->initial_loc); 156 const u8 *opcode = frame->opcodes; 157 int l; 158 159 if (use_sdata8) { 160 loc = (u64)&frame->initial_loc64 + frame->initial_loc64; 161 opcode = frame->opcodes64; 162 size -= 8; 163 } 164 165 // assume single byte uleb128_t for augmentation data size 166 if (*opcode & BIT(7)) 167 return EDYNSCS_INVALID_FDE_AUGM_DATA_SIZE; 168 169 l = *opcode++; 170 opcode += l; 171 size -= l + 1; 172 173 /* 174 * Starting from 'loc', apply the CFA opcodes that advance the location 175 * pointer, and identify the locations of the PAC instructions. 176 */ 177 while (size-- > 0) { 178 switch (*opcode++) { 179 case DW_CFA_nop: 180 case DW_CFA_remember_state: 181 case DW_CFA_restore_state: 182 break; 183 184 case DW_CFA_advance_loc1: 185 loc += *opcode++ * code_alignment_factor; 186 size--; 187 break; 188 189 case DW_CFA_advance_loc2: 190 loc += *opcode++ * code_alignment_factor; 191 loc += (*opcode++ << 8) * code_alignment_factor; 192 size -= 2; 193 break; 194 195 case DW_CFA_def_cfa: 196 case DW_CFA_offset_extended: 197 size = skip_xleb128(&opcode, size); 198 fallthrough; 199 case DW_CFA_def_cfa_offset: 200 case DW_CFA_def_cfa_offset_sf: 201 case DW_CFA_def_cfa_register: 202 case DW_CFA_same_value: 203 case DW_CFA_restore_extended: 204 case 0x80 ... 0xbf: 205 size = skip_xleb128(&opcode, size); 206 break; 207 208 case DW_CFA_negate_ra_state: 209 if (!dry_run) 210 scs_patch_loc(loc - 4); 211 break; 212 213 case 0x40 ... 0x7f: 214 // advance loc 215 loc += (opcode[-1] & 0x3f) * code_alignment_factor; 216 break; 217 218 case 0xc0 ... 0xff: 219 break; 220 221 default: 222 return EDYNSCS_INVALID_CFA_OPCODE; 223 } 224 } 225 return 0; 226 } 227 228 int scs_patch(const u8 eh_frame[], int size) 229 { 230 int code_alignment_factor = 1; 231 bool fde_use_sdata8 = false; 232 const u8 *p = eh_frame; 233 234 while (size > 4) { 235 const struct eh_frame *frame = (const void *)p; 236 int ret; 237 238 if (frame->size == 0 || 239 frame->size == U32_MAX || 240 frame->size > size) 241 break; 242 243 if (frame->cie_id_or_pointer == 0) { 244 /* 245 * Require presence of augmentation data (z) with a 246 * specifier for the size of the FDE initial_loc and 247 * range fields (R), and nothing else. 248 */ 249 if (strcmp(frame->augmentation_string, "zR")) 250 return EDYNSCS_INVALID_CIE_HEADER; 251 252 /* 253 * The code alignment factor is a uleb128 encoded field 254 * but given that the only sensible values are 1 or 4, 255 * there is no point in decoding the whole thing. Also 256 * sanity check the size of the data alignment factor 257 * field, and the values of the return address register 258 * and augmentation data size fields. 259 */ 260 if ((frame->code_alignment_factor & BIT(7)) || 261 (frame->data_alignment_factor & BIT(7)) || 262 frame->return_address_register != 30 || 263 frame->augmentation_data_size != 1) 264 return EDYNSCS_INVALID_CIE_HEADER; 265 266 code_alignment_factor = frame->code_alignment_factor; 267 268 switch (frame->fde_pointer_format) { 269 case DW_EH_PE_pcrel | DW_EH_PE_sdata4: 270 fde_use_sdata8 = false; 271 break; 272 case DW_EH_PE_pcrel | DW_EH_PE_sdata8: 273 fde_use_sdata8 = true; 274 break; 275 default: 276 return EDYNSCS_INVALID_CIE_SDATA_SIZE; 277 } 278 } else { 279 ret = scs_handle_fde_frame(frame, code_alignment_factor, 280 fde_use_sdata8, true); 281 if (ret) 282 return ret; 283 scs_handle_fde_frame(frame, code_alignment_factor, 284 fde_use_sdata8, false); 285 } 286 287 p += sizeof(frame->size) + frame->size; 288 size -= sizeof(frame->size) + frame->size; 289 } 290 return 0; 291 } 292