1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2022 - Google LLC 4 * Author: Ard Biesheuvel <ardb@google.com> 5 */ 6 7 #include <linux/errno.h> 8 #include <linux/init.h> 9 #include <linux/linkage.h> 10 #include <linux/types.h> 11 12 #include <asm/scs.h> 13 14 #include "pi.h" 15 16 bool dynamic_scs_is_enabled; 17 18 // 19 // This minimal DWARF CFI parser is partially based on the code in 20 // arch/arc/kernel/unwind.c, and on the document below: 21 // https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html 22 // 23 24 #define DW_CFA_nop 0x00 25 #define DW_CFA_set_loc 0x01 26 #define DW_CFA_advance_loc1 0x02 27 #define DW_CFA_advance_loc2 0x03 28 #define DW_CFA_advance_loc4 0x04 29 #define DW_CFA_offset_extended 0x05 30 #define DW_CFA_restore_extended 0x06 31 #define DW_CFA_undefined 0x07 32 #define DW_CFA_same_value 0x08 33 #define DW_CFA_register 0x09 34 #define DW_CFA_remember_state 0x0a 35 #define DW_CFA_restore_state 0x0b 36 #define DW_CFA_def_cfa 0x0c 37 #define DW_CFA_def_cfa_register 0x0d 38 #define DW_CFA_def_cfa_offset 0x0e 39 #define DW_CFA_def_cfa_expression 0x0f 40 #define DW_CFA_expression 0x10 41 #define DW_CFA_offset_extended_sf 0x11 42 #define DW_CFA_def_cfa_sf 0x12 43 #define DW_CFA_def_cfa_offset_sf 0x13 44 #define DW_CFA_val_offset 0x14 45 #define DW_CFA_val_offset_sf 0x15 46 #define DW_CFA_val_expression 0x16 47 #define DW_CFA_lo_user 0x1c 48 #define DW_CFA_negate_ra_state 0x2d 49 #define DW_CFA_GNU_args_size 0x2e 50 #define DW_CFA_GNU_negative_offset_extended 0x2f 51 #define DW_CFA_hi_user 0x3f 52 53 enum { 54 PACIASP = 0xd503233f, 55 AUTIASP = 0xd50323bf, 56 SCS_PUSH = 0xf800865e, 57 SCS_POP = 0xf85f8e5e, 58 }; 59 60 static void __always_inline scs_patch_loc(u64 loc) 61 { 62 u32 insn = le32_to_cpup((void *)loc); 63 64 switch (insn) { 65 case PACIASP: 66 *(u32 *)loc = cpu_to_le32(SCS_PUSH); 67 break; 68 case AUTIASP: 69 *(u32 *)loc = cpu_to_le32(SCS_POP); 70 break; 71 default: 72 /* 73 * While the DW_CFA_negate_ra_state directive is guaranteed to 74 * appear right after a PACIASP/AUTIASP instruction, it may 75 * also appear after a DW_CFA_restore_state directive that 76 * restores a state that is only partially accurate, and is 77 * followed by DW_CFA_negate_ra_state directive to toggle the 78 * PAC bit again. So we permit other instructions here, and ignore 79 * them. 80 */ 81 return; 82 } 83 if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_CLEAN_CACHE)) 84 asm("dc civac, %0" :: "r"(loc)); 85 else 86 asm(ALTERNATIVE("dc cvau, %0", "nop", ARM64_HAS_CACHE_IDC) 87 :: "r"(loc)); 88 } 89 90 /* 91 * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes 92 * except the last one have bit #7 set. 93 */ 94 static int __always_inline skip_xleb128(const u8 **opcode, int size) 95 { 96 u8 c; 97 98 do { 99 c = *(*opcode)++; 100 size--; 101 } while (c & BIT(7)); 102 103 return size; 104 } 105 106 struct eh_frame { 107 /* 108 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list. 109 */ 110 u32 size; 111 112 /* 113 * The first frame is a Common Information Entry (CIE) frame, followed 114 * by one or more Frame Description Entry (FDE) frames. In the former 115 * case, this field is 0, otherwise it is the negated offset relative 116 * to the associated CIE frame. 117 */ 118 u32 cie_id_or_pointer; 119 120 union { 121 struct { // CIE 122 u8 version; 123 u8 augmentation_string[]; 124 }; 125 126 struct { // FDE 127 s32 initial_loc; 128 s32 range; 129 u8 opcodes[]; 130 }; 131 }; 132 }; 133 134 static int scs_handle_fde_frame(const struct eh_frame *frame, 135 bool fde_has_augmentation_data, 136 int code_alignment_factor, 137 bool dry_run) 138 { 139 int size = frame->size - offsetof(struct eh_frame, opcodes) + 4; 140 u64 loc = (u64)offset_to_ptr(&frame->initial_loc); 141 const u8 *opcode = frame->opcodes; 142 143 if (fde_has_augmentation_data) { 144 int l; 145 146 // assume single byte uleb128_t 147 if (WARN_ON(*opcode & BIT(7))) 148 return -ENOEXEC; 149 150 l = *opcode++; 151 opcode += l; 152 size -= l + 1; 153 } 154 155 /* 156 * Starting from 'loc', apply the CFA opcodes that advance the location 157 * pointer, and identify the locations of the PAC instructions. 158 */ 159 while (size-- > 0) { 160 switch (*opcode++) { 161 case DW_CFA_nop: 162 case DW_CFA_remember_state: 163 case DW_CFA_restore_state: 164 break; 165 166 case DW_CFA_advance_loc1: 167 loc += *opcode++ * code_alignment_factor; 168 size--; 169 break; 170 171 case DW_CFA_advance_loc2: 172 loc += *opcode++ * code_alignment_factor; 173 loc += (*opcode++ << 8) * code_alignment_factor; 174 size -= 2; 175 break; 176 177 case DW_CFA_def_cfa: 178 case DW_CFA_offset_extended: 179 size = skip_xleb128(&opcode, size); 180 fallthrough; 181 case DW_CFA_def_cfa_offset: 182 case DW_CFA_def_cfa_offset_sf: 183 case DW_CFA_def_cfa_register: 184 case DW_CFA_same_value: 185 case DW_CFA_restore_extended: 186 case 0x80 ... 0xbf: 187 size = skip_xleb128(&opcode, size); 188 break; 189 190 case DW_CFA_negate_ra_state: 191 if (!dry_run) 192 scs_patch_loc(loc - 4); 193 break; 194 195 case 0x40 ... 0x7f: 196 // advance loc 197 loc += (opcode[-1] & 0x3f) * code_alignment_factor; 198 break; 199 200 case 0xc0 ... 0xff: 201 break; 202 203 default: 204 return -ENOEXEC; 205 } 206 } 207 return 0; 208 } 209 210 int scs_patch(const u8 eh_frame[], int size) 211 { 212 const u8 *p = eh_frame; 213 214 while (size > 4) { 215 const struct eh_frame *frame = (const void *)p; 216 bool fde_has_augmentation_data = true; 217 int code_alignment_factor = 1; 218 int ret; 219 220 if (frame->size == 0 || 221 frame->size == U32_MAX || 222 frame->size > size) 223 break; 224 225 if (frame->cie_id_or_pointer == 0) { 226 const u8 *p = frame->augmentation_string; 227 228 /* a 'z' in the augmentation string must come first */ 229 fde_has_augmentation_data = *p == 'z'; 230 231 /* 232 * The code alignment factor is a uleb128 encoded field 233 * but given that the only sensible values are 1 or 4, 234 * there is no point in decoding the whole thing. 235 */ 236 p += strlen(p) + 1; 237 if (!WARN_ON(*p & BIT(7))) 238 code_alignment_factor = *p; 239 } else { 240 ret = scs_handle_fde_frame(frame, 241 fde_has_augmentation_data, 242 code_alignment_factor, 243 true); 244 if (ret) 245 return ret; 246 scs_handle_fde_frame(frame, fde_has_augmentation_data, 247 code_alignment_factor, false); 248 } 249 250 p += sizeof(frame->size) + frame->size; 251 size -= sizeof(frame->size) + frame->size; 252 } 253 return 0; 254 } 255