1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/errno.h> 3 #include <linux/kernel.h> 4 #include <linux/sched.h> 5 #include <linux/sched/task_stack.h> 6 #include <linux/perf_event.h> 7 #include <linux/bug.h> 8 #include <linux/stddef.h> 9 #include <asm/perf_regs.h> 10 #include <asm/ptrace.h> 11 12 #ifdef CONFIG_X86_32 13 #define PERF_REG_X86_MAX PERF_REG_X86_32_MAX 14 #else 15 #define PERF_REG_X86_MAX PERF_REG_X86_64_MAX 16 #endif 17 18 #define PT_REGS_OFFSET(id, r) [id] = offsetof(struct pt_regs, r) 19 20 static unsigned int pt_regs_offset[PERF_REG_X86_MAX] = { 21 PT_REGS_OFFSET(PERF_REG_X86_AX, ax), 22 PT_REGS_OFFSET(PERF_REG_X86_BX, bx), 23 PT_REGS_OFFSET(PERF_REG_X86_CX, cx), 24 PT_REGS_OFFSET(PERF_REG_X86_DX, dx), 25 PT_REGS_OFFSET(PERF_REG_X86_SI, si), 26 PT_REGS_OFFSET(PERF_REG_X86_DI, di), 27 PT_REGS_OFFSET(PERF_REG_X86_BP, bp), 28 PT_REGS_OFFSET(PERF_REG_X86_SP, sp), 29 PT_REGS_OFFSET(PERF_REG_X86_IP, ip), 30 PT_REGS_OFFSET(PERF_REG_X86_FLAGS, flags), 31 PT_REGS_OFFSET(PERF_REG_X86_CS, cs), 32 PT_REGS_OFFSET(PERF_REG_X86_SS, ss), 33 #ifdef CONFIG_X86_32 34 PT_REGS_OFFSET(PERF_REG_X86_DS, ds), 35 PT_REGS_OFFSET(PERF_REG_X86_ES, es), 36 PT_REGS_OFFSET(PERF_REG_X86_FS, fs), 37 PT_REGS_OFFSET(PERF_REG_X86_GS, gs), 38 #else 39 /* 40 * The pt_regs struct does not store 41 * ds, es, fs, gs in 64 bit mode. 42 */ 43 (unsigned int) -1, 44 (unsigned int) -1, 45 (unsigned int) -1, 46 (unsigned int) -1, 47 #endif 48 #ifdef CONFIG_X86_64 49 PT_REGS_OFFSET(PERF_REG_X86_R8, r8), 50 PT_REGS_OFFSET(PERF_REG_X86_R9, r9), 51 PT_REGS_OFFSET(PERF_REG_X86_R10, r10), 52 PT_REGS_OFFSET(PERF_REG_X86_R11, r11), 53 PT_REGS_OFFSET(PERF_REG_X86_R12, r12), 54 PT_REGS_OFFSET(PERF_REG_X86_R13, r13), 55 PT_REGS_OFFSET(PERF_REG_X86_R14, r14), 56 PT_REGS_OFFSET(PERF_REG_X86_R15, r15), 57 #endif 58 }; 59 60 u64 perf_reg_value(struct pt_regs *regs, int idx) 61 { 62 struct x86_perf_regs *perf_regs; 63 64 if (idx > PERF_REG_X86_R15) { 65 perf_regs = container_of(regs, struct x86_perf_regs, regs); 66 if (perf_regs->abi == PERF_SAMPLE_REGS_ABI_NONE) 67 return 0; 68 69 if (perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD) { 70 if (idx <= PERF_REG_X86_R31) { 71 if (!perf_regs->egpr_regs) 72 return 0; 73 return perf_regs->egpr_regs[idx - PERF_REG_X86_R16]; 74 } 75 if (idx == PERF_REG_X86_SSP) { 76 if (!perf_regs->ssp) 77 return 0; 78 return *perf_regs->ssp; 79 } 80 } else { 81 if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) { 82 if (!perf_regs->xmm_regs) 83 return 0; 84 return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0]; 85 } 86 } 87 } 88 89 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset))) 90 return 0; 91 92 return regs_get_register(regs, pt_regs_offset[idx]); 93 } 94 95 #define PERF_X86_YMMH_QWORDS (PERF_X86_YMM_QWORDS / 2) 96 #define PERF_X86_ZMMH_QWORDS (PERF_X86_ZMM_QWORDS / 2) 97 98 u64 perf_simd_reg_value(struct pt_regs *regs, int idx, 99 u16 qwords_idx, bool pred) 100 { 101 struct x86_perf_regs *perf_regs = 102 container_of(regs, struct x86_perf_regs, regs); 103 104 if (!(perf_regs->abi & PERF_SAMPLE_REGS_ABI_SIMD)) 105 return 0; 106 107 if (pred) { 108 if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_PRED_REGS_MAX || 109 qwords_idx >= PERF_X86_OPMASK_QWORDS)) 110 return 0; 111 if (!perf_regs->opmask_regs) 112 return 0; 113 return perf_regs->opmask_regs[idx]; 114 } 115 116 if (WARN_ON_ONCE(idx >= PERF_X86_SIMD_VEC_REGS_MAX || 117 qwords_idx >= PERF_X86_SIMD_QWORDS_MAX)) 118 return 0; 119 120 if (idx >= PERF_X86_H16ZMM_BASE) { 121 if (!perf_regs->h16zmm_regs) 122 return 0; 123 return perf_regs->h16zmm_regs[(idx - PERF_X86_H16ZMM_BASE) * 124 PERF_X86_ZMM_QWORDS + qwords_idx]; 125 } 126 127 if (qwords_idx < PERF_X86_XMM_QWORDS) { 128 if (!perf_regs->xmm_regs) 129 return 0; 130 return perf_regs->xmm_regs[idx * PERF_X86_XMM_QWORDS + 131 qwords_idx]; 132 } else if (qwords_idx < PERF_X86_YMM_QWORDS) { 133 if (!perf_regs->ymmh_regs) 134 return 0; 135 return perf_regs->ymmh_regs[idx * PERF_X86_YMMH_QWORDS + 136 qwords_idx - PERF_X86_XMM_QWORDS]; 137 } else if (qwords_idx < PERF_X86_ZMM_QWORDS) { 138 if (!perf_regs->zmmh_regs) 139 return 0; 140 return perf_regs->zmmh_regs[idx * PERF_X86_ZMMH_QWORDS + 141 qwords_idx - PERF_X86_YMM_QWORDS]; 142 } 143 144 return 0; 145 } 146 147 int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, 148 u16 pred_qwords, u32 pred_mask) 149 { 150 unsigned long mask; 151 u64 size; 152 153 if (!vec_qwords && !pred_qwords) { 154 if (vec_mask || pred_mask) 155 return -EINVAL; 156 } 157 158 if (vec_qwords) { 159 if (vec_qwords != PERF_X86_XMM_QWORDS && 160 vec_qwords != PERF_X86_YMM_QWORDS && 161 vec_qwords != PERF_X86_ZMM_QWORDS) 162 return -EINVAL; 163 if (vec_mask & ~PERF_X86_SIMD_VEC_MASK) 164 return -EINVAL; 165 /* Only full-register sampling is allowed. */ 166 mask = vec_mask; 167 if (vec_qwords == PERF_X86_XMM_QWORDS && mask && 168 !bitmap_full(&mask, PERF_X86_SIMD_XMM_REGS)) 169 return -EINVAL; 170 if (vec_qwords == PERF_X86_YMM_QWORDS && mask && 171 !bitmap_full(&mask, PERF_X86_SIMD_YMM_REGS)) 172 return -EINVAL; 173 if (vec_qwords == PERF_X86_ZMM_QWORDS && mask && 174 !bitmap_full(&mask, PERF_X86_SIMD_ZMM_REGS)) 175 return -EINVAL; 176 } 177 178 if (pred_qwords) { 179 if (pred_qwords != PERF_X86_OPMASK_QWORDS) 180 return -EINVAL; 181 if (pred_mask & ~PERF_X86_SIMD_PRED_MASK) 182 return -EINVAL; 183 /* Only full-register sampling is allowed. */ 184 mask = pred_mask; 185 if (pred_qwords == PERF_X86_OPMASK_QWORDS && mask && 186 !bitmap_full(&mask, PERF_X86_SIMD_OPMASK_REGS)) 187 return -EINVAL; 188 } 189 190 size = sizeof(u64) * 4; 191 size += (hweight64(vec_mask) * vec_qwords + 192 hweight32(pred_mask) * pred_qwords) * sizeof(u64); 193 /* 194 * INTR_REGS and USR_REGS could be sampled simultaneously, 195 * so roughly restrict the size to half of U16_MAX. 196 */ 197 if (size >= U16_MAX / 2) 198 return -EINVAL; 199 200 return 0; 201 } 202 203 #define PERF_REG_X86_RESERVED (GENMASK_ULL(PERF_REG_X86_XMM0 - 1, PERF_REG_X86_AX) & \ 204 ~GENMASK_ULL(PERF_REG_X86_R15, PERF_REG_X86_AX)) 205 #define PERF_REG_X86_EXT_RESERVED (~GENMASK_ULL(PERF_REG_MISC_MAX - 1, PERF_REG_X86_AX)) 206 207 #ifdef CONFIG_X86_32 208 #define REG_NOSUPPORT GENMASK_ULL(PERF_REG_X86_R15, PERF_REG_X86_R8) 209 210 int perf_reg_validate(u64 mask, bool simd_enabled) 211 { 212 if (!simd_enabled && 213 (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))) 214 return -EINVAL; 215 216 /* The mask could be 0 if only the SIMD registers are interested */ 217 if (simd_enabled && 218 (mask & ~GENMASK_ULL(PERF_REG_X86_GS, PERF_REG_X86_AX))) 219 return -EINVAL; 220 221 return 0; 222 } 223 224 u64 perf_reg_abi(struct task_struct *task) 225 { 226 return PERF_SAMPLE_REGS_ABI_32; 227 } 228 229 #else /* CONFIG_X86_64 */ 230 #define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) | \ 231 (1ULL << PERF_REG_X86_ES) | \ 232 (1ULL << PERF_REG_X86_FS) | \ 233 (1ULL << PERF_REG_X86_GS)) 234 235 int perf_reg_validate(u64 mask, bool simd_enabled) 236 { 237 if (!simd_enabled && 238 (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))) 239 return -EINVAL; 240 241 /* The mask could be 0 if only the SIMD registers are interested */ 242 if (simd_enabled && 243 (mask & (REG_NOSUPPORT | PERF_REG_X86_EXT_RESERVED))) 244 return -EINVAL; 245 246 return 0; 247 } 248 249 u64 perf_reg_abi(struct task_struct *task) 250 { 251 if (!user_64bit_mode(task_pt_regs(task))) 252 return PERF_SAMPLE_REGS_ABI_32; 253 else 254 return PERF_SAMPLE_REGS_ABI_64; 255 } 256 257 static DEFINE_PER_CPU(struct pt_regs, nmi_user_regs); 258 259 void perf_get_regs_user(struct perf_regs *regs_user, 260 struct pt_regs *regs) 261 { 262 struct pt_regs *regs_user_copy = this_cpu_ptr(&nmi_user_regs); 263 struct pt_regs *user_regs = task_pt_regs(current); 264 265 if (!in_nmi()) { 266 regs_user->regs = user_regs; 267 regs_user->abi = perf_reg_abi(current); 268 return; 269 } 270 271 /* 272 * If we're in an NMI that interrupted task_pt_regs setup, then 273 * we can't sample user regs at all. This check isn't really 274 * sufficient, though, as we could be in an NMI inside an interrupt 275 * that happened during task_pt_regs setup. 276 */ 277 if (regs->sp > (unsigned long)&user_regs->r11 && 278 regs->sp <= (unsigned long)(user_regs + 1)) { 279 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; 280 regs_user->regs = NULL; 281 return; 282 } 283 284 /* 285 * These registers are always saved on 64-bit syscall entry. 286 * On 32-bit entry points, they are saved too except r8..r11. 287 */ 288 regs_user_copy->ip = user_regs->ip; 289 regs_user_copy->ax = user_regs->ax; 290 regs_user_copy->cx = user_regs->cx; 291 regs_user_copy->dx = user_regs->dx; 292 regs_user_copy->si = user_regs->si; 293 regs_user_copy->di = user_regs->di; 294 regs_user_copy->r8 = user_regs->r8; 295 regs_user_copy->r9 = user_regs->r9; 296 regs_user_copy->r10 = user_regs->r10; 297 regs_user_copy->r11 = user_regs->r11; 298 regs_user_copy->orig_ax = user_regs->orig_ax; 299 regs_user_copy->flags = user_regs->flags; 300 regs_user_copy->sp = user_regs->sp; 301 regs_user_copy->cs = user_regs->cs; 302 regs_user_copy->ss = user_regs->ss; 303 /* 304 * Store user space frame-pointer value on sample 305 * to facilitate stack unwinding for cases when 306 * user space executable code has such support 307 * enabled at compile time: 308 */ 309 regs_user_copy->bp = user_regs->bp; 310 311 regs_user_copy->bx = -1; 312 regs_user_copy->r12 = -1; 313 regs_user_copy->r13 = -1; 314 regs_user_copy->r14 = -1; 315 regs_user_copy->r15 = -1; 316 /* 317 * For this to be at all useful, we need a reasonable guess for 318 * the ABI. Be careful: we're in NMI context, and we're 319 * considering current to be the current task, so we should 320 * be careful not to look at any other percpu variables that might 321 * change during context switches. 322 */ 323 regs_user->abi = user_64bit_mode(user_regs) ? 324 PERF_SAMPLE_REGS_ABI_64 : PERF_SAMPLE_REGS_ABI_32; 325 326 regs_user->regs = regs_user_copy; 327 } 328 #endif /* CONFIG_X86_32 */ 329