1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Based on arch/arm/kernel/ptrace.c 4 * 5 * By Ross Biro 1/23/92 6 * edited by Linus Torvalds 7 * ARM modifications Copyright (C) 2000 Russell King 8 * Copyright (C) 2012 ARM Ltd. 9 */ 10 11 #include <linux/audit.h> 12 #include <linux/compat.h> 13 #include <linux/kernel.h> 14 #include <linux/sched/signal.h> 15 #include <linux/sched/task_stack.h> 16 #include <linux/mm.h> 17 #include <linux/nospec.h> 18 #include <linux/smp.h> 19 #include <linux/ptrace.h> 20 #include <linux/user.h> 21 #include <linux/seccomp.h> 22 #include <linux/security.h> 23 #include <linux/init.h> 24 #include <linux/signal.h> 25 #include <linux/string.h> 26 #include <linux/uaccess.h> 27 #include <linux/perf_event.h> 28 #include <linux/hw_breakpoint.h> 29 #include <linux/regset.h> 30 #include <linux/elf.h> 31 #include <linux/rseq.h> 32 33 #include <asm/compat.h> 34 #include <asm/cpufeature.h> 35 #include <asm/debug-monitors.h> 36 #include <asm/fpsimd.h> 37 #include <asm/gcs.h> 38 #include <asm/mte.h> 39 #include <asm/pointer_auth.h> 40 #include <asm/stacktrace.h> 41 #include <asm/syscall.h> 42 #include <asm/traps.h> 43 #include <asm/system_misc.h> 44 45 #define CREATE_TRACE_POINTS 46 #include <trace/events/syscalls.h> 47 48 struct pt_regs_offset { 49 const char *name; 50 int offset; 51 }; 52 53 #define REG_OFFSET_NAME(r) {.name = #r, .offset = offsetof(struct pt_regs, r)} 54 #define REG_OFFSET_END {.name = NULL, .offset = 0} 55 #define GPR_OFFSET_NAME(r) \ 56 {.name = "x" #r, .offset = offsetof(struct pt_regs, regs[r])} 57 58 static const struct pt_regs_offset regoffset_table[] = { 59 GPR_OFFSET_NAME(0), 60 GPR_OFFSET_NAME(1), 61 GPR_OFFSET_NAME(2), 62 GPR_OFFSET_NAME(3), 63 GPR_OFFSET_NAME(4), 64 GPR_OFFSET_NAME(5), 65 GPR_OFFSET_NAME(6), 66 GPR_OFFSET_NAME(7), 67 GPR_OFFSET_NAME(8), 68 GPR_OFFSET_NAME(9), 69 GPR_OFFSET_NAME(10), 70 GPR_OFFSET_NAME(11), 71 GPR_OFFSET_NAME(12), 72 GPR_OFFSET_NAME(13), 73 GPR_OFFSET_NAME(14), 74 GPR_OFFSET_NAME(15), 75 GPR_OFFSET_NAME(16), 76 GPR_OFFSET_NAME(17), 77 GPR_OFFSET_NAME(18), 78 GPR_OFFSET_NAME(19), 79 GPR_OFFSET_NAME(20), 80 GPR_OFFSET_NAME(21), 81 GPR_OFFSET_NAME(22), 82 GPR_OFFSET_NAME(23), 83 GPR_OFFSET_NAME(24), 84 GPR_OFFSET_NAME(25), 85 GPR_OFFSET_NAME(26), 86 GPR_OFFSET_NAME(27), 87 GPR_OFFSET_NAME(28), 88 GPR_OFFSET_NAME(29), 89 GPR_OFFSET_NAME(30), 90 {.name = "lr", .offset = offsetof(struct pt_regs, regs[30])}, 91 REG_OFFSET_NAME(sp), 92 REG_OFFSET_NAME(pc), 93 REG_OFFSET_NAME(pstate), 94 REG_OFFSET_END, 95 }; 96 97 /** 98 * regs_query_register_offset() - query register offset from its name 99 * @name: the name of a register 100 * 101 * regs_query_register_offset() returns the offset of a register in struct 102 * pt_regs from its name. If the name is invalid, this returns -EINVAL; 103 */ 104 int regs_query_register_offset(const char *name) 105 { 106 const struct pt_regs_offset *roff; 107 108 for (roff = regoffset_table; roff->name != NULL; roff++) 109 if (!strcmp(roff->name, name)) 110 return roff->offset; 111 return -EINVAL; 112 } 113 114 /** 115 * regs_within_kernel_stack() - check the address in the stack 116 * @regs: pt_regs which contains kernel stack pointer. 117 * @addr: address which is checked. 118 * 119 * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). 120 * If @addr is within the kernel stack, it returns true. If not, returns false. 121 */ 122 static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) 123 { 124 return ((addr & ~(THREAD_SIZE - 1)) == 125 (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) || 126 on_irq_stack(addr, sizeof(unsigned long)); 127 } 128 129 /** 130 * regs_get_kernel_stack_nth() - get Nth entry of the stack 131 * @regs: pt_regs which contains kernel stack pointer. 132 * @n: stack entry number. 133 * 134 * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which 135 * is specified by @regs. If the @n th entry is NOT in the kernel stack, 136 * this returns 0. 137 */ 138 unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) 139 { 140 unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); 141 142 addr += n; 143 if (regs_within_kernel_stack(regs, (unsigned long)addr)) 144 return READ_ONCE_NOCHECK(*addr); 145 else 146 return 0; 147 } 148 149 /* 150 * TODO: does not yet catch signals sent when the child dies. 151 * in exit.c or in signal.c. 152 */ 153 154 /* 155 * Called by kernel/ptrace.c when detaching.. 156 */ 157 void ptrace_disable(struct task_struct *child) 158 { 159 /* 160 * This would be better off in core code, but PTRACE_DETACH has 161 * grown its fair share of arch-specific worts and changing it 162 * is likely to cause regressions on obscure architectures. 163 */ 164 user_disable_single_step(child); 165 } 166 167 #ifdef CONFIG_HAVE_HW_BREAKPOINT 168 /* 169 * Handle hitting a HW-breakpoint. 170 */ 171 static void ptrace_hbptriggered(struct perf_event *bp, 172 struct perf_sample_data *data, 173 struct pt_regs *regs) 174 { 175 struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp); 176 const char *desc = "Hardware breakpoint trap (ptrace)"; 177 178 if (is_compat_task()) { 179 int si_errno = 0; 180 int i; 181 182 for (i = 0; i < ARM_MAX_BRP; ++i) { 183 if (current->thread.debug.hbp_break[i] == bp) { 184 si_errno = (i << 1) + 1; 185 break; 186 } 187 } 188 189 for (i = 0; i < ARM_MAX_WRP; ++i) { 190 if (current->thread.debug.hbp_watch[i] == bp) { 191 si_errno = -((i << 1) + 1); 192 break; 193 } 194 } 195 arm64_force_sig_ptrace_errno_trap(si_errno, bkpt->trigger, 196 desc); 197 return; 198 } 199 200 arm64_force_sig_fault(SIGTRAP, TRAP_HWBKPT, bkpt->trigger, desc); 201 } 202 203 /* 204 * Unregister breakpoints from this task and reset the pointers in 205 * the thread_struct. 206 */ 207 void flush_ptrace_hw_breakpoint(struct task_struct *tsk) 208 { 209 int i; 210 struct thread_struct *t = &tsk->thread; 211 212 for (i = 0; i < ARM_MAX_BRP; i++) { 213 if (t->debug.hbp_break[i]) { 214 unregister_hw_breakpoint(t->debug.hbp_break[i]); 215 t->debug.hbp_break[i] = NULL; 216 } 217 } 218 219 for (i = 0; i < ARM_MAX_WRP; i++) { 220 if (t->debug.hbp_watch[i]) { 221 unregister_hw_breakpoint(t->debug.hbp_watch[i]); 222 t->debug.hbp_watch[i] = NULL; 223 } 224 } 225 } 226 227 void ptrace_hw_copy_thread(struct task_struct *tsk) 228 { 229 memset(&tsk->thread.debug, 0, sizeof(struct debug_info)); 230 } 231 232 static struct perf_event *ptrace_hbp_get_event(unsigned int note_type, 233 struct task_struct *tsk, 234 unsigned long idx) 235 { 236 struct perf_event *bp = ERR_PTR(-EINVAL); 237 238 switch (note_type) { 239 case NT_ARM_HW_BREAK: 240 if (idx >= ARM_MAX_BRP) 241 goto out; 242 idx = array_index_nospec(idx, ARM_MAX_BRP); 243 bp = tsk->thread.debug.hbp_break[idx]; 244 break; 245 case NT_ARM_HW_WATCH: 246 if (idx >= ARM_MAX_WRP) 247 goto out; 248 idx = array_index_nospec(idx, ARM_MAX_WRP); 249 bp = tsk->thread.debug.hbp_watch[idx]; 250 break; 251 } 252 253 out: 254 return bp; 255 } 256 257 static int ptrace_hbp_set_event(unsigned int note_type, 258 struct task_struct *tsk, 259 unsigned long idx, 260 struct perf_event *bp) 261 { 262 int err = -EINVAL; 263 264 switch (note_type) { 265 case NT_ARM_HW_BREAK: 266 if (idx >= ARM_MAX_BRP) 267 goto out; 268 idx = array_index_nospec(idx, ARM_MAX_BRP); 269 tsk->thread.debug.hbp_break[idx] = bp; 270 err = 0; 271 break; 272 case NT_ARM_HW_WATCH: 273 if (idx >= ARM_MAX_WRP) 274 goto out; 275 idx = array_index_nospec(idx, ARM_MAX_WRP); 276 tsk->thread.debug.hbp_watch[idx] = bp; 277 err = 0; 278 break; 279 } 280 281 out: 282 return err; 283 } 284 285 static struct perf_event *ptrace_hbp_create(unsigned int note_type, 286 struct task_struct *tsk, 287 unsigned long idx) 288 { 289 struct perf_event *bp; 290 struct perf_event_attr attr; 291 int err, type; 292 293 switch (note_type) { 294 case NT_ARM_HW_BREAK: 295 type = HW_BREAKPOINT_X; 296 break; 297 case NT_ARM_HW_WATCH: 298 type = HW_BREAKPOINT_RW; 299 break; 300 default: 301 return ERR_PTR(-EINVAL); 302 } 303 304 ptrace_breakpoint_init(&attr); 305 306 /* 307 * Initialise fields to sane defaults 308 * (i.e. values that will pass validation). 309 */ 310 attr.bp_addr = 0; 311 attr.bp_len = HW_BREAKPOINT_LEN_4; 312 attr.bp_type = type; 313 attr.disabled = 1; 314 315 bp = register_user_hw_breakpoint(&attr, ptrace_hbptriggered, NULL, tsk); 316 if (IS_ERR(bp)) 317 return bp; 318 319 err = ptrace_hbp_set_event(note_type, tsk, idx, bp); 320 if (err) 321 return ERR_PTR(err); 322 323 return bp; 324 } 325 326 static int ptrace_hbp_fill_attr_ctrl(unsigned int note_type, 327 struct arch_hw_breakpoint_ctrl ctrl, 328 struct perf_event_attr *attr) 329 { 330 int err, len, type, offset, disabled = !ctrl.enabled; 331 332 attr->disabled = disabled; 333 if (disabled) 334 return 0; 335 336 err = arch_bp_generic_fields(ctrl, &len, &type, &offset); 337 if (err) 338 return err; 339 340 switch (note_type) { 341 case NT_ARM_HW_BREAK: 342 if ((type & HW_BREAKPOINT_X) != type) 343 return -EINVAL; 344 break; 345 case NT_ARM_HW_WATCH: 346 if ((type & HW_BREAKPOINT_RW) != type) 347 return -EINVAL; 348 break; 349 default: 350 return -EINVAL; 351 } 352 353 attr->bp_len = len; 354 attr->bp_type = type; 355 attr->bp_addr += offset; 356 357 return 0; 358 } 359 360 static int ptrace_hbp_get_resource_info(unsigned int note_type, u32 *info) 361 { 362 u8 num; 363 u32 reg = 0; 364 365 switch (note_type) { 366 case NT_ARM_HW_BREAK: 367 num = hw_breakpoint_slots(TYPE_INST); 368 break; 369 case NT_ARM_HW_WATCH: 370 num = hw_breakpoint_slots(TYPE_DATA); 371 break; 372 default: 373 return -EINVAL; 374 } 375 376 reg |= debug_monitors_arch(); 377 reg <<= 8; 378 reg |= num; 379 380 *info = reg; 381 return 0; 382 } 383 384 static int ptrace_hbp_get_ctrl(unsigned int note_type, 385 struct task_struct *tsk, 386 unsigned long idx, 387 u32 *ctrl) 388 { 389 struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx); 390 391 if (IS_ERR(bp)) 392 return PTR_ERR(bp); 393 394 *ctrl = bp ? encode_ctrl_reg(counter_arch_bp(bp)->ctrl) : 0; 395 return 0; 396 } 397 398 static int ptrace_hbp_get_addr(unsigned int note_type, 399 struct task_struct *tsk, 400 unsigned long idx, 401 u64 *addr) 402 { 403 struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx); 404 405 if (IS_ERR(bp)) 406 return PTR_ERR(bp); 407 408 *addr = bp ? counter_arch_bp(bp)->address : 0; 409 return 0; 410 } 411 412 static struct perf_event *ptrace_hbp_get_initialised_bp(unsigned int note_type, 413 struct task_struct *tsk, 414 unsigned long idx) 415 { 416 struct perf_event *bp = ptrace_hbp_get_event(note_type, tsk, idx); 417 418 if (!bp) 419 bp = ptrace_hbp_create(note_type, tsk, idx); 420 421 return bp; 422 } 423 424 static int ptrace_hbp_set_ctrl(unsigned int note_type, 425 struct task_struct *tsk, 426 unsigned long idx, 427 u32 uctrl) 428 { 429 int err; 430 struct perf_event *bp; 431 struct perf_event_attr attr; 432 struct arch_hw_breakpoint_ctrl ctrl; 433 434 bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx); 435 if (IS_ERR(bp)) { 436 err = PTR_ERR(bp); 437 return err; 438 } 439 440 attr = bp->attr; 441 decode_ctrl_reg(uctrl, &ctrl); 442 err = ptrace_hbp_fill_attr_ctrl(note_type, ctrl, &attr); 443 if (err) 444 return err; 445 446 return modify_user_hw_breakpoint(bp, &attr); 447 } 448 449 static int ptrace_hbp_set_addr(unsigned int note_type, 450 struct task_struct *tsk, 451 unsigned long idx, 452 u64 addr) 453 { 454 int err; 455 struct perf_event *bp; 456 struct perf_event_attr attr; 457 458 bp = ptrace_hbp_get_initialised_bp(note_type, tsk, idx); 459 if (IS_ERR(bp)) { 460 err = PTR_ERR(bp); 461 return err; 462 } 463 464 attr = bp->attr; 465 attr.bp_addr = addr; 466 err = modify_user_hw_breakpoint(bp, &attr); 467 return err; 468 } 469 470 #define PTRACE_HBP_ADDR_SZ sizeof(u64) 471 #define PTRACE_HBP_CTRL_SZ sizeof(u32) 472 #define PTRACE_HBP_PAD_SZ sizeof(u32) 473 474 static int hw_break_get(struct task_struct *target, 475 const struct user_regset *regset, 476 struct membuf to) 477 { 478 unsigned int note_type = regset->core_note_type; 479 int ret, idx = 0; 480 u32 info, ctrl; 481 u64 addr; 482 483 /* Resource info */ 484 ret = ptrace_hbp_get_resource_info(note_type, &info); 485 if (ret) 486 return ret; 487 488 membuf_write(&to, &info, sizeof(info)); 489 membuf_zero(&to, sizeof(u32)); 490 /* (address, ctrl) registers */ 491 while (to.left) { 492 ret = ptrace_hbp_get_addr(note_type, target, idx, &addr); 493 if (ret) 494 return ret; 495 ret = ptrace_hbp_get_ctrl(note_type, target, idx, &ctrl); 496 if (ret) 497 return ret; 498 membuf_store(&to, addr); 499 membuf_store(&to, ctrl); 500 membuf_zero(&to, sizeof(u32)); 501 idx++; 502 } 503 return 0; 504 } 505 506 static int hw_break_set(struct task_struct *target, 507 const struct user_regset *regset, 508 unsigned int pos, unsigned int count, 509 const void *kbuf, const void __user *ubuf) 510 { 511 unsigned int note_type = regset->core_note_type; 512 int ret, idx = 0, offset, limit; 513 u32 ctrl; 514 u64 addr; 515 516 /* Resource info and pad */ 517 offset = offsetof(struct user_hwdebug_state, dbg_regs); 518 user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 0, offset); 519 520 /* (address, ctrl) registers */ 521 limit = regset->n * regset->size; 522 while (count && offset < limit) { 523 if (count < PTRACE_HBP_ADDR_SZ) 524 return -EINVAL; 525 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &addr, 526 offset, offset + PTRACE_HBP_ADDR_SZ); 527 if (ret) 528 return ret; 529 ret = ptrace_hbp_set_addr(note_type, target, idx, addr); 530 if (ret) 531 return ret; 532 offset += PTRACE_HBP_ADDR_SZ; 533 534 if (!count) 535 break; 536 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, 537 offset, offset + PTRACE_HBP_CTRL_SZ); 538 if (ret) 539 return ret; 540 ret = ptrace_hbp_set_ctrl(note_type, target, idx, ctrl); 541 if (ret) 542 return ret; 543 offset += PTRACE_HBP_CTRL_SZ; 544 545 user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 546 offset, offset + PTRACE_HBP_PAD_SZ); 547 offset += PTRACE_HBP_PAD_SZ; 548 idx++; 549 } 550 551 return 0; 552 } 553 #endif /* CONFIG_HAVE_HW_BREAKPOINT */ 554 555 static int gpr_get(struct task_struct *target, 556 const struct user_regset *regset, 557 struct membuf to) 558 { 559 struct user_pt_regs *uregs = &task_pt_regs(target)->user_regs; 560 return membuf_write(&to, uregs, sizeof(*uregs)); 561 } 562 563 static void update_syscall_orig_x0_after_ptrace(struct task_struct *target) 564 { 565 struct pt_regs *regs = task_pt_regs(target); 566 struct kernel_siginfo *info = target->last_siginfo; 567 568 /* 569 * Skip the update for NO_SYSCALL (set either by the user or the 570 * tracer), as regs[0] holds the return value (see the comment in 571 * el0_svc_common()) and can be unwound using syscall_rollback(). 572 */ 573 if (regs->syscallno == NO_SYSCALL) 574 return; 575 576 /* We should only be called when target is in a ptrace stop */ 577 if (WARN_ON_ONCE(!info)) 578 return; 579 580 /* 581 * For compat tasks, orig_r0 is provided directly through GPR index 582 * 17. 583 */ 584 if (is_compat_thread(task_thread_info(target))) 585 return; 586 587 /* 588 * Don't update orig_x0 for a syscall-exit-stop, as x0 now contains the 589 * return value of the system call. 590 */ 591 if ((info->si_code & ~0x80) == SIGTRAP && 592 target->ptrace_message == PTRACE_EVENTMSG_SYSCALL_EXIT) { 593 return; 594 } 595 596 regs->orig_x0 = regs->regs[0]; 597 } 598 599 static int gpr_set(struct task_struct *target, const struct user_regset *regset, 600 unsigned int pos, unsigned int count, 601 const void *kbuf, const void __user *ubuf) 602 { 603 int ret; 604 struct user_pt_regs newregs = task_pt_regs(target)->user_regs; 605 606 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newregs, 0, -1); 607 if (ret) 608 return ret; 609 610 if (!valid_user_regs(&newregs, target)) 611 return -EINVAL; 612 613 task_pt_regs(target)->user_regs = newregs; 614 615 /* 616 * Keep orig_x0 authoritative so that seccomp (via 617 * syscall_get_arguments()), audit and the restart path all see the same 618 * first argument the syscall is dispatched with, even if it has been 619 * updated by a tracer. 620 */ 621 update_syscall_orig_x0_after_ptrace(target); 622 return 0; 623 } 624 625 static int fpr_active(struct task_struct *target, const struct user_regset *regset) 626 { 627 if (!system_supports_fpsimd()) 628 return -ENODEV; 629 return regset->n; 630 } 631 632 /* 633 * TODO: update fp accessors for lazy context switching (sync/flush hwstate) 634 */ 635 static int __fpr_get(struct task_struct *target, 636 const struct user_regset *regset, 637 struct membuf to) 638 { 639 struct user_fpsimd_state *uregs; 640 641 fpsimd_sync_from_effective_state(target); 642 643 uregs = &target->thread.uw.fpsimd_state; 644 645 return membuf_write(&to, uregs, sizeof(*uregs)); 646 } 647 648 static int fpr_get(struct task_struct *target, const struct user_regset *regset, 649 struct membuf to) 650 { 651 if (!system_supports_fpsimd()) 652 return -EINVAL; 653 654 if (target == current) 655 fpsimd_preserve_current_state(); 656 657 return __fpr_get(target, regset, to); 658 } 659 660 static int __fpr_set(struct task_struct *target, 661 const struct user_regset *regset, 662 unsigned int pos, unsigned int count, 663 const void *kbuf, const void __user *ubuf, 664 unsigned int start_pos) 665 { 666 int ret; 667 struct user_fpsimd_state newstate; 668 669 /* 670 * Ensure target->thread.uw.fpsimd_state is up to date, so that a 671 * short copyin can't resurrect stale data. 672 */ 673 fpsimd_sync_from_effective_state(target); 674 675 newstate = target->thread.uw.fpsimd_state; 676 677 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate, 678 start_pos, start_pos + sizeof(newstate)); 679 if (ret) 680 return ret; 681 682 target->thread.uw.fpsimd_state = newstate; 683 684 return ret; 685 } 686 687 static int fpr_set(struct task_struct *target, const struct user_regset *regset, 688 unsigned int pos, unsigned int count, 689 const void *kbuf, const void __user *ubuf) 690 { 691 int ret; 692 693 if (!system_supports_fpsimd()) 694 return -EINVAL; 695 696 ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, 0); 697 if (ret) 698 return ret; 699 700 fpsimd_sync_to_effective_state_zeropad(target); 701 fpsimd_flush_task_state(target); 702 703 return ret; 704 } 705 706 static int tls_get(struct task_struct *target, const struct user_regset *regset, 707 struct membuf to) 708 { 709 int ret; 710 711 if (target == current) 712 tls_preserve_current_state(); 713 714 ret = membuf_store(&to, target->thread.uw.tp_value); 715 if (system_supports_tpidr2()) 716 ret = membuf_store(&to, target->thread.tpidr2_el0); 717 else 718 ret = membuf_zero(&to, sizeof(u64)); 719 720 return ret; 721 } 722 723 static int tls_set(struct task_struct *target, const struct user_regset *regset, 724 unsigned int pos, unsigned int count, 725 const void *kbuf, const void __user *ubuf) 726 { 727 int ret; 728 unsigned long tls[2]; 729 730 tls[0] = target->thread.uw.tp_value; 731 if (system_supports_tpidr2()) 732 tls[1] = target->thread.tpidr2_el0; 733 734 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, tls, 0, count); 735 if (ret) 736 return ret; 737 738 target->thread.uw.tp_value = tls[0]; 739 if (system_supports_tpidr2()) 740 target->thread.tpidr2_el0 = tls[1]; 741 742 return ret; 743 } 744 745 static int fpmr_get(struct task_struct *target, const struct user_regset *regset, 746 struct membuf to) 747 { 748 if (!system_supports_fpmr()) 749 return -EINVAL; 750 751 if (target == current) 752 fpsimd_preserve_current_state(); 753 754 return membuf_store(&to, target->thread.uw.fpmr); 755 } 756 757 static int fpmr_set(struct task_struct *target, const struct user_regset *regset, 758 unsigned int pos, unsigned int count, 759 const void *kbuf, const void __user *ubuf) 760 { 761 int ret; 762 unsigned long fpmr; 763 764 if (!system_supports_fpmr()) 765 return -EINVAL; 766 767 fpmr = target->thread.uw.fpmr; 768 769 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fpmr, 0, count); 770 if (ret) 771 return ret; 772 773 target->thread.uw.fpmr = fpmr; 774 775 fpsimd_flush_task_state(target); 776 777 return 0; 778 } 779 780 static int system_call_get(struct task_struct *target, 781 const struct user_regset *regset, 782 struct membuf to) 783 { 784 return membuf_store(&to, task_pt_regs(target)->syscallno); 785 } 786 787 static int system_call_set(struct task_struct *target, 788 const struct user_regset *regset, 789 unsigned int pos, unsigned int count, 790 const void *kbuf, const void __user *ubuf) 791 { 792 int syscallno = task_pt_regs(target)->syscallno; 793 int ret; 794 795 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &syscallno, 0, -1); 796 if (ret) 797 return ret; 798 799 task_pt_regs(target)->syscallno = syscallno; 800 801 /* 802 * Re-sync orig_x0 in case the syscall number has been changed 803 * from NO_SYSCALL. 804 */ 805 update_syscall_orig_x0_after_ptrace(target); 806 return ret; 807 } 808 809 #ifdef CONFIG_ARM64_SVE 810 811 static void sve_init_header_from_task(struct user_sve_header *header, 812 struct task_struct *target, 813 enum vec_type type) 814 { 815 unsigned int vq; 816 bool active; 817 enum vec_type task_type; 818 819 memset(header, 0, sizeof(*header)); 820 821 /* Check if the requested registers are active for the task */ 822 if (thread_sm_enabled(&target->thread)) 823 task_type = ARM64_VEC_SME; 824 else 825 task_type = ARM64_VEC_SVE; 826 active = (task_type == type); 827 828 if (active && target->thread.fp_type == FP_STATE_SVE) 829 header->flags = SVE_PT_REGS_SVE; 830 else 831 header->flags = SVE_PT_REGS_FPSIMD; 832 833 switch (type) { 834 case ARM64_VEC_SVE: 835 if (test_tsk_thread_flag(target, TIF_SVE_VL_INHERIT)) 836 header->flags |= SVE_PT_VL_INHERIT; 837 break; 838 case ARM64_VEC_SME: 839 if (test_tsk_thread_flag(target, TIF_SME_VL_INHERIT)) 840 header->flags |= SVE_PT_VL_INHERIT; 841 break; 842 default: 843 WARN_ON_ONCE(1); 844 return; 845 } 846 847 header->vl = task_get_vl(target, type); 848 vq = sve_vq_from_vl(header->vl); 849 850 header->max_vl = vec_max_vl(type); 851 if (active) 852 header->size = SVE_PT_SIZE(vq, header->flags); 853 else 854 header->size = sizeof(*header); 855 header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl), 856 SVE_PT_REGS_SVE); 857 } 858 859 static unsigned int sve_size_from_header(struct user_sve_header const *header) 860 { 861 return ALIGN(header->size, SVE_VQ_BYTES); 862 } 863 864 static int sve_get_common(struct task_struct *target, 865 const struct user_regset *regset, 866 struct membuf to, 867 enum vec_type type) 868 { 869 struct user_sve_header header; 870 unsigned int vq; 871 unsigned long start, end; 872 873 if (target == current) 874 fpsimd_preserve_current_state(); 875 876 /* Header */ 877 sve_init_header_from_task(&header, target, type); 878 vq = sve_vq_from_vl(header.vl); 879 880 membuf_write(&to, &header, sizeof(header)); 881 882 BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header)); 883 BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header)); 884 885 /* 886 * When the requested vector type is not active, do not present data 887 * from the other mode to userspace. 888 */ 889 if (header.size == sizeof(header)) 890 return to.left; 891 892 switch ((header.flags & SVE_PT_REGS_MASK)) { 893 case SVE_PT_REGS_FPSIMD: 894 return __fpr_get(target, regset, to); 895 896 case SVE_PT_REGS_SVE: 897 start = SVE_PT_SVE_OFFSET; 898 end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq); 899 membuf_write(&to, target->thread.sve_state, end - start); 900 901 start = end; 902 end = SVE_PT_SVE_FPSR_OFFSET(vq); 903 membuf_zero(&to, end - start); 904 905 /* 906 * Copy fpsr, and fpcr which must follow contiguously in 907 * struct fpsimd_state: 908 */ 909 start = end; 910 end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE; 911 membuf_write(&to, &target->thread.uw.fpsimd_state.fpsr, 912 end - start); 913 914 start = end; 915 end = sve_size_from_header(&header); 916 return membuf_zero(&to, end - start); 917 918 default: 919 BUILD_BUG(); 920 } 921 } 922 923 static int sve_get(struct task_struct *target, 924 const struct user_regset *regset, 925 struct membuf to) 926 { 927 if (!system_supports_sve()) 928 return -EINVAL; 929 930 return sve_get_common(target, regset, to, ARM64_VEC_SVE); 931 } 932 933 static int sve_set_common(struct task_struct *target, 934 const struct user_regset *regset, 935 unsigned int pos, unsigned int count, 936 const void *kbuf, const void __user *ubuf, 937 enum vec_type type) 938 { 939 int ret; 940 struct user_sve_header header; 941 unsigned int vq; 942 unsigned long start, end; 943 bool fpsimd; 944 945 fpsimd_flush_task_state(target); 946 947 /* Header */ 948 if (count < sizeof(header)) 949 return -EINVAL; 950 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header, 951 0, sizeof(header)); 952 if (ret) 953 return ret; 954 955 /* 956 * Streaming SVE data is always stored and presented in SVE format. 957 * Require the user to provide SVE formatted data for consistency, and 958 * to avoid the risk that we configure the task into an invalid state. 959 */ 960 fpsimd = (header.flags & SVE_PT_REGS_MASK) == SVE_PT_REGS_FPSIMD; 961 if (fpsimd && type == ARM64_VEC_SME) 962 return -EINVAL; 963 964 /* 965 * On systems without SVE we accept FPSIMD format writes with 966 * a VL of 0 to allow exiting streaming mode, otherwise a VL 967 * is required. 968 */ 969 if (header.vl) { 970 /* 971 * If the system does not support SVE we can't 972 * configure a SVE VL. 973 */ 974 if (!system_supports_sve() && type == ARM64_VEC_SVE) 975 return -EINVAL; 976 977 /* 978 * Apart from SVE_PT_REGS_MASK, all SVE_PT_* flags are 979 * consumed by vec_set_vector_length(), which will 980 * also validate them for us: 981 */ 982 ret = vec_set_vector_length(target, type, header.vl, 983 ((unsigned long)header.flags & ~SVE_PT_REGS_MASK) << 16); 984 if (ret) 985 return ret; 986 } else { 987 /* If the system supports SVE we require a VL. */ 988 if (system_supports_sve()) 989 return -EINVAL; 990 991 /* 992 * Only FPSIMD formatted data with no flags set is 993 * supported. 994 */ 995 if (header.flags != SVE_PT_REGS_FPSIMD) 996 return -EINVAL; 997 } 998 999 /* Allocate SME storage if necessary, preserving any existing ZA/ZT state */ 1000 if (type == ARM64_VEC_SME) { 1001 sme_alloc(target, false); 1002 if (!target->thread.sme_state) 1003 return -ENOMEM; 1004 } 1005 1006 /* Allocate SVE storage if necessary, zeroing any existing SVE state */ 1007 if (!fpsimd) { 1008 sve_alloc(target, true); 1009 if (!target->thread.sve_state) 1010 return -ENOMEM; 1011 } 1012 1013 /* 1014 * Actual VL set may be different from what the user asked 1015 * for, or we may have configured the _ONEXEC VL not the 1016 * current VL: 1017 */ 1018 vq = sve_vq_from_vl(task_get_vl(target, type)); 1019 1020 /* Enter/exit streaming mode */ 1021 switch (type) { 1022 case ARM64_VEC_SVE: 1023 target->thread.svcr &= ~SVCR_SM_MASK; 1024 set_tsk_thread_flag(target, TIF_SVE); 1025 break; 1026 case ARM64_VEC_SME: 1027 target->thread.svcr |= SVCR_SM_MASK; 1028 set_tsk_thread_flag(target, TIF_SME); 1029 break; 1030 default: 1031 WARN_ON_ONCE(1); 1032 return -EINVAL; 1033 } 1034 1035 /* Always zero V regs, FPSR, and FPCR */ 1036 memset(&target->thread.uw.fpsimd_state, 0, 1037 sizeof(target->thread.uw.fpsimd_state)); 1038 1039 /* Registers: FPSIMD-only case */ 1040 1041 BUILD_BUG_ON(SVE_PT_FPSIMD_OFFSET != sizeof(header)); 1042 if (fpsimd) { 1043 clear_tsk_thread_flag(target, TIF_SVE); 1044 target->thread.fp_type = FP_STATE_FPSIMD; 1045 ret = __fpr_set(target, regset, pos, count, kbuf, ubuf, 1046 SVE_PT_FPSIMD_OFFSET); 1047 return ret; 1048 } 1049 1050 /* Otherwise: no registers or full SVE case. */ 1051 1052 target->thread.fp_type = FP_STATE_SVE; 1053 1054 /* 1055 * If setting a different VL from the requested VL and there is 1056 * register data, the data layout will be wrong: don't even 1057 * try to set the registers in this case. 1058 */ 1059 if (count && vq != sve_vq_from_vl(header.vl)) 1060 return -EIO; 1061 1062 BUILD_BUG_ON(SVE_PT_SVE_OFFSET != sizeof(header)); 1063 start = SVE_PT_SVE_OFFSET; 1064 end = SVE_PT_SVE_FFR_OFFSET(vq) + SVE_PT_SVE_FFR_SIZE(vq); 1065 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, 1066 target->thread.sve_state, 1067 start, end); 1068 if (ret) 1069 return ret; 1070 1071 start = end; 1072 end = SVE_PT_SVE_FPSR_OFFSET(vq); 1073 user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, start, end); 1074 1075 /* 1076 * Copy fpsr, and fpcr which must follow contiguously in 1077 * struct fpsimd_state: 1078 */ 1079 start = end; 1080 end = SVE_PT_SVE_FPCR_OFFSET(vq) + SVE_PT_SVE_FPCR_SIZE; 1081 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, 1082 &target->thread.uw.fpsimd_state.fpsr, 1083 start, end); 1084 1085 return ret; 1086 } 1087 1088 static int sve_set(struct task_struct *target, 1089 const struct user_regset *regset, 1090 unsigned int pos, unsigned int count, 1091 const void *kbuf, const void __user *ubuf) 1092 { 1093 if (!system_supports_sve() && !system_supports_sme()) 1094 return -EINVAL; 1095 1096 return sve_set_common(target, regset, pos, count, kbuf, ubuf, 1097 ARM64_VEC_SVE); 1098 } 1099 1100 #endif /* CONFIG_ARM64_SVE */ 1101 1102 #ifdef CONFIG_ARM64_SME 1103 1104 static int ssve_get(struct task_struct *target, 1105 const struct user_regset *regset, 1106 struct membuf to) 1107 { 1108 if (!system_supports_sme()) 1109 return -EINVAL; 1110 1111 return sve_get_common(target, regset, to, ARM64_VEC_SME); 1112 } 1113 1114 static int ssve_set(struct task_struct *target, 1115 const struct user_regset *regset, 1116 unsigned int pos, unsigned int count, 1117 const void *kbuf, const void __user *ubuf) 1118 { 1119 if (!system_supports_sme()) 1120 return -EINVAL; 1121 1122 return sve_set_common(target, regset, pos, count, kbuf, ubuf, 1123 ARM64_VEC_SME); 1124 } 1125 1126 static int za_get(struct task_struct *target, 1127 const struct user_regset *regset, 1128 struct membuf to) 1129 { 1130 struct user_za_header header; 1131 unsigned int vq; 1132 unsigned long start, end; 1133 1134 if (!system_supports_sme()) 1135 return -EINVAL; 1136 1137 /* Header */ 1138 memset(&header, 0, sizeof(header)); 1139 1140 if (test_tsk_thread_flag(target, TIF_SME_VL_INHERIT)) 1141 header.flags |= ZA_PT_VL_INHERIT; 1142 1143 header.vl = task_get_sme_vl(target); 1144 vq = sve_vq_from_vl(header.vl); 1145 header.max_vl = sme_max_vl(); 1146 header.max_size = ZA_PT_SIZE(vq); 1147 1148 /* If ZA is not active there is only the header */ 1149 if (thread_za_enabled(&target->thread)) 1150 header.size = ZA_PT_SIZE(vq); 1151 else 1152 header.size = ZA_PT_ZA_OFFSET; 1153 1154 membuf_write(&to, &header, sizeof(header)); 1155 1156 BUILD_BUG_ON(ZA_PT_ZA_OFFSET != sizeof(header)); 1157 end = ZA_PT_ZA_OFFSET; 1158 1159 if (target == current) 1160 fpsimd_preserve_current_state(); 1161 1162 /* Any register data to include? */ 1163 if (thread_za_enabled(&target->thread)) { 1164 start = end; 1165 end = ZA_PT_SIZE(vq); 1166 membuf_write(&to, target->thread.sme_state, end - start); 1167 } 1168 1169 /* Zero any trailing padding */ 1170 start = end; 1171 end = ALIGN(header.size, SVE_VQ_BYTES); 1172 return membuf_zero(&to, end - start); 1173 } 1174 1175 static int za_set(struct task_struct *target, 1176 const struct user_regset *regset, 1177 unsigned int pos, unsigned int count, 1178 const void *kbuf, const void __user *ubuf) 1179 { 1180 int ret; 1181 struct user_za_header header; 1182 unsigned int vq; 1183 unsigned long start, end; 1184 1185 if (!system_supports_sme()) 1186 return -EINVAL; 1187 1188 /* Header */ 1189 if (count < sizeof(header)) 1190 return -EINVAL; 1191 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &header, 1192 0, sizeof(header)); 1193 if (ret) 1194 goto out; 1195 1196 /* 1197 * All current ZA_PT_* flags are consumed by 1198 * vec_set_vector_length(), which will also validate them for 1199 * us: 1200 */ 1201 ret = vec_set_vector_length(target, ARM64_VEC_SME, header.vl, 1202 ((unsigned long)header.flags) << 16); 1203 if (ret) 1204 goto out; 1205 1206 /* 1207 * Actual VL set may be different from what the user asked 1208 * for, or we may have configured the _ONEXEC rather than 1209 * current VL: 1210 */ 1211 vq = sve_vq_from_vl(task_get_sme_vl(target)); 1212 1213 /* Ensure there is some SVE storage for streaming mode */ 1214 if (!target->thread.sve_state) { 1215 sve_alloc(target, false); 1216 if (!target->thread.sve_state) { 1217 ret = -ENOMEM; 1218 goto out; 1219 } 1220 } 1221 1222 /* 1223 * Only flush the storage if PSTATE.ZA was not already set, 1224 * otherwise preserve any existing data. 1225 */ 1226 sme_alloc(target, !thread_za_enabled(&target->thread)); 1227 if (!target->thread.sme_state) 1228 return -ENOMEM; 1229 1230 /* If there is no data then disable ZA */ 1231 if (!count) { 1232 target->thread.svcr &= ~SVCR_ZA_MASK; 1233 goto out; 1234 } 1235 1236 /* 1237 * If setting a different VL from the requested VL and there is 1238 * register data, the data layout will be wrong: don't even 1239 * try to set the registers in this case. 1240 */ 1241 if (vq != sve_vq_from_vl(header.vl)) { 1242 ret = -EIO; 1243 goto out; 1244 } 1245 1246 BUILD_BUG_ON(ZA_PT_ZA_OFFSET != sizeof(header)); 1247 start = ZA_PT_ZA_OFFSET; 1248 end = ZA_PT_SIZE(vq); 1249 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, 1250 target->thread.sme_state, 1251 start, end); 1252 if (ret) 1253 goto out; 1254 1255 /* Mark ZA as active and let userspace use it */ 1256 set_tsk_thread_flag(target, TIF_SME); 1257 target->thread.svcr |= SVCR_ZA_MASK; 1258 1259 out: 1260 fpsimd_flush_task_state(target); 1261 return ret; 1262 } 1263 1264 static int zt_get(struct task_struct *target, 1265 const struct user_regset *regset, 1266 struct membuf to) 1267 { 1268 if (!system_supports_sme2()) 1269 return -EINVAL; 1270 1271 /* 1272 * If PSTATE.ZA is not set then ZT will be zeroed when it is 1273 * enabled so report the current register value as zero. 1274 */ 1275 if (thread_za_enabled(&target->thread)) 1276 membuf_write(&to, thread_zt_state(&target->thread), 1277 ZT_SIG_REG_BYTES); 1278 else 1279 membuf_zero(&to, ZT_SIG_REG_BYTES); 1280 1281 return 0; 1282 } 1283 1284 static int zt_set(struct task_struct *target, 1285 const struct user_regset *regset, 1286 unsigned int pos, unsigned int count, 1287 const void *kbuf, const void __user *ubuf) 1288 { 1289 int ret; 1290 1291 if (!system_supports_sme2()) 1292 return -EINVAL; 1293 1294 /* Ensure SVE storage in case this is first use of SME */ 1295 sve_alloc(target, false); 1296 if (!target->thread.sve_state) 1297 return -ENOMEM; 1298 1299 if (!thread_za_enabled(&target->thread)) { 1300 sme_alloc(target, true); 1301 if (!target->thread.sme_state) 1302 return -ENOMEM; 1303 } 1304 1305 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, 1306 thread_zt_state(&target->thread), 1307 0, ZT_SIG_REG_BYTES); 1308 if (ret == 0) { 1309 target->thread.svcr |= SVCR_ZA_MASK; 1310 set_tsk_thread_flag(target, TIF_SME); 1311 } 1312 1313 fpsimd_flush_task_state(target); 1314 1315 return ret; 1316 } 1317 1318 #endif /* CONFIG_ARM64_SME */ 1319 1320 #ifdef CONFIG_ARM64_PTR_AUTH 1321 static int pac_mask_get(struct task_struct *target, 1322 const struct user_regset *regset, 1323 struct membuf to) 1324 { 1325 /* 1326 * The PAC bits can differ across data and instruction pointers 1327 * depending on TCR_EL1.TBID*, which we may make use of in future, so 1328 * we expose separate masks. 1329 */ 1330 unsigned long mask = ptrauth_user_pac_mask(); 1331 struct user_pac_mask uregs = { 1332 .data_mask = mask, 1333 .insn_mask = mask, 1334 }; 1335 1336 if (!system_supports_address_auth()) 1337 return -EINVAL; 1338 1339 return membuf_write(&to, &uregs, sizeof(uregs)); 1340 } 1341 1342 static int pac_enabled_keys_get(struct task_struct *target, 1343 const struct user_regset *regset, 1344 struct membuf to) 1345 { 1346 long enabled_keys = ptrauth_get_enabled_keys(target); 1347 1348 if (IS_ERR_VALUE(enabled_keys)) 1349 return enabled_keys; 1350 1351 return membuf_write(&to, &enabled_keys, sizeof(enabled_keys)); 1352 } 1353 1354 static int pac_enabled_keys_set(struct task_struct *target, 1355 const struct user_regset *regset, 1356 unsigned int pos, unsigned int count, 1357 const void *kbuf, const void __user *ubuf) 1358 { 1359 int ret; 1360 long enabled_keys = ptrauth_get_enabled_keys(target); 1361 1362 if (IS_ERR_VALUE(enabled_keys)) 1363 return enabled_keys; 1364 1365 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &enabled_keys, 0, 1366 sizeof(long)); 1367 if (ret) 1368 return ret; 1369 1370 return ptrauth_set_enabled_keys(target, PR_PAC_ENABLED_KEYS_MASK, 1371 enabled_keys); 1372 } 1373 1374 #ifdef CONFIG_CHECKPOINT_RESTORE 1375 static __uint128_t pac_key_to_user(const struct ptrauth_key *key) 1376 { 1377 return (__uint128_t)key->hi << 64 | key->lo; 1378 } 1379 1380 static struct ptrauth_key pac_key_from_user(__uint128_t ukey) 1381 { 1382 struct ptrauth_key key = { 1383 .lo = (unsigned long)ukey, 1384 .hi = (unsigned long)(ukey >> 64), 1385 }; 1386 1387 return key; 1388 } 1389 1390 static void pac_address_keys_to_user(struct user_pac_address_keys *ukeys, 1391 const struct ptrauth_keys_user *keys) 1392 { 1393 ukeys->apiakey = pac_key_to_user(&keys->apia); 1394 ukeys->apibkey = pac_key_to_user(&keys->apib); 1395 ukeys->apdakey = pac_key_to_user(&keys->apda); 1396 ukeys->apdbkey = pac_key_to_user(&keys->apdb); 1397 } 1398 1399 static void pac_address_keys_from_user(struct ptrauth_keys_user *keys, 1400 const struct user_pac_address_keys *ukeys) 1401 { 1402 keys->apia = pac_key_from_user(ukeys->apiakey); 1403 keys->apib = pac_key_from_user(ukeys->apibkey); 1404 keys->apda = pac_key_from_user(ukeys->apdakey); 1405 keys->apdb = pac_key_from_user(ukeys->apdbkey); 1406 } 1407 1408 static int pac_address_keys_get(struct task_struct *target, 1409 const struct user_regset *regset, 1410 struct membuf to) 1411 { 1412 struct ptrauth_keys_user *keys = &target->thread.keys_user; 1413 struct user_pac_address_keys user_keys; 1414 1415 if (!system_supports_address_auth()) 1416 return -EINVAL; 1417 1418 pac_address_keys_to_user(&user_keys, keys); 1419 1420 return membuf_write(&to, &user_keys, sizeof(user_keys)); 1421 } 1422 1423 static int pac_address_keys_set(struct task_struct *target, 1424 const struct user_regset *regset, 1425 unsigned int pos, unsigned int count, 1426 const void *kbuf, const void __user *ubuf) 1427 { 1428 struct ptrauth_keys_user *keys = &target->thread.keys_user; 1429 struct user_pac_address_keys user_keys; 1430 int ret; 1431 1432 if (!system_supports_address_auth()) 1433 return -EINVAL; 1434 1435 pac_address_keys_to_user(&user_keys, keys); 1436 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, 1437 &user_keys, 0, -1); 1438 if (ret) 1439 return ret; 1440 pac_address_keys_from_user(keys, &user_keys); 1441 1442 return 0; 1443 } 1444 1445 static void pac_generic_keys_to_user(struct user_pac_generic_keys *ukeys, 1446 const struct ptrauth_keys_user *keys) 1447 { 1448 ukeys->apgakey = pac_key_to_user(&keys->apga); 1449 } 1450 1451 static void pac_generic_keys_from_user(struct ptrauth_keys_user *keys, 1452 const struct user_pac_generic_keys *ukeys) 1453 { 1454 keys->apga = pac_key_from_user(ukeys->apgakey); 1455 } 1456 1457 static int pac_generic_keys_get(struct task_struct *target, 1458 const struct user_regset *regset, 1459 struct membuf to) 1460 { 1461 struct ptrauth_keys_user *keys = &target->thread.keys_user; 1462 struct user_pac_generic_keys user_keys; 1463 1464 if (!system_supports_generic_auth()) 1465 return -EINVAL; 1466 1467 pac_generic_keys_to_user(&user_keys, keys); 1468 1469 return membuf_write(&to, &user_keys, sizeof(user_keys)); 1470 } 1471 1472 static int pac_generic_keys_set(struct task_struct *target, 1473 const struct user_regset *regset, 1474 unsigned int pos, unsigned int count, 1475 const void *kbuf, const void __user *ubuf) 1476 { 1477 struct ptrauth_keys_user *keys = &target->thread.keys_user; 1478 struct user_pac_generic_keys user_keys; 1479 int ret; 1480 1481 if (!system_supports_generic_auth()) 1482 return -EINVAL; 1483 1484 pac_generic_keys_to_user(&user_keys, keys); 1485 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, 1486 &user_keys, 0, -1); 1487 if (ret) 1488 return ret; 1489 pac_generic_keys_from_user(keys, &user_keys); 1490 1491 return 0; 1492 } 1493 #endif /* CONFIG_CHECKPOINT_RESTORE */ 1494 #endif /* CONFIG_ARM64_PTR_AUTH */ 1495 1496 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI 1497 static int tagged_addr_ctrl_get(struct task_struct *target, 1498 const struct user_regset *regset, 1499 struct membuf to) 1500 { 1501 long ctrl = get_tagged_addr_ctrl(target); 1502 1503 if (WARN_ON_ONCE(IS_ERR_VALUE(ctrl))) 1504 return ctrl; 1505 1506 return membuf_write(&to, &ctrl, sizeof(ctrl)); 1507 } 1508 1509 static int tagged_addr_ctrl_set(struct task_struct *target, const struct 1510 user_regset *regset, unsigned int pos, 1511 unsigned int count, const void *kbuf, const 1512 void __user *ubuf) 1513 { 1514 int ret; 1515 long ctrl; 1516 1517 ctrl = get_tagged_addr_ctrl(target); 1518 if (WARN_ON_ONCE(IS_ERR_VALUE(ctrl))) 1519 return ctrl; 1520 1521 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, 0, -1); 1522 if (ret) 1523 return ret; 1524 1525 return set_tagged_addr_ctrl(target, ctrl); 1526 } 1527 #endif 1528 1529 #ifdef CONFIG_ARM64_POE 1530 static int poe_get(struct task_struct *target, 1531 const struct user_regset *regset, 1532 struct membuf to) 1533 { 1534 if (!system_supports_poe()) 1535 return -EINVAL; 1536 1537 if (target == current) 1538 current->thread.por_el0 = read_sysreg_s(SYS_POR_EL0); 1539 1540 return membuf_write(&to, &target->thread.por_el0, 1541 sizeof(target->thread.por_el0)); 1542 } 1543 1544 static int poe_set(struct task_struct *target, const struct 1545 user_regset *regset, unsigned int pos, 1546 unsigned int count, const void *kbuf, const 1547 void __user *ubuf) 1548 { 1549 int ret; 1550 long ctrl; 1551 1552 if (!system_supports_poe()) 1553 return -EINVAL; 1554 1555 ctrl = target->thread.por_el0; 1556 1557 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, 0, -1); 1558 if (ret) 1559 return ret; 1560 1561 target->thread.por_el0 = ctrl; 1562 1563 return 0; 1564 } 1565 #endif 1566 1567 #ifdef CONFIG_ARM64_GCS 1568 static void task_gcs_to_user(struct user_gcs *user_gcs, 1569 const struct task_struct *target) 1570 { 1571 user_gcs->features_enabled = target->thread.gcs_el0_mode; 1572 user_gcs->features_locked = target->thread.gcs_el0_locked; 1573 user_gcs->gcspr_el0 = target->thread.gcspr_el0; 1574 } 1575 1576 static void task_gcs_from_user(struct task_struct *target, 1577 const struct user_gcs *user_gcs) 1578 { 1579 target->thread.gcs_el0_mode = user_gcs->features_enabled; 1580 target->thread.gcs_el0_locked = user_gcs->features_locked; 1581 target->thread.gcspr_el0 = user_gcs->gcspr_el0; 1582 } 1583 1584 static int gcs_get(struct task_struct *target, 1585 const struct user_regset *regset, 1586 struct membuf to) 1587 { 1588 struct user_gcs user_gcs; 1589 1590 if (!system_supports_gcs()) 1591 return -EINVAL; 1592 1593 if (target == current) 1594 gcs_preserve_current_state(); 1595 1596 task_gcs_to_user(&user_gcs, target); 1597 1598 return membuf_write(&to, &user_gcs, sizeof(user_gcs)); 1599 } 1600 1601 static int gcs_set(struct task_struct *target, const struct 1602 user_regset *regset, unsigned int pos, 1603 unsigned int count, const void *kbuf, const 1604 void __user *ubuf) 1605 { 1606 int ret; 1607 struct user_gcs user_gcs; 1608 1609 if (!system_supports_gcs()) 1610 return -EINVAL; 1611 1612 task_gcs_to_user(&user_gcs, target); 1613 1614 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &user_gcs, 0, -1); 1615 if (ret) 1616 return ret; 1617 1618 if (user_gcs.features_enabled & ~PR_SHADOW_STACK_SUPPORTED_STATUS_MASK) 1619 return -EINVAL; 1620 1621 task_gcs_from_user(target, &user_gcs); 1622 1623 return 0; 1624 } 1625 #endif 1626 1627 enum aarch64_regset { 1628 REGSET_GPR, 1629 REGSET_FPR, 1630 REGSET_TLS, 1631 #ifdef CONFIG_HAVE_HW_BREAKPOINT 1632 REGSET_HW_BREAK, 1633 REGSET_HW_WATCH, 1634 #endif 1635 REGSET_FPMR, 1636 REGSET_SYSTEM_CALL, 1637 #ifdef CONFIG_ARM64_SVE 1638 REGSET_SVE, 1639 #endif 1640 #ifdef CONFIG_ARM64_SME 1641 REGSET_SSVE, 1642 REGSET_ZA, 1643 REGSET_ZT, 1644 #endif 1645 #ifdef CONFIG_ARM64_PTR_AUTH 1646 REGSET_PAC_MASK, 1647 REGSET_PAC_ENABLED_KEYS, 1648 #ifdef CONFIG_CHECKPOINT_RESTORE 1649 REGSET_PACA_KEYS, 1650 REGSET_PACG_KEYS, 1651 #endif 1652 #endif 1653 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI 1654 REGSET_TAGGED_ADDR_CTRL, 1655 #endif 1656 #ifdef CONFIG_ARM64_POE 1657 REGSET_POE, 1658 #endif 1659 #ifdef CONFIG_ARM64_GCS 1660 REGSET_GCS, 1661 #endif 1662 }; 1663 1664 static const struct user_regset aarch64_regsets[] = { 1665 [REGSET_GPR] = { 1666 USER_REGSET_NOTE_TYPE(PRSTATUS), 1667 .n = sizeof(struct user_pt_regs) / sizeof(u64), 1668 .size = sizeof(u64), 1669 .align = sizeof(u64), 1670 .regset_get = gpr_get, 1671 .set = gpr_set 1672 }, 1673 [REGSET_FPR] = { 1674 USER_REGSET_NOTE_TYPE(PRFPREG), 1675 .n = sizeof(struct user_fpsimd_state) / sizeof(u32), 1676 /* 1677 * We pretend we have 32-bit registers because the fpsr and 1678 * fpcr are 32-bits wide. 1679 */ 1680 .size = sizeof(u32), 1681 .align = sizeof(u32), 1682 .active = fpr_active, 1683 .regset_get = fpr_get, 1684 .set = fpr_set 1685 }, 1686 [REGSET_TLS] = { 1687 USER_REGSET_NOTE_TYPE(ARM_TLS), 1688 .n = 2, 1689 .size = sizeof(void *), 1690 .align = sizeof(void *), 1691 .regset_get = tls_get, 1692 .set = tls_set, 1693 }, 1694 #ifdef CONFIG_HAVE_HW_BREAKPOINT 1695 [REGSET_HW_BREAK] = { 1696 USER_REGSET_NOTE_TYPE(ARM_HW_BREAK), 1697 .n = sizeof(struct user_hwdebug_state) / sizeof(u32), 1698 .size = sizeof(u32), 1699 .align = sizeof(u32), 1700 .regset_get = hw_break_get, 1701 .set = hw_break_set, 1702 }, 1703 [REGSET_HW_WATCH] = { 1704 USER_REGSET_NOTE_TYPE(ARM_HW_WATCH), 1705 .n = sizeof(struct user_hwdebug_state) / sizeof(u32), 1706 .size = sizeof(u32), 1707 .align = sizeof(u32), 1708 .regset_get = hw_break_get, 1709 .set = hw_break_set, 1710 }, 1711 #endif 1712 [REGSET_SYSTEM_CALL] = { 1713 USER_REGSET_NOTE_TYPE(ARM_SYSTEM_CALL), 1714 .n = 1, 1715 .size = sizeof(int), 1716 .align = sizeof(int), 1717 .regset_get = system_call_get, 1718 .set = system_call_set, 1719 }, 1720 [REGSET_FPMR] = { 1721 USER_REGSET_NOTE_TYPE(ARM_FPMR), 1722 .n = 1, 1723 .size = sizeof(u64), 1724 .align = sizeof(u64), 1725 .regset_get = fpmr_get, 1726 .set = fpmr_set, 1727 }, 1728 #ifdef CONFIG_ARM64_SVE 1729 [REGSET_SVE] = { /* Scalable Vector Extension */ 1730 USER_REGSET_NOTE_TYPE(ARM_SVE), 1731 .n = DIV_ROUND_UP(SVE_PT_SIZE(ARCH_SVE_VQ_MAX, 1732 SVE_PT_REGS_SVE), 1733 SVE_VQ_BYTES), 1734 .size = SVE_VQ_BYTES, 1735 .align = SVE_VQ_BYTES, 1736 .regset_get = sve_get, 1737 .set = sve_set, 1738 }, 1739 #endif 1740 #ifdef CONFIG_ARM64_SME 1741 [REGSET_SSVE] = { /* Streaming mode SVE */ 1742 USER_REGSET_NOTE_TYPE(ARM_SSVE), 1743 .n = DIV_ROUND_UP(SVE_PT_SIZE(SME_VQ_MAX, SVE_PT_REGS_SVE), 1744 SVE_VQ_BYTES), 1745 .size = SVE_VQ_BYTES, 1746 .align = SVE_VQ_BYTES, 1747 .regset_get = ssve_get, 1748 .set = ssve_set, 1749 }, 1750 [REGSET_ZA] = { /* SME ZA */ 1751 USER_REGSET_NOTE_TYPE(ARM_ZA), 1752 /* 1753 * ZA is a single register but it's variably sized and 1754 * the ptrace core requires that the size of any data 1755 * be an exact multiple of the configured register 1756 * size so report as though we had SVE_VQ_BYTES 1757 * registers. These values aren't exposed to 1758 * userspace. 1759 */ 1760 .n = DIV_ROUND_UP(ZA_PT_SIZE(SME_VQ_MAX), SVE_VQ_BYTES), 1761 .size = SVE_VQ_BYTES, 1762 .align = SVE_VQ_BYTES, 1763 .regset_get = za_get, 1764 .set = za_set, 1765 }, 1766 [REGSET_ZT] = { /* SME ZT */ 1767 USER_REGSET_NOTE_TYPE(ARM_ZT), 1768 .n = 1, 1769 .size = ZT_SIG_REG_BYTES, 1770 .align = sizeof(u64), 1771 .regset_get = zt_get, 1772 .set = zt_set, 1773 }, 1774 #endif 1775 #ifdef CONFIG_ARM64_PTR_AUTH 1776 [REGSET_PAC_MASK] = { 1777 USER_REGSET_NOTE_TYPE(ARM_PAC_MASK), 1778 .n = sizeof(struct user_pac_mask) / sizeof(u64), 1779 .size = sizeof(u64), 1780 .align = sizeof(u64), 1781 .regset_get = pac_mask_get, 1782 /* this cannot be set dynamically */ 1783 }, 1784 [REGSET_PAC_ENABLED_KEYS] = { 1785 USER_REGSET_NOTE_TYPE(ARM_PAC_ENABLED_KEYS), 1786 .n = 1, 1787 .size = sizeof(long), 1788 .align = sizeof(long), 1789 .regset_get = pac_enabled_keys_get, 1790 .set = pac_enabled_keys_set, 1791 }, 1792 #ifdef CONFIG_CHECKPOINT_RESTORE 1793 [REGSET_PACA_KEYS] = { 1794 USER_REGSET_NOTE_TYPE(ARM_PACA_KEYS), 1795 .n = sizeof(struct user_pac_address_keys) / sizeof(__uint128_t), 1796 .size = sizeof(__uint128_t), 1797 .align = sizeof(__uint128_t), 1798 .regset_get = pac_address_keys_get, 1799 .set = pac_address_keys_set, 1800 }, 1801 [REGSET_PACG_KEYS] = { 1802 USER_REGSET_NOTE_TYPE(ARM_PACG_KEYS), 1803 .n = sizeof(struct user_pac_generic_keys) / sizeof(__uint128_t), 1804 .size = sizeof(__uint128_t), 1805 .align = sizeof(__uint128_t), 1806 .regset_get = pac_generic_keys_get, 1807 .set = pac_generic_keys_set, 1808 }, 1809 #endif 1810 #endif 1811 #ifdef CONFIG_ARM64_TAGGED_ADDR_ABI 1812 [REGSET_TAGGED_ADDR_CTRL] = { 1813 USER_REGSET_NOTE_TYPE(ARM_TAGGED_ADDR_CTRL), 1814 .n = 1, 1815 .size = sizeof(long), 1816 .align = sizeof(long), 1817 .regset_get = tagged_addr_ctrl_get, 1818 .set = tagged_addr_ctrl_set, 1819 }, 1820 #endif 1821 #ifdef CONFIG_ARM64_POE 1822 [REGSET_POE] = { 1823 USER_REGSET_NOTE_TYPE(ARM_POE), 1824 .n = 1, 1825 .size = sizeof(long), 1826 .align = sizeof(long), 1827 .regset_get = poe_get, 1828 .set = poe_set, 1829 }, 1830 #endif 1831 #ifdef CONFIG_ARM64_GCS 1832 [REGSET_GCS] = { 1833 USER_REGSET_NOTE_TYPE(ARM_GCS), 1834 .n = sizeof(struct user_gcs) / sizeof(u64), 1835 .size = sizeof(u64), 1836 .align = sizeof(u64), 1837 .regset_get = gcs_get, 1838 .set = gcs_set, 1839 }, 1840 #endif 1841 }; 1842 1843 static const struct user_regset_view user_aarch64_view = { 1844 .name = "aarch64", .e_machine = EM_AARCH64, 1845 .regsets = aarch64_regsets, .n = ARRAY_SIZE(aarch64_regsets) 1846 }; 1847 1848 enum compat_regset { 1849 REGSET_COMPAT_GPR, 1850 REGSET_COMPAT_VFP, 1851 }; 1852 1853 static inline compat_ulong_t compat_get_user_reg(struct task_struct *task, int idx) 1854 { 1855 struct pt_regs *regs = task_pt_regs(task); 1856 1857 switch (idx) { 1858 case 15: 1859 return regs->pc; 1860 case 16: 1861 return pstate_to_compat_psr(regs->pstate); 1862 case 17: 1863 return regs->orig_x0; 1864 default: 1865 return regs->regs[idx]; 1866 } 1867 } 1868 1869 static int compat_gpr_get(struct task_struct *target, 1870 const struct user_regset *regset, 1871 struct membuf to) 1872 { 1873 int i = 0; 1874 1875 while (to.left) 1876 membuf_store(&to, compat_get_user_reg(target, i++)); 1877 return 0; 1878 } 1879 1880 static int compat_gpr_set(struct task_struct *target, 1881 const struct user_regset *regset, 1882 unsigned int pos, unsigned int count, 1883 const void *kbuf, const void __user *ubuf) 1884 { 1885 struct pt_regs newregs; 1886 int ret = 0; 1887 unsigned int i, start, num_regs; 1888 1889 /* Calculate the number of AArch32 registers contained in count */ 1890 num_regs = count / regset->size; 1891 1892 /* Convert pos into an register number */ 1893 start = pos / regset->size; 1894 1895 if (start + num_regs > regset->n) 1896 return -EIO; 1897 1898 newregs = *task_pt_regs(target); 1899 1900 for (i = 0; i < num_regs; ++i) { 1901 unsigned int idx = start + i; 1902 compat_ulong_t reg; 1903 1904 if (kbuf) { 1905 memcpy(®, kbuf, sizeof(reg)); 1906 kbuf += sizeof(reg); 1907 } else { 1908 ret = copy_from_user(®, ubuf, sizeof(reg)); 1909 if (ret) { 1910 ret = -EFAULT; 1911 break; 1912 } 1913 1914 ubuf += sizeof(reg); 1915 } 1916 1917 switch (idx) { 1918 case 15: 1919 newregs.pc = reg; 1920 break; 1921 case 16: 1922 reg = compat_psr_to_pstate(reg); 1923 newregs.pstate = reg; 1924 break; 1925 case 17: 1926 newregs.orig_x0 = reg; 1927 break; 1928 default: 1929 newregs.regs[idx] = reg; 1930 } 1931 1932 } 1933 1934 if (valid_user_regs(&newregs.user_regs, target)) 1935 *task_pt_regs(target) = newregs; 1936 else 1937 ret = -EINVAL; 1938 1939 return ret; 1940 } 1941 1942 static int compat_vfp_get(struct task_struct *target, 1943 const struct user_regset *regset, 1944 struct membuf to) 1945 { 1946 struct user_fpsimd_state *uregs; 1947 compat_ulong_t fpscr; 1948 1949 if (!system_supports_fpsimd()) 1950 return -EINVAL; 1951 1952 uregs = &target->thread.uw.fpsimd_state; 1953 1954 if (target == current) 1955 fpsimd_preserve_current_state(); 1956 1957 /* 1958 * The VFP registers are packed into the fpsimd_state, so they all sit 1959 * nicely together for us. We just need to create the fpscr separately. 1960 */ 1961 membuf_write(&to, uregs, VFP_STATE_SIZE - sizeof(compat_ulong_t)); 1962 fpscr = (uregs->fpsr & VFP_FPSCR_STAT_MASK) | 1963 (uregs->fpcr & VFP_FPSCR_CTRL_MASK); 1964 return membuf_store(&to, fpscr); 1965 } 1966 1967 static int compat_vfp_set(struct task_struct *target, 1968 const struct user_regset *regset, 1969 unsigned int pos, unsigned int count, 1970 const void *kbuf, const void __user *ubuf) 1971 { 1972 struct user_fpsimd_state *uregs; 1973 compat_ulong_t fpscr; 1974 int ret, vregs_end_pos; 1975 1976 if (!system_supports_fpsimd()) 1977 return -EINVAL; 1978 1979 uregs = &target->thread.uw.fpsimd_state; 1980 1981 vregs_end_pos = VFP_STATE_SIZE - sizeof(compat_ulong_t); 1982 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, uregs, 0, 1983 vregs_end_pos); 1984 1985 if (count && !ret) { 1986 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &fpscr, 1987 vregs_end_pos, VFP_STATE_SIZE); 1988 if (!ret) { 1989 uregs->fpsr = fpscr & VFP_FPSCR_STAT_MASK; 1990 uregs->fpcr = fpscr & VFP_FPSCR_CTRL_MASK; 1991 } 1992 } 1993 1994 fpsimd_flush_task_state(target); 1995 return ret; 1996 } 1997 1998 static int compat_tls_get(struct task_struct *target, 1999 const struct user_regset *regset, 2000 struct membuf to) 2001 { 2002 return membuf_store(&to, (compat_ulong_t)target->thread.uw.tp_value); 2003 } 2004 2005 static int compat_tls_set(struct task_struct *target, 2006 const struct user_regset *regset, unsigned int pos, 2007 unsigned int count, const void *kbuf, 2008 const void __user *ubuf) 2009 { 2010 int ret; 2011 compat_ulong_t tls = target->thread.uw.tp_value; 2012 2013 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1); 2014 if (ret) 2015 return ret; 2016 2017 target->thread.uw.tp_value = tls; 2018 return ret; 2019 } 2020 2021 static const struct user_regset aarch32_regsets[] = { 2022 [REGSET_COMPAT_GPR] = { 2023 USER_REGSET_NOTE_TYPE(PRSTATUS), 2024 .n = COMPAT_ELF_NGREG, 2025 .size = sizeof(compat_elf_greg_t), 2026 .align = sizeof(compat_elf_greg_t), 2027 .regset_get = compat_gpr_get, 2028 .set = compat_gpr_set 2029 }, 2030 [REGSET_COMPAT_VFP] = { 2031 USER_REGSET_NOTE_TYPE(ARM_VFP), 2032 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t), 2033 .size = sizeof(compat_ulong_t), 2034 .align = sizeof(compat_ulong_t), 2035 .active = fpr_active, 2036 .regset_get = compat_vfp_get, 2037 .set = compat_vfp_set 2038 }, 2039 }; 2040 2041 static const struct user_regset_view user_aarch32_view = { 2042 .name = "aarch32", .e_machine = EM_ARM, 2043 .regsets = aarch32_regsets, .n = ARRAY_SIZE(aarch32_regsets) 2044 }; 2045 2046 static const struct user_regset aarch32_ptrace_regsets[] = { 2047 [REGSET_GPR] = { 2048 USER_REGSET_NOTE_TYPE(PRSTATUS), 2049 .n = COMPAT_ELF_NGREG, 2050 .size = sizeof(compat_elf_greg_t), 2051 .align = sizeof(compat_elf_greg_t), 2052 .regset_get = compat_gpr_get, 2053 .set = compat_gpr_set 2054 }, 2055 [REGSET_FPR] = { 2056 USER_REGSET_NOTE_TYPE(ARM_VFP), 2057 .n = VFP_STATE_SIZE / sizeof(compat_ulong_t), 2058 .size = sizeof(compat_ulong_t), 2059 .align = sizeof(compat_ulong_t), 2060 .regset_get = compat_vfp_get, 2061 .set = compat_vfp_set 2062 }, 2063 [REGSET_TLS] = { 2064 USER_REGSET_NOTE_TYPE(ARM_TLS), 2065 .n = 1, 2066 .size = sizeof(compat_ulong_t), 2067 .align = sizeof(compat_ulong_t), 2068 .regset_get = compat_tls_get, 2069 .set = compat_tls_set, 2070 }, 2071 #ifdef CONFIG_HAVE_HW_BREAKPOINT 2072 [REGSET_HW_BREAK] = { 2073 USER_REGSET_NOTE_TYPE(ARM_HW_BREAK), 2074 .n = sizeof(struct user_hwdebug_state) / sizeof(u32), 2075 .size = sizeof(u32), 2076 .align = sizeof(u32), 2077 .regset_get = hw_break_get, 2078 .set = hw_break_set, 2079 }, 2080 [REGSET_HW_WATCH] = { 2081 USER_REGSET_NOTE_TYPE(ARM_HW_WATCH), 2082 .n = sizeof(struct user_hwdebug_state) / sizeof(u32), 2083 .size = sizeof(u32), 2084 .align = sizeof(u32), 2085 .regset_get = hw_break_get, 2086 .set = hw_break_set, 2087 }, 2088 #endif 2089 [REGSET_SYSTEM_CALL] = { 2090 USER_REGSET_NOTE_TYPE(ARM_SYSTEM_CALL), 2091 .n = 1, 2092 .size = sizeof(int), 2093 .align = sizeof(int), 2094 .regset_get = system_call_get, 2095 .set = system_call_set, 2096 }, 2097 }; 2098 2099 static const struct user_regset_view user_aarch32_ptrace_view = { 2100 .name = "aarch32", .e_machine = EM_ARM, 2101 .regsets = aarch32_ptrace_regsets, .n = ARRAY_SIZE(aarch32_ptrace_regsets) 2102 }; 2103 2104 #ifdef CONFIG_COMPAT 2105 static int compat_ptrace_read_user(struct task_struct *tsk, compat_ulong_t off, 2106 compat_ulong_t __user *ret) 2107 { 2108 compat_ulong_t tmp; 2109 2110 if (off & 3) 2111 return -EIO; 2112 2113 if (off == COMPAT_PT_TEXT_ADDR) 2114 tmp = tsk->mm->start_code; 2115 else if (off == COMPAT_PT_DATA_ADDR) 2116 tmp = tsk->mm->start_data; 2117 else if (off == COMPAT_PT_TEXT_END_ADDR) 2118 tmp = tsk->mm->end_code; 2119 else if (off < sizeof(compat_elf_gregset_t)) 2120 tmp = compat_get_user_reg(tsk, off >> 2); 2121 else if (off >= COMPAT_USER_SZ) 2122 return -EIO; 2123 else 2124 tmp = 0; 2125 2126 return put_user(tmp, ret); 2127 } 2128 2129 static int compat_ptrace_write_user(struct task_struct *tsk, compat_ulong_t off, 2130 compat_ulong_t val) 2131 { 2132 struct pt_regs newregs = *task_pt_regs(tsk); 2133 unsigned int idx = off / 4; 2134 2135 if (off & 3 || off >= COMPAT_USER_SZ) 2136 return -EIO; 2137 2138 if (off >= sizeof(compat_elf_gregset_t)) 2139 return 0; 2140 2141 switch (idx) { 2142 case 15: 2143 newregs.pc = val; 2144 break; 2145 case 16: 2146 newregs.pstate = compat_psr_to_pstate(val); 2147 break; 2148 case 17: 2149 newregs.orig_x0 = val; 2150 break; 2151 default: 2152 newregs.regs[idx] = val; 2153 } 2154 2155 if (!valid_user_regs(&newregs.user_regs, tsk)) 2156 return -EINVAL; 2157 2158 *task_pt_regs(tsk) = newregs; 2159 return 0; 2160 } 2161 2162 #ifdef CONFIG_HAVE_HW_BREAKPOINT 2163 2164 /* 2165 * Convert a virtual register number into an index for a thread_info 2166 * breakpoint array. Breakpoints are identified using positive numbers 2167 * whilst watchpoints are negative. The registers are laid out as pairs 2168 * of (address, control), each pair mapping to a unique hw_breakpoint struct. 2169 * Register 0 is reserved for describing resource information. 2170 */ 2171 static int compat_ptrace_hbp_num_to_idx(compat_long_t num) 2172 { 2173 return (abs(num) - 1) >> 1; 2174 } 2175 2176 static int compat_ptrace_hbp_get_resource_info(u32 *kdata) 2177 { 2178 u8 num_brps, num_wrps, debug_arch, wp_len; 2179 u32 reg = 0; 2180 2181 num_brps = hw_breakpoint_slots(TYPE_INST); 2182 num_wrps = hw_breakpoint_slots(TYPE_DATA); 2183 2184 debug_arch = debug_monitors_arch(); 2185 wp_len = 8; 2186 reg |= debug_arch; 2187 reg <<= 8; 2188 reg |= wp_len; 2189 reg <<= 8; 2190 reg |= num_wrps; 2191 reg <<= 8; 2192 reg |= num_brps; 2193 2194 *kdata = reg; 2195 return 0; 2196 } 2197 2198 static int compat_ptrace_hbp_get(unsigned int note_type, 2199 struct task_struct *tsk, 2200 compat_long_t num, 2201 u32 *kdata) 2202 { 2203 u64 addr = 0; 2204 u32 ctrl = 0; 2205 2206 int err, idx = compat_ptrace_hbp_num_to_idx(num); 2207 2208 if (num & 1) { 2209 err = ptrace_hbp_get_addr(note_type, tsk, idx, &addr); 2210 *kdata = (u32)addr; 2211 } else { 2212 err = ptrace_hbp_get_ctrl(note_type, tsk, idx, &ctrl); 2213 *kdata = ctrl; 2214 } 2215 2216 return err; 2217 } 2218 2219 static int compat_ptrace_hbp_set(unsigned int note_type, 2220 struct task_struct *tsk, 2221 compat_long_t num, 2222 u32 *kdata) 2223 { 2224 u64 addr; 2225 u32 ctrl; 2226 2227 int err, idx = compat_ptrace_hbp_num_to_idx(num); 2228 2229 if (num & 1) { 2230 addr = *kdata; 2231 err = ptrace_hbp_set_addr(note_type, tsk, idx, addr); 2232 } else { 2233 ctrl = *kdata; 2234 err = ptrace_hbp_set_ctrl(note_type, tsk, idx, ctrl); 2235 } 2236 2237 return err; 2238 } 2239 2240 static int compat_ptrace_gethbpregs(struct task_struct *tsk, compat_long_t num, 2241 compat_ulong_t __user *data) 2242 { 2243 int ret; 2244 u32 kdata; 2245 2246 /* Watchpoint */ 2247 if (num < 0) { 2248 ret = compat_ptrace_hbp_get(NT_ARM_HW_WATCH, tsk, num, &kdata); 2249 /* Resource info */ 2250 } else if (num == 0) { 2251 ret = compat_ptrace_hbp_get_resource_info(&kdata); 2252 /* Breakpoint */ 2253 } else { 2254 ret = compat_ptrace_hbp_get(NT_ARM_HW_BREAK, tsk, num, &kdata); 2255 } 2256 2257 if (!ret) 2258 ret = put_user(kdata, data); 2259 2260 return ret; 2261 } 2262 2263 static int compat_ptrace_sethbpregs(struct task_struct *tsk, compat_long_t num, 2264 compat_ulong_t __user *data) 2265 { 2266 int ret; 2267 u32 kdata = 0; 2268 2269 if (num == 0) 2270 return 0; 2271 2272 ret = get_user(kdata, data); 2273 if (ret) 2274 return ret; 2275 2276 if (num < 0) 2277 ret = compat_ptrace_hbp_set(NT_ARM_HW_WATCH, tsk, num, &kdata); 2278 else 2279 ret = compat_ptrace_hbp_set(NT_ARM_HW_BREAK, tsk, num, &kdata); 2280 2281 return ret; 2282 } 2283 #endif /* CONFIG_HAVE_HW_BREAKPOINT */ 2284 2285 long compat_arch_ptrace(struct task_struct *child, compat_long_t request, 2286 compat_ulong_t caddr, compat_ulong_t cdata) 2287 { 2288 unsigned long addr = caddr; 2289 unsigned long data = cdata; 2290 void __user *datap = compat_ptr(data); 2291 int ret; 2292 2293 switch (request) { 2294 case PTRACE_PEEKUSR: 2295 ret = compat_ptrace_read_user(child, addr, datap); 2296 break; 2297 2298 case PTRACE_POKEUSR: 2299 ret = compat_ptrace_write_user(child, addr, data); 2300 break; 2301 2302 case COMPAT_PTRACE_GETREGS: 2303 ret = copy_regset_to_user(child, 2304 &user_aarch32_view, 2305 REGSET_COMPAT_GPR, 2306 0, sizeof(compat_elf_gregset_t), 2307 datap); 2308 break; 2309 2310 case COMPAT_PTRACE_SETREGS: 2311 ret = copy_regset_from_user(child, 2312 &user_aarch32_view, 2313 REGSET_COMPAT_GPR, 2314 0, sizeof(compat_elf_gregset_t), 2315 datap); 2316 break; 2317 2318 case COMPAT_PTRACE_GET_THREAD_AREA: 2319 ret = put_user((compat_ulong_t)child->thread.uw.tp_value, 2320 (compat_ulong_t __user *)datap); 2321 break; 2322 2323 case COMPAT_PTRACE_SET_SYSCALL: 2324 task_pt_regs(child)->syscallno = data; 2325 ret = 0; 2326 break; 2327 2328 case COMPAT_PTRACE_GETVFPREGS: 2329 ret = copy_regset_to_user(child, 2330 &user_aarch32_view, 2331 REGSET_COMPAT_VFP, 2332 0, VFP_STATE_SIZE, 2333 datap); 2334 break; 2335 2336 case COMPAT_PTRACE_SETVFPREGS: 2337 ret = copy_regset_from_user(child, 2338 &user_aarch32_view, 2339 REGSET_COMPAT_VFP, 2340 0, VFP_STATE_SIZE, 2341 datap); 2342 break; 2343 2344 #ifdef CONFIG_HAVE_HW_BREAKPOINT 2345 case COMPAT_PTRACE_GETHBPREGS: 2346 ret = compat_ptrace_gethbpregs(child, addr, datap); 2347 break; 2348 2349 case COMPAT_PTRACE_SETHBPREGS: 2350 ret = compat_ptrace_sethbpregs(child, addr, datap); 2351 break; 2352 #endif 2353 2354 default: 2355 ret = compat_ptrace_request(child, request, addr, 2356 data); 2357 break; 2358 } 2359 2360 return ret; 2361 } 2362 #endif /* CONFIG_COMPAT */ 2363 2364 const struct user_regset_view *task_user_regset_view(struct task_struct *task) 2365 { 2366 /* 2367 * Core dumping of 32-bit tasks or compat ptrace requests must use the 2368 * user_aarch32_view compatible with arm32. Native ptrace requests on 2369 * 32-bit children use an extended user_aarch32_ptrace_view to allow 2370 * access to the TLS register. 2371 */ 2372 if (is_compat_task()) 2373 return &user_aarch32_view; 2374 else if (is_compat_thread(task_thread_info(task))) 2375 return &user_aarch32_ptrace_view; 2376 2377 return &user_aarch64_view; 2378 } 2379 2380 long arch_ptrace(struct task_struct *child, long request, 2381 unsigned long addr, unsigned long data) 2382 { 2383 switch (request) { 2384 case PTRACE_PEEKMTETAGS: 2385 case PTRACE_POKEMTETAGS: 2386 return mte_ptrace_copy_tags(child, request, addr, data); 2387 } 2388 2389 return ptrace_request(child, request, addr, data); 2390 } 2391 2392 enum ptrace_syscall_dir { 2393 PTRACE_SYSCALL_ENTER = 0, 2394 PTRACE_SYSCALL_EXIT, 2395 }; 2396 2397 static __always_inline unsigned long ptrace_save_reg(struct pt_regs *regs, 2398 enum ptrace_syscall_dir dir, 2399 int *regno) 2400 { 2401 unsigned long saved_reg; 2402 2403 /* 2404 * We have some ABI weirdness here in the way that we handle syscall 2405 * exit stops because we indicate whether or not the stop has been 2406 * signalled from syscall entry or syscall exit by clobbering a general 2407 * purpose register (ip/r12 for AArch32, x7 for AArch64) in the tracee 2408 * and restoring its old value after the stop. This means that: 2409 * 2410 * - Any writes by the tracer to this register during the stop are 2411 * ignored/discarded. 2412 * 2413 * - The actual value of the register is not available during the stop, 2414 * so the tracer cannot save it and restore it later. 2415 * 2416 * - Syscall stops behave differently to seccomp and pseudo-step traps 2417 * (the latter do not nobble any registers). 2418 */ 2419 *regno = (is_compat_task() ? 12 : 7); 2420 saved_reg = regs->regs[*regno]; 2421 regs->regs[*regno] = dir; 2422 2423 return saved_reg; 2424 } 2425 2426 static int report_syscall_entry(struct pt_regs *regs) 2427 { 2428 unsigned long saved_reg; 2429 int regno, ret; 2430 2431 saved_reg = ptrace_save_reg(regs, PTRACE_SYSCALL_ENTER, ®no); 2432 ret = !ptrace_report_syscall_permit_entry(regs); 2433 if (ret) 2434 forget_syscall(regs); 2435 regs->regs[regno] = saved_reg; 2436 2437 return ret; 2438 } 2439 2440 static void report_syscall_exit(struct pt_regs *regs) 2441 { 2442 unsigned long saved_reg; 2443 int regno; 2444 2445 saved_reg = ptrace_save_reg(regs, PTRACE_SYSCALL_EXIT, ®no); 2446 if (!test_thread_flag(TIF_SINGLESTEP)) { 2447 ptrace_report_syscall_exit(regs, 0); 2448 regs->regs[regno] = saved_reg; 2449 } else { 2450 regs->regs[regno] = saved_reg; 2451 2452 /* 2453 * Signal a pseudo-step exception since we are stepping but 2454 * tracer modifications to the registers may have rewound the 2455 * state machine. 2456 */ 2457 ptrace_report_syscall_exit(regs, 1); 2458 } 2459 } 2460 2461 int syscall_trace_enter(struct pt_regs *regs) 2462 { 2463 unsigned long flags = read_thread_flags(); 2464 int ret; 2465 2466 if (flags & (_TIF_SYSCALL_EMU | _TIF_SYSCALL_TRACE)) { 2467 ret = report_syscall_entry(regs); 2468 if (ret || (flags & _TIF_SYSCALL_EMU)) 2469 return NO_SYSCALL; 2470 } 2471 2472 /* Do the secure computing after ptrace; failures should be fast. */ 2473 if (!seccomp_permit_syscall()) 2474 return NO_SYSCALL; 2475 2476 if (test_thread_flag(TIF_SYSCALL_TRACEPOINT)) 2477 trace_sys_enter(regs, regs->syscallno); 2478 2479 audit_syscall_entry(regs->syscallno, regs->orig_x0, regs->regs[1], 2480 regs->regs[2], regs->regs[3]); 2481 2482 return regs->syscallno; 2483 } 2484 2485 void syscall_trace_exit(struct pt_regs *regs) 2486 { 2487 unsigned long flags = read_thread_flags(); 2488 2489 audit_syscall_exit(regs); 2490 2491 if (flags & _TIF_SYSCALL_TRACEPOINT) 2492 trace_sys_exit(regs, syscall_get_return_value(current, regs)); 2493 2494 if (flags & (_TIF_SYSCALL_TRACE | _TIF_SINGLESTEP)) 2495 report_syscall_exit(regs); 2496 2497 rseq_syscall(regs); 2498 } 2499 2500 /* 2501 * SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487D.a. 2502 * We permit userspace to set SSBS (AArch64 bit 12, AArch32 bit 23) which is 2503 * not described in ARM DDI 0487D.a. 2504 * We treat PAN and UAO as RES0 bits, as they are meaningless at EL0, and may 2505 * be allocated an EL0 meaning in future. 2506 * Userspace cannot use these until they have an architectural meaning. 2507 * Note that this follows the SPSR_ELx format, not the AArch32 PSR format. 2508 * We also reserve IL for the kernel; SS is handled dynamically. 2509 */ 2510 #define SPSR_EL1_AARCH64_RES0_BITS \ 2511 (GENMASK_ULL(63, 32) | GENMASK_ULL(27, 26) | GENMASK_ULL(23, 22) | \ 2512 GENMASK_ULL(20, 13) | GENMASK_ULL(5, 5)) 2513 #define SPSR_EL1_AARCH32_RES0_BITS \ 2514 (GENMASK_ULL(63, 32) | GENMASK_ULL(22, 22) | GENMASK_ULL(20, 20)) 2515 2516 static int valid_compat_regs(struct user_pt_regs *regs) 2517 { 2518 regs->pstate &= ~SPSR_EL1_AARCH32_RES0_BITS; 2519 2520 if (!system_supports_mixed_endian_el0()) { 2521 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) 2522 regs->pstate |= PSR_AA32_E_BIT; 2523 else 2524 regs->pstate &= ~PSR_AA32_E_BIT; 2525 } 2526 2527 if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) && 2528 (regs->pstate & PSR_AA32_A_BIT) == 0 && 2529 (regs->pstate & PSR_AA32_I_BIT) == 0 && 2530 (regs->pstate & PSR_AA32_F_BIT) == 0) { 2531 return 1; 2532 } 2533 2534 /* 2535 * Force PSR to a valid 32-bit EL0t, preserving the same bits as 2536 * arch/arm. 2537 */ 2538 regs->pstate &= PSR_AA32_N_BIT | PSR_AA32_Z_BIT | 2539 PSR_AA32_C_BIT | PSR_AA32_V_BIT | 2540 PSR_AA32_Q_BIT | PSR_AA32_IT_MASK | 2541 PSR_AA32_GE_MASK | PSR_AA32_E_BIT | 2542 PSR_AA32_T_BIT; 2543 regs->pstate |= PSR_MODE32_BIT; 2544 2545 return 0; 2546 } 2547 2548 static int valid_native_regs(struct user_pt_regs *regs) 2549 { 2550 regs->pstate &= ~SPSR_EL1_AARCH64_RES0_BITS; 2551 2552 if (user_mode(regs) && !(regs->pstate & PSR_MODE32_BIT) && 2553 (regs->pstate & PSR_D_BIT) == 0 && 2554 (regs->pstate & PSR_A_BIT) == 0 && 2555 (regs->pstate & PSR_I_BIT) == 0 && 2556 (regs->pstate & PSR_F_BIT) == 0) { 2557 return 1; 2558 } 2559 2560 /* Force PSR to a valid 64-bit EL0t */ 2561 regs->pstate &= PSR_N_BIT | PSR_Z_BIT | PSR_C_BIT | PSR_V_BIT; 2562 2563 return 0; 2564 } 2565 2566 /* 2567 * Are the current registers suitable for user mode? (used to maintain 2568 * security in signal handlers) 2569 */ 2570 int valid_user_regs(struct user_pt_regs *regs, struct task_struct *task) 2571 { 2572 /* https://lore.kernel.org/lkml/20191118131525.GA4180@willie-the-truck */ 2573 user_regs_reset_single_step(regs, task); 2574 2575 if (is_compat_thread(task_thread_info(task))) 2576 return valid_compat_regs(regs); 2577 else 2578 return valid_native_regs(regs); 2579 } 2580