1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 ARM Ltd. 4 */ 5 6 #include <linux/bitops.h> 7 #include <linux/kernel.h> 8 #include <linux/mm.h> 9 #include <linux/prctl.h> 10 #include <linux/sched.h> 11 #include <linux/sched/mm.h> 12 #include <linux/string.h> 13 #include <linux/swap.h> 14 #include <linux/swapops.h> 15 #include <linux/thread_info.h> 16 #include <linux/types.h> 17 #include <linux/uio.h> 18 19 #include <asm/barrier.h> 20 #include <asm/cpufeature.h> 21 #include <asm/mte.h> 22 #include <asm/mte-kasan.h> 23 #include <asm/ptrace.h> 24 #include <asm/sysreg.h> 25 26 u64 gcr_kernel_excl __ro_after_init; 27 28 static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap) 29 { 30 pte_t old_pte = READ_ONCE(*ptep); 31 32 if (check_swap && is_swap_pte(old_pte)) { 33 swp_entry_t entry = pte_to_swp_entry(old_pte); 34 35 if (!non_swap_entry(entry) && mte_restore_tags(entry, page)) 36 return; 37 } 38 39 page_kasan_tag_reset(page); 40 /* 41 * We need smp_wmb() in between setting the flags and clearing the 42 * tags because if another thread reads page->flags and builds a 43 * tagged address out of it, there is an actual dependency to the 44 * memory access, but on the current thread we do not guarantee that 45 * the new page->flags are visible before the tags were updated. 46 */ 47 smp_wmb(); 48 mte_clear_page_tags(page_address(page)); 49 } 50 51 void mte_sync_tags(pte_t *ptep, pte_t pte) 52 { 53 struct page *page = pte_page(pte); 54 long i, nr_pages = compound_nr(page); 55 bool check_swap = nr_pages == 1; 56 57 /* if PG_mte_tagged is set, tags have already been initialised */ 58 for (i = 0; i < nr_pages; i++, page++) { 59 if (!test_and_set_bit(PG_mte_tagged, &page->flags)) 60 mte_sync_page_tags(page, ptep, check_swap); 61 } 62 } 63 64 int memcmp_pages(struct page *page1, struct page *page2) 65 { 66 char *addr1, *addr2; 67 int ret; 68 69 addr1 = page_address(page1); 70 addr2 = page_address(page2); 71 ret = memcmp(addr1, addr2, PAGE_SIZE); 72 73 if (!system_supports_mte() || ret) 74 return ret; 75 76 /* 77 * If the page content is identical but at least one of the pages is 78 * tagged, return non-zero to avoid KSM merging. If only one of the 79 * pages is tagged, set_pte_at() may zero or change the tags of the 80 * other page via mte_sync_tags(). 81 */ 82 if (test_bit(PG_mte_tagged, &page1->flags) || 83 test_bit(PG_mte_tagged, &page2->flags)) 84 return addr1 != addr2; 85 86 return ret; 87 } 88 89 u8 mte_get_mem_tag(void *addr) 90 { 91 if (!system_supports_mte()) 92 return 0xFF; 93 94 asm(__MTE_PREAMBLE "ldg %0, [%0]" 95 : "+r" (addr)); 96 97 return mte_get_ptr_tag(addr); 98 } 99 100 u8 mte_get_random_tag(void) 101 { 102 void *addr; 103 104 if (!system_supports_mte()) 105 return 0xFF; 106 107 asm(__MTE_PREAMBLE "irg %0, %0" 108 : "+r" (addr)); 109 110 return mte_get_ptr_tag(addr); 111 } 112 113 void *mte_set_mem_tag_range(void *addr, size_t size, u8 tag) 114 { 115 void *ptr = addr; 116 117 if ((!system_supports_mte()) || (size == 0)) 118 return addr; 119 120 /* Make sure that size is MTE granule aligned. */ 121 WARN_ON(size & (MTE_GRANULE_SIZE - 1)); 122 123 /* Make sure that the address is MTE granule aligned. */ 124 WARN_ON((u64)addr & (MTE_GRANULE_SIZE - 1)); 125 126 tag = 0xF0 | tag; 127 ptr = (void *)__tag_set(ptr, tag); 128 129 mte_assign_mem_tag_range(ptr, size); 130 131 return ptr; 132 } 133 134 void mte_init_tags(u64 max_tag) 135 { 136 static bool gcr_kernel_excl_initialized; 137 138 if (!gcr_kernel_excl_initialized) { 139 /* 140 * The format of the tags in KASAN is 0xFF and in MTE is 0xF. 141 * This conversion extracts an MTE tag from a KASAN tag. 142 */ 143 u64 incl = GENMASK(FIELD_GET(MTE_TAG_MASK >> MTE_TAG_SHIFT, 144 max_tag), 0); 145 146 gcr_kernel_excl = ~incl & SYS_GCR_EL1_EXCL_MASK; 147 gcr_kernel_excl_initialized = true; 148 } 149 150 /* Enable the kernel exclude mask for random tags generation. */ 151 write_sysreg_s(SYS_GCR_EL1_RRND | gcr_kernel_excl, SYS_GCR_EL1); 152 } 153 154 void mte_enable_kernel(void) 155 { 156 /* Enable MTE Sync Mode for EL1. */ 157 sysreg_clear_set(sctlr_el1, SCTLR_ELx_TCF_MASK, SCTLR_ELx_TCF_SYNC); 158 isb(); 159 } 160 161 static void update_sctlr_el1_tcf0(u64 tcf0) 162 { 163 /* ISB required for the kernel uaccess routines */ 164 sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF0_MASK, tcf0); 165 isb(); 166 } 167 168 static void set_sctlr_el1_tcf0(u64 tcf0) 169 { 170 /* 171 * mte_thread_switch() checks current->thread.sctlr_tcf0 as an 172 * optimisation. Disable preemption so that it does not see 173 * the variable update before the SCTLR_EL1.TCF0 one. 174 */ 175 preempt_disable(); 176 current->thread.sctlr_tcf0 = tcf0; 177 update_sctlr_el1_tcf0(tcf0); 178 preempt_enable(); 179 } 180 181 static void update_gcr_el1_excl(u64 excl) 182 { 183 184 /* 185 * Note that the mask controlled by the user via prctl() is an 186 * include while GCR_EL1 accepts an exclude mask. 187 * No need for ISB since this only affects EL0 currently, implicit 188 * with ERET. 189 */ 190 sysreg_clear_set_s(SYS_GCR_EL1, SYS_GCR_EL1_EXCL_MASK, excl); 191 } 192 193 static void set_gcr_el1_excl(u64 excl) 194 { 195 current->thread.gcr_user_excl = excl; 196 197 /* 198 * SYS_GCR_EL1 will be set to current->thread.gcr_user_excl value 199 * by mte_set_user_gcr() in kernel_exit, 200 */ 201 } 202 203 void flush_mte_state(void) 204 { 205 if (!system_supports_mte()) 206 return; 207 208 /* clear any pending asynchronous tag fault */ 209 dsb(ish); 210 write_sysreg_s(0, SYS_TFSRE0_EL1); 211 clear_thread_flag(TIF_MTE_ASYNC_FAULT); 212 /* disable tag checking */ 213 set_sctlr_el1_tcf0(SCTLR_EL1_TCF0_NONE); 214 /* reset tag generation mask */ 215 set_gcr_el1_excl(SYS_GCR_EL1_EXCL_MASK); 216 } 217 218 void mte_thread_switch(struct task_struct *next) 219 { 220 if (!system_supports_mte()) 221 return; 222 223 /* avoid expensive SCTLR_EL1 accesses if no change */ 224 if (current->thread.sctlr_tcf0 != next->thread.sctlr_tcf0) 225 update_sctlr_el1_tcf0(next->thread.sctlr_tcf0); 226 } 227 228 void mte_suspend_exit(void) 229 { 230 if (!system_supports_mte()) 231 return; 232 233 update_gcr_el1_excl(gcr_kernel_excl); 234 } 235 236 long set_mte_ctrl(struct task_struct *task, unsigned long arg) 237 { 238 u64 tcf0; 239 u64 gcr_excl = ~((arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT) & 240 SYS_GCR_EL1_EXCL_MASK; 241 242 if (!system_supports_mte()) 243 return 0; 244 245 switch (arg & PR_MTE_TCF_MASK) { 246 case PR_MTE_TCF_NONE: 247 tcf0 = SCTLR_EL1_TCF0_NONE; 248 break; 249 case PR_MTE_TCF_SYNC: 250 tcf0 = SCTLR_EL1_TCF0_SYNC; 251 break; 252 case PR_MTE_TCF_ASYNC: 253 tcf0 = SCTLR_EL1_TCF0_ASYNC; 254 break; 255 default: 256 return -EINVAL; 257 } 258 259 if (task != current) { 260 task->thread.sctlr_tcf0 = tcf0; 261 task->thread.gcr_user_excl = gcr_excl; 262 } else { 263 set_sctlr_el1_tcf0(tcf0); 264 set_gcr_el1_excl(gcr_excl); 265 } 266 267 return 0; 268 } 269 270 long get_mte_ctrl(struct task_struct *task) 271 { 272 unsigned long ret; 273 u64 incl = ~task->thread.gcr_user_excl & SYS_GCR_EL1_EXCL_MASK; 274 275 if (!system_supports_mte()) 276 return 0; 277 278 ret = incl << PR_MTE_TAG_SHIFT; 279 280 switch (task->thread.sctlr_tcf0) { 281 case SCTLR_EL1_TCF0_NONE: 282 ret |= PR_MTE_TCF_NONE; 283 break; 284 case SCTLR_EL1_TCF0_SYNC: 285 ret |= PR_MTE_TCF_SYNC; 286 break; 287 case SCTLR_EL1_TCF0_ASYNC: 288 ret |= PR_MTE_TCF_ASYNC; 289 break; 290 } 291 292 return ret; 293 } 294 295 /* 296 * Access MTE tags in another process' address space as given in mm. Update 297 * the number of tags copied. Return 0 if any tags copied, error otherwise. 298 * Inspired by __access_remote_vm(). 299 */ 300 static int __access_remote_tags(struct mm_struct *mm, unsigned long addr, 301 struct iovec *kiov, unsigned int gup_flags) 302 { 303 struct vm_area_struct *vma; 304 void __user *buf = kiov->iov_base; 305 size_t len = kiov->iov_len; 306 int ret; 307 int write = gup_flags & FOLL_WRITE; 308 309 if (!access_ok(buf, len)) 310 return -EFAULT; 311 312 if (mmap_read_lock_killable(mm)) 313 return -EIO; 314 315 while (len) { 316 unsigned long tags, offset; 317 void *maddr; 318 struct page *page = NULL; 319 320 ret = get_user_pages_remote(mm, addr, 1, gup_flags, &page, 321 &vma, NULL); 322 if (ret <= 0) 323 break; 324 325 /* 326 * Only copy tags if the page has been mapped as PROT_MTE 327 * (PG_mte_tagged set). Otherwise the tags are not valid and 328 * not accessible to user. Moreover, an mprotect(PROT_MTE) 329 * would cause the existing tags to be cleared if the page 330 * was never mapped with PROT_MTE. 331 */ 332 if (!test_bit(PG_mte_tagged, &page->flags)) { 333 ret = -EOPNOTSUPP; 334 put_page(page); 335 break; 336 } 337 338 /* limit access to the end of the page */ 339 offset = offset_in_page(addr); 340 tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE); 341 342 maddr = page_address(page); 343 if (write) { 344 tags = mte_copy_tags_from_user(maddr + offset, buf, tags); 345 set_page_dirty_lock(page); 346 } else { 347 tags = mte_copy_tags_to_user(buf, maddr + offset, tags); 348 } 349 put_page(page); 350 351 /* error accessing the tracer's buffer */ 352 if (!tags) 353 break; 354 355 len -= tags; 356 buf += tags; 357 addr += tags * MTE_GRANULE_SIZE; 358 } 359 mmap_read_unlock(mm); 360 361 /* return an error if no tags copied */ 362 kiov->iov_len = buf - kiov->iov_base; 363 if (!kiov->iov_len) { 364 /* check for error accessing the tracee's address space */ 365 if (ret <= 0) 366 return -EIO; 367 else 368 return -EFAULT; 369 } 370 371 return 0; 372 } 373 374 /* 375 * Copy MTE tags in another process' address space at 'addr' to/from tracer's 376 * iovec buffer. Return 0 on success. Inspired by ptrace_access_vm(). 377 */ 378 static int access_remote_tags(struct task_struct *tsk, unsigned long addr, 379 struct iovec *kiov, unsigned int gup_flags) 380 { 381 struct mm_struct *mm; 382 int ret; 383 384 mm = get_task_mm(tsk); 385 if (!mm) 386 return -EPERM; 387 388 if (!tsk->ptrace || (current != tsk->parent) || 389 ((get_dumpable(mm) != SUID_DUMP_USER) && 390 !ptracer_capable(tsk, mm->user_ns))) { 391 mmput(mm); 392 return -EPERM; 393 } 394 395 ret = __access_remote_tags(mm, addr, kiov, gup_flags); 396 mmput(mm); 397 398 return ret; 399 } 400 401 int mte_ptrace_copy_tags(struct task_struct *child, long request, 402 unsigned long addr, unsigned long data) 403 { 404 int ret; 405 struct iovec kiov; 406 struct iovec __user *uiov = (void __user *)data; 407 unsigned int gup_flags = FOLL_FORCE; 408 409 if (!system_supports_mte()) 410 return -EIO; 411 412 if (get_user(kiov.iov_base, &uiov->iov_base) || 413 get_user(kiov.iov_len, &uiov->iov_len)) 414 return -EFAULT; 415 416 if (request == PTRACE_POKEMTETAGS) 417 gup_flags |= FOLL_WRITE; 418 419 /* align addr to the MTE tag granule */ 420 addr &= MTE_GRANULE_MASK; 421 422 ret = access_remote_tags(child, addr, &kiov, gup_flags); 423 if (!ret) 424 ret = put_user(kiov.iov_len, &uiov->iov_len); 425 426 return ret; 427 } 428