1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2024-2026 Arm Ltd
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/param.h>
29 #include <sys/kernel.h>
30 #include <sys/libkern.h>
31 #include <sys/proc.h>
32
33 #include <machine/cpu_feat.h>
34 #include <machine/pcb.h>
35 #include <machine/pte.h>
36 #include <machine/sysarch.h>
37 #include <vm/vm.h>
38 #include <vm/vm_page.h>
39
40 /* Version of MTE implemented. 0 == unimplemented */
41 static u_int __read_mostly mte_version = 0;
42
43 /*
44 * FEAT_MTE (mte_version == 1) has userspace instructions, but no tag
45 * checking. May of the registers/fields need FEAT_MTE2 to be implemented
46 * before we can access them.
47 */
48 #define MTE_HAS_TAG_CHECK (mte_version >= 2)
49
50 struct thread *mte_switch(struct thread *);
51
52 #define load_tags(addr) ({ \
53 uint64_t __val; \
54 asm volatile( \
55 ".arch_extension memtag \n" \
56 "ldgm %0, [%1] \n" \
57 ".arch_extension nomemtag" : "=r" (__val) : "r" (addr)); \
58 __val; \
59 })
60
61 #define set_tags(tags, addr) do { \
62 asm volatile( \
63 ".arch_extension memtag \n" \
64 "stgm %0, [%1] \n" \
65 ".arch_extension nomemtag" : "=r" (tags) : "r" (addr)); \
66 } while (0)
67
68 /* Fetch the block size used by tag load and store instructions */
69 static inline size_t
mte_block_size(void)70 mte_block_size(void)
71 {
72 return (sizeof(int) << GMID_BS_SIZE(READ_SPECIALREG(GMID_EL1_REG)));
73 }
74
75 static void
mte_update_sctlr(struct thread * td,uint64_t sctlr)76 mte_update_sctlr(struct thread *td, uint64_t sctlr)
77 {
78 MPASS((sctlr & ~(SCTLR_ATA0 | SCTLR_TCF0_MASK)) == 0);
79 td->td_md.md_sctlr &= ~(SCTLR_ATA0 | SCTLR_TCF0_MASK);
80 td->td_md.md_sctlr |= sctlr;
81 }
82
83 /**
84 * Clear/sync the allocation tags for a given page. This should be done on
85 * allocation of a page to ensure a tag check fault does not occur immediately
86 * after accessing newly tagged memory.
87 */
88 void
mte_sync_tags(vm_page_t page)89 mte_sync_tags(vm_page_t page)
90 {
91 char *addr;
92 size_t block_size;
93
94 if (!MTE_HAS_TAG_CHECK)
95 return;
96
97 /* don't clear the tags on a page that's already setup for mte */
98 if ((page->md.pv_flags & PV_MTE_TAGGED) != 0)
99 return;
100
101 block_size = mte_block_size();
102 addr = PHYS_TO_DMAP(page->phys_addr);
103
104 for (size_t count = 0; count < PAGE_SIZE;
105 count += block_size, addr += block_size)
106 asm volatile(
107 ".arch_extension memtag \n"
108 "stgm xzr, [%0] \n"
109 ".arch_extension nomemtag" : : "r" (addr));
110
111 page->md.pv_flags |= PV_MTE_TAGGED;
112 }
113
114 /**
115 * Copy the allocation tags from given target to destination page. This is called
116 * on a copy-on-write and anything that causes a pmap_copy_page call.
117 */
118 void
mte_copy_tags(vm_page_t srcpage,vm_page_t dstpage,char * src,char * dst)119 mte_copy_tags(vm_page_t srcpage, vm_page_t dstpage, char *src, char *dst)
120 {
121 size_t block_size;
122 uint64_t tags;
123
124 MPASS((srcpage->md.pv_flags & PV_MTE_TAGGED) != 0);
125
126 /*
127 * Copy the tags from the source page to the destination page,
128 * incrementing by the block count read from GMID_EL1
129 */
130 block_size = mte_block_size();
131 for (size_t count = 0; count < PAGE_SIZE;
132 count += block_size, src += block_size, dst += block_size) {
133 tags = load_tags(src);
134 set_tags(tags, dst);
135 }
136 dstpage->md.pv_flags |= PV_MTE_TAGGED;
137 }
138
139 void
mte_fork(struct thread * new_td,struct thread * orig_td)140 mte_fork(struct thread *new_td, struct thread *orig_td)
141 {
142 if (!MTE_HAS_TAG_CHECK)
143 return;
144
145 mte_update_sctlr(new_td,
146 orig_td->td_md.md_sctlr & SCTLR_TCF0_MASK);
147 new_td->td_md.md_gcr = orig_td->td_md.md_gcr;
148 }
149
150 void
mte_exec(struct thread * td)151 mte_exec(struct thread *td)
152 {
153 if (!MTE_HAS_TAG_CHECK)
154 return;
155
156 mte_update_sctlr(td, SCTLR_TCF0_NONE);
157 td->td_md.md_gcr = GCR_RRND;
158 }
159
160 void
mte_copy_thread(struct thread * new_td,struct thread * orig_td)161 mte_copy_thread(struct thread *new_td, struct thread *orig_td)
162 {
163 if (!MTE_HAS_TAG_CHECK)
164 return;
165
166 mte_update_sctlr(new_td,
167 orig_td->td_md.md_sctlr & SCTLR_TCF0_MASK);
168 new_td->td_md.md_gcr = orig_td->td_md.md_gcr;
169 }
170
171 /* Only for kernel threads */
172 void
mte_thread_alloc(struct thread * td)173 mte_thread_alloc(struct thread *td)
174 {
175 }
176
177 /* Only for a kernel thread */
178 void
mte_thread0(struct thread * td)179 mte_thread0(struct thread *td)
180 {
181 }
182
183
184 struct thread *
mte_switch(struct thread * td)185 mte_switch(struct thread *td)
186 {
187 if (MTE_HAS_TAG_CHECK) {
188 WRITE_SPECIALREG(GCR_EL1_REG, td->td_md.md_gcr);
189 }
190 return (td);
191 }
192