1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2025 Juniper Networks, Inc.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/systm.h>
29 #include <sys/bus.h>
30 #include <sys/conf.h>
31 #include <sys/interrupt.h>
32 #include <sys/kernel.h>
33 #include <sys/kexec.h>
34 #include <vm/vm.h>
35 #include <vm/vm_extern.h>
36 #include <vm/vm_object.h>
37 #include <vm/vm_phys.h>
38 #include <vm/pmap.h>
39 #include <vm/vm_page.h>
40 #include <vm/vm_radix.h>
41
42 #include <machine/intr_machdep.h>
43 #include <machine/kexec.h>
44 #include <machine/md_var.h>
45 #include <machine/pmap.h>
46 #include <x86/apicvar.h>
47
48 /*
49 * Idea behind this:
50 *
51 * kexec_load_md():
52 * - Update boot page tables (identity map) to include all pages needed before
53 * disabling MMU.
54 *
55 * kexec_reboot_md():
56 * - Copy pages into target(s)
57 * - Do "other stuff"
58 * - Does not return
59 */
60
61 /*
62 * do_pte: Create PTE entries (4k pages). If false, create 2MB superpages.
63 * identity: This is for an identity map, treat `start` as a physical address.
64 * Only valid here if do_pte is false.
65 */
66 static void
kexec_generate_page_tables(pml4_entry_t * root,vm_offset_t start,vm_size_t size,bool do_pte,bool identity,struct pctrie_iter * pages)67 kexec_generate_page_tables(pml4_entry_t *root, vm_offset_t start,
68 vm_size_t size, bool do_pte, bool identity, struct pctrie_iter *pages)
69 {
70 vm_paddr_t mpa;
71 vm_offset_t pg;
72 vm_size_t stride = do_pte ? PAGE_SIZE : NBPDR;
73 vm_page_t m;
74 vm_pindex_t i, j, k, l;
75
76 pg = start & ~(stride - 1);
77 i = pmap_pml4e_index(pg);
78 j = pmap_pdpe_index(pg);
79 k = pmap_pde_index(pg);
80 l = pmap_pte_index(pg);
81 for (; pg < start + size; i++, j = 0, k = 0, l = 0) {
82 /*
83 * Walk linearly, as above, but one fell swoop, one page at a
84 * time.
85 */
86 if (root[i] == 0) {
87 m = vm_radix_iter_next(pages);
88 mpa = VM_PAGE_TO_PHYS(m);
89 root[i] = mpa | PG_RW | PG_V;
90 }
91 pdp_entry_t *pdp = PHYS_TO_DMAP(root[i] & PG_FRAME);
92 for (; j < NPDPEPG && pg < start + size; j++, k = 0, l = 0) {
93 if (pdp[j] == 0) {
94 m = vm_radix_iter_next(pages);
95 mpa = VM_PAGE_TO_PHYS(m);
96 pdp[j] = mpa | PG_RW | PG_V;
97 }
98 pd_entry_t *pde = PHYS_TO_DMAP(pdp[j] & PG_FRAME);
99 for (; k < NPDEPG && pg < start + size; k++, l = 0) {
100 if (pde[k] == 0) {
101 if (!do_pte) {
102 pde[k] =
103 (identity ? pg : pmap_kextract(pg)) |
104 PG_RW | PG_PS | PG_V;
105 pg += NBPDR;
106 continue;
107 }
108 m = vm_radix_iter_next(pages);
109 mpa = VM_PAGE_TO_PHYS(m);
110 pde[k] = mpa | PG_V | PG_RW;
111 } else if ((pde[k] & PG_PS) != 0) {
112 pg += NBPDR;
113 continue;
114 }
115 /* Populate the PTEs. */
116 for (; l < NPTEPG && pg < start + size;
117 l++, pg += PAGE_SIZE) {
118 pt_entry_t *pte =
119 PHYS_TO_DMAP(pde[pmap_pde_index(pg)] & PG_FRAME);
120 pte[pmap_pte_index(pg)] =
121 pmap_kextract(pg) | PG_RW | PG_V;
122 }
123 }
124 }
125 }
126 }
127
128 void
kexec_reboot_md(struct kexec_image * image)129 kexec_reboot_md(struct kexec_image *image)
130 {
131 void (*kexec_do_tramp)(void) = image->md_image;
132
133 intr_disable_all();
134 lapic_disable();
135 kexec_do_reboot_trampoline(VM_PAGE_TO_PHYS(image->first_md_page),
136 kexec_do_tramp);
137
138 for (;;)
139 ;
140 }
141
142 int
kexec_load_md(struct kexec_image * image)143 kexec_load_md(struct kexec_image *image)
144 {
145 struct pctrie_iter pct_iter;
146 pml4_entry_t *PT4;
147 pdp_entry_t *PDP_l;
148 pd_entry_t *PD_l0;
149 vm_offset_t va;
150 int i;
151
152 /*
153 * Start building the page table.
154 * First part of the page table is standard for all.
155 */
156 vm_offset_t pa_pdp_l, pa_pd_l0, pa_pd_l1, pa_pd_l2, pa_pd_l3;
157 vm_page_t m;
158
159 if (la57)
160 return (EINVAL);
161
162 vm_radix_iter_init(&pct_iter, &image->map_obj->rtree);
163 /* Working in linear space in the mapped space, `va` is our tracker. */
164 m = vm_radix_iter_lookup(&pct_iter, image->first_md_page->pindex);
165 va = (vm_offset_t)image->map_addr + ptoa(m->pindex);
166 /* We'll find a place for these later */
167 PT4 = (void *)va;
168 va += PAGE_SIZE;
169 m = vm_radix_iter_next(&pct_iter);
170 pa_pdp_l = VM_PAGE_TO_PHYS(m);
171 PDP_l = (void *)va;
172 va += PAGE_SIZE;
173 m = vm_radix_iter_next(&pct_iter);
174 pa_pd_l0 = VM_PAGE_TO_PHYS(m);
175 PD_l0 = (void *)va;
176 va += PAGE_SIZE;
177 m = vm_radix_iter_next(&pct_iter);
178 pa_pd_l1 = VM_PAGE_TO_PHYS(m);
179 m = vm_radix_iter_next(&pct_iter);
180 pa_pd_l2 = VM_PAGE_TO_PHYS(m);
181 m = vm_radix_iter_next(&pct_iter);
182 pa_pd_l3 = VM_PAGE_TO_PHYS(m);
183 m = vm_radix_iter_next(&pct_iter);
184
185 /* 1:1 mapping of lower 4G */
186 PT4[0] = (pml4_entry_t)pa_pdp_l | PG_V | PG_RW;
187 PDP_l[0] = (pdp_entry_t)pa_pd_l0 | PG_V | PG_RW;
188 PDP_l[1] = (pdp_entry_t)pa_pd_l1 | PG_V | PG_RW;
189 PDP_l[2] = (pdp_entry_t)pa_pd_l2 | PG_V | PG_RW;
190 PDP_l[3] = (pdp_entry_t)pa_pd_l3 | PG_V | PG_RW;
191 for (i = 0; i < 4 * NPDEPG; i++) { /* we overflow PD_l0 into _l1, etc */
192 PD_l0[i] = ((pd_entry_t)i << PDRSHIFT) | PG_V |
193 PG_RW | PG_PS;
194 }
195
196 /* Map the target(s) in 2MB chunks. */
197 for (i = 0; i < KEXEC_SEGMENT_MAX; i++) {
198 struct kexec_segment_stage *s = &image->segments[i];
199
200 if (s->size == 0)
201 break;
202 kexec_generate_page_tables(PT4, s->target, s->size, false,
203 true, &pct_iter);
204 }
205 /* Now create the source page tables */
206 kexec_generate_page_tables(PT4, image->map_addr, image->map_size, true,
207 false, &pct_iter);
208 kexec_generate_page_tables(PT4,
209 trunc_page((vm_offset_t)kexec_do_reboot_trampoline),
210 PAGE_SIZE, true, false, &pct_iter);
211 KASSERT(m != NULL, ("kexec_load_md: Missing trampoline page!\n"));
212
213 /* MD control pages start at this next page. */
214 image->md_image = (void *)(image->map_addr + ptoa(m->pindex));
215 bcopy(kexec_do_reboot, image->md_image, kexec_do_reboot_size);
216
217 /* Save the image into the MD page(s) right after the trampoline */
218 bcopy(image, (void *)((vm_offset_t)image->md_image +
219 (vm_offset_t)&kexec_saved_image - (vm_offset_t)&kexec_do_reboot),
220 sizeof(*image));
221
222 return (0);
223 }
224
225 /*
226 * Required pages:
227 * - L4 (1) (root)
228 * - L3 (PDPE) - 2 (bottom 512GB, bottom 4 used, top range for kernel map)
229 * - L2 (PDP) - 5 (2MB superpage mappings, 1GB each, for bottom 4GB, top 1)
230 * - L1 (PDR) - 1 (kexec trampoline page, first MD page)
231 * - kexec_do_reboot trampoline - 1
232 * - Slop pages for staging (in case it's not aligned nicely) - 3 (worst case)
233 *
234 * Minimum 9 pages for the direct map.
235 */
236 int
kexec_md_pages(struct kexec_segment * seg_in)237 kexec_md_pages(struct kexec_segment *seg_in)
238 {
239 struct kexec_segment *segs = seg_in;
240 vm_size_t pages = 13; /* Minimum number of starting pages */
241 vm_paddr_t cur_addr = (1UL << 32) - 1; /* Bottom 4G will be identity mapped in full */
242 vm_size_t source_total = 0;
243
244 for (int i = 0; i < KEXEC_SEGMENT_MAX; i++) {
245 vm_offset_t start, end;
246 if (segs[i].memsz == 0)
247 break;
248
249 end = round_2mpage((vm_offset_t)segs[i].mem + segs[i].memsz);
250 start = trunc_2mpage((vm_offset_t)segs[i].mem);
251 start = max(start, cur_addr + 1);
252 /*
253 * Round to cover the full range of page table pages for each
254 * segment.
255 */
256 source_total += round_2mpage(end - start);
257
258 /*
259 * Bottom 4GB are identity mapped already in the count, so skip
260 * any segments that end up there, this will short-circuit that.
261 */
262 if (end <= cur_addr + 1)
263 continue;
264
265 if (pmap_pml4e_index(end) != pmap_pml4e_index(cur_addr)) {
266 /* Need a new 512GB mapping page */
267 pages++;
268 pages += howmany(end - (start & ~PML4MASK), NBPML4);
269 pages += howmany(end - (start & ~PDPMASK), NBPDP);
270 pages += howmany(end - (start & ~PDRMASK), NBPDR);
271
272 } else if (pmap_pdpe_index(end) != pmap_pdpe_index(cur_addr)) {
273 pages++;
274 pages += howmany(end - (start & ~PDPMASK), NBPDP) - 1;
275 pages += howmany(end - (start & ~PDRMASK), NBPDR);
276 }
277
278 }
279 /* Be pessimistic when totaling up source pages. We likely
280 * can't use superpages, so need to map each page individually.
281 */
282 pages += howmany(source_total, NBPDR);
283 pages += howmany(source_total, NBPDP);
284 pages += howmany(source_total, NBPML4);
285
286 /*
287 * Be intentionally sloppy adding in the extra page table pages. It's
288 * better to go over than under.
289 */
290 pages += howmany(pages * PAGE_SIZE, NBPDR);
291 pages += howmany(pages * PAGE_SIZE, NBPDP);
292 pages += howmany(pages * PAGE_SIZE, NBPML4);
293
294 /* Add in the trampoline pages */
295 pages += howmany(kexec_do_reboot_size, PAGE_SIZE);
296
297 return (pages);
298 }
299