1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2020-2021 Ruslan Bukin <br@bsdpad.com> 5 * Copyright (c) 2014-2021 Andrew Turner 6 * Copyright (c) 2014-2016 The FreeBSD Foundation 7 * All rights reserved. 8 * 9 * This work was supported by Innovate UK project 105694, "Digital Security 10 * by Design (DSbD) Technology Platform Prototype". 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 /* 36 * Manages physical address maps for ARM SMMUv3 and ARM Mali GPU. 37 */ 38 39 #include "opt_vm.h" 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/ktr.h> 44 #include <sys/lock.h> 45 #include <sys/mutex.h> 46 #include <sys/rwlock.h> 47 48 #include <vm/vm.h> 49 #include <vm/vm_param.h> 50 #include <vm/vm_page.h> 51 #include <vm/vm_map.h> 52 #include <vm/vm_object.h> 53 #include <vm/vm_pageout.h> 54 #include <vm/vm_radix.h> 55 56 #include <machine/machdep.h> 57 58 #include <arm64/iommu/iommu_pmap.h> 59 #include <arm64/iommu/iommu_pte.h> 60 61 #define IOMMU_PAGE_SIZE 4096 62 63 #define SMMU_PMAP_LOCK(pmap) mtx_lock(&(pmap)->sp_mtx) 64 #define SMMU_PMAP_UNLOCK(pmap) mtx_unlock(&(pmap)->sp_mtx) 65 #define SMMU_PMAP_LOCK_ASSERT(pmap, type) \ 66 mtx_assert(&(pmap)->sp_mtx, (type)) 67 68 #define NL0PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) 69 #define NL1PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) 70 #define NL2PG (IOMMU_PAGE_SIZE/(sizeof (pd_entry_t))) 71 #define NL3PG (IOMMU_PAGE_SIZE/(sizeof (pt_entry_t))) 72 73 #define NUL0E IOMMU_L0_ENTRIES 74 #define NUL1E (NUL0E * NL1PG) 75 #define NUL2E (NUL1E * NL2PG) 76 77 #define smmu_l0_pindex(v) (NUL2E + NUL1E + ((v) >> IOMMU_L0_SHIFT)) 78 #define smmu_l1_pindex(v) (NUL2E + ((v) >> IOMMU_L1_SHIFT)) 79 #define smmu_l2_pindex(v) ((v) >> IOMMU_L2_SHIFT) 80 81 #define smmu_l0_index(va) (((va) >> IOMMU_L0_SHIFT) & IOMMU_L0_ADDR_MASK) 82 #define smmu_l1_index(va) (((va) >> IOMMU_L1_SHIFT) & IOMMU_Ln_ADDR_MASK) 83 #define smmu_l2_index(va) (((va) >> IOMMU_L2_SHIFT) & IOMMU_Ln_ADDR_MASK) 84 #define smmu_l3_index(va) (((va) >> IOMMU_L3_SHIFT) & IOMMU_Ln_ADDR_MASK) 85 86 static vm_page_t _pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex); 87 static void _smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, 88 vm_page_t m, struct spglist *free); 89 90 /* 91 * These load the old table data and store the new value. 92 * They need to be atomic as the System MMU may write to the table at 93 * the same time as the CPU. 94 */ 95 #define smmu_pmap_load(table) (*table) 96 #define smmu_pmap_clear(table) atomic_store_64(table, 0) 97 #define smmu_pmap_store(table, entry) atomic_store_64(table, entry) 98 99 /********************/ 100 /* Inline functions */ 101 /********************/ 102 103 static __inline pd_entry_t * 104 smmu_pmap_l0(struct smmu_pmap *pmap, vm_offset_t va) 105 { 106 107 return (&pmap->sp_l0[smmu_l0_index(va)]); 108 } 109 110 static __inline pd_entry_t * 111 smmu_pmap_l0_to_l1(pd_entry_t *l0, vm_offset_t va) 112 { 113 pd_entry_t *l1; 114 115 l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) & ~ATTR_MASK); 116 return (&l1[smmu_l1_index(va)]); 117 } 118 119 static __inline pd_entry_t * 120 smmu_pmap_l1(struct smmu_pmap *pmap, vm_offset_t va) 121 { 122 pd_entry_t *l0; 123 124 l0 = smmu_pmap_l0(pmap, va); 125 if ((smmu_pmap_load(l0) & ATTR_DESCR_MASK) != IOMMU_L0_TABLE) 126 return (NULL); 127 128 return (smmu_pmap_l0_to_l1(l0, va)); 129 } 130 131 static __inline pd_entry_t * 132 smmu_pmap_l1_to_l2(pd_entry_t *l1p, vm_offset_t va) 133 { 134 pd_entry_t l1, *l2p; 135 136 l1 = smmu_pmap_load(l1p); 137 138 /* 139 * The valid bit may be clear if pmap_update_entry() is concurrently 140 * modifying the entry, so for KVA only the entry type may be checked. 141 */ 142 KASSERT(va >= VM_MAX_USER_ADDRESS || (l1 & ATTR_DESCR_VALID) != 0, 143 ("%s: L1 entry %#lx for %#lx is invalid", __func__, l1, va)); 144 KASSERT((l1 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, 145 ("%s: L1 entry %#lx for %#lx is a leaf", __func__, l1, va)); 146 l2p = (pd_entry_t *)PHYS_TO_DMAP(l1 & ~ATTR_MASK); 147 return (&l2p[smmu_l2_index(va)]); 148 } 149 150 static __inline pd_entry_t * 151 smmu_pmap_l2(struct smmu_pmap *pmap, vm_offset_t va) 152 { 153 pd_entry_t *l1; 154 155 l1 = smmu_pmap_l1(pmap, va); 156 if ((smmu_pmap_load(l1) & ATTR_DESCR_MASK) != IOMMU_L1_TABLE) 157 return (NULL); 158 159 return (smmu_pmap_l1_to_l2(l1, va)); 160 } 161 162 static __inline pt_entry_t * 163 smmu_pmap_l2_to_l3(pd_entry_t *l2p, vm_offset_t va) 164 { 165 pd_entry_t l2; 166 pt_entry_t *l3p; 167 168 l2 = smmu_pmap_load(l2p); 169 170 /* 171 * The valid bit may be clear if pmap_update_entry() is concurrently 172 * modifying the entry, so for KVA only the entry type may be checked. 173 */ 174 KASSERT(va >= VM_MAX_USER_ADDRESS || (l2 & ATTR_DESCR_VALID) != 0, 175 ("%s: L2 entry %#lx for %#lx is invalid", __func__, l2, va)); 176 KASSERT((l2 & ATTR_DESCR_TYPE_MASK) == ATTR_DESCR_TYPE_TABLE, 177 ("%s: L2 entry %#lx for %#lx is a leaf", __func__, l2, va)); 178 l3p = (pt_entry_t *)PHYS_TO_DMAP(l2 & ~ATTR_MASK); 179 return (&l3p[smmu_l3_index(va)]); 180 } 181 182 /* 183 * Returns the lowest valid pde for a given virtual address. 184 * The next level may or may not point to a valid page or block. 185 */ 186 static __inline pd_entry_t * 187 smmu_pmap_pde(struct smmu_pmap *pmap, vm_offset_t va, int *level) 188 { 189 pd_entry_t *l0, *l1, *l2, desc; 190 191 l0 = smmu_pmap_l0(pmap, va); 192 desc = smmu_pmap_load(l0) & ATTR_DESCR_MASK; 193 if (desc != IOMMU_L0_TABLE) { 194 *level = -1; 195 return (NULL); 196 } 197 198 l1 = smmu_pmap_l0_to_l1(l0, va); 199 desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK; 200 if (desc != IOMMU_L1_TABLE) { 201 *level = 0; 202 return (l0); 203 } 204 205 l2 = smmu_pmap_l1_to_l2(l1, va); 206 desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK; 207 if (desc != IOMMU_L2_TABLE) { 208 *level = 1; 209 return (l1); 210 } 211 212 *level = 2; 213 return (l2); 214 } 215 216 /* 217 * Returns the lowest valid pte block or table entry for a given virtual 218 * address. If there are no valid entries return NULL and set the level to 219 * the first invalid level. 220 */ 221 static __inline pt_entry_t * 222 smmu_pmap_pte(struct smmu_pmap *pmap, vm_offset_t va, int *level) 223 { 224 pd_entry_t *l1, *l2, desc; 225 pt_entry_t *l3; 226 227 l1 = smmu_pmap_l1(pmap, va); 228 if (l1 == NULL) { 229 *level = 0; 230 return (NULL); 231 } 232 desc = smmu_pmap_load(l1) & ATTR_DESCR_MASK; 233 if (desc == IOMMU_L1_BLOCK) { 234 *level = 1; 235 return (l1); 236 } 237 238 if (desc != IOMMU_L1_TABLE) { 239 *level = 1; 240 return (NULL); 241 } 242 243 l2 = smmu_pmap_l1_to_l2(l1, va); 244 desc = smmu_pmap_load(l2) & ATTR_DESCR_MASK; 245 if (desc == IOMMU_L2_BLOCK) { 246 *level = 2; 247 return (l2); 248 } 249 250 if (desc != IOMMU_L2_TABLE) { 251 *level = 2; 252 return (NULL); 253 } 254 255 *level = 3; 256 l3 = smmu_pmap_l2_to_l3(l2, va); 257 if ((smmu_pmap_load(l3) & ATTR_DESCR_MASK) != IOMMU_L3_PAGE) 258 return (NULL); 259 260 return (l3); 261 } 262 263 static __inline int 264 smmu_pmap_l3_valid(pt_entry_t l3) 265 { 266 267 return ((l3 & ATTR_DESCR_MASK) == IOMMU_L3_PAGE); 268 } 269 270 CTASSERT(IOMMU_L1_BLOCK == IOMMU_L2_BLOCK); 271 272 #ifdef INVARIANTS 273 static __inline void 274 smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count) 275 { 276 277 SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); 278 pmap->sp_resident_count += count; 279 } 280 281 static __inline void 282 smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count) 283 { 284 285 SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); 286 KASSERT(pmap->sp_resident_count >= count, 287 ("pmap %p resident count underflow %ld %d", pmap, 288 pmap->sp_resident_count, count)); 289 pmap->sp_resident_count -= count; 290 } 291 #else 292 static __inline void 293 smmu_pmap_resident_count_inc(struct smmu_pmap *pmap, int count) 294 { 295 } 296 297 static __inline void 298 smmu_pmap_resident_count_dec(struct smmu_pmap *pmap, int count) 299 { 300 } 301 #endif 302 303 /*************************************************** 304 * Page table page management routines..... 305 ***************************************************/ 306 /* 307 * Schedule the specified unused page table page to be freed. Specifically, 308 * add the page to the specified list of pages that will be released to the 309 * physical memory manager after the TLB has been updated. 310 */ 311 static __inline void 312 smmu_pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, 313 boolean_t set_PG_ZERO) 314 { 315 316 if (set_PG_ZERO) 317 m->flags |= PG_ZERO; 318 else 319 m->flags &= ~PG_ZERO; 320 SLIST_INSERT_HEAD(free, m, plinks.s.ss); 321 } 322 323 /*************************************************** 324 * Low level mapping routines..... 325 ***************************************************/ 326 327 /* 328 * Decrements a page table page's reference count, which is used to record the 329 * number of valid page table entries within the page. If the reference count 330 * drops to zero, then the page table page is unmapped. Returns TRUE if the 331 * page table page was unmapped and FALSE otherwise. 332 */ 333 static inline boolean_t 334 smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m, 335 struct spglist *free) 336 { 337 338 --m->ref_count; 339 if (m->ref_count == 0) { 340 _smmu_pmap_unwire_l3(pmap, va, m, free); 341 return (TRUE); 342 } else 343 return (FALSE); 344 } 345 346 static void 347 _smmu_pmap_unwire_l3(struct smmu_pmap *pmap, vm_offset_t va, vm_page_t m, 348 struct spglist *free) 349 { 350 351 SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); 352 /* 353 * unmap the page table page 354 */ 355 if (m->pindex >= (NUL2E + NUL1E)) { 356 /* l1 page */ 357 pd_entry_t *l0; 358 359 l0 = smmu_pmap_l0(pmap, va); 360 smmu_pmap_clear(l0); 361 } else if (m->pindex >= NUL2E) { 362 /* l2 page */ 363 pd_entry_t *l1; 364 365 l1 = smmu_pmap_l1(pmap, va); 366 smmu_pmap_clear(l1); 367 } else { 368 /* l3 page */ 369 pd_entry_t *l2; 370 371 l2 = smmu_pmap_l2(pmap, va); 372 smmu_pmap_clear(l2); 373 } 374 smmu_pmap_resident_count_dec(pmap, 1); 375 if (m->pindex < NUL2E) { 376 /* We just released an l3, unhold the matching l2 */ 377 pd_entry_t *l1, tl1; 378 vm_page_t l2pg; 379 380 l1 = smmu_pmap_l1(pmap, va); 381 tl1 = smmu_pmap_load(l1); 382 l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); 383 smmu_pmap_unwire_l3(pmap, va, l2pg, free); 384 } else if (m->pindex < (NUL2E + NUL1E)) { 385 /* We just released an l2, unhold the matching l1 */ 386 pd_entry_t *l0, tl0; 387 vm_page_t l1pg; 388 389 l0 = smmu_pmap_l0(pmap, va); 390 tl0 = smmu_pmap_load(l0); 391 l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); 392 smmu_pmap_unwire_l3(pmap, va, l1pg, free); 393 } 394 395 /* 396 * Put page on a list so that it is released after 397 * *ALL* TLB shootdown is done 398 */ 399 smmu_pmap_add_delayed_free_list(m, free, TRUE); 400 } 401 402 int 403 smmu_pmap_pinit(struct smmu_pmap *pmap) 404 { 405 vm_page_t m; 406 407 /* 408 * allocate the l0 page 409 */ 410 m = vm_page_alloc_noobj(VM_ALLOC_WAITOK | VM_ALLOC_WIRED | 411 VM_ALLOC_ZERO); 412 pmap->sp_l0_paddr = VM_PAGE_TO_PHYS(m); 413 pmap->sp_l0 = (pd_entry_t *)PHYS_TO_DMAP(pmap->sp_l0_paddr); 414 415 #ifdef INVARIANTS 416 pmap->sp_resident_count = 0; 417 #endif 418 mtx_init(&pmap->sp_mtx, "smmu pmap", NULL, MTX_DEF); 419 420 return (1); 421 } 422 423 /* 424 * This routine is called if the desired page table page does not exist. 425 * 426 * If page table page allocation fails, this routine may sleep before 427 * returning NULL. It sleeps only if a lock pointer was given. 428 * 429 * Note: If a page allocation fails at page table level two or three, 430 * one or two pages may be held during the wait, only to be released 431 * afterwards. This conservative approach is easily argued to avoid 432 * race conditions. 433 */ 434 static vm_page_t 435 _pmap_alloc_l3(struct smmu_pmap *pmap, vm_pindex_t ptepindex) 436 { 437 vm_page_t m, l1pg, l2pg; 438 439 SMMU_PMAP_LOCK_ASSERT(pmap, MA_OWNED); 440 441 /* 442 * Allocate a page table page. 443 */ 444 if ((m = vm_page_alloc_noobj(VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { 445 /* 446 * Indicate the need to retry. While waiting, the page table 447 * page may have been allocated. 448 */ 449 return (NULL); 450 } 451 m->pindex = ptepindex; 452 453 /* 454 * Because of AArch64's weak memory consistency model, we must have a 455 * barrier here to ensure that the stores for zeroing "m", whether by 456 * pmap_zero_page() or an earlier function, are visible before adding 457 * "m" to the page table. Otherwise, a page table walk by another 458 * processor's MMU could see the mapping to "m" and a stale, non-zero 459 * PTE within "m". 460 */ 461 dmb(ishst); 462 463 /* 464 * Map the pagetable page into the process address space, if 465 * it isn't already there. 466 */ 467 468 if (ptepindex >= (NUL2E + NUL1E)) { 469 pd_entry_t *l0; 470 vm_pindex_t l0index; 471 472 l0index = ptepindex - (NUL2E + NUL1E); 473 l0 = &pmap->sp_l0[l0index]; 474 smmu_pmap_store(l0, VM_PAGE_TO_PHYS(m) | IOMMU_L0_TABLE); 475 } else if (ptepindex >= NUL2E) { 476 vm_pindex_t l0index, l1index; 477 pd_entry_t *l0, *l1; 478 pd_entry_t tl0; 479 480 l1index = ptepindex - NUL2E; 481 l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT; 482 483 l0 = &pmap->sp_l0[l0index]; 484 tl0 = smmu_pmap_load(l0); 485 if (tl0 == 0) { 486 /* recurse for allocating page dir */ 487 if (_pmap_alloc_l3(pmap, NUL2E + NUL1E + l0index) 488 == NULL) { 489 vm_page_unwire_noq(m); 490 vm_page_free_zero(m); 491 return (NULL); 492 } 493 } else { 494 l1pg = PHYS_TO_VM_PAGE(tl0 & ~ATTR_MASK); 495 l1pg->ref_count++; 496 } 497 498 l1 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l0) &~ATTR_MASK); 499 l1 = &l1[ptepindex & Ln_ADDR_MASK]; 500 smmu_pmap_store(l1, VM_PAGE_TO_PHYS(m) | IOMMU_L1_TABLE); 501 } else { 502 vm_pindex_t l0index, l1index; 503 pd_entry_t *l0, *l1, *l2; 504 pd_entry_t tl0, tl1; 505 506 l1index = ptepindex >> IOMMU_Ln_ENTRIES_SHIFT; 507 l0index = l1index >> IOMMU_L0_ENTRIES_SHIFT; 508 509 l0 = &pmap->sp_l0[l0index]; 510 tl0 = smmu_pmap_load(l0); 511 if (tl0 == 0) { 512 /* recurse for allocating page dir */ 513 if (_pmap_alloc_l3(pmap, NUL2E + l1index) == NULL) { 514 vm_page_unwire_noq(m); 515 vm_page_free_zero(m); 516 return (NULL); 517 } 518 tl0 = smmu_pmap_load(l0); 519 l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); 520 l1 = &l1[l1index & Ln_ADDR_MASK]; 521 } else { 522 l1 = (pd_entry_t *)PHYS_TO_DMAP(tl0 & ~ATTR_MASK); 523 l1 = &l1[l1index & Ln_ADDR_MASK]; 524 tl1 = smmu_pmap_load(l1); 525 if (tl1 == 0) { 526 /* recurse for allocating page dir */ 527 if (_pmap_alloc_l3(pmap, NUL2E + l1index) 528 == NULL) { 529 vm_page_unwire_noq(m); 530 vm_page_free_zero(m); 531 return (NULL); 532 } 533 } else { 534 l2pg = PHYS_TO_VM_PAGE(tl1 & ~ATTR_MASK); 535 l2pg->ref_count++; 536 } 537 } 538 539 l2 = (pd_entry_t *)PHYS_TO_DMAP(smmu_pmap_load(l1) &~ATTR_MASK); 540 l2 = &l2[ptepindex & Ln_ADDR_MASK]; 541 smmu_pmap_store(l2, VM_PAGE_TO_PHYS(m) | IOMMU_L2_TABLE); 542 } 543 544 smmu_pmap_resident_count_inc(pmap, 1); 545 546 return (m); 547 } 548 549 /*************************************************** 550 * Pmap allocation/deallocation routines. 551 ***************************************************/ 552 553 /* 554 * Release any resources held by the given physical map. 555 * Called when a pmap initialized by pmap_pinit is being released. 556 * Should only be called if the map contains no valid mappings. 557 */ 558 void 559 smmu_pmap_release(struct smmu_pmap *pmap) 560 { 561 vm_page_t m; 562 563 KASSERT(pmap->sp_resident_count == 0, 564 ("pmap_release: pmap resident count %ld != 0", 565 pmap->sp_resident_count)); 566 567 m = PHYS_TO_VM_PAGE(pmap->sp_l0_paddr); 568 vm_page_unwire_noq(m); 569 vm_page_free_zero(m); 570 mtx_destroy(&pmap->sp_mtx); 571 } 572 573 /*************************************************** 574 * page management routines. 575 ***************************************************/ 576 577 /* 578 * Add a single Mali GPU entry. This function does not sleep. 579 */ 580 int 581 pmap_gpu_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa, 582 vm_prot_t prot, u_int flags) 583 { 584 pd_entry_t *pde; 585 pt_entry_t new_l3; 586 pt_entry_t orig_l3 __diagused; 587 pt_entry_t *l3; 588 vm_page_t mpte; 589 pd_entry_t *l1p; 590 pd_entry_t *l2p; 591 int lvl; 592 int rv; 593 594 KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space")); 595 KASSERT((va & PAGE_MASK) == 0, ("va is misaligned")); 596 KASSERT((pa & PAGE_MASK) == 0, ("pa is misaligned")); 597 598 new_l3 = (pt_entry_t)(pa | ATTR_SH(ATTR_SH_IS) | IOMMU_L3_BLOCK); 599 600 if ((prot & VM_PROT_WRITE) != 0) 601 new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_WRITE); 602 if ((prot & VM_PROT_READ) != 0) 603 new_l3 |= ATTR_S2_S2AP(ATTR_S2_S2AP_READ); 604 if ((prot & VM_PROT_EXECUTE) == 0) 605 new_l3 |= ATTR_S2_XN(ATTR_S2_XN_ALL); 606 607 CTR2(KTR_PMAP, "pmap_gpu_enter: %.16lx -> %.16lx", va, pa); 608 609 SMMU_PMAP_LOCK(pmap); 610 611 /* 612 * In the case that a page table page is not 613 * resident, we are creating it here. 614 */ 615 retry: 616 pde = smmu_pmap_pde(pmap, va, &lvl); 617 if (pde != NULL && lvl == 2) { 618 l3 = smmu_pmap_l2_to_l3(pde, va); 619 } else { 620 mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va)); 621 if (mpte == NULL) { 622 CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); 623 rv = KERN_RESOURCE_SHORTAGE; 624 goto out; 625 } 626 627 /* 628 * Ensure newly created l1, l2 are visible to GPU. 629 * l0 is already visible by similar call in panfrost driver. 630 * The cache entry for l3 handled below. 631 */ 632 633 l1p = smmu_pmap_l1(pmap, va); 634 l2p = smmu_pmap_l2(pmap, va); 635 cpu_dcache_wb_range(l1p, sizeof(pd_entry_t)); 636 cpu_dcache_wb_range(l2p, sizeof(pd_entry_t)); 637 638 goto retry; 639 } 640 641 orig_l3 = smmu_pmap_load(l3); 642 KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid")); 643 644 /* New mapping */ 645 smmu_pmap_store(l3, new_l3); 646 647 cpu_dcache_wb_range(l3, sizeof(pt_entry_t)); 648 649 smmu_pmap_resident_count_inc(pmap, 1); 650 dsb(ishst); 651 652 rv = KERN_SUCCESS; 653 out: 654 SMMU_PMAP_UNLOCK(pmap); 655 656 return (rv); 657 } 658 659 /* 660 * Remove a single Mali GPU entry. 661 */ 662 int 663 pmap_gpu_remove(struct smmu_pmap *pmap, vm_offset_t va) 664 { 665 pd_entry_t *pde; 666 pt_entry_t *pte; 667 int lvl; 668 int rc; 669 670 KASSERT((va & PAGE_MASK) == 0, ("va is misaligned")); 671 672 SMMU_PMAP_LOCK(pmap); 673 674 pde = smmu_pmap_pde(pmap, va, &lvl); 675 if (pde == NULL || lvl != 2) { 676 rc = KERN_FAILURE; 677 goto out; 678 } 679 680 pte = smmu_pmap_l2_to_l3(pde, va); 681 682 smmu_pmap_resident_count_dec(pmap, 1); 683 smmu_pmap_clear(pte); 684 cpu_dcache_wb_range(pte, sizeof(pt_entry_t)); 685 rc = KERN_SUCCESS; 686 687 out: 688 SMMU_PMAP_UNLOCK(pmap); 689 690 return (rc); 691 } 692 693 /* 694 * Add a single SMMU entry. This function does not sleep. 695 */ 696 int 697 smmu_pmap_enter(struct smmu_pmap *pmap, vm_offset_t va, vm_paddr_t pa, 698 vm_prot_t prot, u_int flags) 699 { 700 pd_entry_t *pde; 701 pt_entry_t new_l3; 702 pt_entry_t orig_l3 __diagused; 703 pt_entry_t *l3; 704 vm_page_t mpte; 705 int lvl; 706 int rv; 707 708 KASSERT(va < VM_MAXUSER_ADDRESS, ("wrong address space")); 709 710 va = trunc_page(va); 711 new_l3 = (pt_entry_t)(pa | ATTR_DEFAULT | 712 ATTR_S1_IDX(VM_MEMATTR_DEVICE) | IOMMU_L3_PAGE); 713 if ((prot & VM_PROT_WRITE) == 0) 714 new_l3 |= ATTR_S1_AP(ATTR_S1_AP_RO); 715 new_l3 |= ATTR_S1_XN; /* Execute never. */ 716 new_l3 |= ATTR_S1_AP(ATTR_S1_AP_USER); 717 new_l3 |= ATTR_S1_nG; /* Non global. */ 718 719 CTR2(KTR_PMAP, "pmap_senter: %.16lx -> %.16lx", va, pa); 720 721 SMMU_PMAP_LOCK(pmap); 722 723 /* 724 * In the case that a page table page is not 725 * resident, we are creating it here. 726 */ 727 retry: 728 pde = smmu_pmap_pde(pmap, va, &lvl); 729 if (pde != NULL && lvl == 2) { 730 l3 = smmu_pmap_l2_to_l3(pde, va); 731 } else { 732 mpte = _pmap_alloc_l3(pmap, smmu_l2_pindex(va)); 733 if (mpte == NULL) { 734 CTR0(KTR_PMAP, "pmap_enter: mpte == NULL"); 735 rv = KERN_RESOURCE_SHORTAGE; 736 goto out; 737 } 738 goto retry; 739 } 740 741 orig_l3 = smmu_pmap_load(l3); 742 KASSERT(!smmu_pmap_l3_valid(orig_l3), ("l3 is valid")); 743 744 /* New mapping */ 745 smmu_pmap_store(l3, new_l3); 746 smmu_pmap_resident_count_inc(pmap, 1); 747 dsb(ishst); 748 749 rv = KERN_SUCCESS; 750 out: 751 SMMU_PMAP_UNLOCK(pmap); 752 753 return (rv); 754 } 755 756 /* 757 * Remove a single SMMU entry. 758 */ 759 int 760 smmu_pmap_remove(struct smmu_pmap *pmap, vm_offset_t va) 761 { 762 pt_entry_t *pte; 763 int lvl; 764 int rc; 765 766 SMMU_PMAP_LOCK(pmap); 767 768 pte = smmu_pmap_pte(pmap, va, &lvl); 769 KASSERT(lvl == 3, 770 ("Invalid SMMU pagetable level: %d != 3", lvl)); 771 772 if (pte != NULL) { 773 smmu_pmap_resident_count_dec(pmap, 1); 774 smmu_pmap_clear(pte); 775 rc = KERN_SUCCESS; 776 } else 777 rc = KERN_FAILURE; 778 779 SMMU_PMAP_UNLOCK(pmap); 780 781 return (rc); 782 } 783 784 /* 785 * Remove all the allocated L1, L2 pages from SMMU pmap. 786 * All the L3 entires must be cleared in advance, otherwise 787 * this function panics. 788 */ 789 void 790 smmu_pmap_remove_pages(struct smmu_pmap *pmap) 791 { 792 pd_entry_t l0e, *l1, l1e, *l2, l2e; 793 pt_entry_t *l3, l3e; 794 vm_page_t m, m0, m1; 795 vm_paddr_t pa; 796 vm_paddr_t pa0; 797 vm_paddr_t pa1; 798 int i, j, k, l; 799 800 SMMU_PMAP_LOCK(pmap); 801 802 for (i = 0; i < IOMMU_L0_ENTRIES; i++) { 803 l0e = pmap->sp_l0[i]; 804 if ((l0e & ATTR_DESCR_VALID) == 0) { 805 continue; 806 } 807 pa0 = l0e & ~ATTR_MASK; 808 m0 = PHYS_TO_VM_PAGE(pa0); 809 l1 = (pd_entry_t *)PHYS_TO_DMAP(pa0); 810 811 for (j = 0; j < IOMMU_Ln_ENTRIES; j++) { 812 l1e = l1[j]; 813 if ((l1e & ATTR_DESCR_VALID) == 0) { 814 continue; 815 } 816 if ((l1e & ATTR_DESCR_MASK) == IOMMU_L1_BLOCK) { 817 continue; 818 } 819 pa1 = l1e & ~ATTR_MASK; 820 m1 = PHYS_TO_VM_PAGE(pa1); 821 l2 = (pd_entry_t *)PHYS_TO_DMAP(pa1); 822 823 for (k = 0; k < IOMMU_Ln_ENTRIES; k++) { 824 l2e = l2[k]; 825 if ((l2e & ATTR_DESCR_VALID) == 0) { 826 continue; 827 } 828 pa = l2e & ~ATTR_MASK; 829 m = PHYS_TO_VM_PAGE(pa); 830 l3 = (pt_entry_t *)PHYS_TO_DMAP(pa); 831 832 for (l = 0; l < IOMMU_Ln_ENTRIES; l++) { 833 l3e = l3[l]; 834 if ((l3e & ATTR_DESCR_VALID) == 0) 835 continue; 836 panic( 837 "%s: l3e found (indexes %d %d %d %d)", 838 __func__, i, j, k, l); 839 } 840 841 vm_page_unwire_noq(m1); 842 vm_page_unwire_noq(m); 843 smmu_pmap_resident_count_dec(pmap, 1); 844 vm_page_free(m); 845 smmu_pmap_clear(&l2[k]); 846 } 847 848 vm_page_unwire_noq(m0); 849 smmu_pmap_resident_count_dec(pmap, 1); 850 vm_page_free(m1); 851 smmu_pmap_clear(&l1[j]); 852 } 853 854 smmu_pmap_resident_count_dec(pmap, 1); 855 vm_page_free(m0); 856 smmu_pmap_clear(&pmap->sp_l0[i]); 857 } 858 859 KASSERT(pmap->sp_resident_count == 0, 860 ("Invalid resident count %jd", pmap->sp_resident_count)); 861 862 SMMU_PMAP_UNLOCK(pmap); 863 } 864