1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2013, 2014 The FreeBSD Foundation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/systm.h> 32 #include <sys/lock.h> 33 #include <sys/memdesc.h> 34 #include <sys/mutex.h> 35 #include <sys/sf_buf.h> 36 #include <sys/sysctl.h> 37 #include <sys/proc.h> 38 #include <sys/sched.h> 39 #include <sys/rwlock.h> 40 #include <sys/taskqueue.h> 41 #include <sys/tree.h> 42 #include <vm/vm.h> 43 #include <vm/vm_page.h> 44 #include <vm/vm_object.h> 45 #include <dev/pci/pcireg.h> 46 #include <machine/atomic.h> 47 #include <machine/bus.h> 48 #include <machine/cpu.h> 49 #include <x86/include/busdma_impl.h> 50 #include <dev/iommu/busdma_iommu.h> 51 #include <dev/iommu/iommu.h> 52 #include <x86/iommu/x86_iommu.h> 53 54 vm_page_t 55 iommu_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags) 56 { 57 vm_page_t m; 58 int zeroed, aflags; 59 60 zeroed = (flags & IOMMU_PGF_ZERO) != 0 ? VM_ALLOC_ZERO : 0; 61 aflags = zeroed | VM_ALLOC_NOBUSY | VM_ALLOC_SYSTEM | VM_ALLOC_NODUMP | 62 ((flags & IOMMU_PGF_WAITOK) != 0 ? VM_ALLOC_WAITFAIL : 63 VM_ALLOC_NOWAIT); 64 for (;;) { 65 if ((flags & IOMMU_PGF_OBJL) == 0) 66 VM_OBJECT_WLOCK(obj); 67 m = vm_page_lookup(obj, idx); 68 if ((flags & IOMMU_PGF_NOALLOC) != 0 || m != NULL) { 69 if ((flags & IOMMU_PGF_OBJL) == 0) 70 VM_OBJECT_WUNLOCK(obj); 71 break; 72 } 73 m = vm_page_alloc_contig(obj, idx, aflags, 1, 0, 74 iommu_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); 75 if ((flags & IOMMU_PGF_OBJL) == 0) 76 VM_OBJECT_WUNLOCK(obj); 77 if (m != NULL) { 78 if (zeroed && (m->flags & PG_ZERO) == 0) 79 pmap_zero_page(m); 80 atomic_add_int(&iommu_tbl_pagecnt, 1); 81 break; 82 } 83 if ((flags & IOMMU_PGF_WAITOK) == 0) 84 break; 85 } 86 return (m); 87 } 88 89 void 90 iommu_pgfree(vm_object_t obj, vm_pindex_t idx, int flags) 91 { 92 vm_page_t m; 93 94 if ((flags & IOMMU_PGF_OBJL) == 0) 95 VM_OBJECT_WLOCK(obj); 96 m = vm_page_grab(obj, idx, VM_ALLOC_NOCREAT); 97 if (m != NULL) { 98 vm_page_free(m); 99 atomic_subtract_int(&iommu_tbl_pagecnt, 1); 100 } 101 if ((flags & IOMMU_PGF_OBJL) == 0) 102 VM_OBJECT_WUNLOCK(obj); 103 } 104 105 void * 106 iommu_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags, 107 struct sf_buf **sf) 108 { 109 vm_page_t m; 110 bool allocated; 111 112 if ((flags & IOMMU_PGF_OBJL) == 0) 113 VM_OBJECT_WLOCK(obj); 114 m = vm_page_lookup(obj, idx); 115 if (m == NULL && (flags & IOMMU_PGF_ALLOC) != 0) { 116 m = iommu_pgalloc(obj, idx, flags | IOMMU_PGF_OBJL); 117 allocated = true; 118 } else 119 allocated = false; 120 if (m == NULL) { 121 if ((flags & IOMMU_PGF_OBJL) == 0) 122 VM_OBJECT_WUNLOCK(obj); 123 return (NULL); 124 } 125 /* Sleepable allocations cannot fail. */ 126 if ((flags & IOMMU_PGF_WAITOK) != 0) 127 VM_OBJECT_WUNLOCK(obj); 128 sched_pin(); 129 *sf = sf_buf_alloc(m, SFB_CPUPRIVATE | ((flags & IOMMU_PGF_WAITOK) 130 == 0 ? SFB_NOWAIT : 0)); 131 if (*sf == NULL) { 132 sched_unpin(); 133 if (allocated) { 134 VM_OBJECT_ASSERT_WLOCKED(obj); 135 iommu_pgfree(obj, m->pindex, flags | IOMMU_PGF_OBJL); 136 } 137 if ((flags & IOMMU_PGF_OBJL) == 0) 138 VM_OBJECT_WUNLOCK(obj); 139 return (NULL); 140 } 141 if ((flags & (IOMMU_PGF_WAITOK | IOMMU_PGF_OBJL)) == 142 (IOMMU_PGF_WAITOK | IOMMU_PGF_OBJL)) 143 VM_OBJECT_WLOCK(obj); 144 else if ((flags & (IOMMU_PGF_WAITOK | IOMMU_PGF_OBJL)) == 0) 145 VM_OBJECT_WUNLOCK(obj); 146 return ((void *)sf_buf_kva(*sf)); 147 } 148 149 void 150 iommu_unmap_pgtbl(struct sf_buf *sf) 151 { 152 153 sf_buf_free(sf); 154 sched_unpin(); 155 } 156 157 iommu_haddr_t iommu_high; 158 int iommu_tbl_pagecnt; 159 160 SYSCTL_NODE(_hw_iommu, OID_AUTO, dmar, CTLFLAG_RD | CTLFLAG_MPSAFE, 161 NULL, ""); 162 SYSCTL_INT(_hw_iommu_dmar, OID_AUTO, tbl_pagecnt, CTLFLAG_RD, 163 &iommu_tbl_pagecnt, 0, 164 "Count of pages used for DMAR pagetables"); 165