1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2023 Chelsio Communications, Inc. 5 * Written by: John Baldwin <jhb@FreeBSD.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/mbuf.h> 31 #include <sys/memdesc.h> 32 #include <sys/systm.h> 33 #include <sys/uio.h> 34 #include <vm/vm.h> 35 #include <vm/pmap.h> 36 #include <vm/vm_param.h> 37 #include <machine/bus.h> 38 39 static void 40 phys_copyback(vm_paddr_t pa, int off, int size, const void *src) 41 { 42 const char *cp; 43 u_int page_off; 44 int todo; 45 void *p; 46 47 KASSERT(PMAP_HAS_DMAP, ("direct-map required")); 48 49 cp = src; 50 pa += off; 51 page_off = pa & PAGE_MASK; 52 while (size > 0) { 53 todo = min(PAGE_SIZE - page_off, size); 54 p = (void *)PHYS_TO_DMAP(pa); 55 memcpy(p, cp, todo); 56 size -= todo; 57 cp += todo; 58 pa += todo; 59 page_off = 0; 60 } 61 } 62 63 static void 64 vlist_copyback(struct bus_dma_segment *vlist, int sglist_cnt, int off, 65 int size, const void *src) 66 { 67 const char *p; 68 int todo; 69 70 while (vlist->ds_len <= off) { 71 KASSERT(sglist_cnt > 1, ("out of sglist entries")); 72 73 off -= vlist->ds_len; 74 vlist++; 75 sglist_cnt--; 76 } 77 78 p = src; 79 while (size > 0) { 80 KASSERT(sglist_cnt >= 1, ("out of sglist entries")); 81 82 todo = size; 83 if (todo > vlist->ds_len - off) 84 todo = vlist->ds_len - off; 85 86 memcpy((char *)(uintptr_t)vlist->ds_addr + off, p, todo); 87 off = 0; 88 vlist++; 89 sglist_cnt--; 90 size -= todo; 91 p += todo; 92 } 93 } 94 95 static void 96 plist_copyback(struct bus_dma_segment *plist, int sglist_cnt, int off, 97 int size, const void *src) 98 { 99 const char *p; 100 int todo; 101 102 while (plist->ds_len <= off) { 103 KASSERT(sglist_cnt > 1, ("out of sglist entries")); 104 105 off -= plist->ds_len; 106 plist++; 107 sglist_cnt--; 108 } 109 110 p = src; 111 while (size > 0) { 112 KASSERT(sglist_cnt >= 1, ("out of sglist entries")); 113 114 todo = size; 115 if (todo > plist->ds_len - off) 116 todo = plist->ds_len - off; 117 118 phys_copyback(plist->ds_addr, off, todo, p); 119 off = 0; 120 plist++; 121 sglist_cnt--; 122 size -= todo; 123 p += todo; 124 } 125 } 126 127 static void 128 vmpages_copyback(vm_page_t *m, int off, int size, const void *src) 129 { 130 struct iovec iov[1]; 131 struct uio uio; 132 int error __diagused; 133 134 iov[0].iov_base = __DECONST(void *, src); 135 iov[0].iov_len = size; 136 uio.uio_iov = iov; 137 uio.uio_iovcnt = 1; 138 uio.uio_offset = 0; 139 uio.uio_resid = size; 140 uio.uio_segflg = UIO_SYSSPACE; 141 uio.uio_rw = UIO_WRITE; 142 error = uiomove_fromphys(m, off, size, &uio); 143 KASSERT(error == 0 && uio.uio_resid == 0, ("copy failed")); 144 } 145 146 void 147 memdesc_copyback(struct memdesc *mem, int off, int size, const void *src) 148 { 149 KASSERT(off >= 0, ("%s: invalid offset %d", __func__, off)); 150 KASSERT(size >= 0, ("%s: invalid size %d", __func__, off)); 151 152 switch (mem->md_type) { 153 case MEMDESC_VADDR: 154 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); 155 memcpy((char *)mem->u.md_vaddr + off, src, size); 156 break; 157 case MEMDESC_PADDR: 158 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); 159 phys_copyback(mem->u.md_paddr, off, size, src); 160 break; 161 case MEMDESC_VLIST: 162 vlist_copyback(mem->u.md_list, mem->md_nseg, off, size, src); 163 break; 164 case MEMDESC_PLIST: 165 plist_copyback(mem->u.md_list, mem->md_nseg, off, size, src); 166 break; 167 case MEMDESC_UIO: 168 panic("Use uiomove instead"); 169 break; 170 case MEMDESC_MBUF: 171 m_copyback(mem->u.md_mbuf, off, size, src); 172 break; 173 case MEMDESC_VMPAGES: 174 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); 175 vmpages_copyback(mem->u.md_ma, mem->md_offset + off, size, 176 src); 177 break; 178 default: 179 __assert_unreachable(); 180 } 181 } 182 183 static void 184 phys_copydata(vm_paddr_t pa, int off, int size, void *dst) 185 { 186 char *cp; 187 u_int page_off; 188 int todo; 189 const void *p; 190 191 KASSERT(PMAP_HAS_DMAP, ("direct-map required")); 192 193 cp = dst; 194 pa += off; 195 page_off = pa & PAGE_MASK; 196 while (size > 0) { 197 todo = min(PAGE_SIZE - page_off, size); 198 p = (const void *)PHYS_TO_DMAP(pa); 199 memcpy(cp, p, todo); 200 size -= todo; 201 cp += todo; 202 pa += todo; 203 page_off = 0; 204 } 205 } 206 207 static void 208 vlist_copydata(struct bus_dma_segment *vlist, int sglist_cnt, int off, 209 int size, void *dst) 210 { 211 char *p; 212 int todo; 213 214 while (vlist->ds_len <= off) { 215 KASSERT(sglist_cnt > 1, ("out of sglist entries")); 216 217 off -= vlist->ds_len; 218 vlist++; 219 sglist_cnt--; 220 } 221 222 p = dst; 223 while (size > 0) { 224 KASSERT(sglist_cnt >= 1, ("out of sglist entries")); 225 226 todo = size; 227 if (todo > vlist->ds_len - off) 228 todo = vlist->ds_len - off; 229 230 memcpy(p, (char *)(uintptr_t)vlist->ds_addr + off, todo); 231 off = 0; 232 vlist++; 233 sglist_cnt--; 234 size -= todo; 235 p += todo; 236 } 237 } 238 239 static void 240 plist_copydata(struct bus_dma_segment *plist, int sglist_cnt, int off, 241 int size, void *dst) 242 { 243 char *p; 244 int todo; 245 246 while (plist->ds_len <= off) { 247 KASSERT(sglist_cnt > 1, ("out of sglist entries")); 248 249 off -= plist->ds_len; 250 plist++; 251 sglist_cnt--; 252 } 253 254 p = dst; 255 while (size > 0) { 256 KASSERT(sglist_cnt >= 1, ("out of sglist entries")); 257 258 todo = size; 259 if (todo > plist->ds_len - off) 260 todo = plist->ds_len - off; 261 262 phys_copydata(plist->ds_addr, off, todo, p); 263 off = 0; 264 plist++; 265 sglist_cnt--; 266 size -= todo; 267 p += todo; 268 } 269 } 270 271 static void 272 vmpages_copydata(vm_page_t *m, int off, int size, void *dst) 273 { 274 struct iovec iov[1]; 275 struct uio uio; 276 int error __diagused; 277 278 iov[0].iov_base = dst; 279 iov[0].iov_len = size; 280 uio.uio_iov = iov; 281 uio.uio_iovcnt = 1; 282 uio.uio_offset = 0; 283 uio.uio_resid = size; 284 uio.uio_segflg = UIO_SYSSPACE; 285 uio.uio_rw = UIO_READ; 286 error = uiomove_fromphys(m, off, size, &uio); 287 KASSERT(error == 0 && uio.uio_resid == 0, ("copy failed")); 288 } 289 290 void 291 memdesc_copydata(struct memdesc *mem, int off, int size, void *dst) 292 { 293 KASSERT(off >= 0, ("%s: invalid offset %d", __func__, off)); 294 KASSERT(size >= 0, ("%s: invalid size %d", __func__, off)); 295 296 switch (mem->md_type) { 297 case MEMDESC_VADDR: 298 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); 299 memcpy(dst, (const char *)mem->u.md_vaddr + off, size); 300 break; 301 case MEMDESC_PADDR: 302 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); 303 phys_copydata(mem->u.md_paddr, off, size, dst); 304 break; 305 case MEMDESC_VLIST: 306 vlist_copydata(mem->u.md_list, mem->md_nseg, off, size, dst); 307 break; 308 case MEMDESC_PLIST: 309 plist_copydata(mem->u.md_list, mem->md_nseg, off, size, dst); 310 break; 311 case MEMDESC_UIO: 312 panic("Use uiomove instead"); 313 break; 314 case MEMDESC_MBUF: 315 m_copydata(mem->u.md_mbuf, off, size, dst); 316 break; 317 case MEMDESC_VMPAGES: 318 KASSERT(off + size <= mem->md_len, ("copy out of bounds")); 319 vmpages_copydata(mem->u.md_ma, mem->md_offset + off, size, 320 dst); 321 break; 322 default: 323 __assert_unreachable(); 324 } 325 } 326