1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 * 26 * Copyright 2018 Joyent, Inc. 27 */ 28 29 #include <sys/debug.h> 30 #include <sys/types.h> 31 #include <sys/param.h> 32 #include <sys/time.h> 33 #include <sys/buf.h> 34 #include <sys/errno.h> 35 #include <sys/systm.h> 36 #include <sys/conf.h> 37 #include <sys/signal.h> 38 #include <sys/file.h> 39 #include <sys/uio.h> 40 #include <sys/ioctl.h> 41 #include <sys/map.h> 42 #include <sys/proc.h> 43 #include <sys/user.h> 44 #include <sys/mman.h> 45 #include <sys/cred.h> 46 #include <sys/open.h> 47 #include <sys/stat.h> 48 #include <sys/utsname.h> 49 #include <sys/kmem.h> 50 #include <sys/cmn_err.h> 51 #include <sys/vnode.h> 52 #include <vm/page.h> 53 #include <vm/as.h> 54 #include <vm/hat.h> 55 #include <vm/seg.h> 56 #include <vm/seg_kmem.h> 57 #include <vm/hat_i86.h> 58 #include <sys/vmsystm.h> 59 #include <sys/ddi.h> 60 #include <sys/devops.h> 61 #include <sys/sunddi.h> 62 #include <sys/ddi_impldefs.h> 63 #include <sys/fs/snode.h> 64 #include <sys/pci.h> 65 #include <sys/modctl.h> 66 #include <sys/uio.h> 67 #include <sys/visual_io.h> 68 #include <sys/fbio.h> 69 #include <sys/ddidmareq.h> 70 #include <sys/tnf_probe.h> 71 #include <sys/kstat.h> 72 #include <sys/callb.h> 73 #include <sys/promif.h> 74 #include <sys/atomic.h> 75 #include <sys/gfx_private.h> 76 77 #ifdef __xpv 78 #include <sys/hypervisor.h> 79 #endif 80 81 /* 82 * Create a kva mapping for a pa (start..start+size) with 83 * the specified cache attributes (mode). 84 */ 85 gfxp_kva_t 86 gfxp_map_kernel_space(uint64_t start, size_t size, uint32_t mode) 87 { 88 uint_t pgoffset; 89 uint64_t base; 90 pgcnt_t npages; 91 caddr_t cvaddr; 92 int hat_flags; 93 uint_t hat_attr; 94 pfn_t pfn; 95 96 if (size == 0) 97 return (0); 98 99 #ifdef __xpv 100 /* 101 * The hypervisor doesn't allow r/w mappings to some pages, such as 102 * page tables, gdt, etc. Detect %cr3 to notify users of this interface. 103 */ 104 if (start == mmu_ptob(mmu_btop(getcr3_pa()))) 105 return (0); 106 #endif 107 108 if (mode == GFXP_MEMORY_CACHED) 109 hat_attr = HAT_STORECACHING_OK; 110 else if (mode == GFXP_MEMORY_WRITECOMBINED) 111 hat_attr = HAT_MERGING_OK | HAT_PLAT_NOCACHE; 112 else /* GFXP_MEMORY_UNCACHED */ 113 hat_attr = HAT_STRICTORDER | HAT_PLAT_NOCACHE; 114 hat_flags = HAT_LOAD_LOCK; 115 pgoffset = start & PAGEOFFSET; 116 base = start - pgoffset; 117 npages = btopr(size + pgoffset); 118 cvaddr = vmem_alloc(heap_arena, ptob(npages), VM_NOSLEEP); 119 if (cvaddr == NULL) 120 return (NULL); 121 122 #ifdef __xpv 123 ASSERT(DOMAIN_IS_INITDOMAIN(xen_info)); 124 pfn = xen_assign_pfn(mmu_btop(base)); 125 #else 126 pfn = btop(base); 127 #endif 128 129 hat_devload(kas.a_hat, cvaddr, ptob(npages), pfn, 130 PROT_READ|PROT_WRITE|hat_attr, hat_flags); 131 return (cvaddr + pgoffset); 132 } 133 134 /* 135 * Destroy the mapping created by gfxp_map_kernel_space(). 136 * Physical memory is not reclaimed. 137 */ 138 void 139 gfxp_unmap_kernel_space(gfxp_kva_t address, size_t size) 140 { 141 uint_t pgoffset; 142 caddr_t base; 143 pgcnt_t npages; 144 145 if (size == 0 || address == NULL) 146 return; 147 148 pgoffset = (uintptr_t)address & PAGEOFFSET; 149 base = (caddr_t)address - pgoffset; 150 npages = btopr(size + pgoffset); 151 hat_unload(kas.a_hat, base, ptob(npages), HAT_UNLOAD_UNLOCK); 152 vmem_free(heap_arena, base, ptob(npages)); 153 } 154 155 /* 156 * For a VA return the pfn 157 */ 158 int 159 gfxp_va2pa(struct as *as, caddr_t addr, uint64_t *pa) 160 { 161 #ifdef __xpv 162 ASSERT(DOMAIN_IS_INITDOMAIN(xen_info)); 163 *pa = pa_to_ma(pfn_to_pa(hat_getpfnum(as->a_hat, addr))); 164 #else 165 *pa = pfn_to_pa(hat_getpfnum(as->a_hat, addr)); 166 #endif 167 return (0); 168 } 169 170 /* 171 * NOP now 172 */ 173 /* ARGSUSED */ 174 void 175 gfxp_fix_mem_cache_attrs(caddr_t kva_start, size_t length, int cache_attr) 176 { 177 } 178 179 int 180 gfxp_ddi_dma_mem_alloc(ddi_dma_handle_t handle, size_t length, 181 ddi_device_acc_attr_t *accattrp, uint_t flags, int (*waitfp) (caddr_t), 182 caddr_t arg, caddr_t *kaddrp, size_t *real_length, 183 ddi_acc_handle_t *handlep) 184 { 185 uint_t l_flags = flags & ~IOMEM_DATA_MASK; /* clear cache attrs */ 186 int e; 187 188 /* 189 * Set an appropriate attribute from devacc_attr_dataorder 190 * to keep compatibility. The cache attributes are igonred 191 * if specified. 192 */ 193 if (accattrp != NULL) { 194 if (accattrp->devacc_attr_dataorder == DDI_STRICTORDER_ACC) { 195 l_flags |= IOMEM_DATA_UNCACHED; 196 } else if (accattrp->devacc_attr_dataorder == 197 DDI_MERGING_OK_ACC) { 198 l_flags |= IOMEM_DATA_UC_WR_COMBINE; 199 } else { 200 l_flags |= IOMEM_DATA_CACHED; 201 } 202 } 203 204 e = ddi_dma_mem_alloc(handle, length, accattrp, l_flags, waitfp, 205 arg, kaddrp, real_length, handlep); 206 return (e); 207 } 208 209 int 210 gfxp_mlock_user_memory(caddr_t address, size_t length) 211 { 212 struct as *as = ttoproc(curthread)->p_as; 213 int error = 0; 214 215 if (((uintptr_t)address & PAGEOFFSET) != 0 || length == 0) 216 return (set_errno(EINVAL)); 217 218 if (valid_usr_range(address, length, 0, as, as->a_userlimit) != 219 RANGE_OKAY) 220 return (set_errno(ENOMEM)); 221 222 error = as_ctl(as, address, length, MC_LOCK, 0, 0, NULL, 0); 223 if (error) 224 (void) set_errno(error); 225 226 return (error); 227 } 228 229 int 230 gfxp_munlock_user_memory(caddr_t address, size_t length) 231 { 232 struct as *as = ttoproc(curthread)->p_as; 233 int error = 0; 234 235 if (((uintptr_t)address & PAGEOFFSET) != 0 || length == 0) 236 return (set_errno(EINVAL)); 237 238 if (valid_usr_range(address, length, 0, as, as->a_userlimit) != 239 RANGE_OKAY) 240 return (set_errno(ENOMEM)); 241 242 error = as_ctl(as, address, length, MC_UNLOCK, 0, 0, NULL, 0); 243 if (error) 244 (void) set_errno(error); 245 246 return (error); 247 } 248 249 gfx_maddr_t 250 gfxp_convert_addr(paddr_t paddr) 251 { 252 #ifdef __xpv 253 ASSERT(DOMAIN_IS_INITDOMAIN(xen_info)); 254 return (pfn_to_pa(xen_assign_pfn(btop(paddr)))); 255 #else 256 return ((gfx_maddr_t)paddr); 257 #endif 258 } 259 260 /* 261 * Support getting VA space separately from pages 262 */ 263 264 /* 265 * A little like gfxp_map_kernel_space, but 266 * just the vmem_alloc part. 267 */ 268 caddr_t 269 gfxp_alloc_kernel_space(size_t size) 270 { 271 caddr_t cvaddr; 272 pgcnt_t npages; 273 274 npages = btopr(size); 275 cvaddr = vmem_alloc(heap_arena, ptob(npages), VM_NOSLEEP); 276 return (cvaddr); 277 } 278 279 /* 280 * Like gfxp_unmap_kernel_space, but 281 * just the vmem_free part. 282 */ 283 void 284 gfxp_free_kernel_space(caddr_t address, size_t size) 285 { 286 287 uint_t pgoffset; 288 caddr_t base; 289 pgcnt_t npages; 290 291 if (size == 0 || address == NULL) 292 return; 293 294 pgoffset = (uintptr_t)address & PAGEOFFSET; 295 base = (caddr_t)address - pgoffset; 296 npages = btopr(size + pgoffset); 297 vmem_free(heap_arena, base, ptob(npages)); 298 } 299 300 /* 301 * Like gfxp_map_kernel_space, but 302 * just the hat_devload part. 303 */ 304 void 305 gfxp_load_kernel_space(uint64_t start, size_t size, 306 uint32_t mode, caddr_t cvaddr) 307 { 308 uint_t pgoffset; 309 uint64_t base; 310 pgcnt_t npages; 311 int hat_flags; 312 uint_t hat_attr; 313 pfn_t pfn; 314 315 if (size == 0) 316 return; 317 318 #ifdef __xpv 319 /* 320 * The hypervisor doesn't allow r/w mappings to some pages, such as 321 * page tables, gdt, etc. Detect %cr3 to notify users of this interface. 322 */ 323 if (start == mmu_ptob(mmu_btop(getcr3_pa()))) 324 return; 325 #endif 326 327 if (mode == GFXP_MEMORY_CACHED) 328 hat_attr = HAT_STORECACHING_OK; 329 else if (mode == GFXP_MEMORY_WRITECOMBINED) 330 hat_attr = HAT_MERGING_OK | HAT_PLAT_NOCACHE; 331 else /* GFXP_MEMORY_UNCACHED */ 332 hat_attr = HAT_STRICTORDER | HAT_PLAT_NOCACHE; 333 hat_flags = HAT_LOAD_LOCK; 334 335 pgoffset = start & PAGEOFFSET; 336 base = start - pgoffset; 337 npages = btopr(size + pgoffset); 338 339 #ifdef __xpv 340 ASSERT(DOMAIN_IS_INITDOMAIN(xen_info)); 341 pfn = xen_assign_pfn(mmu_btop(base)); 342 #else 343 pfn = btop(base); 344 #endif 345 346 hat_devload(kas.a_hat, cvaddr, ptob(npages), pfn, 347 PROT_READ|PROT_WRITE|hat_attr, hat_flags); 348 } 349 350 /* 351 * Like gfxp_unmap_kernel_space, but 352 * just the had_unload part. 353 */ 354 void 355 gfxp_unload_kernel_space(caddr_t address, size_t size) 356 { 357 uint_t pgoffset; 358 caddr_t base; 359 pgcnt_t npages; 360 361 if (size == 0 || address == NULL) 362 return; 363 364 pgoffset = (uintptr_t)address & PAGEOFFSET; 365 base = (caddr_t)address - pgoffset; 366 npages = btopr(size + pgoffset); 367 hat_unload(kas.a_hat, base, ptob(npages), HAT_UNLOAD_UNLOCK); 368 } 369 370 /* 371 * Note that "mempool" is optional and normally disabled in drm_gem.c 372 * (see HAS_MEM_POOL). Let's just stub these out so we can reduce 373 * changes from the upstream in the DRM driver code. 374 */ 375 376 void 377 gfxp_mempool_init(void) 378 { 379 } 380 381 void 382 gfxp_mempool_destroy(void) 383 { 384 } 385 386 /* ARGSUSED */ 387 int 388 gfxp_alloc_from_mempool(struct gfxp_pmem_cookie *cookie, caddr_t *kva, 389 pfn_t *pgarray, pgcnt_t alen, int flags) 390 { 391 return (-1); 392 } 393 394 /* ARGSUSED */ 395 void 396 gfxp_free_mempool(struct gfxp_pmem_cookie *cookie, caddr_t kva, size_t len) 397 { 398 } 399