1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31 #define pr_fmt(fmt) "[TTM] " fmt 32 33 #include <ttm/ttm_module.h> 34 #include <ttm/ttm_bo_driver.h> 35 #include <ttm/ttm_placement.h> 36 #include <drm/drm_vma_manager.h> 37 #include <linux/mm.h> 38 #include <linux/rbtree.h> 39 #include <linux/module.h> 40 #include <linux/uaccess.h> 41 42 #define TTM_BO_VM_NUM_PREFAULT 16 43 44 static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, 45 struct vm_area_struct *vma, 46 struct vm_fault *vmf) 47 { 48 int ret = 0; 49 50 if (likely(!test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags))) 51 goto out_unlock; 52 53 /* 54 * Quick non-stalling check for idle. 55 */ 56 ret = ttm_bo_wait(bo, false, false, true); 57 if (likely(ret == 0)) 58 goto out_unlock; 59 60 /* 61 * If possible, avoid waiting for GPU with mmap_sem 62 * held. 63 */ 64 if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { 65 ret = VM_FAULT_RETRY; 66 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 67 goto out_unlock; 68 69 up_read(&vma->vm_mm->mmap_sem); 70 (void) ttm_bo_wait(bo, false, true, false); 71 goto out_unlock; 72 } 73 74 /* 75 * Ordinary wait. 76 */ 77 ret = ttm_bo_wait(bo, false, true, false); 78 if (unlikely(ret != 0)) 79 ret = (ret != -ERESTARTSYS) ? VM_FAULT_SIGBUS : 80 VM_FAULT_NOPAGE; 81 82 out_unlock: 83 return ret; 84 } 85 86 static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 87 { 88 struct ttm_buffer_object *bo = (struct ttm_buffer_object *) 89 vma->vm_private_data; 90 struct ttm_bo_device *bdev = bo->bdev; 91 unsigned long page_offset; 92 unsigned long page_last; 93 unsigned long pfn; 94 struct ttm_tt *ttm = NULL; 95 struct page *page; 96 int ret; 97 int i; 98 unsigned long address = (unsigned long)vmf->virtual_address; 99 int retval = VM_FAULT_NOPAGE; 100 struct ttm_mem_type_manager *man = 101 &bdev->man[bo->mem.mem_type]; 102 struct vm_area_struct cvma; 103 104 /* 105 * Work around locking order reversal in fault / nopfn 106 * between mmap_sem and bo_reserve: Perform a trylock operation 107 * for reserve, and if it fails, retry the fault after waiting 108 * for the buffer to become unreserved. 109 */ 110 ret = ttm_bo_reserve(bo, true, true, false, NULL); 111 if (unlikely(ret != 0)) { 112 if (ret != -EBUSY) 113 return VM_FAULT_NOPAGE; 114 115 if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) { 116 if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { 117 up_read(&vma->vm_mm->mmap_sem); 118 (void) ttm_bo_wait_unreserved(bo); 119 } 120 121 return VM_FAULT_RETRY; 122 } 123 124 /* 125 * If we'd want to change locking order to 126 * mmap_sem -> bo::reserve, we'd use a blocking reserve here 127 * instead of retrying the fault... 128 */ 129 return VM_FAULT_NOPAGE; 130 } 131 132 /* 133 * Refuse to fault imported pages. This should be handled 134 * (if at all) by redirecting mmap to the exporter. 135 */ 136 if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { 137 retval = VM_FAULT_SIGBUS; 138 goto out_unlock; 139 } 140 141 if (bdev->driver->fault_reserve_notify) { 142 ret = bdev->driver->fault_reserve_notify(bo); 143 switch (ret) { 144 case 0: 145 break; 146 case -EBUSY: 147 case -ERESTARTSYS: 148 retval = VM_FAULT_NOPAGE; 149 goto out_unlock; 150 default: 151 retval = VM_FAULT_SIGBUS; 152 goto out_unlock; 153 } 154 } 155 156 /* 157 * Wait for buffer data in transit, due to a pipelined 158 * move. 159 */ 160 ret = ttm_bo_vm_fault_idle(bo, vma, vmf); 161 if (unlikely(ret != 0)) { 162 retval = ret; 163 goto out_unlock; 164 } 165 166 ret = ttm_mem_io_lock(man, true); 167 if (unlikely(ret != 0)) { 168 retval = VM_FAULT_NOPAGE; 169 goto out_unlock; 170 } 171 ret = ttm_mem_io_reserve_vm(bo); 172 if (unlikely(ret != 0)) { 173 retval = VM_FAULT_SIGBUS; 174 goto out_io_unlock; 175 } 176 177 page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + 178 vma->vm_pgoff - drm_vma_node_start(&bo->vma_node); 179 page_last = vma_pages(vma) + vma->vm_pgoff - 180 drm_vma_node_start(&bo->vma_node); 181 182 if (unlikely(page_offset >= bo->num_pages)) { 183 retval = VM_FAULT_SIGBUS; 184 goto out_io_unlock; 185 } 186 187 /* 188 * Make a local vma copy to modify the page_prot member 189 * and vm_flags if necessary. The vma parameter is protected 190 * by mmap_sem in write mode. 191 */ 192 cvma = *vma; 193 cvma.vm_page_prot = vm_get_page_prot(cvma.vm_flags); 194 195 if (bo->mem.bus.is_iomem) { 196 cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, 197 cvma.vm_page_prot); 198 } else { 199 ttm = bo->ttm; 200 cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, 201 cvma.vm_page_prot); 202 203 /* Allocate all page at once, most common usage */ 204 if (ttm->bdev->driver->ttm_tt_populate(ttm)) { 205 retval = VM_FAULT_OOM; 206 goto out_io_unlock; 207 } 208 } 209 210 /* 211 * Speculatively prefault a number of pages. Only error on 212 * first page. 213 */ 214 for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { 215 if (bo->mem.bus.is_iomem) 216 pfn = ((bo->mem.bus.base + bo->mem.bus.offset) >> PAGE_SHIFT) + page_offset; 217 else { 218 page = ttm->pages[page_offset]; 219 if (unlikely(!page && i == 0)) { 220 retval = VM_FAULT_OOM; 221 goto out_io_unlock; 222 } else if (unlikely(!page)) { 223 break; 224 } 225 page->mapping = vma->vm_file->f_mapping; 226 page->index = drm_vma_node_start(&bo->vma_node) + 227 page_offset; 228 pfn = page_to_pfn(page); 229 } 230 231 if (vma->vm_flags & VM_MIXEDMAP) 232 ret = vm_insert_mixed(&cvma, address, pfn); 233 else 234 ret = vm_insert_pfn(&cvma, address, pfn); 235 236 /* 237 * Somebody beat us to this PTE or prefaulting to 238 * an already populated PTE, or prefaulting error. 239 */ 240 241 if (unlikely((ret == -EBUSY) || (ret != 0 && i > 0))) 242 break; 243 else if (unlikely(ret != 0)) { 244 retval = 245 (ret == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS; 246 goto out_io_unlock; 247 } 248 249 address += PAGE_SIZE; 250 if (unlikely(++page_offset >= page_last)) 251 break; 252 } 253 out_io_unlock: 254 ttm_mem_io_unlock(man); 255 out_unlock: 256 ttm_bo_unreserve(bo); 257 return retval; 258 } 259 260 static void ttm_bo_vm_open(struct vm_area_struct *vma) 261 { 262 struct ttm_buffer_object *bo = 263 (struct ttm_buffer_object *)vma->vm_private_data; 264 265 WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping); 266 267 (void)ttm_bo_reference(bo); 268 } 269 270 static void ttm_bo_vm_close(struct vm_area_struct *vma) 271 { 272 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)vma->vm_private_data; 273 274 ttm_bo_unref(&bo); 275 vma->vm_private_data = NULL; 276 } 277 278 static const struct vm_operations_struct ttm_bo_vm_ops = { 279 .fault = ttm_bo_vm_fault, 280 .open = ttm_bo_vm_open, 281 .close = ttm_bo_vm_close 282 }; 283 284 static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, 285 unsigned long offset, 286 unsigned long pages) 287 { 288 struct drm_vma_offset_node *node; 289 struct ttm_buffer_object *bo = NULL; 290 291 drm_vma_offset_lock_lookup(&bdev->vma_manager); 292 293 node = drm_vma_offset_lookup_locked(&bdev->vma_manager, offset, pages); 294 if (likely(node)) { 295 bo = container_of(node, struct ttm_buffer_object, vma_node); 296 if (!kref_get_unless_zero(&bo->kref)) 297 bo = NULL; 298 } 299 300 drm_vma_offset_unlock_lookup(&bdev->vma_manager); 301 302 if (!bo) 303 pr_err("Could not find buffer object to map\n"); 304 305 return bo; 306 } 307 308 int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, 309 struct ttm_bo_device *bdev) 310 { 311 struct ttm_bo_driver *driver; 312 struct ttm_buffer_object *bo; 313 int ret; 314 315 bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); 316 if (unlikely(!bo)) 317 return -EINVAL; 318 319 driver = bo->bdev->driver; 320 if (unlikely(!driver->verify_access)) { 321 ret = -EPERM; 322 goto out_unref; 323 } 324 ret = driver->verify_access(bo, filp); 325 if (unlikely(ret != 0)) 326 goto out_unref; 327 328 vma->vm_ops = &ttm_bo_vm_ops; 329 330 /* 331 * Note: We're transferring the bo reference to 332 * vma->vm_private_data here. 333 */ 334 335 vma->vm_private_data = bo; 336 337 /* 338 * We'd like to use VM_PFNMAP on shared mappings, where 339 * (vma->vm_flags & VM_SHARED) != 0, for performance reasons, 340 * but for some reason VM_PFNMAP + x86 PAT + write-combine is very 341 * bad for performance. Until that has been sorted out, use 342 * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719 343 */ 344 vma->vm_flags |= VM_MIXEDMAP; 345 vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; 346 return 0; 347 out_unref: 348 ttm_bo_unref(&bo); 349 return ret; 350 } 351 EXPORT_SYMBOL(ttm_bo_mmap); 352 353 int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) 354 { 355 if (vma->vm_pgoff != 0) 356 return -EACCES; 357 358 vma->vm_ops = &ttm_bo_vm_ops; 359 vma->vm_private_data = ttm_bo_reference(bo); 360 vma->vm_flags |= VM_MIXEDMAP; 361 vma->vm_flags |= VM_IO | VM_DONTEXPAND; 362 return 0; 363 } 364 EXPORT_SYMBOL(ttm_fbdev_mmap); 365