/* * SPDX-License-Identifier: MIT * * Copyright © 2014-2016 Intel Corporation */ #include #include #include #include #include #include "gt/intel_gt.h" #include "gt/intel_gt_requests.h" #include "i915_drv.h" #include "i915_gem_evict.h" #include "i915_gem_gtt.h" #include "i915_gem_ioctls.h" #include "i915_gem_object.h" #include "i915_gem_mman.h" #include "i915_mm.h" #include "i915_trace.h" #include "i915_user_extensions.h" #include "i915_gem_ttm.h" #include "i915_vma.h" static inline bool __vma_matches(struct vm_area_struct *vma, struct file *filp, unsigned long addr, unsigned long size) { if (vma->vm_file != filp) return false; return vma->vm_start == addr && (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size); } /** * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address * it is mapped to. * @dev: drm device * @data: ioctl data blob * @file: drm file * * While the mapping holds a reference on the contents of the object, it doesn't * imply a ref on the object itself. * * IMPORTANT: * * DRM driver writers who look a this function as an example for how to do GEM * mmap support, please don't implement mmap support like here. The modern way * to implement DRM mmap support is with an mmap offset ioctl (like * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly. * That way debug tooling like valgrind will understand what's going on, hiding * the mmap call in a driver private ioctl will break that. The i915 driver only * does cpu mmaps this way because we didn't know better. */ int i915_gem_mmap_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_mmap *args = data; struct drm_i915_gem_object *obj; unsigned long addr; /* * mmap ioctl is disallowed for all discrete platforms, * and for all platforms with GRAPHICS_VER > 12. */ if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) > IP_VER(12, 0)) return -EOPNOTSUPP; if (args->flags & ~(I915_MMAP_WC)) return -EINVAL; if (args->flags & I915_MMAP_WC && !pat_enabled()) return -ENODEV; obj = i915_gem_object_lookup(file, args->handle); if (!obj) return -ENOENT; /* prime objects have no backing filp to GEM mmap * pages from. */ if (!obj->base.filp) { addr = -ENXIO; goto err; } if (range_overflows(args->offset, args->size, (u64)obj->base.size)) { addr = -EINVAL; goto err; } addr = vm_mmap(obj->base.filp, 0, args->size, PROT_READ | PROT_WRITE, MAP_SHARED, args->offset); if (IS_ERR_VALUE(addr)) goto err; if (args->flags & I915_MMAP_WC) { struct mm_struct *mm = current->mm; struct vm_area_struct *vma; if (mmap_write_lock_killable(mm)) { addr = -EINTR; goto err; } vma = find_vma(mm, addr); if (vma && __vma_matches(vma, obj->base.filp, addr, args->size)) vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); else addr = -ENOMEM; mmap_write_unlock(mm); if (IS_ERR_VALUE(addr)) goto err; } i915_gem_object_put(obj); args->addr_ptr = (u64)addr; return 0; err: i915_gem_object_put(obj); return addr; } static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj) { return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT; } /** * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps * * A history of the GTT mmap interface: * * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to * aligned and suitable for fencing, and still fit into the available * mappable space left by the pinned display objects. A classic problem * we called the page-fault-of-doom where we would ping-pong between * two objects that could not fit inside the GTT and so the memcpy * would page one object in at the expense of the other between every * single byte. * * 1 - Objects can be any size, and have any compatible fencing (X Y, or none * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the * object is too large for the available space (or simply too large * for the mappable aperture!), a view is created instead and faulted * into userspace. (This view is aligned and sized appropriately for * fenced access.) * * 2 - Recognise WC as a separate cache domain so that we can flush the * delayed writes via GTT before performing direct access via WC. * * 3 - Remove implicit set-domain(GTT) and synchronisation on initial * pagefault; swapin remains transparent. * * 4 - Support multiple fault handlers per object depending on object's * backing storage (a.k.a. MMAP_OFFSET). * * Restrictions: * * * snoopable objects cannot be accessed via the GTT. It can cause machine * hangs on some architectures, corruption on others. An attempt to service * a GTT page fault from a snoopable object will generate a SIGBUS. * * * the object must be able to fit into RAM (physical memory, though no * limited to the mappable aperture). * * * Caveats: * * * a new GTT page fault will synchronize rendering from the GPU and flush * all data to system memory. Subsequent access will not be synchronized. * * * all mappings are revoked on runtime device suspend. * * * there are only 8, 16 or 32 fence registers to share between all users * (older machines require fence register for display and blitter access * as well). Contention of the fence registers will cause the previous users * to be unmapped and any new access will generate new page faults. * * * running out of memory while servicing a fault may generate a SIGBUS, * rather than the expected SIGSEGV. */ int i915_gem_mmap_gtt_version(void) { return 4; } static inline struct i915_gtt_view compute_partial_view(const struct drm_i915_gem_object *obj, pgoff_t page_offset, unsigned int chunk) { struct i915_gtt_view view; if (i915_gem_object_is_tiled(obj)) chunk = roundup(chunk, tile_row_pages(obj) ?: 1); view.type = I915_GTT_VIEW_PARTIAL; view.partial.offset = rounddown(page_offset, chunk); view.partial.size = min_t(unsigned int, chunk, (obj->base.size >> PAGE_SHIFT) - view.partial.offset); /* If the partial covers the entire object, just create a normal VMA. */ if (chunk >= obj->base.size >> PAGE_SHIFT) view.type = I915_GTT_VIEW_NORMAL; return view; } static vm_fault_t i915_error_to_vmf_fault(int err) { switch (err) { default: WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err); fallthrough; case -EIO: /* shmemfs failure from swap device */ case -EFAULT: /* purged object */ case -ENODEV: /* bad object, how did you get here! */ case -ENXIO: /* unable to access backing store (on device) */ return VM_FAULT_SIGBUS; case -ENOMEM: /* our allocation failure */ return VM_FAULT_OOM; case 0: case -EAGAIN: case -ENOSPC: /* transient failure to evict? */ case -ENOBUFS: /* temporarily out of fences? */ case -ERESTARTSYS: case -EINTR: case -EBUSY: /* * EBUSY is ok: this just means that another thread * already did the job. */ return VM_FAULT_NOPAGE; } } static vm_fault_t vm_fault_cpu(struct vm_fault *vmf) { struct vm_area_struct *area = vmf->vma; struct i915_mmap_offset *mmo = area->vm_private_data; struct drm_i915_gem_object *obj = mmo->obj; unsigned long obj_offset; resource_size_t iomap; int err; /* Sanity check that we allow writing into this object */ if (unlikely(i915_gem_object_is_readonly(obj) && area->vm_flags & VM_WRITE)) return VM_FAULT_SIGBUS; if (i915_gem_object_lock_interruptible(obj, NULL)) return VM_FAULT_NOPAGE; err = i915_gem_object_pin_pages(obj); if (err) goto out; iomap = -1; if (!i915_gem_object_has_struct_page(obj)) { iomap = obj->mm.region->iomap.base; iomap -= obj->mm.region->region.start; } obj_offset = area->vm_pgoff - drm_vma_node_start(&mmo->vma_node); /* PTEs are revoked in obj->ops->put_pages() */ err = remap_io_sg(area, area->vm_start, area->vm_end - area->vm_start, obj->mm.pages->sgl, obj_offset, iomap); if (area->vm_flags & VM_WRITE) { GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); obj->mm.dirty = true; } i915_gem_object_unpin_pages(obj); out: i915_gem_object_unlock(obj); return i915_error_to_vmf_fault(err); } static void set_address_limits(struct vm_area_struct *area, struct i915_vma *vma, unsigned long obj_offset, resource_size_t gmadr_start, unsigned long *start_vaddr, unsigned long *end_vaddr, unsigned long *pfn) { unsigned long vm_start, vm_end, vma_size; /* user's memory parameters */ long start, end; /* memory boundaries */ /* * Let's move into the ">> PAGE_SHIFT" * domain to be sure not to lose bits */ vm_start = area->vm_start >> PAGE_SHIFT; vm_end = area->vm_end >> PAGE_SHIFT; vma_size = vma->size >> PAGE_SHIFT; /* * Calculate the memory boundaries by considering the offset * provided by the user during memory mapping and the offset * provided for the partial mapping. */ start = vm_start; start -= obj_offset; start += vma->gtt_view.partial.offset; end = start + vma_size; start = max_t(long, start, vm_start); end = min_t(long, end, vm_end); /* Let's move back into the "<< PAGE_SHIFT" domain */ *start_vaddr = (unsigned long)start << PAGE_SHIFT; *end_vaddr = (unsigned long)end << PAGE_SHIFT; *pfn = (gmadr_start + i915_ggtt_offset(vma)) >> PAGE_SHIFT; *pfn += (*start_vaddr - area->vm_start) >> PAGE_SHIFT; *pfn += obj_offset - vma->gtt_view.partial.offset; } static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) { #define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT) struct vm_area_struct *area = vmf->vma; struct i915_mmap_offset *mmo = area->vm_private_data; struct drm_i915_gem_object *obj = mmo->obj; struct drm_device *dev = obj->base.dev; struct drm_i915_private *i915 = to_i915(dev); struct intel_runtime_pm *rpm = &i915->runtime_pm; struct i915_ggtt *ggtt = to_gt(i915)->ggtt; bool write = area->vm_flags & VM_WRITE; struct i915_gem_ww_ctx ww; unsigned long obj_offset; unsigned long start, end; /* memory boundaries */ intel_wakeref_t wakeref; struct i915_vma *vma; pgoff_t page_offset; unsigned long pfn; int srcu; int ret; obj_offset = area->vm_pgoff - drm_vma_node_start(&mmo->vma_node); page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT; page_offset += obj_offset; trace_i915_gem_object_fault(obj, page_offset, true, write); wakeref = intel_runtime_pm_get(rpm); i915_gem_ww_ctx_init(&ww, true); retry: ret = i915_gem_object_lock(obj, &ww); if (ret) goto err_rpm; /* Sanity check that we allow writing into this object */ if (i915_gem_object_is_readonly(obj) && write) { ret = -EFAULT; goto err_rpm; } ret = i915_gem_object_pin_pages(obj); if (ret) goto err_rpm; ret = intel_gt_reset_lock_interruptible(ggtt->vm.gt, &srcu); if (ret) goto err_pages; /* Now pin it into the GTT as needed */ vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0, PIN_MAPPABLE | PIN_NONBLOCK /* NOWARN */ | PIN_NOEVICT); if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { /* Use a partial view if it is bigger than available space */ struct i915_gtt_view view = compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES); unsigned int flags; flags = PIN_MAPPABLE | PIN_NOSEARCH; if (view.type == I915_GTT_VIEW_NORMAL) flags |= PIN_NONBLOCK; /* avoid warnings for pinned */ /* * Userspace is now writing through an untracked VMA, abandon * all hope that the hardware is able to track future writes. */ vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) { flags = PIN_MAPPABLE; view.type = I915_GTT_VIEW_PARTIAL; vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); } /* * The entire mappable GGTT is pinned? Unexpected! * Try to evict the object we locked too, as normally we skip it * due to lack of short term pinning inside execbuf. */ if (vma == ERR_PTR(-ENOSPC)) { ret = mutex_lock_interruptible(&ggtt->vm.mutex); if (!ret) { ret = i915_gem_evict_vm(&ggtt->vm, &ww, NULL); mutex_unlock(&ggtt->vm.mutex); } if (ret) goto err_reset; vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags); } } if (IS_ERR(vma)) { ret = PTR_ERR(vma); goto err_reset; } /* Access to snoopable pages through the GTT is incoherent. */ /* * For objects created by userspace through GEM_CREATE with pat_index * set by set_pat extension, coherency is managed by userspace, make * sure we don't fail handling the vm fault by calling * i915_gem_object_has_cache_level() which always return true for such * objects. Otherwise this helper function would fall back to checking * whether the object is un-cached. */ if (!(i915_gem_object_has_cache_level(obj, I915_CACHE_NONE) || HAS_LLC(i915))) { ret = -EFAULT; goto err_unpin; } ret = i915_vma_pin_fence(vma); if (ret) goto err_unpin; /* * Dump all the necessary parameters in this function to perform the * arithmetic calculation for the virtual address start and end and * the PFN (Page Frame Number). */ set_address_limits(area, vma, obj_offset, ggtt->gmadr.start, &start, &end, &pfn); /* Finally, remap it using the new GTT offset */ ret = remap_io_mapping(area, start, pfn, end - start, &ggtt->iomap); if (ret) goto err_fence; assert_rpm_wakelock_held(rpm); /* Mark as being mmapped into userspace for later revocation */ mutex_lock(&to_gt(i915)->ggtt->vm.mutex); if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) list_add(&obj->userfault_link, &to_gt(i915)->ggtt->userfault_list); mutex_unlock(&to_gt(i915)->ggtt->vm.mutex); /* Track the mmo associated with the fenced vma */ vma->mmo = mmo; if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref, msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); if (write) { GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); i915_vma_set_ggtt_write(vma); obj->mm.dirty = true; } err_fence: i915_vma_unpin_fence(vma); err_unpin: __i915_vma_unpin(vma); err_reset: intel_gt_reset_unlock(ggtt->vm.gt, srcu); err_pages: i915_gem_object_unpin_pages(obj); err_rpm: if (ret == -EDEADLK) { ret = i915_gem_ww_ctx_backoff(&ww); if (!ret) goto retry; } i915_gem_ww_ctx_fini(&ww); intel_runtime_pm_put(rpm, wakeref); return i915_error_to_vmf_fault(ret); } static int vm_access(struct vm_area_struct *area, unsigned long addr, void *buf, int len, int write) { struct i915_mmap_offset *mmo = area->vm_private_data; struct drm_i915_gem_object *obj = mmo->obj; struct i915_gem_ww_ctx ww; void *vaddr; int err = 0; if (i915_gem_object_is_readonly(obj) && write) return -EACCES; addr -= area->vm_start; if (range_overflows_t(u64, addr, len, obj->base.size)) return -EINVAL; i915_gem_ww_ctx_init(&ww, true); retry: err = i915_gem_object_lock(obj, &ww); if (err) goto out; /* As this is primarily for debugging, let's focus on simplicity */ vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC); if (IS_ERR(vaddr)) { err = PTR_ERR(vaddr); goto out; } if (write) { memcpy(vaddr + addr, buf, len); __i915_gem_object_flush_map(obj, addr, len); } else { memcpy(buf, vaddr + addr, len); } i915_gem_object_unpin_map(obj); out: if (err == -EDEADLK) { err = i915_gem_ww_ctx_backoff(&ww); if (!err) goto retry; } i915_gem_ww_ctx_fini(&ww); if (err) return err; return len; } void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) { struct i915_vma *vma; GEM_BUG_ON(!obj->userfault_count); for_each_ggtt_vma(vma, obj) i915_vma_revoke_mmap(vma); GEM_BUG_ON(obj->userfault_count); } /* * It is vital that we remove the page mapping if we have mapped a tiled * object through the GTT and then lose the fence register due to * resource pressure. Similarly if the object has been moved out of the * aperture, than pages mapped into userspace must be revoked. Removing the * mapping will then trigger a page fault on the next user access, allowing * fixup by vm_fault_gtt(). */ void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) { struct drm_i915_private *i915 = to_i915(obj->base.dev); intel_wakeref_t wakeref; /* * Serialisation between user GTT access and our code depends upon * revoking the CPU's PTE whilst the mutex is held. The next user * pagefault then has to wait until we release the mutex. * * Note that RPM complicates somewhat by adding an additional * requirement that operations to the GGTT be made holding the RPM * wakeref. */ wakeref = intel_runtime_pm_get(&i915->runtime_pm); mutex_lock(&to_gt(i915)->ggtt->vm.mutex); if (!obj->userfault_count) goto out; __i915_gem_object_release_mmap_gtt(obj); /* * Ensure that the CPU's PTE are revoked and there are not outstanding * memory transactions from userspace before we return. The TLB * flushing implied above by changing the PTE above *should* be * sufficient, an extra barrier here just provides us with a bit * of paranoid documentation about our requirement to serialise * memory writes before touching registers / GSM. */ wmb(); out: mutex_unlock(&to_gt(i915)->ggtt->vm.mutex); intel_runtime_pm_put(&i915->runtime_pm, wakeref); } void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj) { struct ttm_buffer_object *bo = i915_gem_to_ttm(obj); struct ttm_device *bdev = bo->bdev; drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); /* * We have exclusive access here via runtime suspend. All other callers * must first grab the rpm wakeref. */ GEM_BUG_ON(!obj->userfault_count); list_del(&obj->userfault_link); obj->userfault_count = 0; } void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj) { struct i915_mmap_offset *mmo, *mn; if (obj->ops->unmap_virtual) obj->ops->unmap_virtual(obj); spin_lock(&obj->mmo.lock); rbtree_postorder_for_each_entry_safe(mmo, mn, &obj->mmo.offsets, offset) { /* * vma_node_unmap for GTT mmaps handled already in * __i915_gem_object_release_mmap_gtt */ if (mmo->mmap_type == I915_MMAP_TYPE_GTT) continue; spin_unlock(&obj->mmo.lock); drm_vma_node_unmap(&mmo->vma_node, obj->base.dev->anon_inode->i_mapping); spin_lock(&obj->mmo.lock); } spin_unlock(&obj->mmo.lock); } static struct i915_mmap_offset * lookup_mmo(struct drm_i915_gem_object *obj, enum i915_mmap_type mmap_type) { struct rb_node *rb; spin_lock(&obj->mmo.lock); rb = obj->mmo.offsets.rb_node; while (rb) { struct i915_mmap_offset *mmo = rb_entry(rb, typeof(*mmo), offset); if (mmo->mmap_type == mmap_type) { spin_unlock(&obj->mmo.lock); return mmo; } if (mmo->mmap_type < mmap_type) rb = rb->rb_right; else rb = rb->rb_left; } spin_unlock(&obj->mmo.lock); return NULL; } static struct i915_mmap_offset * insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo) { struct rb_node *rb, **p; spin_lock(&obj->mmo.lock); rb = NULL; p = &obj->mmo.offsets.rb_node; while (*p) { struct i915_mmap_offset *pos; rb = *p; pos = rb_entry(rb, typeof(*pos), offset); if (pos->mmap_type == mmo->mmap_type) { spin_unlock(&obj->mmo.lock); drm_vma_offset_remove(obj->base.dev->vma_offset_manager, &mmo->vma_node); kfree(mmo); return pos; } if (pos->mmap_type < mmo->mmap_type) p = &rb->rb_right; else p = &rb->rb_left; } rb_link_node(&mmo->offset, rb, p); rb_insert_color(&mmo->offset, &obj->mmo.offsets); spin_unlock(&obj->mmo.lock); return mmo; } static struct i915_mmap_offset * mmap_offset_attach(struct drm_i915_gem_object *obj, enum i915_mmap_type mmap_type, struct drm_file *file) { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct i915_mmap_offset *mmo; int err; GEM_BUG_ON(obj->ops->mmap_offset || obj->ops->mmap_ops); mmo = lookup_mmo(obj, mmap_type); if (mmo) goto out; mmo = kmalloc(sizeof(*mmo), GFP_KERNEL); if (!mmo) return ERR_PTR(-ENOMEM); mmo->obj = obj; mmo->mmap_type = mmap_type; drm_vma_node_reset(&mmo->vma_node); err = drm_vma_offset_add(obj->base.dev->vma_offset_manager, &mmo->vma_node, obj->base.size / PAGE_SIZE); if (likely(!err)) goto insert; /* Attempt to reap some mmap space from dead objects */ err = intel_gt_retire_requests_timeout(to_gt(i915), MAX_SCHEDULE_TIMEOUT, NULL); if (err) goto err; i915_gem_drain_freed_objects(i915); err = drm_vma_offset_add(obj->base.dev->vma_offset_manager, &mmo->vma_node, obj->base.size / PAGE_SIZE); if (err) goto err; insert: mmo = insert_mmo(obj, mmo); GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo); out: if (file) drm_vma_node_allow_once(&mmo->vma_node, file); return mmo; err: kfree(mmo); return ERR_PTR(err); } static int __assign_mmap_offset(struct drm_i915_gem_object *obj, enum i915_mmap_type mmap_type, u64 *offset, struct drm_file *file) { struct i915_mmap_offset *mmo; if (i915_gem_object_never_mmap(obj)) return -ENODEV; if (obj->ops->mmap_offset) { if (mmap_type != I915_MMAP_TYPE_FIXED) return -ENODEV; *offset = obj->ops->mmap_offset(obj); return 0; } if (mmap_type == I915_MMAP_TYPE_FIXED) return -ENODEV; if (mmap_type != I915_MMAP_TYPE_GTT && !i915_gem_object_has_struct_page(obj) && !i915_gem_object_has_iomem(obj)) return -ENODEV; mmo = mmap_offset_attach(obj, mmap_type, file); if (IS_ERR(mmo)) return PTR_ERR(mmo); *offset = drm_vma_node_offset_addr(&mmo->vma_node); return 0; } static int __assign_mmap_offset_handle(struct drm_file *file, u32 handle, enum i915_mmap_type mmap_type, u64 *offset) { struct drm_i915_gem_object *obj; int err; obj = i915_gem_object_lookup(file, handle); if (!obj) return -ENOENT; err = i915_gem_object_lock_interruptible(obj, NULL); if (err) goto out_put; err = __assign_mmap_offset(obj, mmap_type, offset, file); i915_gem_object_unlock(obj); out_put: i915_gem_object_put(obj); return err; } int i915_gem_dumb_mmap_offset(struct drm_file *file, struct drm_device *dev, u32 handle, u64 *offset) { struct drm_i915_private *i915 = to_i915(dev); enum i915_mmap_type mmap_type; if (HAS_LMEM(to_i915(dev))) mmap_type = I915_MMAP_TYPE_FIXED; else if (pat_enabled()) mmap_type = I915_MMAP_TYPE_WC; else if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return -ENODEV; else mmap_type = I915_MMAP_TYPE_GTT; return __assign_mmap_offset_handle(file, handle, mmap_type, offset); } /** * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing * @dev: DRM device * @data: GTT mapping ioctl data * @file: GEM object info * * Simply returns the fake offset to userspace so it can mmap it. * The mmap call will end up in drm_gem_mmap(), which will set things * up so we can get faults in the handler above. * * The fault handler will take care of binding the object into the GTT * (since it may have been evicted to make room for something), allocating * a fence register, and mapping the appropriate aperture address into * userspace. */ int i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_private *i915 = to_i915(dev); struct drm_i915_gem_mmap_offset *args = data; enum i915_mmap_type type; int err; /* * Historically we failed to check args.pad and args.offset * and so we cannot use those fields for user input and we cannot * add -EINVAL for them as the ABI is fixed, i.e. old userspace * may be feeding in garbage in those fields. * * if (args->pad) return -EINVAL; is verbotten! */ err = i915_user_extensions(u64_to_user_ptr(args->extensions), NULL, 0, NULL); if (err) return err; switch (args->flags) { case I915_MMAP_OFFSET_GTT: if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return -ENODEV; type = I915_MMAP_TYPE_GTT; break; case I915_MMAP_OFFSET_WC: if (!pat_enabled()) return -ENODEV; type = I915_MMAP_TYPE_WC; break; case I915_MMAP_OFFSET_WB: type = I915_MMAP_TYPE_WB; break; case I915_MMAP_OFFSET_UC: if (!pat_enabled()) return -ENODEV; type = I915_MMAP_TYPE_UC; break; case I915_MMAP_OFFSET_FIXED: type = I915_MMAP_TYPE_FIXED; break; default: return -EINVAL; } return __assign_mmap_offset_handle(file, args->handle, type, &args->offset); } static void vm_open(struct vm_area_struct *vma) { struct i915_mmap_offset *mmo = vma->vm_private_data; struct drm_i915_gem_object *obj = mmo->obj; GEM_BUG_ON(!obj); i915_gem_object_get(obj); } static void vm_close(struct vm_area_struct *vma) { struct i915_mmap_offset *mmo = vma->vm_private_data; struct drm_i915_gem_object *obj = mmo->obj; GEM_BUG_ON(!obj); i915_gem_object_put(obj); } static const struct vm_operations_struct vm_ops_gtt = { .fault = vm_fault_gtt, .access = vm_access, .open = vm_open, .close = vm_close, }; static const struct vm_operations_struct vm_ops_cpu = { .fault = vm_fault_cpu, .access = vm_access, .open = vm_open, .close = vm_close, }; static int singleton_release(struct inode *inode, struct file *file) { struct drm_i915_private *i915 = file->private_data; cmpxchg(&i915->gem.mmap_singleton, file, NULL); drm_dev_put(&i915->drm); return 0; } static const struct file_operations singleton_fops = { .owner = THIS_MODULE, .release = singleton_release, }; static struct file *mmap_singleton(struct drm_i915_private *i915) { struct file *file; file = get_file_active(&i915->gem.mmap_singleton); if (file) return file; file = anon_inode_getfile("i915.gem", &singleton_fops, i915, O_RDWR); if (IS_ERR(file)) return file; /* Everyone shares a single global address space */ file->f_mapping = i915->drm.anon_inode->i_mapping; smp_store_mb(i915->gem.mmap_singleton, file); drm_dev_get(&i915->drm); return file; } static int i915_gem_object_mmap(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo, struct vm_area_struct *vma) { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct drm_device *dev = &i915->drm; struct file *anon; if (i915_gem_object_is_readonly(obj)) { if (vma->vm_flags & VM_WRITE) { i915_gem_object_put(obj); return -EINVAL; } vm_flags_clear(vma, VM_MAYWRITE); } anon = mmap_singleton(to_i915(dev)); if (IS_ERR(anon)) { i915_gem_object_put(obj); return PTR_ERR(anon); } vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO); /* * We keep the ref on mmo->obj, not vm_file, but we require * vma->vm_file->f_mapping, see vma_link(), for later revocation. * Our userspace is accustomed to having per-file resource cleanup * (i.e. contexts, objects and requests) on their close(fd), which * requires avoiding extraneous references to their filp, hence why * we prefer to use an anonymous file for their mmaps. */ vma_set_file(vma, anon); /* Drop the initial creation reference, the vma is now holding one. */ fput(anon); if (obj->ops->mmap_ops) { vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags)); vma->vm_ops = obj->ops->mmap_ops; vma->vm_private_data = obj->base.vma_node.driver_private; return 0; } vma->vm_private_data = mmo; switch (mmo->mmap_type) { case I915_MMAP_TYPE_WC: vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); vma->vm_ops = &vm_ops_cpu; break; case I915_MMAP_TYPE_FIXED: GEM_WARN_ON(1); fallthrough; case I915_MMAP_TYPE_WB: vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); vma->vm_ops = &vm_ops_cpu; break; case I915_MMAP_TYPE_UC: vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags)); vma->vm_ops = &vm_ops_cpu; break; case I915_MMAP_TYPE_GTT: vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags)); vma->vm_ops = &vm_ops_gtt; break; } vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); return 0; } /* * This overcomes the limitation in drm_gem_mmap's assignment of a * drm_gem_object as the vma->vm_private_data. Since we need to * be able to resolve multiple mmap offsets which could be tied * to a single gem object. */ int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma) { struct drm_vma_offset_node *node; struct drm_file *priv = filp->private_data; struct drm_device *dev = priv->minor->dev; struct drm_i915_gem_object *obj = NULL; struct i915_mmap_offset *mmo = NULL; if (drm_dev_is_unplugged(dev)) return -ENODEV; rcu_read_lock(); drm_vma_offset_lock_lookup(dev->vma_offset_manager); node = drm_vma_offset_lookup_locked(dev->vma_offset_manager, vma->vm_pgoff, vma_pages(vma)); if (node && drm_vma_node_is_allowed(node, priv)) { /* * Skip 0-refcnted objects as it is in the process of being * destroyed and will be invalid when the vma manager lock * is released. */ if (!node->driver_private) { mmo = container_of(node, struct i915_mmap_offset, vma_node); obj = i915_gem_object_get_rcu(mmo->obj); GEM_BUG_ON(obj && obj->ops->mmap_ops); } else { obj = i915_gem_object_get_rcu (container_of(node, struct drm_i915_gem_object, base.vma_node)); GEM_BUG_ON(obj && !obj->ops->mmap_ops); } } drm_vma_offset_unlock_lookup(dev->vma_offset_manager); rcu_read_unlock(); if (!obj) return node ? -EACCES : -EINVAL; return i915_gem_object_mmap(obj, mmo, vma); } int i915_gem_fb_mmap(struct drm_i915_gem_object *obj, struct vm_area_struct *vma) { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct drm_device *dev = &i915->drm; struct i915_mmap_offset *mmo = NULL; enum i915_mmap_type mmap_type; struct i915_ggtt *ggtt = to_gt(i915)->ggtt; if (drm_dev_is_unplugged(dev)) return -ENODEV; /* handle ttm object */ if (obj->ops->mmap_ops) { /* * ttm fault handler, ttm_bo_vm_fault_reserved() uses fake offset * to calculate page offset so set that up. */ vma->vm_pgoff += drm_vma_node_start(&obj->base.vma_node); } else { /* handle stolen and smem objects */ mmap_type = i915_ggtt_has_aperture(ggtt) ? I915_MMAP_TYPE_GTT : I915_MMAP_TYPE_WC; mmo = mmap_offset_attach(obj, mmap_type, NULL); if (IS_ERR(mmo)) return PTR_ERR(mmo); vma->vm_pgoff += drm_vma_node_start(&mmo->vma_node); } /* * When we install vm_ops for mmap we are too late for * the vm_ops->open() which increases the ref_count of * this obj and then it gets decreased by the vm_ops->close(). * To balance this increase the obj ref_count here. */ obj = i915_gem_object_get(obj); return i915_gem_object_mmap(obj, mmo, vma); } #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) #include "selftests/i915_gem_mman.c" #endif