1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2016 Intel Corporation 4 */ 5 6 #include <drm/drm_cache.h> 7 8 #include "i915_config.h" 9 #include "i915_drv.h" 10 #include "i915_gem_clflush.h" 11 #include "i915_gem_object_frontbuffer.h" 12 #include "i915_sw_fence_work.h" 13 #include "i915_trace.h" 14 15 struct clflush { 16 struct dma_fence_work base; 17 struct drm_i915_gem_object *obj; 18 }; 19 20 static void __do_clflush(struct drm_i915_gem_object *obj) 21 { 22 GEM_BUG_ON(!i915_gem_object_has_pages(obj)); 23 drm_clflush_sg(obj->mm.pages); 24 25 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); 26 } 27 28 static void clflush_work(struct dma_fence_work *base) 29 { 30 struct clflush *clflush = container_of(base, typeof(*clflush), base); 31 32 __do_clflush(clflush->obj); 33 } 34 35 static void clflush_release(struct dma_fence_work *base) 36 { 37 struct clflush *clflush = container_of(base, typeof(*clflush), base); 38 39 i915_gem_object_unpin_pages(clflush->obj); 40 i915_gem_object_put(clflush->obj); 41 } 42 43 static const struct dma_fence_work_ops clflush_ops = { 44 .name = "clflush", 45 .work = clflush_work, 46 .release = clflush_release, 47 }; 48 49 static struct clflush *clflush_work_create(struct drm_i915_gem_object *obj) 50 { 51 struct clflush *clflush; 52 53 GEM_BUG_ON(!obj->cache_dirty); 54 55 clflush = kmalloc(sizeof(*clflush), GFP_KERNEL); 56 if (!clflush) 57 return NULL; 58 59 if (__i915_gem_object_get_pages(obj) < 0) { 60 kfree(clflush); 61 return NULL; 62 } 63 64 dma_fence_work_init(&clflush->base, &clflush_ops); 65 clflush->obj = i915_gem_object_get(obj); /* obj <-> clflush cycle */ 66 67 return clflush; 68 } 69 70 bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, 71 unsigned int flags) 72 { 73 struct drm_i915_private *i915 = to_i915(obj->base.dev); 74 struct clflush *clflush; 75 76 assert_object_held(obj); 77 78 if (IS_DGFX(i915)) { 79 WARN_ON_ONCE(obj->cache_dirty); 80 return false; 81 } 82 83 /* 84 * Stolen memory is always coherent with the GPU as it is explicitly 85 * marked as wc by the system, or the system is cache-coherent. 86 * Similarly, we only access struct pages through the CPU cache, so 87 * anything not backed by physical memory we consider to be always 88 * coherent and not need clflushing. 89 */ 90 if (!i915_gem_object_has_struct_page(obj)) { 91 obj->cache_dirty = false; 92 return false; 93 } 94 95 /* If the GPU is snooping the contents of the CPU cache, 96 * we do not need to manually clear the CPU cache lines. However, 97 * the caches are only snooped when the render cache is 98 * flushed/invalidated. As we always have to emit invalidations 99 * and flushes when moving into and out of the RENDER domain, correct 100 * snooping behaviour occurs naturally as the result of our domain 101 * tracking. 102 */ 103 if (!(flags & I915_CLFLUSH_FORCE) && 104 obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ) 105 return false; 106 107 trace_i915_gem_object_clflush(obj); 108 109 clflush = NULL; 110 if (!(flags & I915_CLFLUSH_SYNC) && 111 dma_resv_reserve_fences(obj->base.resv, 1) == 0) 112 clflush = clflush_work_create(obj); 113 if (clflush) { 114 i915_sw_fence_await_reservation(&clflush->base.chain, 115 obj->base.resv, true, 116 i915_fence_timeout(i915), 117 I915_FENCE_GFP); 118 dma_resv_add_fence(obj->base.resv, &clflush->base.dma, 119 DMA_RESV_USAGE_KERNEL); 120 dma_fence_work_commit(&clflush->base); 121 /* 122 * We must have successfully populated the pages(since we are 123 * holding a pin on the pages as per the flush worker) to reach 124 * this point, which must mean we have already done the required 125 * flush-on-acquire, hence resetting cache_dirty here should be 126 * safe. 127 */ 128 obj->cache_dirty = false; 129 } else if (obj->mm.pages) { 130 __do_clflush(obj); 131 obj->cache_dirty = false; 132 } else { 133 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU); 134 } 135 136 return true; 137 } 138