1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_hw_fence.h" 7 8 #include <linux/device.h> 9 #include <linux/slab.h> 10 11 #include "xe_device_types.h" 12 #include "xe_hw_engine.h" 13 #include "xe_macros.h" 14 #include "xe_map.h" 15 #include "xe_trace.h" 16 17 static struct kmem_cache *xe_hw_fence_slab; 18 19 int __init xe_hw_fence_module_init(void) 20 { 21 xe_hw_fence_slab = kmem_cache_create("xe_hw_fence", 22 sizeof(struct xe_hw_fence), 0, 23 SLAB_HWCACHE_ALIGN, NULL); 24 if (!xe_hw_fence_slab) 25 return -ENOMEM; 26 27 return 0; 28 } 29 30 void xe_hw_fence_module_exit(void) 31 { 32 rcu_barrier(); 33 kmem_cache_destroy(xe_hw_fence_slab); 34 } 35 36 static struct xe_hw_fence *fence_alloc(void) 37 { 38 return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL); 39 } 40 41 static void fence_free(struct rcu_head *rcu) 42 { 43 struct xe_hw_fence *fence = 44 container_of(rcu, struct xe_hw_fence, dma.rcu); 45 46 if (!WARN_ON_ONCE(!fence)) 47 kmem_cache_free(xe_hw_fence_slab, fence); 48 } 49 50 static void hw_fence_irq_run_cb(struct irq_work *work) 51 { 52 struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work); 53 struct xe_hw_fence *fence, *next; 54 bool tmp; 55 56 tmp = dma_fence_begin_signalling(); 57 spin_lock(&irq->lock); 58 if (irq->enabled) { 59 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { 60 struct dma_fence *dma_fence = &fence->dma; 61 62 trace_xe_hw_fence_try_signal(fence); 63 if (dma_fence_is_signaled_locked(dma_fence)) { 64 trace_xe_hw_fence_signal(fence); 65 list_del_init(&fence->irq_link); 66 dma_fence_put(dma_fence); 67 } 68 } 69 } 70 spin_unlock(&irq->lock); 71 dma_fence_end_signalling(tmp); 72 } 73 74 void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq) 75 { 76 spin_lock_init(&irq->lock); 77 init_irq_work(&irq->work, hw_fence_irq_run_cb); 78 INIT_LIST_HEAD(&irq->pending); 79 irq->enabled = true; 80 } 81 82 void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq) 83 { 84 struct xe_hw_fence *fence, *next; 85 unsigned long flags; 86 bool tmp; 87 88 if (XE_WARN_ON(!list_empty(&irq->pending))) { 89 tmp = dma_fence_begin_signalling(); 90 spin_lock_irqsave(&irq->lock, flags); 91 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { 92 list_del_init(&fence->irq_link); 93 XE_WARN_ON(dma_fence_check_and_signal_locked(&fence->dma)); 94 dma_fence_put(&fence->dma); 95 } 96 spin_unlock_irqrestore(&irq->lock, flags); 97 dma_fence_end_signalling(tmp); 98 } 99 100 /* Safe release of the irq->lock used in dma_fence_init. */ 101 synchronize_rcu(); 102 } 103 104 void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq) 105 { 106 irq_work_queue(&irq->work); 107 } 108 109 void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt, 110 struct xe_hw_fence_irq *irq, const char *name) 111 { 112 ctx->gt = gt; 113 ctx->irq = irq; 114 ctx->dma_fence_ctx = dma_fence_context_alloc(1); 115 ctx->next_seqno = XE_FENCE_INITIAL_SEQNO; 116 snprintf(ctx->name, sizeof(ctx->name), "%s", name); 117 } 118 119 void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx) 120 { 121 } 122 123 static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence); 124 125 static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence) 126 { 127 return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock); 128 } 129 130 static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence) 131 { 132 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 133 134 return dev_name(fence->xe->drm.dev); 135 } 136 137 static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence) 138 { 139 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 140 141 return fence->name; 142 } 143 144 static bool xe_hw_fence_signaled(struct dma_fence *dma_fence) 145 { 146 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 147 struct xe_device *xe = fence->xe; 148 u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32); 149 150 return dma_fence->error || 151 !__dma_fence_is_later(dma_fence, dma_fence->seqno, seqno); 152 } 153 154 static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence) 155 { 156 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 157 struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence); 158 159 dma_fence_get(dma_fence); 160 list_add_tail(&fence->irq_link, &irq->pending); 161 162 /* SW completed (no HW IRQ) so kick handler to signal fence */ 163 if (xe_hw_fence_signaled(dma_fence)) 164 xe_hw_fence_irq_run(irq); 165 166 return true; 167 } 168 169 static void xe_hw_fence_release(struct dma_fence *dma_fence) 170 { 171 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 172 173 XE_WARN_ON(!list_empty(&fence->irq_link)); 174 call_rcu(&dma_fence->rcu, fence_free); 175 } 176 177 static const struct dma_fence_ops xe_hw_fence_ops = { 178 .get_driver_name = xe_hw_fence_get_driver_name, 179 .get_timeline_name = xe_hw_fence_get_timeline_name, 180 .enable_signaling = xe_hw_fence_enable_signaling, 181 .signaled = xe_hw_fence_signaled, 182 .release = xe_hw_fence_release, 183 }; 184 185 static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence) 186 { 187 if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops)) 188 return NULL; 189 190 return container_of(fence, struct xe_hw_fence, dma); 191 } 192 193 /** 194 * xe_hw_fence_alloc() - Allocate an hw fence. 195 * 196 * Allocate but don't initialize an hw fence. 197 * 198 * Return: Pointer to the allocated fence or 199 * negative error pointer on error. 200 */ 201 struct dma_fence *xe_hw_fence_alloc(void) 202 { 203 struct xe_hw_fence *hw_fence = fence_alloc(); 204 205 if (!hw_fence) 206 return ERR_PTR(-ENOMEM); 207 208 return &hw_fence->dma; 209 } 210 211 /** 212 * xe_hw_fence_free() - Free an hw fence. 213 * @fence: Pointer to the fence to free. 214 * 215 * Frees an hw fence that hasn't yet been 216 * initialized. 217 */ 218 void xe_hw_fence_free(struct dma_fence *fence) 219 { 220 fence_free(&fence->rcu); 221 } 222 223 /** 224 * xe_hw_fence_init() - Initialize an hw fence. 225 * @fence: Pointer to the fence to initialize. 226 * @ctx: Pointer to the struct xe_hw_fence_ctx fence context. 227 * @seqno_map: Pointer to the map into where the seqno is blitted. 228 * 229 * Initializes a pre-allocated hw fence. 230 * After initialization, the fence is subject to normal 231 * dma-fence refcounting. 232 */ 233 void xe_hw_fence_init(struct dma_fence *fence, struct xe_hw_fence_ctx *ctx, 234 struct iosys_map seqno_map) 235 { 236 struct xe_hw_fence *hw_fence = 237 container_of(fence, typeof(*hw_fence), dma); 238 239 hw_fence->xe = gt_to_xe(ctx->gt); 240 snprintf(hw_fence->name, sizeof(hw_fence->name), "%s", ctx->name); 241 hw_fence->seqno_map = seqno_map; 242 INIT_LIST_HEAD(&hw_fence->irq_link); 243 244 dma_fence_init(fence, &xe_hw_fence_ops, &ctx->irq->lock, 245 ctx->dma_fence_ctx, ctx->next_seqno++); 246 247 trace_xe_hw_fence_create(hw_fence); 248 } 249