1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_hw_fence.h" 7 8 #include <linux/device.h> 9 #include <linux/slab.h> 10 11 #include "xe_bo.h" 12 #include "xe_device.h" 13 #include "xe_gt.h" 14 #include "xe_hw_engine.h" 15 #include "xe_macros.h" 16 #include "xe_map.h" 17 #include "xe_trace.h" 18 19 static struct kmem_cache *xe_hw_fence_slab; 20 21 int __init xe_hw_fence_module_init(void) 22 { 23 xe_hw_fence_slab = kmem_cache_create("xe_hw_fence", 24 sizeof(struct xe_hw_fence), 0, 25 SLAB_HWCACHE_ALIGN, NULL); 26 if (!xe_hw_fence_slab) 27 return -ENOMEM; 28 29 return 0; 30 } 31 32 void xe_hw_fence_module_exit(void) 33 { 34 rcu_barrier(); 35 kmem_cache_destroy(xe_hw_fence_slab); 36 } 37 38 static struct xe_hw_fence *fence_alloc(void) 39 { 40 return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL); 41 } 42 43 static void fence_free(struct rcu_head *rcu) 44 { 45 struct xe_hw_fence *fence = 46 container_of(rcu, struct xe_hw_fence, dma.rcu); 47 48 if (!WARN_ON_ONCE(!fence)) 49 kmem_cache_free(xe_hw_fence_slab, fence); 50 } 51 52 static void hw_fence_irq_run_cb(struct irq_work *work) 53 { 54 struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work); 55 struct xe_hw_fence *fence, *next; 56 bool tmp; 57 58 tmp = dma_fence_begin_signalling(); 59 spin_lock(&irq->lock); 60 if (irq->enabled) { 61 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { 62 struct dma_fence *dma_fence = &fence->dma; 63 64 trace_xe_hw_fence_try_signal(fence); 65 if (dma_fence_is_signaled_locked(dma_fence)) { 66 trace_xe_hw_fence_signal(fence); 67 list_del_init(&fence->irq_link); 68 dma_fence_put(dma_fence); 69 } 70 } 71 } 72 spin_unlock(&irq->lock); 73 dma_fence_end_signalling(tmp); 74 } 75 76 void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq) 77 { 78 spin_lock_init(&irq->lock); 79 init_irq_work(&irq->work, hw_fence_irq_run_cb); 80 INIT_LIST_HEAD(&irq->pending); 81 irq->enabled = true; 82 } 83 84 void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq) 85 { 86 struct xe_hw_fence *fence, *next; 87 unsigned long flags; 88 bool tmp; 89 90 if (XE_WARN_ON(!list_empty(&irq->pending))) { 91 tmp = dma_fence_begin_signalling(); 92 spin_lock_irqsave(&irq->lock, flags); 93 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { 94 list_del_init(&fence->irq_link); 95 XE_WARN_ON(dma_fence_check_and_signal_locked(&fence->dma)); 96 dma_fence_put(&fence->dma); 97 } 98 spin_unlock_irqrestore(&irq->lock, flags); 99 dma_fence_end_signalling(tmp); 100 } 101 102 /* Safe release of the irq->lock used in dma_fence_init. */ 103 synchronize_rcu(); 104 } 105 106 void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq) 107 { 108 irq_work_queue(&irq->work); 109 } 110 111 void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq) 112 { 113 spin_lock_irq(&irq->lock); 114 irq->enabled = false; 115 spin_unlock_irq(&irq->lock); 116 } 117 118 void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq) 119 { 120 spin_lock_irq(&irq->lock); 121 irq->enabled = true; 122 spin_unlock_irq(&irq->lock); 123 124 irq_work_queue(&irq->work); 125 } 126 127 void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt, 128 struct xe_hw_fence_irq *irq, const char *name) 129 { 130 ctx->gt = gt; 131 ctx->irq = irq; 132 ctx->dma_fence_ctx = dma_fence_context_alloc(1); 133 ctx->next_seqno = XE_FENCE_INITIAL_SEQNO; 134 snprintf(ctx->name, sizeof(ctx->name), "%s", name); 135 } 136 137 void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx) 138 { 139 } 140 141 static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence); 142 143 static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence) 144 { 145 return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock); 146 } 147 148 static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence) 149 { 150 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 151 152 return dev_name(fence->xe->drm.dev); 153 } 154 155 static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence) 156 { 157 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 158 159 return fence->name; 160 } 161 162 static bool xe_hw_fence_signaled(struct dma_fence *dma_fence) 163 { 164 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 165 struct xe_device *xe = fence->xe; 166 u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32); 167 168 return dma_fence->error || 169 !__dma_fence_is_later(dma_fence, dma_fence->seqno, seqno); 170 } 171 172 static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence) 173 { 174 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 175 struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence); 176 177 dma_fence_get(dma_fence); 178 list_add_tail(&fence->irq_link, &irq->pending); 179 180 /* SW completed (no HW IRQ) so kick handler to signal fence */ 181 if (xe_hw_fence_signaled(dma_fence)) 182 xe_hw_fence_irq_run(irq); 183 184 return true; 185 } 186 187 static void xe_hw_fence_release(struct dma_fence *dma_fence) 188 { 189 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 190 191 XE_WARN_ON(!list_empty(&fence->irq_link)); 192 call_rcu(&dma_fence->rcu, fence_free); 193 } 194 195 static const struct dma_fence_ops xe_hw_fence_ops = { 196 .get_driver_name = xe_hw_fence_get_driver_name, 197 .get_timeline_name = xe_hw_fence_get_timeline_name, 198 .enable_signaling = xe_hw_fence_enable_signaling, 199 .signaled = xe_hw_fence_signaled, 200 .release = xe_hw_fence_release, 201 }; 202 203 static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence) 204 { 205 if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops)) 206 return NULL; 207 208 return container_of(fence, struct xe_hw_fence, dma); 209 } 210 211 /** 212 * xe_hw_fence_alloc() - Allocate an hw fence. 213 * 214 * Allocate but don't initialize an hw fence. 215 * 216 * Return: Pointer to the allocated fence or 217 * negative error pointer on error. 218 */ 219 struct dma_fence *xe_hw_fence_alloc(void) 220 { 221 struct xe_hw_fence *hw_fence = fence_alloc(); 222 223 if (!hw_fence) 224 return ERR_PTR(-ENOMEM); 225 226 return &hw_fence->dma; 227 } 228 229 /** 230 * xe_hw_fence_free() - Free an hw fence. 231 * @fence: Pointer to the fence to free. 232 * 233 * Frees an hw fence that hasn't yet been 234 * initialized. 235 */ 236 void xe_hw_fence_free(struct dma_fence *fence) 237 { 238 fence_free(&fence->rcu); 239 } 240 241 /** 242 * xe_hw_fence_init() - Initialize an hw fence. 243 * @fence: Pointer to the fence to initialize. 244 * @ctx: Pointer to the struct xe_hw_fence_ctx fence context. 245 * @seqno_map: Pointer to the map into where the seqno is blitted. 246 * 247 * Initializes a pre-allocated hw fence. 248 * After initialization, the fence is subject to normal 249 * dma-fence refcounting. 250 */ 251 void xe_hw_fence_init(struct dma_fence *fence, struct xe_hw_fence_ctx *ctx, 252 struct iosys_map seqno_map) 253 { 254 struct xe_hw_fence *hw_fence = 255 container_of(fence, typeof(*hw_fence), dma); 256 257 hw_fence->xe = gt_to_xe(ctx->gt); 258 snprintf(hw_fence->name, sizeof(hw_fence->name), "%s", ctx->name); 259 hw_fence->seqno_map = seqno_map; 260 INIT_LIST_HEAD(&hw_fence->irq_link); 261 262 dma_fence_init(fence, &xe_hw_fence_ops, &ctx->irq->lock, 263 ctx->dma_fence_ctx, ctx->next_seqno++); 264 265 trace_xe_hw_fence_create(hw_fence); 266 } 267