1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_hw_fence.h" 7 8 #include <linux/device.h> 9 #include <linux/slab.h> 10 11 #include "xe_bo.h" 12 #include "xe_device.h" 13 #include "xe_gt.h" 14 #include "xe_hw_engine.h" 15 #include "xe_macros.h" 16 #include "xe_map.h" 17 #include "xe_trace.h" 18 19 static struct kmem_cache *xe_hw_fence_slab; 20 21 int __init xe_hw_fence_module_init(void) 22 { 23 xe_hw_fence_slab = kmem_cache_create("xe_hw_fence", 24 sizeof(struct xe_hw_fence), 0, 25 SLAB_HWCACHE_ALIGN, NULL); 26 if (!xe_hw_fence_slab) 27 return -ENOMEM; 28 29 return 0; 30 } 31 32 void xe_hw_fence_module_exit(void) 33 { 34 rcu_barrier(); 35 kmem_cache_destroy(xe_hw_fence_slab); 36 } 37 38 static struct xe_hw_fence *fence_alloc(void) 39 { 40 return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL); 41 } 42 43 static void fence_free(struct rcu_head *rcu) 44 { 45 struct xe_hw_fence *fence = 46 container_of(rcu, struct xe_hw_fence, dma.rcu); 47 48 if (!WARN_ON_ONCE(!fence)) 49 kmem_cache_free(xe_hw_fence_slab, fence); 50 } 51 52 static void hw_fence_irq_run_cb(struct irq_work *work) 53 { 54 struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work); 55 struct xe_hw_fence *fence, *next; 56 bool tmp; 57 58 tmp = dma_fence_begin_signalling(); 59 spin_lock(&irq->lock); 60 if (irq->enabled) { 61 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { 62 struct dma_fence *dma_fence = &fence->dma; 63 64 trace_xe_hw_fence_try_signal(fence); 65 if (dma_fence_is_signaled_locked(dma_fence)) { 66 trace_xe_hw_fence_signal(fence); 67 list_del_init(&fence->irq_link); 68 dma_fence_put(dma_fence); 69 } 70 } 71 } 72 spin_unlock(&irq->lock); 73 dma_fence_end_signalling(tmp); 74 } 75 76 void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq) 77 { 78 spin_lock_init(&irq->lock); 79 init_irq_work(&irq->work, hw_fence_irq_run_cb); 80 INIT_LIST_HEAD(&irq->pending); 81 irq->enabled = true; 82 } 83 84 void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq) 85 { 86 struct xe_hw_fence *fence, *next; 87 unsigned long flags; 88 int err; 89 bool tmp; 90 91 if (XE_WARN_ON(!list_empty(&irq->pending))) { 92 tmp = dma_fence_begin_signalling(); 93 spin_lock_irqsave(&irq->lock, flags); 94 list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { 95 list_del_init(&fence->irq_link); 96 err = dma_fence_signal_locked(&fence->dma); 97 dma_fence_put(&fence->dma); 98 XE_WARN_ON(err); 99 } 100 spin_unlock_irqrestore(&irq->lock, flags); 101 dma_fence_end_signalling(tmp); 102 } 103 104 /* Safe release of the irq->lock used in dma_fence_init. */ 105 synchronize_rcu(); 106 } 107 108 void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq) 109 { 110 irq_work_queue(&irq->work); 111 } 112 113 void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq) 114 { 115 spin_lock_irq(&irq->lock); 116 irq->enabled = false; 117 spin_unlock_irq(&irq->lock); 118 } 119 120 void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq) 121 { 122 spin_lock_irq(&irq->lock); 123 irq->enabled = true; 124 spin_unlock_irq(&irq->lock); 125 126 irq_work_queue(&irq->work); 127 } 128 129 void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt, 130 struct xe_hw_fence_irq *irq, const char *name) 131 { 132 ctx->gt = gt; 133 ctx->irq = irq; 134 ctx->dma_fence_ctx = dma_fence_context_alloc(1); 135 ctx->next_seqno = XE_FENCE_INITIAL_SEQNO; 136 snprintf(ctx->name, sizeof(ctx->name), "%s", name); 137 } 138 139 void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx) 140 { 141 } 142 143 static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence); 144 145 static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence) 146 { 147 return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock); 148 } 149 150 static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence) 151 { 152 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 153 154 return dev_name(fence->xe->drm.dev); 155 } 156 157 static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence) 158 { 159 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 160 161 return fence->name; 162 } 163 164 static bool xe_hw_fence_signaled(struct dma_fence *dma_fence) 165 { 166 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 167 struct xe_device *xe = fence->xe; 168 u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32); 169 170 return dma_fence->error || 171 !__dma_fence_is_later(dma_fence, dma_fence->seqno, seqno); 172 } 173 174 static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence) 175 { 176 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 177 struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence); 178 179 dma_fence_get(dma_fence); 180 list_add_tail(&fence->irq_link, &irq->pending); 181 182 /* SW completed (no HW IRQ) so kick handler to signal fence */ 183 if (xe_hw_fence_signaled(dma_fence)) 184 xe_hw_fence_irq_run(irq); 185 186 return true; 187 } 188 189 static void xe_hw_fence_release(struct dma_fence *dma_fence) 190 { 191 struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); 192 193 XE_WARN_ON(!list_empty(&fence->irq_link)); 194 call_rcu(&dma_fence->rcu, fence_free); 195 } 196 197 static const struct dma_fence_ops xe_hw_fence_ops = { 198 .get_driver_name = xe_hw_fence_get_driver_name, 199 .get_timeline_name = xe_hw_fence_get_timeline_name, 200 .enable_signaling = xe_hw_fence_enable_signaling, 201 .signaled = xe_hw_fence_signaled, 202 .release = xe_hw_fence_release, 203 }; 204 205 static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence) 206 { 207 if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops)) 208 return NULL; 209 210 return container_of(fence, struct xe_hw_fence, dma); 211 } 212 213 /** 214 * xe_hw_fence_alloc() - Allocate an hw fence. 215 * 216 * Allocate but don't initialize an hw fence. 217 * 218 * Return: Pointer to the allocated fence or 219 * negative error pointer on error. 220 */ 221 struct dma_fence *xe_hw_fence_alloc(void) 222 { 223 struct xe_hw_fence *hw_fence = fence_alloc(); 224 225 if (!hw_fence) 226 return ERR_PTR(-ENOMEM); 227 228 return &hw_fence->dma; 229 } 230 231 /** 232 * xe_hw_fence_free() - Free an hw fence. 233 * @fence: Pointer to the fence to free. 234 * 235 * Frees an hw fence that hasn't yet been 236 * initialized. 237 */ 238 void xe_hw_fence_free(struct dma_fence *fence) 239 { 240 fence_free(&fence->rcu); 241 } 242 243 /** 244 * xe_hw_fence_init() - Initialize an hw fence. 245 * @fence: Pointer to the fence to initialize. 246 * @ctx: Pointer to the struct xe_hw_fence_ctx fence context. 247 * @seqno_map: Pointer to the map into where the seqno is blitted. 248 * 249 * Initializes a pre-allocated hw fence. 250 * After initialization, the fence is subject to normal 251 * dma-fence refcounting. 252 */ 253 void xe_hw_fence_init(struct dma_fence *fence, struct xe_hw_fence_ctx *ctx, 254 struct iosys_map seqno_map) 255 { 256 struct xe_hw_fence *hw_fence = 257 container_of(fence, typeof(*hw_fence), dma); 258 259 hw_fence->xe = gt_to_xe(ctx->gt); 260 snprintf(hw_fence->name, sizeof(hw_fence->name), "%s", ctx->name); 261 hw_fence->seqno_map = seqno_map; 262 INIT_LIST_HEAD(&hw_fence->irq_link); 263 264 dma_fence_init(fence, &xe_hw_fence_ops, &ctx->irq->lock, 265 ctx->dma_fence_ctx, ctx->next_seqno++); 266 267 trace_xe_hw_fence_create(hw_fence); 268 } 269