1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_preempt_fence.h" 7 8 #include <linux/slab.h> 9 10 #include "xe_exec_queue.h" 11 #include "xe_gt_printk.h" 12 #include "xe_guc_exec_queue_types.h" 13 #include "xe_vm.h" 14 15 static void preempt_fence_work_func(struct work_struct *w) 16 { 17 bool cookie = dma_fence_begin_signalling(); 18 struct xe_preempt_fence *pfence = 19 container_of(w, typeof(*pfence), preempt_work); 20 struct xe_exec_queue *q = pfence->q; 21 22 if (pfence->error) { 23 dma_fence_set_error(&pfence->base, pfence->error); 24 } else if (!q->ops->reset_status(q)) { 25 int err = q->ops->suspend_wait(q); 26 27 if (err == -EAGAIN) { 28 xe_gt_dbg(q->gt, "PREEMPT FENCE RETRY guc_id=%d", 29 q->guc->id); 30 queue_work(q->vm->xe->preempt_fence_wq, 31 &pfence->preempt_work); 32 dma_fence_end_signalling(cookie); 33 return; 34 } 35 36 if (err) 37 dma_fence_set_error(&pfence->base, err); 38 } else { 39 dma_fence_set_error(&pfence->base, -ENOENT); 40 } 41 42 dma_fence_signal(&pfence->base); 43 /* 44 * Opt for keep everything in the fence critical section. This looks really strange since we 45 * have just signalled the fence, however the preempt fences are all signalled via single 46 * global ordered-wq, therefore anything that happens in this callback can easily block 47 * progress on the entire wq, which itself may prevent other published preempt fences from 48 * ever signalling. Therefore try to keep everything here in the callback in the fence 49 * critical section. For example if something below grabs a scary lock like vm->lock, 50 * lockdep should complain since we also hold that lock whilst waiting on preempt fences to 51 * complete. 52 */ 53 xe_vm_queue_rebind_worker(q->vm); 54 xe_exec_queue_put(q); 55 dma_fence_end_signalling(cookie); 56 } 57 58 static const char * 59 preempt_fence_get_driver_name(struct dma_fence *fence) 60 { 61 return "xe"; 62 } 63 64 static const char * 65 preempt_fence_get_timeline_name(struct dma_fence *fence) 66 { 67 return "preempt"; 68 } 69 70 static bool preempt_fence_enable_signaling(struct dma_fence *fence) 71 { 72 struct xe_preempt_fence *pfence = 73 container_of(fence, typeof(*pfence), base); 74 struct xe_exec_queue *q = pfence->q; 75 76 pfence->error = q->ops->suspend(q); 77 /* 78 * Record a successful suspend so the rebind worker only resumes queues 79 * that were actually suspended; a failed suspend() leaves the queue 80 * un-suspended and must not be paired with a resume(). 81 */ 82 if (!pfence->error) 83 WRITE_ONCE(q->lr.suspended, true); 84 queue_work(q->vm->xe->preempt_fence_wq, &pfence->preempt_work); 85 return true; 86 } 87 88 static const struct dma_fence_ops preempt_fence_ops = { 89 .get_driver_name = preempt_fence_get_driver_name, 90 .get_timeline_name = preempt_fence_get_timeline_name, 91 .enable_signaling = preempt_fence_enable_signaling, 92 }; 93 94 /** 95 * xe_preempt_fence_alloc() - Allocate a preempt fence with minimal 96 * initialization 97 * 98 * Allocate a preempt fence, and initialize its list head. 99 * If the preempt_fence allocated has been armed with 100 * xe_preempt_fence_arm(), it must be freed using dma_fence_put(). If not, 101 * it must be freed using xe_preempt_fence_free(). 102 * 103 * Return: A struct xe_preempt_fence pointer used for calling into 104 * xe_preempt_fence_arm() or xe_preempt_fence_free(). 105 * An error pointer on error. 106 */ 107 struct xe_preempt_fence *xe_preempt_fence_alloc(void) 108 { 109 struct xe_preempt_fence *pfence; 110 111 pfence = kmalloc_obj(*pfence); 112 if (!pfence) 113 return ERR_PTR(-ENOMEM); 114 115 INIT_LIST_HEAD(&pfence->link); 116 INIT_WORK(&pfence->preempt_work, preempt_fence_work_func); 117 118 return pfence; 119 } 120 121 /** 122 * xe_preempt_fence_free() - Free a preempt fence allocated using 123 * xe_preempt_fence_alloc(). 124 * @pfence: pointer obtained from xe_preempt_fence_alloc(); 125 * 126 * Free a preempt fence that has not yet been armed. 127 */ 128 void xe_preempt_fence_free(struct xe_preempt_fence *pfence) 129 { 130 list_del(&pfence->link); 131 kfree(pfence); 132 } 133 134 /** 135 * xe_preempt_fence_arm() - Arm a preempt fence allocated using 136 * xe_preempt_fence_alloc(). 137 * @pfence: The struct xe_preempt_fence pointer returned from 138 * xe_preempt_fence_alloc(). 139 * @q: The struct xe_exec_queue used for arming. 140 * @context: The dma-fence context used for arming. 141 * @seqno: The dma-fence seqno used for arming. 142 * 143 * Inserts the preempt fence into @context's timeline, takes @link off any 144 * list, and registers the struct xe_exec_queue as the xe_engine to be preempted. 145 * 146 * Return: A pointer to a struct dma_fence embedded into the preempt fence. 147 * This function doesn't error. 148 */ 149 struct dma_fence * 150 xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_exec_queue *q, 151 u64 context, u32 seqno) 152 { 153 list_del_init(&pfence->link); 154 pfence->q = xe_exec_queue_get(q); 155 spin_lock_init(&pfence->lock); 156 dma_fence_init(&pfence->base, &preempt_fence_ops, 157 &pfence->lock, context, seqno); 158 159 return &pfence->base; 160 } 161 162 /** 163 * xe_preempt_fence_create() - Helper to create and arm a preempt fence. 164 * @q: The struct xe_exec_queue used for arming. 165 * @context: The dma-fence context used for arming. 166 * @seqno: The dma-fence seqno used for arming. 167 * 168 * Allocates and inserts the preempt fence into @context's timeline, 169 * and registers @e as the struct xe_exec_queue to be preempted. 170 * 171 * Return: A pointer to the resulting struct dma_fence on success. An error 172 * pointer on error. In particular if allocation fails it returns 173 * ERR_PTR(-ENOMEM); 174 */ 175 struct dma_fence * 176 xe_preempt_fence_create(struct xe_exec_queue *q, 177 u64 context, u32 seqno) 178 { 179 struct xe_preempt_fence *pfence; 180 181 pfence = xe_preempt_fence_alloc(); 182 if (IS_ERR(pfence)) 183 return ERR_CAST(pfence); 184 185 return xe_preempt_fence_arm(pfence, q, context, seqno); 186 } 187 188 bool xe_fence_is_xe_preempt(const struct dma_fence *fence) 189 { 190 return fence->ops == &preempt_fence_ops; 191 } 192