1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_preempt_fence.h" 7 8 #include <linux/slab.h> 9 10 #include "xe_exec_queue.h" 11 #include "xe_vm.h" 12 13 static void preempt_fence_work_func(struct work_struct *w) 14 { 15 bool cookie = dma_fence_begin_signalling(); 16 struct xe_preempt_fence *pfence = 17 container_of(w, typeof(*pfence), preempt_work); 18 struct xe_exec_queue *q = pfence->q; 19 20 if (pfence->error) { 21 dma_fence_set_error(&pfence->base, pfence->error); 22 } else if (!q->ops->reset_status(q)) { 23 int err = q->ops->suspend_wait(q); 24 25 if (err) 26 dma_fence_set_error(&pfence->base, err); 27 } else { 28 dma_fence_set_error(&pfence->base, -ENOENT); 29 } 30 31 dma_fence_signal(&pfence->base); 32 /* 33 * Opt for keep everything in the fence critical section. This looks really strange since we 34 * have just signalled the fence, however the preempt fences are all signalled via single 35 * global ordered-wq, therefore anything that happens in this callback can easily block 36 * progress on the entire wq, which itself may prevent other published preempt fences from 37 * ever signalling. Therefore try to keep everything here in the callback in the fence 38 * critical section. For example if something below grabs a scary lock like vm->lock, 39 * lockdep should complain since we also hold that lock whilst waiting on preempt fences to 40 * complete. 41 */ 42 xe_vm_queue_rebind_worker(q->vm); 43 xe_exec_queue_put(q); 44 dma_fence_end_signalling(cookie); 45 } 46 47 static const char * 48 preempt_fence_get_driver_name(struct dma_fence *fence) 49 { 50 return "xe"; 51 } 52 53 static const char * 54 preempt_fence_get_timeline_name(struct dma_fence *fence) 55 { 56 return "preempt"; 57 } 58 59 static bool preempt_fence_enable_signaling(struct dma_fence *fence) 60 { 61 struct xe_preempt_fence *pfence = 62 container_of(fence, typeof(*pfence), base); 63 struct xe_exec_queue *q = pfence->q; 64 65 pfence->error = q->ops->suspend(q); 66 queue_work(q->vm->xe->preempt_fence_wq, &pfence->preempt_work); 67 return true; 68 } 69 70 static const struct dma_fence_ops preempt_fence_ops = { 71 .get_driver_name = preempt_fence_get_driver_name, 72 .get_timeline_name = preempt_fence_get_timeline_name, 73 .enable_signaling = preempt_fence_enable_signaling, 74 }; 75 76 /** 77 * xe_preempt_fence_alloc() - Allocate a preempt fence with minimal 78 * initialization 79 * 80 * Allocate a preempt fence, and initialize its list head. 81 * If the preempt_fence allocated has been armed with 82 * xe_preempt_fence_arm(), it must be freed using dma_fence_put(). If not, 83 * it must be freed using xe_preempt_fence_free(). 84 * 85 * Return: A struct xe_preempt_fence pointer used for calling into 86 * xe_preempt_fence_arm() or xe_preempt_fence_free(). 87 * An error pointer on error. 88 */ 89 struct xe_preempt_fence *xe_preempt_fence_alloc(void) 90 { 91 struct xe_preempt_fence *pfence; 92 93 pfence = kmalloc(sizeof(*pfence), GFP_KERNEL); 94 if (!pfence) 95 return ERR_PTR(-ENOMEM); 96 97 INIT_LIST_HEAD(&pfence->link); 98 INIT_WORK(&pfence->preempt_work, preempt_fence_work_func); 99 100 return pfence; 101 } 102 103 /** 104 * xe_preempt_fence_free() - Free a preempt fence allocated using 105 * xe_preempt_fence_alloc(). 106 * @pfence: pointer obtained from xe_preempt_fence_alloc(); 107 * 108 * Free a preempt fence that has not yet been armed. 109 */ 110 void xe_preempt_fence_free(struct xe_preempt_fence *pfence) 111 { 112 list_del(&pfence->link); 113 kfree(pfence); 114 } 115 116 /** 117 * xe_preempt_fence_arm() - Arm a preempt fence allocated using 118 * xe_preempt_fence_alloc(). 119 * @pfence: The struct xe_preempt_fence pointer returned from 120 * xe_preempt_fence_alloc(). 121 * @q: The struct xe_exec_queue used for arming. 122 * @context: The dma-fence context used for arming. 123 * @seqno: The dma-fence seqno used for arming. 124 * 125 * Inserts the preempt fence into @context's timeline, takes @link off any 126 * list, and registers the struct xe_exec_queue as the xe_engine to be preempted. 127 * 128 * Return: A pointer to a struct dma_fence embedded into the preempt fence. 129 * This function doesn't error. 130 */ 131 struct dma_fence * 132 xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_exec_queue *q, 133 u64 context, u32 seqno) 134 { 135 list_del_init(&pfence->link); 136 pfence->q = xe_exec_queue_get(q); 137 spin_lock_init(&pfence->lock); 138 dma_fence_init(&pfence->base, &preempt_fence_ops, 139 &pfence->lock, context, seqno); 140 141 return &pfence->base; 142 } 143 144 /** 145 * xe_preempt_fence_create() - Helper to create and arm a preempt fence. 146 * @q: The struct xe_exec_queue used for arming. 147 * @context: The dma-fence context used for arming. 148 * @seqno: The dma-fence seqno used for arming. 149 * 150 * Allocates and inserts the preempt fence into @context's timeline, 151 * and registers @e as the struct xe_exec_queue to be preempted. 152 * 153 * Return: A pointer to the resulting struct dma_fence on success. An error 154 * pointer on error. In particular if allocation fails it returns 155 * ERR_PTR(-ENOMEM); 156 */ 157 struct dma_fence * 158 xe_preempt_fence_create(struct xe_exec_queue *q, 159 u64 context, u32 seqno) 160 { 161 struct xe_preempt_fence *pfence; 162 163 pfence = xe_preempt_fence_alloc(); 164 if (IS_ERR(pfence)) 165 return ERR_CAST(pfence); 166 167 return xe_preempt_fence_arm(pfence, q, context, seqno); 168 } 169 170 bool xe_fence_is_xe_preempt(const struct dma_fence *fence) 171 { 172 return fence->ops == &preempt_fence_ops; 173 } 174