1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_preempt_fence.h" 7 8 #include <linux/slab.h> 9 10 #include "xe_exec_queue.h" 11 #include "xe_vm.h" 12 13 static void preempt_fence_work_func(struct work_struct *w) 14 { 15 bool cookie = dma_fence_begin_signalling(); 16 struct xe_preempt_fence *pfence = 17 container_of(w, typeof(*pfence), preempt_work); 18 struct xe_exec_queue *q = pfence->q; 19 20 if (pfence->error) 21 dma_fence_set_error(&pfence->base, pfence->error); 22 else 23 q->ops->suspend_wait(q); 24 25 dma_fence_signal(&pfence->base); 26 /* 27 * Opt for keep everything in the fence critical section. This looks really strange since we 28 * have just signalled the fence, however the preempt fences are all signalled via single 29 * global ordered-wq, therefore anything that happens in this callback can easily block 30 * progress on the entire wq, which itself may prevent other published preempt fences from 31 * ever signalling. Therefore try to keep everything here in the callback in the fence 32 * critical section. For example if something below grabs a scary lock like vm->lock, 33 * lockdep should complain since we also hold that lock whilst waiting on preempt fences to 34 * complete. 35 */ 36 xe_vm_queue_rebind_worker(q->vm); 37 xe_exec_queue_put(q); 38 dma_fence_end_signalling(cookie); 39 } 40 41 static const char * 42 preempt_fence_get_driver_name(struct dma_fence *fence) 43 { 44 return "xe"; 45 } 46 47 static const char * 48 preempt_fence_get_timeline_name(struct dma_fence *fence) 49 { 50 return "preempt"; 51 } 52 53 static bool preempt_fence_enable_signaling(struct dma_fence *fence) 54 { 55 struct xe_preempt_fence *pfence = 56 container_of(fence, typeof(*pfence), base); 57 struct xe_exec_queue *q = pfence->q; 58 59 pfence->error = q->ops->suspend(q); 60 queue_work(q->vm->xe->preempt_fence_wq, &pfence->preempt_work); 61 return true; 62 } 63 64 static const struct dma_fence_ops preempt_fence_ops = { 65 .get_driver_name = preempt_fence_get_driver_name, 66 .get_timeline_name = preempt_fence_get_timeline_name, 67 .enable_signaling = preempt_fence_enable_signaling, 68 }; 69 70 /** 71 * xe_preempt_fence_alloc() - Allocate a preempt fence with minimal 72 * initialization 73 * 74 * Allocate a preempt fence, and initialize its list head. 75 * If the preempt_fence allocated has been armed with 76 * xe_preempt_fence_arm(), it must be freed using dma_fence_put(). If not, 77 * it must be freed using xe_preempt_fence_free(). 78 * 79 * Return: A struct xe_preempt_fence pointer used for calling into 80 * xe_preempt_fence_arm() or xe_preempt_fence_free(). 81 * An error pointer on error. 82 */ 83 struct xe_preempt_fence *xe_preempt_fence_alloc(void) 84 { 85 struct xe_preempt_fence *pfence; 86 87 pfence = kmalloc(sizeof(*pfence), GFP_KERNEL); 88 if (!pfence) 89 return ERR_PTR(-ENOMEM); 90 91 INIT_LIST_HEAD(&pfence->link); 92 INIT_WORK(&pfence->preempt_work, preempt_fence_work_func); 93 94 return pfence; 95 } 96 97 /** 98 * xe_preempt_fence_free() - Free a preempt fence allocated using 99 * xe_preempt_fence_alloc(). 100 * @pfence: pointer obtained from xe_preempt_fence_alloc(); 101 * 102 * Free a preempt fence that has not yet been armed. 103 */ 104 void xe_preempt_fence_free(struct xe_preempt_fence *pfence) 105 { 106 list_del(&pfence->link); 107 kfree(pfence); 108 } 109 110 /** 111 * xe_preempt_fence_arm() - Arm a preempt fence allocated using 112 * xe_preempt_fence_alloc(). 113 * @pfence: The struct xe_preempt_fence pointer returned from 114 * xe_preempt_fence_alloc(). 115 * @q: The struct xe_exec_queue used for arming. 116 * @context: The dma-fence context used for arming. 117 * @seqno: The dma-fence seqno used for arming. 118 * 119 * Inserts the preempt fence into @context's timeline, takes @link off any 120 * list, and registers the struct xe_exec_queue as the xe_engine to be preempted. 121 * 122 * Return: A pointer to a struct dma_fence embedded into the preempt fence. 123 * This function doesn't error. 124 */ 125 struct dma_fence * 126 xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_exec_queue *q, 127 u64 context, u32 seqno) 128 { 129 list_del_init(&pfence->link); 130 pfence->q = xe_exec_queue_get(q); 131 spin_lock_init(&pfence->lock); 132 dma_fence_init(&pfence->base, &preempt_fence_ops, 133 &pfence->lock, context, seqno); 134 135 return &pfence->base; 136 } 137 138 /** 139 * xe_preempt_fence_create() - Helper to create and arm a preempt fence. 140 * @q: The struct xe_exec_queue used for arming. 141 * @context: The dma-fence context used for arming. 142 * @seqno: The dma-fence seqno used for arming. 143 * 144 * Allocates and inserts the preempt fence into @context's timeline, 145 * and registers @e as the struct xe_exec_queue to be preempted. 146 * 147 * Return: A pointer to the resulting struct dma_fence on success. An error 148 * pointer on error. In particular if allocation fails it returns 149 * ERR_PTR(-ENOMEM); 150 */ 151 struct dma_fence * 152 xe_preempt_fence_create(struct xe_exec_queue *q, 153 u64 context, u32 seqno) 154 { 155 struct xe_preempt_fence *pfence; 156 157 pfence = xe_preempt_fence_alloc(); 158 if (IS_ERR(pfence)) 159 return ERR_CAST(pfence); 160 161 return xe_preempt_fence_arm(pfence, q, context, seqno); 162 } 163 164 bool xe_fence_is_xe_preempt(const struct dma_fence *fence) 165 { 166 return fence->ops == &preempt_fence_ops; 167 } 168