1 /* SPDX-License-Identifier: MIT 2 * 3 * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. 4 */ 5 #include <nvif/chan.h> 6 7 static void 8 nvif_chan_gpfifo_push_kick(struct nvif_push *push) 9 { 10 struct nvif_chan *chan = container_of(push, typeof(*chan), push); 11 u32 put = push->bgn - (u32 *)chan->push.mem.object.map.ptr; 12 u32 cnt; 13 14 if (chan->func->gpfifo.post) { 15 if (push->end - push->cur < chan->func->gpfifo.post_size) 16 push->end = push->cur + chan->func->gpfifo.post_size; 17 18 WARN_ON(nvif_chan_gpfifo_post(chan)); 19 } 20 21 cnt = push->cur - push->bgn; 22 23 chan->func->gpfifo.push(chan, true, chan->push.addr + (put << 2), cnt << 2, false); 24 chan->func->gpfifo.kick(chan); 25 } 26 27 static int 28 nvif_chan_gpfifo_push_wait(struct nvif_push *push, u32 push_nr) 29 { 30 struct nvif_chan *chan = container_of(push, typeof(*chan), push); 31 32 return nvif_chan_gpfifo_wait(chan, 1, push_nr); 33 } 34 35 int 36 nvif_chan_gpfifo_post(struct nvif_chan *chan) 37 { 38 const u32 *map = chan->push.mem.object.map.ptr; 39 const u32 pbptr = (chan->push.cur - map) + chan->func->gpfifo.post_size; 40 const u32 gpptr = (chan->gpfifo.cur + 1) & chan->gpfifo.max; 41 42 if (!chan->func->gpfifo.post) 43 return 0; 44 45 return chan->func->gpfifo.post(chan, gpptr, pbptr); 46 } 47 48 void 49 nvif_chan_gpfifo_push(struct nvif_chan *chan, u64 addr, u32 size, bool no_prefetch) 50 { 51 chan->func->gpfifo.push(chan, false, addr, size, no_prefetch); 52 } 53 54 int 55 nvif_chan_gpfifo_wait(struct nvif_chan *chan, u32 gpfifo_nr, u32 push_nr) 56 { 57 struct nvif_push *push = &chan->push; 58 int ret = 0, time = 1000000; 59 60 if (gpfifo_nr) { 61 /* Account for pushbuf space needed by nvif_chan_gpfifo_post(), 62 * if used after pushing userspace GPFIFO entries. 63 */ 64 if (chan->func->gpfifo.post) 65 push_nr += chan->func->gpfifo.post_size; 66 } 67 68 /* Account for the GPFIFO entry needed to submit pushbuf. */ 69 if (push_nr) 70 gpfifo_nr++; 71 72 /* Wait for space in main push buffer. */ 73 if (push->cur + push_nr > push->end) { 74 ret = nvif_chan_dma_wait(chan, push_nr); 75 if (ret) 76 return ret; 77 } 78 79 /* Wait for GPFIFO space. */ 80 while (chan->gpfifo.free < gpfifo_nr) { 81 chan->gpfifo.free = chan->func->gpfifo.read_get(chan) - chan->gpfifo.cur - 1; 82 if (chan->gpfifo.free < 0) 83 chan->gpfifo.free += chan->gpfifo.max + 1; 84 85 if (chan->gpfifo.free < gpfifo_nr) { 86 if (!time--) 87 return -ETIMEDOUT; 88 udelay(1); 89 } 90 } 91 92 return 0; 93 } 94 95 void 96 nvif_chan_gpfifo_ctor(const struct nvif_chan_func *func, void *userd, void *gpfifo, u32 gpfifo_size, 97 void *push, u64 push_addr, u32 push_size, struct nvif_chan *chan) 98 { 99 chan->func = func; 100 101 chan->userd.map.ptr = userd; 102 103 chan->gpfifo.map.ptr = gpfifo; 104 chan->gpfifo.max = (gpfifo_size >> 3) - 1; 105 chan->gpfifo.free = chan->gpfifo.max; 106 107 chan->push.mem.object.map.ptr = push; 108 chan->push.wait = nvif_chan_gpfifo_push_wait; 109 chan->push.kick = nvif_chan_gpfifo_push_kick; 110 chan->push.addr = push_addr; 111 chan->push.hw.max = push_size >> 2; 112 chan->push.bgn = chan->push.cur = chan->push.end = push; 113 } 114 115 int 116 nvif_chan_dma_wait(struct nvif_chan *chan, u32 nr) 117 { 118 struct nvif_push *push = &chan->push; 119 u32 cur = push->cur - (u32 *)push->mem.object.map.ptr; 120 u32 free, time = 1000000; 121 122 nr += chan->func->gpfifo.post_size; 123 124 do { 125 u32 get = chan->func->push.read_get(chan); 126 127 if (get <= cur) { 128 free = push->hw.max - cur; 129 if (free >= nr) 130 break; 131 132 PUSH_KICK(push); 133 134 while (get == 0) { 135 get = chan->func->push.read_get(chan); 136 if (get == 0) { 137 if (!time--) 138 return -ETIMEDOUT; 139 udelay(1); 140 } 141 } 142 143 cur = 0; 144 } 145 146 free = get - cur - 1; 147 148 if (free < nr) { 149 if (!time--) 150 return -ETIMEDOUT; 151 udelay(1); 152 } 153 } while (free < nr); 154 155 push->bgn = (u32 *)push->mem.object.map.ptr + cur; 156 push->cur = push->bgn; 157 push->end = push->bgn + free - chan->func->gpfifo.post_size; 158 return 0; 159 } 160