1 /* SPDX-License-Identifier: MIT 2 * 3 * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. 4 */ 5 #include <nvif/chan.h> 6 7 static void 8 nvif_chan_gpfifo_push_kick(struct nvif_push *push) 9 { 10 struct nvif_chan *chan = container_of(push, typeof(*chan), push); 11 u32 put = push->bgn - (u32 *)chan->push.mem.object.map.ptr; 12 u32 cnt; 13 14 if (chan->func->gpfifo.post) { 15 if (push->end - push->cur < chan->func->gpfifo.post_size) 16 push->end = push->cur + chan->func->gpfifo.post_size; 17 18 WARN_ON(nvif_chan_gpfifo_post(chan)); 19 } 20 21 cnt = push->cur - push->bgn; 22 23 chan->func->gpfifo.push(chan, true, chan->push.addr + (put << 2), cnt << 2, false); 24 chan->func->gpfifo.kick(chan); 25 } 26 27 static int 28 nvif_chan_gpfifo_push_wait(struct nvif_push *push, u32 push_nr) 29 { 30 struct nvif_chan *chan = container_of(push, typeof(*chan), push); 31 32 return nvif_chan_gpfifo_wait(chan, 1, push_nr); 33 } 34 35 int 36 nvif_chan_gpfifo_post(struct nvif_chan *chan) 37 { 38 const u32 *map = chan->push.mem.object.map.ptr; 39 const u32 pbptr = (chan->push.cur - map) + chan->func->gpfifo.post_size; 40 const u32 gpptr = (chan->gpfifo.cur + 1) & chan->gpfifo.max; 41 42 return chan->func->gpfifo.post(chan, gpptr, pbptr); 43 } 44 45 void 46 nvif_chan_gpfifo_push(struct nvif_chan *chan, u64 addr, u32 size, bool no_prefetch) 47 { 48 chan->func->gpfifo.push(chan, false, addr, size, no_prefetch); 49 } 50 51 int 52 nvif_chan_gpfifo_wait(struct nvif_chan *chan, u32 gpfifo_nr, u32 push_nr) 53 { 54 struct nvif_push *push = &chan->push; 55 int ret = 0, time = 1000000; 56 57 if (gpfifo_nr) { 58 /* Account for pushbuf space needed by nvif_chan_gpfifo_post(), 59 * if used after pushing userspace GPFIFO entries. 60 */ 61 if (chan->func->gpfifo.post) 62 push_nr += chan->func->gpfifo.post_size; 63 } 64 65 /* Account for the GPFIFO entry needed to submit pushbuf. */ 66 if (push_nr) 67 gpfifo_nr++; 68 69 /* Wait for space in main push buffer. */ 70 if (push->cur + push_nr > push->end) { 71 ret = nvif_chan_dma_wait(chan, push_nr); 72 if (ret) 73 return ret; 74 } 75 76 /* Wait for GPFIFO space. */ 77 while (chan->gpfifo.free < gpfifo_nr) { 78 chan->gpfifo.free = chan->func->gpfifo.read_get(chan) - chan->gpfifo.cur - 1; 79 if (chan->gpfifo.free < 0) 80 chan->gpfifo.free += chan->gpfifo.max + 1; 81 82 if (chan->gpfifo.free < gpfifo_nr) { 83 if (!time--) 84 return -ETIMEDOUT; 85 udelay(1); 86 } 87 } 88 89 return 0; 90 } 91 92 void 93 nvif_chan_gpfifo_ctor(const struct nvif_chan_func *func, void *userd, void *gpfifo, u32 gpfifo_size, 94 void *push, u64 push_addr, u32 push_size, struct nvif_chan *chan) 95 { 96 chan->func = func; 97 98 chan->userd.map.ptr = userd; 99 100 chan->gpfifo.map.ptr = gpfifo; 101 chan->gpfifo.max = (gpfifo_size >> 3) - 1; 102 chan->gpfifo.free = chan->gpfifo.max; 103 104 chan->push.mem.object.map.ptr = push; 105 chan->push.wait = nvif_chan_gpfifo_push_wait; 106 chan->push.kick = nvif_chan_gpfifo_push_kick; 107 chan->push.addr = push_addr; 108 chan->push.hw.max = push_size >> 2; 109 chan->push.bgn = chan->push.cur = chan->push.end = push; 110 } 111 112 int 113 nvif_chan_dma_wait(struct nvif_chan *chan, u32 nr) 114 { 115 struct nvif_push *push = &chan->push; 116 u32 cur = push->cur - (u32 *)push->mem.object.map.ptr; 117 u32 free, time = 1000000; 118 119 nr += chan->func->gpfifo.post_size; 120 121 do { 122 u32 get = chan->func->push.read_get(chan); 123 124 if (get <= cur) { 125 free = push->hw.max - cur; 126 if (free >= nr) 127 break; 128 129 PUSH_KICK(push); 130 131 while (get == 0) { 132 get = chan->func->push.read_get(chan); 133 if (get == 0) { 134 if (!time--) 135 return -ETIMEDOUT; 136 udelay(1); 137 } 138 } 139 140 cur = 0; 141 } 142 143 free = get - cur - 1; 144 145 if (free < nr) { 146 if (!time--) 147 return -ETIMEDOUT; 148 udelay(1); 149 } 150 } while (free < nr); 151 152 push->bgn = (u32 *)push->mem.object.map.ptr + cur; 153 push->cur = push->bgn; 154 push->end = push->bgn + free - chan->func->gpfifo.post_size; 155 return 0; 156 } 157