1 /* SPDX-License-Identifier: MIT
2 *
3 * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
4 */
5 #include <nvif/chan.h>
6
7 static void
nvif_chan_gpfifo_push_kick(struct nvif_push * push)8 nvif_chan_gpfifo_push_kick(struct nvif_push *push)
9 {
10 struct nvif_chan *chan = container_of(push, typeof(*chan), push);
11 u32 put = push->bgn - (u32 *)chan->push.mem.object.map.ptr;
12 u32 cnt;
13
14 if (chan->func->gpfifo.post) {
15 if (push->end - push->cur < chan->func->gpfifo.post_size)
16 push->end = push->cur + chan->func->gpfifo.post_size;
17
18 WARN_ON(nvif_chan_gpfifo_post(chan));
19 }
20
21 cnt = push->cur - push->bgn;
22
23 chan->func->gpfifo.push(chan, true, chan->push.addr + (put << 2), cnt << 2, false);
24 chan->func->gpfifo.kick(chan);
25 }
26
27 static int
nvif_chan_gpfifo_push_wait(struct nvif_push * push,u32 push_nr)28 nvif_chan_gpfifo_push_wait(struct nvif_push *push, u32 push_nr)
29 {
30 struct nvif_chan *chan = container_of(push, typeof(*chan), push);
31
32 return nvif_chan_gpfifo_wait(chan, 1, push_nr);
33 }
34
35 int
nvif_chan_gpfifo_post(struct nvif_chan * chan)36 nvif_chan_gpfifo_post(struct nvif_chan *chan)
37 {
38 const u32 *map = chan->push.mem.object.map.ptr;
39 const u32 pbptr = (chan->push.cur - map) + chan->func->gpfifo.post_size;
40 const u32 gpptr = (chan->gpfifo.cur + 1) & chan->gpfifo.max;
41
42 if (!chan->func->gpfifo.post)
43 return 0;
44
45 return chan->func->gpfifo.post(chan, gpptr, pbptr);
46 }
47
48 void
nvif_chan_gpfifo_push(struct nvif_chan * chan,u64 addr,u32 size,bool no_prefetch)49 nvif_chan_gpfifo_push(struct nvif_chan *chan, u64 addr, u32 size, bool no_prefetch)
50 {
51 chan->func->gpfifo.push(chan, false, addr, size, no_prefetch);
52 }
53
54 int
nvif_chan_gpfifo_wait(struct nvif_chan * chan,u32 gpfifo_nr,u32 push_nr)55 nvif_chan_gpfifo_wait(struct nvif_chan *chan, u32 gpfifo_nr, u32 push_nr)
56 {
57 struct nvif_push *push = &chan->push;
58 int ret = 0, time = 1000000;
59
60 if (gpfifo_nr) {
61 /* Account for pushbuf space needed by nvif_chan_gpfifo_post(),
62 * if used after pushing userspace GPFIFO entries.
63 */
64 if (chan->func->gpfifo.post)
65 push_nr += chan->func->gpfifo.post_size;
66 }
67
68 /* Account for the GPFIFO entry needed to submit pushbuf. */
69 if (push_nr)
70 gpfifo_nr++;
71
72 /* Wait for space in main push buffer. */
73 if (push->cur + push_nr > push->end) {
74 ret = nvif_chan_dma_wait(chan, push_nr);
75 if (ret)
76 return ret;
77 }
78
79 /* Wait for GPFIFO space. */
80 while (chan->gpfifo.free < gpfifo_nr) {
81 chan->gpfifo.free = chan->func->gpfifo.read_get(chan) - chan->gpfifo.cur - 1;
82 if (chan->gpfifo.free < 0)
83 chan->gpfifo.free += chan->gpfifo.max + 1;
84
85 if (chan->gpfifo.free < gpfifo_nr) {
86 if (!time--)
87 return -ETIMEDOUT;
88 udelay(1);
89 }
90 }
91
92 return 0;
93 }
94
95 void
nvif_chan_gpfifo_ctor(const struct nvif_chan_func * func,void * userd,void * gpfifo,u32 gpfifo_size,void * push,u64 push_addr,u32 push_size,struct nvif_chan * chan)96 nvif_chan_gpfifo_ctor(const struct nvif_chan_func *func, void *userd, void *gpfifo, u32 gpfifo_size,
97 void *push, u64 push_addr, u32 push_size, struct nvif_chan *chan)
98 {
99 chan->func = func;
100
101 chan->userd.map.ptr = userd;
102
103 chan->gpfifo.map.ptr = gpfifo;
104 chan->gpfifo.max = (gpfifo_size >> 3) - 1;
105 chan->gpfifo.free = chan->gpfifo.max;
106
107 chan->push.mem.object.map.ptr = push;
108 chan->push.wait = nvif_chan_gpfifo_push_wait;
109 chan->push.kick = nvif_chan_gpfifo_push_kick;
110 chan->push.addr = push_addr;
111 chan->push.hw.max = push_size >> 2;
112 chan->push.bgn = chan->push.cur = chan->push.end = push;
113 }
114
115 int
nvif_chan_dma_wait(struct nvif_chan * chan,u32 nr)116 nvif_chan_dma_wait(struct nvif_chan *chan, u32 nr)
117 {
118 struct nvif_push *push = &chan->push;
119 u32 cur = push->cur - (u32 *)push->mem.object.map.ptr;
120 u32 free, time = 1000000;
121
122 nr += chan->func->gpfifo.post_size;
123
124 do {
125 u32 get = chan->func->push.read_get(chan);
126
127 if (get <= cur) {
128 free = push->hw.max - cur;
129 if (free >= nr)
130 break;
131
132 PUSH_KICK(push);
133
134 while (get == 0) {
135 get = chan->func->push.read_get(chan);
136 if (get == 0) {
137 if (!time--)
138 return -ETIMEDOUT;
139 udelay(1);
140 }
141 }
142
143 cur = 0;
144 }
145
146 free = get - cur - 1;
147
148 if (free < nr) {
149 if (!time--)
150 return -ETIMEDOUT;
151 udelay(1);
152 }
153 } while (free < nr);
154
155 push->bgn = (u32 *)push->mem.object.map.ptr + cur;
156 push->cur = push->bgn;
157 push->end = push->bgn + free - chan->func->gpfifo.post_size;
158 return 0;
159 }
160