1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
3
4 #include <linux/dynamic_debug.h>
5
6 #include "core.h"
7
pdsc_process_notifyq(struct pdsc_qcq * qcq)8 static int pdsc_process_notifyq(struct pdsc_qcq *qcq)
9 {
10 union pds_core_notifyq_comp *comp;
11 struct pdsc *pdsc = qcq->pdsc;
12 struct pdsc_cq *cq = &qcq->cq;
13 struct pdsc_cq_info *cq_info;
14 int nq_work = 0;
15 u64 eid;
16
17 cq_info = &cq->info[cq->tail_idx];
18 comp = cq_info->comp;
19 eid = le64_to_cpu(comp->event.eid);
20 while (eid > pdsc->last_eid) {
21 u16 ecode;
22
23 /* Order the payload read after the event id, the field the
24 * driver uses to detect a new completion.
25 */
26 dma_rmb();
27 ecode = le16_to_cpu(comp->event.ecode);
28
29 switch (ecode) {
30 case PDS_EVENT_LINK_CHANGE:
31 dev_info(pdsc->dev, "NotifyQ LINK_CHANGE ecode %d eid %lld\n",
32 ecode, eid);
33 pdsc_notify(PDS_EVENT_LINK_CHANGE, comp);
34 break;
35
36 case PDS_EVENT_RESET:
37 dev_info(pdsc->dev, "NotifyQ RESET ecode %d eid %lld\n",
38 ecode, eid);
39 pdsc_notify(PDS_EVENT_RESET, comp);
40 break;
41
42 case PDS_EVENT_XCVR:
43 dev_info(pdsc->dev, "NotifyQ XCVR ecode %d eid %lld\n",
44 ecode, eid);
45 break;
46
47 default:
48 dev_info(pdsc->dev, "NotifyQ ecode %d eid %lld\n",
49 ecode, eid);
50 break;
51 }
52
53 pdsc->last_eid = eid;
54 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
55 cq_info = &cq->info[cq->tail_idx];
56 comp = cq_info->comp;
57 eid = le64_to_cpu(comp->event.eid);
58
59 nq_work++;
60 }
61
62 qcq->accum_work += nq_work;
63
64 return nq_work;
65 }
66
pdsc_adminq_inc_if_up(struct pdsc * pdsc)67 static bool pdsc_adminq_inc_if_up(struct pdsc *pdsc)
68 {
69 if (pdsc->state & BIT_ULL(PDSC_S_STOPPING_DRIVER) ||
70 pdsc->state & BIT_ULL(PDSC_S_FW_DEAD))
71 return false;
72
73 return refcount_inc_not_zero(&pdsc->adminq_refcnt);
74 }
75
pdsc_process_adminq(struct pdsc_qcq * qcq)76 void pdsc_process_adminq(struct pdsc_qcq *qcq)
77 {
78 union pds_core_adminq_comp *comp;
79 struct pdsc_queue *q = &qcq->q;
80 struct pdsc *pdsc = qcq->pdsc;
81 struct pdsc_cq *cq = &qcq->cq;
82 struct pdsc_q_info *q_info;
83 unsigned long irqflags;
84 int nq_work = 0;
85 int aq_work = 0;
86
87 /* Don't process AdminQ when it's not up */
88 if (!pdsc_adminq_inc_if_up(pdsc)) {
89 dev_err(pdsc->dev, "%s: called while adminq is unavailable\n",
90 __func__);
91 return;
92 }
93
94 /* Check for NotifyQ event */
95 nq_work = pdsc_process_notifyq(&pdsc->notifyqcq);
96
97 /* Check for empty queue, which can happen if the interrupt was
98 * for a NotifyQ event and there are no new AdminQ completions.
99 */
100 if (q->tail_idx == q->head_idx)
101 goto credits;
102
103 /* Find the first completion to clean,
104 * run the callback in the related q_info,
105 * and continue while we still match done color
106 */
107 spin_lock_irqsave(&pdsc->adminq_lock, irqflags);
108 comp = cq->info[cq->tail_idx].comp;
109 while (pdsc_color_match(comp->color, cq->done_color)) {
110 /* Order the payload reads after the color bit, the field the
111 * driver uses to detect a new completion.
112 */
113 dma_rmb();
114 q_info = &q->info[q->tail_idx];
115 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
116
117 if (!completion_done(&q_info->completion)) {
118 memcpy(q_info->dest, comp, sizeof(*comp));
119 complete(&q_info->completion);
120 }
121
122 if (cq->tail_idx == cq->num_descs - 1)
123 cq->done_color = !cq->done_color;
124 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
125 comp = cq->info[cq->tail_idx].comp;
126
127 aq_work++;
128 }
129 spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags);
130
131 qcq->accum_work += aq_work;
132
133 credits:
134 /* Return the interrupt credits, one for each completion */
135 pds_core_intr_credits(&pdsc->intr_ctrl[qcq->intx],
136 nq_work + aq_work,
137 PDS_CORE_INTR_CRED_REARM);
138 refcount_dec(&pdsc->adminq_refcnt);
139 }
140
pdsc_work_thread(struct work_struct * work)141 void pdsc_work_thread(struct work_struct *work)
142 {
143 struct pdsc_qcq *qcq = container_of(work, struct pdsc_qcq, work);
144
145 pdsc_process_adminq(qcq);
146 }
147
pdsc_adminq_isr(int irq,void * data)148 irqreturn_t pdsc_adminq_isr(int irq, void *data)
149 {
150 struct pdsc *pdsc = data;
151 struct pdsc_qcq *qcq;
152
153 /* Don't process AdminQ when it's not up */
154 if (!pdsc_adminq_inc_if_up(pdsc)) {
155 dev_err(pdsc->dev, "%s: called while adminq is unavailable\n",
156 __func__);
157 return IRQ_HANDLED;
158 }
159
160 qcq = &pdsc->adminqcq;
161 queue_work(pdsc->wq, &qcq->work);
162 refcount_dec(&pdsc->adminq_refcnt);
163
164 return IRQ_HANDLED;
165 }
166
__pdsc_adminq_post(struct pdsc * pdsc,struct pdsc_qcq * qcq,union pds_core_adminq_cmd * cmd,union pds_core_adminq_comp * comp)167 static int __pdsc_adminq_post(struct pdsc *pdsc,
168 struct pdsc_qcq *qcq,
169 union pds_core_adminq_cmd *cmd,
170 union pds_core_adminq_comp *comp)
171 {
172 struct pdsc_queue *q = &qcq->q;
173 struct pdsc_q_info *q_info;
174 unsigned long irqflags;
175 unsigned int avail;
176 int index;
177 int ret;
178
179 spin_lock_irqsave(&pdsc->adminq_lock, irqflags);
180
181 /* Check for space in the queue */
182 avail = q->tail_idx;
183 if (q->head_idx >= avail)
184 avail += q->num_descs - q->head_idx - 1;
185 else
186 avail -= q->head_idx + 1;
187 if (!avail) {
188 ret = -ENOSPC;
189 goto err_out_unlock;
190 }
191
192 /* Check that the FW is running */
193 if (!pdsc_is_fw_running(pdsc)) {
194 if (pdsc->info_regs) {
195 u8 fw_status =
196 ioread8(&pdsc->info_regs->fw_status);
197
198 dev_info(pdsc->dev, "%s: post failed - fw not running %#02x:\n",
199 __func__, fw_status);
200 } else {
201 dev_info(pdsc->dev, "%s: post failed - BARs not setup\n",
202 __func__);
203 }
204 ret = -ENXIO;
205
206 goto err_out_unlock;
207 }
208
209 /* Post the request */
210 index = q->head_idx;
211 q_info = &q->info[index];
212 q_info->dest = comp;
213 memcpy(q_info->desc, cmd, sizeof(*cmd));
214 reinit_completion(&q_info->completion);
215
216 dev_dbg(pdsc->dev, "head_idx %d tail_idx %d\n",
217 q->head_idx, q->tail_idx);
218 dev_dbg(pdsc->dev, "post admin queue command:\n");
219 dynamic_hex_dump("cmd ", DUMP_PREFIX_OFFSET, 16, 1,
220 cmd, sizeof(*cmd), true);
221
222 q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);
223
224 pds_core_dbell_ring(pdsc->kern_dbpage,
225 q->hw_type, q->dbval | q->head_idx);
226 ret = index;
227
228 err_out_unlock:
229 spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags);
230 return ret;
231 }
232
pdsc_adminq_post(struct pdsc * pdsc,union pds_core_adminq_cmd * cmd,union pds_core_adminq_comp * comp,bool fast_poll)233 int pdsc_adminq_post(struct pdsc *pdsc,
234 union pds_core_adminq_cmd *cmd,
235 union pds_core_adminq_comp *comp,
236 bool fast_poll)
237 {
238 unsigned long poll_interval = 200;
239 unsigned long poll_jiffies;
240 unsigned long time_limit;
241 unsigned long time_start;
242 unsigned long time_done;
243 unsigned long remaining;
244 struct completion *wc;
245 int err = 0;
246 int index;
247
248 if (!pdsc_adminq_inc_if_up(pdsc)) {
249 dev_dbg(pdsc->dev, "%s: preventing adminq cmd %u\n",
250 __func__, cmd->opcode);
251 return -ENXIO;
252 }
253
254 index = __pdsc_adminq_post(pdsc, &pdsc->adminqcq, cmd, comp);
255 if (index < 0) {
256 err = index;
257 goto err_out;
258 }
259
260 wc = &pdsc->adminqcq.q.info[index].completion;
261 time_start = jiffies;
262 time_limit = time_start + HZ * pdsc->devcmd_timeout;
263 do {
264 /* Timeslice the actual wait to catch IO errors etc early */
265 poll_jiffies = usecs_to_jiffies(poll_interval);
266 remaining = wait_for_completion_timeout(wc, poll_jiffies);
267 if (remaining)
268 break;
269
270 if (!pdsc_is_fw_running(pdsc)) {
271 if (pdsc->info_regs) {
272 u8 fw_status =
273 ioread8(&pdsc->info_regs->fw_status);
274
275 dev_dbg(pdsc->dev, "%s: post wait failed - fw not running %#02x:\n",
276 __func__, fw_status);
277 } else {
278 dev_dbg(pdsc->dev, "%s: post wait failed - BARs not setup\n",
279 __func__);
280 }
281 err = -ENXIO;
282 break;
283 }
284
285 /* When fast_poll is not requested, prevent aggressive polling
286 * on failures due to timeouts by doing exponential back off.
287 */
288 if (!fast_poll && poll_interval < PDSC_ADMINQ_MAX_POLL_INTERVAL)
289 poll_interval <<= 1;
290 } while (time_before(jiffies, time_limit));
291 time_done = jiffies;
292 dev_dbg(pdsc->dev, "%s: elapsed %d msecs\n",
293 __func__, jiffies_to_msecs(time_done - time_start));
294
295 /* Check the results and clear an un-completed timeout */
296 if (time_after_eq(time_done, time_limit) && !completion_done(wc)) {
297 err = -ETIMEDOUT;
298 complete(wc);
299 }
300
301 dev_dbg(pdsc->dev, "read admin queue completion idx %d:\n", index);
302 dynamic_hex_dump("comp ", DUMP_PREFIX_OFFSET, 16, 1,
303 comp, sizeof(*comp), true);
304
305 if (remaining && comp->status)
306 err = pdsc_err_to_errno(comp->status);
307
308 err_out:
309 if (err) {
310 dev_dbg(pdsc->dev, "%s: opcode %d status %d err %pe\n",
311 __func__, cmd->opcode, comp->status, ERR_PTR(err));
312 if (err == -ENXIO || err == -ETIMEDOUT)
313 queue_work(pdsc->wq, &pdsc->health_work);
314 }
315
316 refcount_dec(&pdsc->adminq_refcnt);
317
318 return err;
319 }
320 EXPORT_SYMBOL_GPL(pdsc_adminq_post);
321