1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */
3
4 #include <linux/errno.h>
5 #include <linux/pci.h>
6 #include <linux/utsname.h>
7
8 #include "core.h"
9
pdsc_err_to_errno(enum pds_core_status_code code)10 int pdsc_err_to_errno(enum pds_core_status_code code)
11 {
12 switch (code) {
13 case PDS_RC_SUCCESS:
14 return 0;
15 case PDS_RC_EVERSION:
16 case PDS_RC_EQTYPE:
17 case PDS_RC_EQID:
18 case PDS_RC_EINVAL:
19 case PDS_RC_ENOSUPP:
20 return -EINVAL;
21 case PDS_RC_EPERM:
22 return -EPERM;
23 case PDS_RC_ENOENT:
24 return -ENOENT;
25 case PDS_RC_EAGAIN:
26 return -EAGAIN;
27 case PDS_RC_ENOMEM:
28 return -ENOMEM;
29 case PDS_RC_EFAULT:
30 return -EFAULT;
31 case PDS_RC_EBUSY:
32 return -EBUSY;
33 case PDS_RC_EEXIST:
34 return -EEXIST;
35 case PDS_RC_EVFID:
36 return -ENODEV;
37 case PDS_RC_ECLIENT:
38 return -ECHILD;
39 case PDS_RC_ENOSPC:
40 return -ENOSPC;
41 case PDS_RC_ERANGE:
42 return -ERANGE;
43 case PDS_RC_BAD_ADDR:
44 return -EFAULT;
45 case PDS_RC_BAD_PCI:
46 return -ENXIO;
47 case PDS_RC_EOPCODE:
48 case PDS_RC_EINTR:
49 case PDS_RC_DEV_CMD:
50 case PDS_RC_ERROR:
51 case PDS_RC_ERDMA:
52 case PDS_RC_EIO:
53 default:
54 return -EIO;
55 }
56 }
57
pdsc_is_fw_running(struct pdsc * pdsc)58 bool pdsc_is_fw_running(struct pdsc *pdsc)
59 {
60 if (!pdsc->info_regs)
61 return false;
62
63 pdsc->fw_status = ioread8(&pdsc->info_regs->fw_status);
64 pdsc->last_fw_time = jiffies;
65 pdsc->last_hb = ioread32(&pdsc->info_regs->fw_heartbeat);
66
67 /* Firmware is useful only if the running bit is set and
68 * fw_status != 0xff (bad PCI read)
69 */
70 return (pdsc->fw_status != PDS_RC_BAD_PCI) &&
71 (pdsc->fw_status & PDS_CORE_FW_STS_F_RUNNING);
72 }
73
pdsc_is_fw_good(struct pdsc * pdsc)74 bool pdsc_is_fw_good(struct pdsc *pdsc)
75 {
76 bool fw_running = pdsc_is_fw_running(pdsc);
77 u8 gen;
78
79 /* Make sure to update the cached fw_status by calling
80 * pdsc_is_fw_running() before getting the generation
81 */
82 gen = pdsc->fw_status & PDS_CORE_FW_STS_F_GENERATION;
83
84 return fw_running && gen == pdsc->fw_generation;
85 }
86
pdsc_devcmd_status(struct pdsc * pdsc)87 static u8 pdsc_devcmd_status(struct pdsc *pdsc)
88 {
89 return ioread8(&pdsc->cmd_regs->comp.status);
90 }
91
pdsc_devcmd_done(struct pdsc * pdsc)92 static bool pdsc_devcmd_done(struct pdsc *pdsc)
93 {
94 return ioread32(&pdsc->cmd_regs->done) & PDS_CORE_DEV_CMD_DONE;
95 }
96
pdsc_devcmd_dbell(struct pdsc * pdsc)97 static void pdsc_devcmd_dbell(struct pdsc *pdsc)
98 {
99 iowrite32(0, &pdsc->cmd_regs->done);
100 iowrite32(1, &pdsc->cmd_regs->doorbell);
101 }
102
pdsc_devcmd_clean(struct pdsc * pdsc)103 static void pdsc_devcmd_clean(struct pdsc *pdsc)
104 {
105 iowrite32(0, &pdsc->cmd_regs->doorbell);
106 memset_io(&pdsc->cmd_regs->cmd, 0, sizeof(pdsc->cmd_regs->cmd));
107 }
108
pdsc_devcmd_str(int opcode)109 static const char *pdsc_devcmd_str(int opcode)
110 {
111 switch (opcode) {
112 case PDS_CORE_CMD_NOP:
113 return "PDS_CORE_CMD_NOP";
114 case PDS_CORE_CMD_IDENTIFY:
115 return "PDS_CORE_CMD_IDENTIFY";
116 case PDS_CORE_CMD_RESET:
117 return "PDS_CORE_CMD_RESET";
118 case PDS_CORE_CMD_INIT:
119 return "PDS_CORE_CMD_INIT";
120 case PDS_CORE_CMD_FW_DOWNLOAD:
121 return "PDS_CORE_CMD_FW_DOWNLOAD";
122 case PDS_CORE_CMD_FW_CONTROL:
123 return "PDS_CORE_CMD_FW_CONTROL";
124 default:
125 return "PDS_CORE_CMD_UNKNOWN";
126 }
127 }
128
__pdsc_devcmd_wait(struct pdsc * pdsc,u8 opcode,int max_seconds,const bool do_msg)129 static int __pdsc_devcmd_wait(struct pdsc *pdsc, u8 opcode, int max_seconds,
130 const bool do_msg)
131 {
132 struct device *dev = pdsc->dev;
133 unsigned long start_time;
134 unsigned long max_wait;
135 unsigned long duration;
136 int timeout = 0;
137 bool running;
138 int done = 0;
139 int err = 0;
140 int status;
141
142 start_time = jiffies;
143 max_wait = start_time + (max_seconds * HZ);
144
145 while (!done && !timeout) {
146 running = pdsc_is_fw_running(pdsc);
147 if (!running)
148 break;
149
150 done = pdsc_devcmd_done(pdsc);
151 if (done)
152 break;
153
154 timeout = time_after(jiffies, max_wait);
155 if (timeout)
156 break;
157
158 usleep_range(100, 200);
159 }
160 duration = jiffies - start_time;
161
162 if (done && duration > HZ)
163 dev_dbg(dev, "DEVCMD %d %s after %ld secs\n",
164 opcode, pdsc_devcmd_str(opcode), duration / HZ);
165
166 if (!running) {
167 dev_err(dev, "DEVCMD %d %s fw not running\n",
168 opcode, pdsc_devcmd_str(opcode));
169 pdsc_devcmd_clean(pdsc);
170 return -ENXIO;
171 }
172
173 if (!done || timeout) {
174 dev_err(dev, "DEVCMD %d %s timeout, done %d timeout %d max_seconds=%d\n",
175 opcode, pdsc_devcmd_str(opcode), done, timeout,
176 max_seconds);
177 pdsc_devcmd_clean(pdsc);
178 return -ETIMEDOUT;
179 }
180
181 status = pdsc_devcmd_status(pdsc);
182 err = pdsc_err_to_errno(status);
183 if (do_msg && err && err != -EAGAIN)
184 dev_err(dev, "DEVCMD %d %s failed, status=%d err %d %pe\n",
185 opcode, pdsc_devcmd_str(opcode), status, err,
186 ERR_PTR(err));
187
188 return err;
189 }
190
__pdsc_devcmd_locked(struct pdsc * pdsc,union pds_core_dev_cmd * cmd,union pds_core_dev_comp * comp,int max_seconds,const bool do_msg)191 static int __pdsc_devcmd_locked(struct pdsc *pdsc, union pds_core_dev_cmd *cmd,
192 union pds_core_dev_comp *comp, int max_seconds,
193 const bool do_msg)
194 {
195 int err;
196
197 if (!pdsc->cmd_regs)
198 return -ENXIO;
199
200 memcpy_toio(&pdsc->cmd_regs->cmd, cmd, sizeof(*cmd));
201 pdsc_devcmd_dbell(pdsc);
202 err = __pdsc_devcmd_wait(pdsc, cmd->opcode, max_seconds, do_msg);
203
204 if ((err == -ENXIO || err == -ETIMEDOUT) && pdsc->wq)
205 queue_work(pdsc->wq, &pdsc->health_work);
206 else
207 memcpy_fromio(comp, &pdsc->cmd_regs->comp, sizeof(*comp));
208
209 if (err != -ETIMEDOUT && err != -EAGAIN)
210 pdsc_deferred_dma_free(pdsc);
211
212 return err;
213 }
214
pdsc_deferred_dma_add(struct pdsc * pdsc,struct pdsc_deferred_dma * entry,dma_addr_t dma_addr,void * va,size_t size,enum dma_data_direction dir)215 void pdsc_deferred_dma_add(struct pdsc *pdsc, struct pdsc_deferred_dma *entry,
216 dma_addr_t dma_addr, void *va, size_t size,
217 enum dma_data_direction dir)
218 {
219 entry->dma_addr = dma_addr;
220 entry->va = va;
221 entry->size = size;
222 entry->dir = dir;
223
224 spin_lock(&pdsc->deferred_dma_lock);
225 list_add_tail(&entry->list, &pdsc->deferred_dma_list);
226 spin_unlock(&pdsc->deferred_dma_lock);
227 }
228
pdsc_deferred_dma_free(struct pdsc * pdsc)229 void pdsc_deferred_dma_free(struct pdsc *pdsc)
230 {
231 struct pdsc_deferred_dma *entry, *tmp;
232 LIST_HEAD(local_list);
233
234 spin_lock(&pdsc->deferred_dma_lock);
235 list_splice_init(&pdsc->deferred_dma_list, &local_list);
236 spin_unlock(&pdsc->deferred_dma_lock);
237
238 list_for_each_entry_safe(entry, tmp, &local_list, list) {
239 dma_unmap_single(pdsc->dev, entry->dma_addr,
240 entry->size, entry->dir);
241 kfree(entry->va);
242 list_del(&entry->list);
243 kfree(entry);
244 }
245 }
246
pdsc_devcmd_locked(struct pdsc * pdsc,union pds_core_dev_cmd * cmd,union pds_core_dev_comp * comp,int max_seconds)247 int pdsc_devcmd_locked(struct pdsc *pdsc, union pds_core_dev_cmd *cmd,
248 union pds_core_dev_comp *comp, int max_seconds)
249 {
250 return __pdsc_devcmd_locked(pdsc, cmd, comp, max_seconds, true);
251 }
252
pdsc_devcmd_locked_nomsg(struct pdsc * pdsc,union pds_core_dev_cmd * cmd,union pds_core_dev_comp * comp,int max_seconds)253 int pdsc_devcmd_locked_nomsg(struct pdsc *pdsc, union pds_core_dev_cmd *cmd,
254 union pds_core_dev_comp *comp, int max_seconds)
255 {
256 return __pdsc_devcmd_locked(pdsc, cmd, comp, max_seconds, false);
257 }
258
pdsc_devcmd(struct pdsc * pdsc,union pds_core_dev_cmd * cmd,union pds_core_dev_comp * comp,int max_seconds)259 int pdsc_devcmd(struct pdsc *pdsc, union pds_core_dev_cmd *cmd,
260 union pds_core_dev_comp *comp, int max_seconds)
261 {
262 int err;
263
264 mutex_lock(&pdsc->devcmd_lock);
265 err = pdsc_devcmd_locked(pdsc, cmd, comp, max_seconds);
266 mutex_unlock(&pdsc->devcmd_lock);
267
268 return err;
269 }
270
__pdsc_devcmd_with_data(struct pdsc * pdsc,union pds_core_dev_cmd * cmd,const void * data,size_t data_len,union pds_core_dev_comp * comp,int max_seconds,bool do_msg)271 static int __pdsc_devcmd_with_data(struct pdsc *pdsc,
272 union pds_core_dev_cmd *cmd,
273 const void *data, size_t data_len,
274 union pds_core_dev_comp *comp,
275 int max_seconds, bool do_msg)
276 {
277 int err;
278
279 mutex_lock(&pdsc->devcmd_lock);
280 if (!pdsc->cmd_regs) {
281 err = -ENXIO;
282 goto unlock;
283 }
284 if (data_len > sizeof(pdsc->cmd_regs->data)) {
285 err = -ENOSPC;
286 goto unlock;
287 }
288 memcpy_toio(&pdsc->cmd_regs->data, data, data_len);
289 err = __pdsc_devcmd_locked(pdsc, cmd, comp, max_seconds, do_msg);
290 unlock:
291 mutex_unlock(&pdsc->devcmd_lock);
292
293 return err;
294 }
295
pdsc_devcmd_with_data(struct pdsc * pdsc,union pds_core_dev_cmd * cmd,const void * data,size_t data_len,union pds_core_dev_comp * comp,int max_seconds)296 int pdsc_devcmd_with_data(struct pdsc *pdsc, union pds_core_dev_cmd *cmd,
297 const void *data, size_t data_len,
298 union pds_core_dev_comp *comp, int max_seconds)
299 {
300 return __pdsc_devcmd_with_data(pdsc, cmd, data, data_len,
301 comp, max_seconds, true);
302 }
303
pdsc_devcmd_with_data_nomsg(struct pdsc * pdsc,union pds_core_dev_cmd * cmd,const void * data,size_t data_len,union pds_core_dev_comp * comp,int max_seconds)304 int pdsc_devcmd_with_data_nomsg(struct pdsc *pdsc, union pds_core_dev_cmd *cmd,
305 const void *data, size_t data_len,
306 union pds_core_dev_comp *comp, int max_seconds)
307 {
308 return __pdsc_devcmd_with_data(pdsc, cmd, data, data_len,
309 comp, max_seconds, false);
310 }
311
pdsc_devcmd_init(struct pdsc * pdsc)312 int pdsc_devcmd_init(struct pdsc *pdsc)
313 {
314 union pds_core_dev_comp comp = {};
315 union pds_core_dev_cmd cmd = {
316 .opcode = PDS_CORE_CMD_INIT,
317 };
318
319 return pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
320 }
321
pdsc_devcmd_reset(struct pdsc * pdsc)322 int pdsc_devcmd_reset(struct pdsc *pdsc)
323 {
324 union pds_core_dev_comp comp = {};
325 union pds_core_dev_cmd cmd = {
326 .reset.opcode = PDS_CORE_CMD_RESET,
327 };
328
329 if (!pdsc_is_fw_running(pdsc))
330 return 0;
331
332 return pdsc_devcmd(pdsc, &cmd, &comp, pdsc->devcmd_timeout);
333 }
334
pdsc_devcmd_identify_locked(struct pdsc * pdsc,u8 drv_ident_ver,bool do_msg)335 static int pdsc_devcmd_identify_locked(struct pdsc *pdsc, u8 drv_ident_ver,
336 bool do_msg)
337 {
338 union pds_core_dev_comp comp = {};
339 union pds_core_dev_cmd cmd = {
340 .identify.opcode = PDS_CORE_CMD_IDENTIFY,
341 .identify.ver = drv_ident_ver,
342 };
343
344 return __pdsc_devcmd_locked(pdsc, &cmd, &comp, pdsc->devcmd_timeout,
345 do_msg);
346 }
347
pdsc_init_devinfo(struct pdsc * pdsc)348 static void pdsc_init_devinfo(struct pdsc *pdsc)
349 {
350 pdsc->dev_info.asic_type = ioread8(&pdsc->info_regs->asic_type);
351 pdsc->dev_info.asic_rev = ioread8(&pdsc->info_regs->asic_rev);
352 pdsc->fw_generation = PDS_CORE_FW_STS_F_GENERATION &
353 ioread8(&pdsc->info_regs->fw_status);
354
355 memcpy_fromio(pdsc->dev_info.fw_version,
356 pdsc->info_regs->fw_version,
357 PDS_CORE_DEVINFO_FWVERS_BUFLEN);
358 pdsc->dev_info.fw_version[PDS_CORE_DEVINFO_FWVERS_BUFLEN] = 0;
359
360 memcpy_fromio(pdsc->dev_info.serial_num,
361 pdsc->info_regs->serial_num,
362 PDS_CORE_DEVINFO_SERIAL_BUFLEN);
363 pdsc->dev_info.serial_num[PDS_CORE_DEVINFO_SERIAL_BUFLEN] = 0;
364
365 dev_dbg(pdsc->dev, "fw_version %s\n", pdsc->dev_info.fw_version);
366 }
367
pdsc_identify_ver(struct pdsc * pdsc,u8 drv_ident_ver)368 static int pdsc_identify_ver(struct pdsc *pdsc, u8 drv_ident_ver)
369 {
370 bool do_msg = drv_ident_ver == PDS_CORE_IDENTITY_VERSION_1;
371 struct pds_core_drv_identity drv = {};
372 size_t sz;
373 int err;
374 int n;
375
376 drv.drv_type = cpu_to_le32(PDS_DRIVER_LINUX);
377 /* Catching the return quiets a Wformat-truncation complaint */
378 n = snprintf(drv.driver_ver_str, sizeof(drv.driver_ver_str),
379 "%s %s", PDS_CORE_DRV_NAME, utsname()->release);
380 if (n > sizeof(drv.driver_ver_str))
381 dev_dbg(pdsc->dev, "release name truncated, don't care\n");
382
383 /* Next let's get some info about the device
384 * We use the devcmd_lock at this level in order to
385 * get safe access to the cmd_regs->data before anyone
386 * else can mess it up
387 */
388 mutex_lock(&pdsc->devcmd_lock);
389
390 sz = min_t(size_t, sizeof(drv), sizeof(pdsc->cmd_regs->data));
391 memcpy_toio(&pdsc->cmd_regs->data, &drv, sz);
392
393 err = pdsc_devcmd_identify_locked(pdsc, drv_ident_ver, do_msg);
394 if (!err) {
395 sz = min_t(size_t, sizeof(pdsc->dev_ident),
396 sizeof(pdsc->cmd_regs->data));
397 memcpy_fromio(&pdsc->dev_ident, &pdsc->cmd_regs->data, sz);
398
399 /* V1 firmware doesn't set capabilities, so the field may
400 * contain garbage from the outgoing driver identity.
401 */
402 if (pdsc->dev_ident.version < PDS_CORE_IDENTITY_VERSION_2)
403 pdsc->dev_ident.capabilities = 0;
404 }
405 mutex_unlock(&pdsc->devcmd_lock);
406
407 if (err)
408 return err;
409
410 if (isprint(pdsc->dev_info.fw_version[0]) &&
411 isascii(pdsc->dev_info.fw_version[0]))
412 dev_info(pdsc->dev, "FW: %.*s\n",
413 (int)(sizeof(pdsc->dev_info.fw_version) - 1),
414 pdsc->dev_info.fw_version);
415 else
416 dev_info(pdsc->dev, "FW: (invalid string) 0x%02x 0x%02x 0x%02x 0x%02x ...\n",
417 (u8)pdsc->dev_info.fw_version[0],
418 (u8)pdsc->dev_info.fw_version[1],
419 (u8)pdsc->dev_info.fw_version[2],
420 (u8)pdsc->dev_info.fw_version[3]);
421
422 return 0;
423 }
424
pdsc_identify(struct pdsc * pdsc)425 static int pdsc_identify(struct pdsc *pdsc)
426 {
427 int err;
428
429 /* Older firmware rejects anything but PDS_CORE_IDENTITY_VERSION_1
430 * with PDS_RC_EVERSION (-EINVAL), so retry with V1 on version
431 * rejection. Don't retry on other errors like -ENXIO/-ETIMEDOUT
432 * which indicate firmware is not running or hung.
433 */
434 err = pdsc_identify_ver(pdsc, PDS_CORE_IDENTITY_VERSION_2);
435 if (err == -EINVAL)
436 err = pdsc_identify_ver(pdsc, PDS_CORE_IDENTITY_VERSION_1);
437
438 if (err)
439 dev_err(pdsc->dev, "Cannot identify device: %pe\n",
440 ERR_PTR(err));
441
442 return err;
443 }
444
pdsc_dev_uninit(struct pdsc * pdsc)445 void pdsc_dev_uninit(struct pdsc *pdsc)
446 {
447 if (pdsc->intr_info) {
448 int i;
449
450 for (i = 0; i < pdsc->nintrs; i++)
451 pdsc_intr_free(pdsc, i);
452
453 kfree(pdsc->intr_info);
454 pdsc->intr_info = NULL;
455 pdsc->nintrs = 0;
456 }
457
458 pci_free_irq_vectors(pdsc->pdev);
459 }
460
pdsc_dev_init(struct pdsc * pdsc)461 int pdsc_dev_init(struct pdsc *pdsc)
462 {
463 unsigned int nintrs;
464 int err;
465
466 /* Initial init and reset of device */
467 pdsc_init_devinfo(pdsc);
468 pdsc->devcmd_timeout = PDS_CORE_DEVCMD_TIMEOUT;
469
470 err = pdsc_devcmd_reset(pdsc);
471 if (err)
472 return err;
473
474 err = pdsc_identify(pdsc);
475 if (err)
476 return err;
477
478 pdsc_debugfs_add_ident(pdsc);
479
480 /* Now we can reserve interrupts */
481 nintrs = le32_to_cpu(pdsc->dev_ident.nintrs);
482 nintrs = min_t(unsigned int, num_online_cpus(), nintrs);
483
484 /* Get intr_info struct array for tracking */
485 pdsc->intr_info = kzalloc_objs(*pdsc->intr_info, nintrs);
486 if (!pdsc->intr_info)
487 return -ENOMEM;
488
489 err = pci_alloc_irq_vectors(pdsc->pdev, nintrs, nintrs, PCI_IRQ_MSIX);
490 if (err != nintrs) {
491 dev_err(pdsc->dev, "Can't get %d intrs from OS: %pe\n",
492 nintrs, ERR_PTR(err));
493 err = -ENOSPC;
494 goto err_out;
495 }
496 pdsc->nintrs = nintrs;
497
498 return 0;
499
500 err_out:
501 kfree(pdsc->intr_info);
502 pdsc->intr_info = NULL;
503
504 return err;
505 }
506