1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Driver for Alibaba Elastic Ethernet Adapter.
4 *
5 * Copyright (C) 2025 Alibaba Inc.
6 */
7
8 #include <linux/etherdevice.h>
9 #include <linux/iopoll.h>
10 #include <linux/utsname.h>
11 #include <linux/version.h>
12
13 #include "eea_adminq.h"
14 #include "eea_net.h"
15 #include "eea_pci.h"
16 #include "eea_ring.h"
17
18 #define EEA_AQ_CMD_CFG_QUERY ((0 << 8) | 0)
19
20 #define EEA_AQ_CMD_QUEUE_CREATE ((1 << 8) | 0)
21 #define EEA_AQ_CMD_QUEUE_DESTROY_ALL ((1 << 8) | 1)
22
23 #define EEA_AQ_CMD_HOST_INFO ((2 << 8) | 0)
24
25 #define EEA_AQ_CMD_DEV_STATUS ((3 << 8) | 0)
26
27 #define EEA_RING_DESC_F_AQ_PHASE (BIT(15) | BIT(7))
28
29 #define EEA_QUEUE_FLAGS_HW_SPLIT_HDR BIT(0)
30 #define EEA_QUEUE_FLAGS_SQCQ BIT(1)
31 #define EEA_QUEUE_FLAGS_HWTS BIT(2)
32
33 struct eea_aq_create {
34 __le32 flags;
35 /* queue index.
36 * rx: 0 == qidx % 2
37 * tx: 1 == qidx % 2
38 */
39 __le16 qidx;
40 /* the depth of the queue */
41 __le16 depth;
42 /* 0: without SPLIT HDR
43 * 1: 128B
44 * 2: 256B
45 * 3: 512B
46 */
47 u8 hdr_buf_size;
48 u8 sq_desc_size;
49 u8 cq_desc_size;
50 u8 reserve0;
51 /* The vector for the irq. rx,tx share the same vector */
52 __le16 msix_vector;
53 __le16 reserve;
54 /* sq ring cfg. */
55 __le32 sq_addr_low;
56 __le32 sq_addr_high;
57 /* cq ring cfg. Just valid when flags include EEA_QUEUE_FLAGS_SQCQ. */
58 __le32 cq_addr_low;
59 __le32 cq_addr_high;
60 };
61
62 struct eea_aq_queue_drv_status {
63 __le16 qidx;
64
65 __le16 sq_head;
66 __le16 cq_head;
67 __le16 reserved;
68 };
69
70 #define EEA_OS_DISTRO 0
71 #define EEA_DRV_TYPE 0
72 #define EEA_OS_LINUX 1
73 #define EEA_SPEC_VER_MAJOR 1
74 #define EEA_SPEC_VER_MINOR 0
75
76 struct eea_aq_host_info_cfg {
77 __le16 os_type;
78 __le16 os_dist;
79 __le16 drv_type;
80
81 __le16 kern_ver_major;
82 __le16 kern_ver_minor;
83 __le16 kern_ver_sub_minor;
84
85 __le16 drv_ver_major;
86 __le16 drv_ver_minor;
87 __le16 drv_ver_sub_minor;
88
89 __le16 spec_ver_major;
90 __le16 spec_ver_minor;
91 __le16 pci_bdf;
92 __le32 pci_domain;
93
94 u8 os_ver_str[64];
95 u8 isa_str[64];
96 };
97
98 #define EEA_HINFO_MAX_REP_LEN 1024
99 #define EEA_HINFO_REP_BAD 2
100
101 struct eea_aq_host_info_rep {
102 u8 op_code;
103 u8 has_reply;
104 u8 reply_str[EEA_HINFO_MAX_REP_LEN];
105 };
106
qid_to_ering(struct eea_net * enet,u32 qid)107 static struct eea_ring *qid_to_ering(struct eea_net *enet, u32 qid)
108 {
109 struct eea_ring *ering;
110
111 if (qid % 2 == 0)
112 ering = enet->rx[qid / 2]->ering;
113 else
114 ering = enet->tx[qid / 2].ering;
115
116 return ering;
117 }
118
119 #define EEA_AQ_TIMEOUT_US (60 * 1000 * 1000)
120
eea_device_broken(struct eea_net * enet)121 static void eea_device_broken(struct eea_net *enet)
122 {
123 if (enet->adminq.broken)
124 return;
125
126 eea_device_reset(enet->edev);
127 enet->adminq.broken = true;
128 }
129
eea_adminq_submit(struct eea_net * enet,u16 cmd,dma_addr_t req_addr,dma_addr_t res_addr,u32 req_size,u32 res_size,u32 * reply_len)130 static int eea_adminq_submit(struct eea_net *enet, u16 cmd,
131 dma_addr_t req_addr, dma_addr_t res_addr,
132 u32 req_size, u32 res_size, u32 *reply_len)
133 {
134 struct eea_aq_cdesc *cdesc;
135 struct eea_aq_desc *desc;
136 int ret;
137
138 if (enet->adminq.broken)
139 return -EIO;
140
141 desc = eea_ering_aq_alloc_desc(enet->adminq.ring);
142
143 desc->classid = cmd >> 8;
144 desc->command = cmd & 0xff;
145
146 desc->data_addr = cpu_to_le64(req_addr);
147 desc->data_len = cpu_to_le32(req_size);
148
149 desc->reply_addr = cpu_to_le64(res_addr);
150 desc->reply_len = cpu_to_le32(res_size);
151
152 /* for update flags */
153 dma_wmb();
154
155 desc->flags = cpu_to_le16(enet->adminq.phase);
156
157 eea_ering_sq_commit_desc(enet->adminq.ring);
158
159 eea_ering_kick(enet->adminq.ring);
160
161 ++enet->adminq.num;
162
163 if ((enet->adminq.num % enet->adminq.ring->num) == 0)
164 enet->adminq.phase ^= EEA_RING_DESC_F_AQ_PHASE;
165
166 ret = read_poll_timeout(eea_ering_cq_get_desc, cdesc, cdesc, 10,
167 EEA_AQ_TIMEOUT_US, false, enet->adminq.ring);
168 if (ret) {
169 netdev_err(enet->netdev,
170 "adminq exec timeout. cmd: %d reset device.\n",
171 cmd);
172 /* The device must be reset before unmapping buffers to avoid
173 * potential DMA writes after the memory is freed.
174 */
175 eea_device_broken(enet);
176 return ret;
177 }
178
179 /* Returns 0 on success, or a negative error code on failure. */
180 ret = le32_to_cpu(cdesc->status);
181
182 eea_ering_cq_ack_desc(enet->adminq.ring, 1);
183
184 if (ret)
185 netdev_err(enet->netdev,
186 "adminq exec failed. cmd: %d ret %d\n", cmd, ret);
187 else
188 *reply_len = le32_to_cpu(cdesc->reply_len);
189
190 return ret;
191 }
192
eea_adminq_exec(struct eea_net * enet,u16 cmd,void * req,u32 req_size,void * res,u32 res_size,u32 * reply)193 static int eea_adminq_exec(struct eea_net *enet, u16 cmd,
194 void *req, u32 req_size,
195 void *res, u32 res_size,
196 u32 *reply)
197 {
198 dma_addr_t req_addr = 0, res_addr = 0;
199 struct device *dma;
200 u32 reply_len = 0;
201 int ret;
202
203 if (reply)
204 *reply = 0;
205
206 dma = enet->edev->dma_dev;
207
208 if (req) {
209 req_addr = dma_map_single(dma, req, req_size, DMA_TO_DEVICE);
210 if (unlikely(dma_mapping_error(dma, req_addr)))
211 return -ENOMEM;
212 }
213
214 if (res) {
215 res_addr = dma_map_single(dma, res, res_size, DMA_FROM_DEVICE);
216 if (unlikely(dma_mapping_error(dma, res_addr))) {
217 ret = -ENOMEM;
218 goto err_unmap_req;
219 }
220 }
221
222 mutex_lock(&enet->adminq.lock);
223 ret = eea_adminq_submit(enet, cmd, req_addr, res_addr,
224 req_size, res_size, &reply_len);
225 mutex_unlock(&enet->adminq.lock);
226 if (res) {
227 dma_unmap_single(dma, res_addr, res_size, DMA_FROM_DEVICE);
228
229 if (ret)
230 memset(res, 0, res_size);
231 else if (res_size > reply_len)
232 memset(res + reply_len, 0, res_size - reply_len);
233
234 if (reply)
235 *reply = reply_len;
236 }
237
238 err_unmap_req:
239 if (req)
240 dma_unmap_single(dma, req_addr, req_size, DMA_TO_DEVICE);
241
242 return ret;
243 }
244
eea_destroy_adminq(struct eea_net * enet)245 void eea_destroy_adminq(struct eea_net *enet)
246 {
247 struct eea_aq *aq;
248
249 aq = &enet->adminq;
250
251 if (aq->ring) {
252 eea_ering_free(aq->ring);
253 aq->ring = NULL;
254 aq->phase = 0;
255 }
256
257 kfree(aq->q_req_buf);
258 kfree(aq->q_res_buf);
259
260 aq->q_req_buf = NULL;
261 aq->q_res_buf = NULL;
262 }
263
eea_create_adminq(struct eea_net * enet,u32 qid)264 int eea_create_adminq(struct eea_net *enet, u32 qid)
265 {
266 u32 db_size, q_size, num;
267 struct eea_ring *ering;
268 struct eea_aq *aq;
269 int err = -ENOMEM;
270
271 num = enet->edev->rx_num + enet->edev->tx_num;
272 aq = &enet->adminq;
273
274 ering = eea_ering_alloc(qid, 64, enet->edev, sizeof(struct eea_aq_desc),
275 sizeof(struct eea_aq_cdesc), "adminq");
276 if (!ering)
277 return -ENOMEM;
278
279 aq->ring = ering;
280
281 err = eea_pci_active_aq(ering, qid / 2 + 1);
282 if (err)
283 goto err;
284
285 aq->phase = BIT(7);
286 aq->num = 0;
287
288 q_size = sizeof(*aq->q_req_buf) * num;
289 db_size = sizeof(*aq->q_res_buf) * num;
290
291 aq->q_req_size = q_size;
292 aq->q_res_size = db_size;
293
294 err = -ENOMEM;
295
296 aq->q_req_buf = kzalloc(q_size, GFP_KERNEL);
297 if (!aq->q_req_buf)
298 goto err;
299
300 aq->q_res_buf = kzalloc(db_size, GFP_KERNEL);
301 if (!aq->q_res_buf)
302 goto err;
303
304 /* Before we set up the AQ, the device remains in an inactive state, so
305 * there will be no DMA operations. If the 'set up AQ' process fails, we
306 * can safely free the DMA-related memory.
307 */
308 err = eea_pci_set_aq_up(enet->edev);
309 if (err)
310 goto err;
311
312 aq->broken = false;
313
314 mutex_init(&aq->lock);
315
316 return 0;
317
318 err:
319 eea_destroy_adminq(enet);
320 return err;
321 }
322
eea_adminq_query_cfg(struct eea_net * enet,struct eea_aq_cfg * cfg)323 int eea_adminq_query_cfg(struct eea_net *enet, struct eea_aq_cfg *cfg)
324 {
325 return eea_adminq_exec(enet, EEA_AQ_CMD_CFG_QUERY, NULL, 0, cfg,
326 sizeof(*cfg), NULL);
327 }
328
qcfg_fill(struct eea_aq_create * qcfg,struct eea_ring * ering,u32 flags)329 static void qcfg_fill(struct eea_aq_create *qcfg, struct eea_ring *ering,
330 u32 flags)
331 {
332 qcfg->flags = cpu_to_le32(flags);
333 qcfg->qidx = cpu_to_le16(ering->index);
334 qcfg->depth = cpu_to_le16(ering->num);
335
336 qcfg->hdr_buf_size = flags & EEA_QUEUE_FLAGS_HW_SPLIT_HDR ? 1 : 0;
337 qcfg->sq_desc_size = ering->sq.desc_size;
338 qcfg->cq_desc_size = ering->cq.desc_size;
339 qcfg->msix_vector = cpu_to_le16(ering->msix_vec);
340
341 qcfg->sq_addr_low = cpu_to_le32(lower_32_bits(ering->sq.dma_addr));
342 qcfg->sq_addr_high = cpu_to_le32(upper_32_bits(ering->sq.dma_addr));
343
344 qcfg->cq_addr_low = cpu_to_le32(lower_32_bits(ering->cq.dma_addr));
345 qcfg->cq_addr_high = cpu_to_le32(upper_32_bits(ering->cq.dma_addr));
346 }
347
eea_adminq_create_q(struct eea_net * enet,u32 num,u32 flags)348 int eea_adminq_create_q(struct eea_net *enet, u32 num, u32 flags)
349 {
350 int i, db_size, q_size, err = -ENOMEM;
351 struct eea_net_cfg *cfg;
352 struct eea_ring *ering;
353 struct eea_aq *aq;
354 u32 reply_len;
355
356 cfg = &enet->cfg;
357 aq = &enet->adminq;
358
359 if (cfg->split_hdr)
360 flags |= EEA_QUEUE_FLAGS_HW_SPLIT_HDR;
361
362 flags |= EEA_QUEUE_FLAGS_SQCQ;
363 flags |= EEA_QUEUE_FLAGS_HWTS;
364
365 q_size = sizeof(*aq->q_req_buf) * num;
366 db_size = sizeof(*aq->q_res_buf) * num;
367
368 for (i = 0; i < num; i++) {
369 ering = qid_to_ering(enet, i);
370 qcfg_fill(aq->q_req_buf + i, ering, flags);
371 }
372
373 err = eea_adminq_exec(enet, EEA_AQ_CMD_QUEUE_CREATE,
374 aq->q_req_buf, q_size,
375 aq->q_res_buf, db_size,
376 &reply_len);
377 if (err)
378 return err;
379
380 if (reply_len != db_size) {
381 eea_adminq_destroy_all_q(enet);
382 netdev_err(enet->netdev, "invalid reply len %u\n", reply_len);
383 return -EINVAL;
384 }
385
386 for (i = 0; i < num; i++) {
387 ering = qid_to_ering(enet, i);
388 ering->db = eea_pci_db_addr(ering->edev,
389 le32_to_cpu(aq->q_res_buf[i]));
390 if (!ering->db) {
391 netdev_err(enet->netdev, "invalid db off %u\n",
392 le32_to_cpu(aq->q_res_buf[i]));
393 goto err;
394 }
395 }
396
397 return err;
398
399 err:
400 eea_adminq_destroy_all_q(enet);
401 for (i = 0; i < num; i++) {
402 ering = qid_to_ering(enet, i);
403 ering->db = NULL;
404 }
405
406 return -EIO;
407 }
408
eea_adminq_destroy_all_q(struct eea_net * enet)409 int eea_adminq_destroy_all_q(struct eea_net *enet)
410 {
411 int err;
412
413 err = eea_adminq_exec(enet, EEA_AQ_CMD_QUEUE_DESTROY_ALL, NULL, 0,
414 NULL, 0, NULL);
415 if (err) {
416 /* The device must be reset before unmapping buffers to avoid
417 * potential DMA writes after the memory is freed.
418 */
419 mutex_lock(&enet->adminq.lock);
420 eea_device_broken(enet);
421 mutex_unlock(&enet->adminq.lock);
422
423 netdev_err(enet->netdev, "QUEUE_DESTROY fail: reset device.\n");
424 }
425
426 return err;
427 }
428
429 /* The caller must ensure that both the 'rx' and 'tx' arrays are valid. */
eea_adminq_dev_status(struct eea_net * enet,struct eea_aq_dev_status * dstatus)430 int eea_adminq_dev_status(struct eea_net *enet,
431 struct eea_aq_dev_status *dstatus)
432 {
433 struct eea_aq_queue_drv_status *drv_status;
434 struct __eea_aq_dev_status *dev_status;
435 int err, i, io_num, size, q_num;
436 struct eea_ring *ering;
437 void *rep, *req;
438
439 q_num = enet->cfg.rx_ring_num + enet->cfg.tx_ring_num + 1;
440 io_num = enet->cfg.rx_ring_num + enet->cfg.tx_ring_num;
441
442 req = kzalloc_objs(struct eea_aq_queue_drv_status, q_num);
443 if (!req)
444 return -ENOMEM;
445
446 size = struct_size(dev_status, q_status, q_num);
447
448 rep = kzalloc(size, GFP_KERNEL);
449 if (!rep) {
450 kfree(req);
451 return -ENOMEM;
452 }
453
454 drv_status = req;
455 for (i = 0; i < io_num; ++i, ++drv_status) {
456 ering = qid_to_ering(enet, i);
457 drv_status->qidx = cpu_to_le16(i);
458 drv_status->cq_head = cpu_to_le16(ering->cq.head);
459 drv_status->sq_head = cpu_to_le16(ering->sq.head);
460 }
461
462 drv_status->qidx = cpu_to_le16(i);
463 drv_status->cq_head = cpu_to_le16(enet->adminq.ring->cq.head);
464 drv_status->sq_head = cpu_to_le16(enet->adminq.ring->sq.head);
465
466 err = eea_adminq_exec(enet, EEA_AQ_CMD_DEV_STATUS, req,
467 q_num * sizeof(struct eea_aq_queue_drv_status),
468 rep, size, NULL);
469 kfree(req);
470 if (err) {
471 kfree(rep);
472 return err;
473 }
474
475 dstatus->num = q_num;
476 dstatus->status = rep;
477
478 return 0;
479 }
480
eea_adminq_config_host_info(struct eea_net * enet)481 void eea_adminq_config_host_info(struct eea_net *enet)
482 {
483 struct device *dev = enet->edev->dma_dev;
484 struct eea_aq_host_info_cfg *cfg;
485 struct eea_aq_host_info_rep *rep;
486 int rc = -ENOMEM;
487
488 cfg = kzalloc_obj(*cfg);
489 if (!cfg)
490 return;
491
492 rep = kzalloc_obj(*rep);
493 if (!rep)
494 goto err_free_cfg;
495
496 cfg->os_type = cpu_to_le16(EEA_OS_LINUX);
497 cfg->os_dist = cpu_to_le16(EEA_OS_DISTRO);
498 cfg->drv_type = cpu_to_le16(EEA_DRV_TYPE);
499
500 cfg->kern_ver_major = cpu_to_le16(LINUX_VERSION_MAJOR);
501 cfg->kern_ver_minor = cpu_to_le16(LINUX_VERSION_PATCHLEVEL);
502 cfg->kern_ver_sub_minor = cpu_to_le16(LINUX_VERSION_SUBLEVEL);
503
504 cfg->drv_ver_major = cpu_to_le16(EEA_VER_MAJOR);
505 cfg->drv_ver_minor = cpu_to_le16(EEA_VER_MINOR);
506 cfg->drv_ver_sub_minor = cpu_to_le16(EEA_VER_SUB_MINOR);
507
508 cfg->spec_ver_major = cpu_to_le16(EEA_SPEC_VER_MAJOR);
509 cfg->spec_ver_minor = cpu_to_le16(EEA_SPEC_VER_MINOR);
510
511 cfg->pci_bdf = cpu_to_le16(eea_pci_bdf(enet->edev));
512 cfg->pci_domain = cpu_to_le32(eea_pci_domain_nr(enet->edev));
513
514 strscpy(cfg->os_ver_str, utsname()->release, sizeof(cfg->os_ver_str));
515 strscpy(cfg->isa_str, utsname()->machine, sizeof(cfg->isa_str));
516
517 rc = eea_adminq_exec(enet, EEA_AQ_CMD_HOST_INFO,
518 cfg, sizeof(*cfg), rep, sizeof(*rep), NULL);
519
520 if (!rc) {
521 if (rep->op_code == EEA_HINFO_REP_BAD)
522 dev_warn(dev, "The hardware-driven state validation may be abnormal.\n");
523
524 if (rep->has_reply) {
525 char buf[EEA_HINFO_MAX_REP_LEN] = {0};
526
527 rep->reply_str[EEA_HINFO_MAX_REP_LEN - 1] = '\0';
528
529 string_escape_str(rep->reply_str, buf, sizeof(buf),
530 ESCAPE_NP, NULL);
531
532 buf[EEA_HINFO_MAX_REP_LEN - 1] = '\0';
533
534 dev_warn(dev, "Device replied: %s\n", buf);
535 }
536 }
537
538 kfree(rep);
539 err_free_cfg:
540 kfree(cfg);
541 }
542