1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Driver for Alibaba Elastic Ethernet Adapter. 4 * 5 * Copyright (C) 2025 Alibaba Inc. 6 */ 7 8 #include <linux/etherdevice.h> 9 #include <linux/iopoll.h> 10 #include <linux/utsname.h> 11 #include <linux/version.h> 12 13 #include "eea_adminq.h" 14 #include "eea_net.h" 15 #include "eea_pci.h" 16 #include "eea_ring.h" 17 18 #define EEA_AQ_CMD_CFG_QUERY ((0 << 8) | 0) 19 20 #define EEA_AQ_CMD_QUEUE_CREATE ((1 << 8) | 0) 21 #define EEA_AQ_CMD_QUEUE_DESTROY_ALL ((1 << 8) | 1) 22 23 #define EEA_AQ_CMD_HOST_INFO ((2 << 8) | 0) 24 25 #define EEA_AQ_CMD_DEV_STATUS ((3 << 8) | 0) 26 27 #define EEA_RING_DESC_F_AQ_PHASE (BIT(15) | BIT(7)) 28 29 #define EEA_QUEUE_FLAGS_HW_SPLIT_HDR BIT(0) 30 #define EEA_QUEUE_FLAGS_SQCQ BIT(1) 31 #define EEA_QUEUE_FLAGS_HWTS BIT(2) 32 33 struct eea_aq_create { 34 __le32 flags; 35 /* queue index. 36 * rx: 0 == qidx % 2 37 * tx: 1 == qidx % 2 38 */ 39 __le16 qidx; 40 /* the depth of the queue */ 41 __le16 depth; 42 /* 0: without SPLIT HDR 43 * 1: 128B 44 * 2: 256B 45 * 3: 512B 46 */ 47 u8 hdr_buf_size; 48 u8 sq_desc_size; 49 u8 cq_desc_size; 50 u8 reserve0; 51 /* The vector for the irq. rx,tx share the same vector */ 52 __le16 msix_vector; 53 __le16 reserve; 54 /* sq ring cfg. */ 55 __le32 sq_addr_low; 56 __le32 sq_addr_high; 57 /* cq ring cfg. Just valid when flags include EEA_QUEUE_FLAGS_SQCQ. */ 58 __le32 cq_addr_low; 59 __le32 cq_addr_high; 60 }; 61 62 struct eea_aq_queue_drv_status { 63 __le16 qidx; 64 65 __le16 sq_head; 66 __le16 cq_head; 67 __le16 reserved; 68 }; 69 70 #define EEA_OS_DISTRO 0 71 #define EEA_DRV_TYPE 0 72 #define EEA_OS_LINUX 1 73 #define EEA_SPEC_VER_MAJOR 1 74 #define EEA_SPEC_VER_MINOR 0 75 76 struct eea_aq_host_info_cfg { 77 __le16 os_type; 78 __le16 os_dist; 79 __le16 drv_type; 80 81 __le16 kern_ver_major; 82 __le16 kern_ver_minor; 83 __le16 kern_ver_sub_minor; 84 85 __le16 drv_ver_major; 86 __le16 drv_ver_minor; 87 __le16 drv_ver_sub_minor; 88 89 __le16 spec_ver_major; 90 __le16 spec_ver_minor; 91 __le16 pci_bdf; 92 __le32 pci_domain; 93 94 u8 os_ver_str[64]; 95 u8 isa_str[64]; 96 }; 97 98 #define EEA_HINFO_MAX_REP_LEN 1024 99 #define EEA_HINFO_REP_BAD 2 100 101 struct eea_aq_host_info_rep { 102 u8 op_code; 103 u8 has_reply; 104 u8 reply_str[EEA_HINFO_MAX_REP_LEN]; 105 }; 106 107 static struct eea_ring *qid_to_ering(struct eea_net *enet, u32 qid) 108 { 109 struct eea_ring *ering; 110 111 if (qid % 2 == 0) 112 ering = enet->rx[qid / 2]->ering; 113 else 114 ering = enet->tx[qid / 2].ering; 115 116 return ering; 117 } 118 119 #define EEA_AQ_TIMEOUT_US (60 * 1000 * 1000) 120 121 static void eea_device_broken(struct eea_net *enet) 122 { 123 if (enet->adminq.broken) 124 return; 125 126 eea_device_reset(enet->edev); 127 enet->adminq.broken = true; 128 } 129 130 static int eea_adminq_submit(struct eea_net *enet, u16 cmd, 131 dma_addr_t req_addr, dma_addr_t res_addr, 132 u32 req_size, u32 res_size, u32 *reply_len) 133 { 134 struct eea_aq_cdesc *cdesc; 135 struct eea_aq_desc *desc; 136 int ret; 137 138 if (enet->adminq.broken) 139 return -EIO; 140 141 desc = eea_ering_aq_alloc_desc(enet->adminq.ring); 142 143 desc->classid = cmd >> 8; 144 desc->command = cmd & 0xff; 145 146 desc->data_addr = cpu_to_le64(req_addr); 147 desc->data_len = cpu_to_le32(req_size); 148 149 desc->reply_addr = cpu_to_le64(res_addr); 150 desc->reply_len = cpu_to_le32(res_size); 151 152 /* for update flags */ 153 dma_wmb(); 154 155 desc->flags = cpu_to_le16(enet->adminq.phase); 156 157 eea_ering_sq_commit_desc(enet->adminq.ring); 158 159 eea_ering_kick(enet->adminq.ring); 160 161 ++enet->adminq.num; 162 163 if ((enet->adminq.num % enet->adminq.ring->num) == 0) 164 enet->adminq.phase ^= EEA_RING_DESC_F_AQ_PHASE; 165 166 ret = read_poll_timeout(eea_ering_cq_get_desc, cdesc, cdesc, 10, 167 EEA_AQ_TIMEOUT_US, false, enet->adminq.ring); 168 if (ret) { 169 netdev_err(enet->netdev, 170 "adminq exec timeout. cmd: %d reset device.\n", 171 cmd); 172 /* The device must be reset before unmapping buffers to avoid 173 * potential DMA writes after the memory is freed. 174 */ 175 eea_device_broken(enet); 176 return ret; 177 } 178 179 /* Returns 0 on success, or a negative error code on failure. */ 180 ret = le32_to_cpu(cdesc->status); 181 182 eea_ering_cq_ack_desc(enet->adminq.ring, 1); 183 184 if (ret) 185 netdev_err(enet->netdev, 186 "adminq exec failed. cmd: %d ret %d\n", cmd, ret); 187 else 188 *reply_len = le32_to_cpu(cdesc->reply_len); 189 190 return ret; 191 } 192 193 static int eea_adminq_exec(struct eea_net *enet, u16 cmd, 194 void *req, u32 req_size, 195 void *res, u32 res_size, 196 u32 *reply) 197 { 198 dma_addr_t req_addr = 0, res_addr = 0; 199 struct device *dma; 200 u32 reply_len = 0; 201 int ret; 202 203 if (reply) 204 *reply = 0; 205 206 dma = enet->edev->dma_dev; 207 208 if (req) { 209 req_addr = dma_map_single(dma, req, req_size, DMA_TO_DEVICE); 210 if (unlikely(dma_mapping_error(dma, req_addr))) 211 return -ENOMEM; 212 } 213 214 if (res) { 215 res_addr = dma_map_single(dma, res, res_size, DMA_FROM_DEVICE); 216 if (unlikely(dma_mapping_error(dma, res_addr))) { 217 ret = -ENOMEM; 218 goto err_unmap_req; 219 } 220 } 221 222 mutex_lock(&enet->adminq.lock); 223 ret = eea_adminq_submit(enet, cmd, req_addr, res_addr, 224 req_size, res_size, &reply_len); 225 mutex_unlock(&enet->adminq.lock); 226 if (res) { 227 dma_unmap_single(dma, res_addr, res_size, DMA_FROM_DEVICE); 228 229 if (ret) 230 memset(res, 0, res_size); 231 else if (res_size > reply_len) 232 memset(res + reply_len, 0, res_size - reply_len); 233 234 if (reply) 235 *reply = reply_len; 236 } 237 238 err_unmap_req: 239 if (req) 240 dma_unmap_single(dma, req_addr, req_size, DMA_TO_DEVICE); 241 242 return ret; 243 } 244 245 void eea_destroy_adminq(struct eea_net *enet) 246 { 247 struct eea_aq *aq; 248 249 aq = &enet->adminq; 250 251 if (aq->ring) { 252 eea_ering_free(aq->ring); 253 aq->ring = NULL; 254 aq->phase = 0; 255 } 256 257 kfree(aq->q_req_buf); 258 kfree(aq->q_res_buf); 259 260 aq->q_req_buf = NULL; 261 aq->q_res_buf = NULL; 262 } 263 264 int eea_create_adminq(struct eea_net *enet, u32 qid) 265 { 266 u32 db_size, q_size, num; 267 struct eea_ring *ering; 268 struct eea_aq *aq; 269 int err = -ENOMEM; 270 271 num = enet->edev->rx_num + enet->edev->tx_num; 272 aq = &enet->adminq; 273 274 ering = eea_ering_alloc(qid, 64, enet->edev, sizeof(struct eea_aq_desc), 275 sizeof(struct eea_aq_cdesc), "adminq"); 276 if (!ering) 277 return -ENOMEM; 278 279 aq->ring = ering; 280 281 err = eea_pci_active_aq(ering, qid / 2 + 1); 282 if (err) 283 goto err; 284 285 aq->phase = BIT(7); 286 aq->num = 0; 287 288 q_size = sizeof(*aq->q_req_buf) * num; 289 db_size = sizeof(*aq->q_res_buf) * num; 290 291 aq->q_req_size = q_size; 292 aq->q_res_size = db_size; 293 294 err = -ENOMEM; 295 296 aq->q_req_buf = kzalloc(q_size, GFP_KERNEL); 297 if (!aq->q_req_buf) 298 goto err; 299 300 aq->q_res_buf = kzalloc(db_size, GFP_KERNEL); 301 if (!aq->q_res_buf) 302 goto err; 303 304 /* Before we set up the AQ, the device remains in an inactive state, so 305 * there will be no DMA operations. If the 'set up AQ' process fails, we 306 * can safely free the DMA-related memory. 307 */ 308 err = eea_pci_set_aq_up(enet->edev); 309 if (err) 310 goto err; 311 312 aq->broken = false; 313 314 mutex_init(&aq->lock); 315 316 return 0; 317 318 err: 319 eea_destroy_adminq(enet); 320 return err; 321 } 322 323 int eea_adminq_query_cfg(struct eea_net *enet, struct eea_aq_cfg *cfg) 324 { 325 return eea_adminq_exec(enet, EEA_AQ_CMD_CFG_QUERY, NULL, 0, cfg, 326 sizeof(*cfg), NULL); 327 } 328 329 static void qcfg_fill(struct eea_aq_create *qcfg, struct eea_ring *ering, 330 u32 flags) 331 { 332 qcfg->flags = cpu_to_le32(flags); 333 qcfg->qidx = cpu_to_le16(ering->index); 334 qcfg->depth = cpu_to_le16(ering->num); 335 336 qcfg->hdr_buf_size = flags & EEA_QUEUE_FLAGS_HW_SPLIT_HDR ? 1 : 0; 337 qcfg->sq_desc_size = ering->sq.desc_size; 338 qcfg->cq_desc_size = ering->cq.desc_size; 339 qcfg->msix_vector = cpu_to_le16(ering->msix_vec); 340 341 qcfg->sq_addr_low = cpu_to_le32(lower_32_bits(ering->sq.dma_addr)); 342 qcfg->sq_addr_high = cpu_to_le32(upper_32_bits(ering->sq.dma_addr)); 343 344 qcfg->cq_addr_low = cpu_to_le32(lower_32_bits(ering->cq.dma_addr)); 345 qcfg->cq_addr_high = cpu_to_le32(upper_32_bits(ering->cq.dma_addr)); 346 } 347 348 int eea_adminq_create_q(struct eea_net *enet, u32 num, u32 flags) 349 { 350 int i, db_size, q_size, err = -ENOMEM; 351 struct eea_net_cfg *cfg; 352 struct eea_ring *ering; 353 struct eea_aq *aq; 354 u32 reply_len; 355 356 cfg = &enet->cfg; 357 aq = &enet->adminq; 358 359 if (cfg->split_hdr) 360 flags |= EEA_QUEUE_FLAGS_HW_SPLIT_HDR; 361 362 flags |= EEA_QUEUE_FLAGS_SQCQ; 363 flags |= EEA_QUEUE_FLAGS_HWTS; 364 365 q_size = sizeof(*aq->q_req_buf) * num; 366 db_size = sizeof(*aq->q_res_buf) * num; 367 368 for (i = 0; i < num; i++) { 369 ering = qid_to_ering(enet, i); 370 qcfg_fill(aq->q_req_buf + i, ering, flags); 371 } 372 373 err = eea_adminq_exec(enet, EEA_AQ_CMD_QUEUE_CREATE, 374 aq->q_req_buf, q_size, 375 aq->q_res_buf, db_size, 376 &reply_len); 377 if (err) 378 return err; 379 380 if (reply_len != db_size) { 381 eea_adminq_destroy_all_q(enet); 382 netdev_err(enet->netdev, "invalid reply len %u\n", reply_len); 383 return -EINVAL; 384 } 385 386 for (i = 0; i < num; i++) { 387 ering = qid_to_ering(enet, i); 388 ering->db = eea_pci_db_addr(ering->edev, 389 le32_to_cpu(aq->q_res_buf[i])); 390 if (!ering->db) { 391 netdev_err(enet->netdev, "invalid db off %u\n", 392 le32_to_cpu(aq->q_res_buf[i])); 393 goto err; 394 } 395 } 396 397 return err; 398 399 err: 400 eea_adminq_destroy_all_q(enet); 401 for (i = 0; i < num; i++) { 402 ering = qid_to_ering(enet, i); 403 ering->db = NULL; 404 } 405 406 return -EIO; 407 } 408 409 int eea_adminq_destroy_all_q(struct eea_net *enet) 410 { 411 int err; 412 413 err = eea_adminq_exec(enet, EEA_AQ_CMD_QUEUE_DESTROY_ALL, NULL, 0, 414 NULL, 0, NULL); 415 if (err) { 416 /* The device must be reset before unmapping buffers to avoid 417 * potential DMA writes after the memory is freed. 418 */ 419 mutex_lock(&enet->adminq.lock); 420 eea_device_broken(enet); 421 mutex_unlock(&enet->adminq.lock); 422 423 netdev_err(enet->netdev, "QUEUE_DESTROY fail: reset device.\n"); 424 } 425 426 return err; 427 } 428 429 /* The caller must ensure that both the 'rx' and 'tx' arrays are valid. */ 430 int eea_adminq_dev_status(struct eea_net *enet, 431 struct eea_aq_dev_status *dstatus) 432 { 433 struct eea_aq_queue_drv_status *drv_status; 434 struct __eea_aq_dev_status *dev_status; 435 int err, i, io_num, size, q_num; 436 struct eea_ring *ering; 437 void *rep, *req; 438 439 q_num = enet->cfg.rx_ring_num + enet->cfg.tx_ring_num + 1; 440 io_num = enet->cfg.rx_ring_num + enet->cfg.tx_ring_num; 441 442 req = kcalloc(q_num, sizeof(struct eea_aq_queue_drv_status), 443 GFP_KERNEL); 444 if (!req) 445 return -ENOMEM; 446 447 size = struct_size(dev_status, q_status, q_num); 448 449 rep = kzalloc(size, GFP_KERNEL); 450 if (!rep) { 451 kfree(req); 452 return -ENOMEM; 453 } 454 455 drv_status = req; 456 for (i = 0; i < io_num; ++i, ++drv_status) { 457 ering = qid_to_ering(enet, i); 458 drv_status->qidx = cpu_to_le16(i); 459 drv_status->cq_head = cpu_to_le16(ering->cq.head); 460 drv_status->sq_head = cpu_to_le16(ering->sq.head); 461 } 462 463 drv_status->qidx = cpu_to_le16(i); 464 drv_status->cq_head = cpu_to_le16(enet->adminq.ring->cq.head); 465 drv_status->sq_head = cpu_to_le16(enet->adminq.ring->sq.head); 466 467 err = eea_adminq_exec(enet, EEA_AQ_CMD_DEV_STATUS, req, 468 q_num * sizeof(struct eea_aq_queue_drv_status), 469 rep, size, NULL); 470 kfree(req); 471 if (err) { 472 kfree(rep); 473 return err; 474 } 475 476 dstatus->num = q_num; 477 dstatus->status = rep; 478 479 return 0; 480 } 481 482 void eea_adminq_config_host_info(struct eea_net *enet) 483 { 484 struct device *dev = enet->edev->dma_dev; 485 struct eea_aq_host_info_cfg *cfg; 486 struct eea_aq_host_info_rep *rep; 487 int rc = -ENOMEM; 488 489 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 490 if (!cfg) 491 return; 492 493 rep = kzalloc(sizeof(*rep), GFP_KERNEL); 494 if (!rep) 495 goto err_free_cfg; 496 497 cfg->os_type = cpu_to_le16(EEA_OS_LINUX); 498 cfg->os_dist = cpu_to_le16(EEA_OS_DISTRO); 499 cfg->drv_type = cpu_to_le16(EEA_DRV_TYPE); 500 501 cfg->kern_ver_major = cpu_to_le16(LINUX_VERSION_MAJOR); 502 cfg->kern_ver_minor = cpu_to_le16(LINUX_VERSION_PATCHLEVEL); 503 cfg->kern_ver_sub_minor = cpu_to_le16(LINUX_VERSION_SUBLEVEL); 504 505 cfg->drv_ver_major = cpu_to_le16(EEA_VER_MAJOR); 506 cfg->drv_ver_minor = cpu_to_le16(EEA_VER_MINOR); 507 cfg->drv_ver_sub_minor = cpu_to_le16(EEA_VER_SUB_MINOR); 508 509 cfg->spec_ver_major = cpu_to_le16(EEA_SPEC_VER_MAJOR); 510 cfg->spec_ver_minor = cpu_to_le16(EEA_SPEC_VER_MINOR); 511 512 cfg->pci_bdf = cpu_to_le16(eea_pci_bdf(enet->edev)); 513 cfg->pci_domain = cpu_to_le32(eea_pci_domain_nr(enet->edev)); 514 515 strscpy(cfg->os_ver_str, utsname()->release, sizeof(cfg->os_ver_str)); 516 strscpy(cfg->isa_str, utsname()->machine, sizeof(cfg->isa_str)); 517 518 rc = eea_adminq_exec(enet, EEA_AQ_CMD_HOST_INFO, 519 cfg, sizeof(*cfg), rep, sizeof(*rep), NULL); 520 521 if (!rc) { 522 if (rep->op_code == EEA_HINFO_REP_BAD) 523 dev_warn(dev, "The hardware-driven state validation may be abnormal.\n"); 524 525 if (rep->has_reply) { 526 char buf[EEA_HINFO_MAX_REP_LEN] = {0}; 527 528 rep->reply_str[EEA_HINFO_MAX_REP_LEN - 1] = '\0'; 529 530 string_escape_str(rep->reply_str, buf, sizeof(buf), 531 ESCAPE_NP, NULL); 532 533 buf[EEA_HINFO_MAX_REP_LEN - 1] = '\0'; 534 535 dev_warn(dev, "Device replied: %s\n", buf); 536 } 537 } 538 539 kfree(rep); 540 err_free_cfg: 541 kfree(cfg); 542 } 543