1 // SPDX-License-Identifier: GPL-2.0-only 2 /**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2012-2013 Solarflare Communications Inc. 5 */ 6 7 #include "net_driver.h" 8 #include "rx_common.h" 9 #include "tx_common.h" 10 #include "ef10_regs.h" 11 #include "io.h" 12 #include "mcdi.h" 13 #include "mcdi_pcol.h" 14 #include "mcdi_port.h" 15 #include "mcdi_port_common.h" 16 #include "mcdi_functions.h" 17 #include "nic.h" 18 #include "mcdi_filters.h" 19 #include "workarounds.h" 20 #include "selftest.h" 21 #include "ef10_sriov.h" 22 #include <linux/in.h> 23 #include <linux/jhash.h> 24 #include <linux/wait.h> 25 #include <linux/workqueue.h> 26 #include <net/udp_tunnel.h> 27 #include "efx_cxl.h" 28 29 /* Hardware control for EF10 architecture including 'Huntington'. */ 30 31 #define EFX_EF10_DRVGEN_EV 7 32 enum { 33 EFX_EF10_TEST = 1, 34 EFX_EF10_REFILL, 35 }; 36 37 /* VLAN list entry */ 38 struct efx_ef10_vlan { 39 struct list_head list; 40 u16 vid; 41 }; 42 43 static int efx_ef10_set_udp_tnl_ports(struct efx_nic *efx, bool unloading); 44 static const struct udp_tunnel_nic_info efx_ef10_udp_tunnels; 45 46 static int efx_ef10_get_warm_boot_count(struct efx_nic *efx) 47 { 48 efx_dword_t reg; 49 50 efx_readd(efx, ®, ER_DZ_BIU_MC_SFT_STATUS); 51 return EFX_DWORD_FIELD(reg, EFX_WORD_1) == 0xb007 ? 52 EFX_DWORD_FIELD(reg, EFX_WORD_0) : -EIO; 53 } 54 55 /* On all EF10s up to and including SFC9220 (Medford1), all PFs use BAR 0 for 56 * I/O space and BAR 2(&3) for memory. On SFC9250 (Medford2), there is no I/O 57 * bar; PFs use BAR 0/1 for memory. 58 */ 59 static unsigned int efx_ef10_pf_mem_bar(struct efx_nic *efx) 60 { 61 switch (efx->pci_dev->device) { 62 case 0x0b03: /* SFC9250 PF */ 63 return 0; 64 default: 65 return 2; 66 } 67 } 68 69 /* All VFs use BAR 0/1 for memory */ 70 static unsigned int efx_ef10_vf_mem_bar(struct efx_nic *efx) 71 { 72 return 0; 73 } 74 75 static unsigned int efx_ef10_mem_map_size(struct efx_nic *efx) 76 { 77 int bar; 78 79 bar = efx->type->mem_bar(efx); 80 return resource_size(&efx->pci_dev->resource[bar]); 81 } 82 83 static bool efx_ef10_is_vf(struct efx_nic *efx) 84 { 85 return efx->type->is_vf; 86 } 87 88 #ifdef CONFIG_SFC_SRIOV 89 static int efx_ef10_get_vf_index(struct efx_nic *efx) 90 { 91 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_FUNCTION_INFO_OUT_LEN); 92 struct efx_ef10_nic_data *nic_data = efx->nic_data; 93 size_t outlen; 94 int rc; 95 96 rc = efx_mcdi_rpc(efx, MC_CMD_GET_FUNCTION_INFO, NULL, 0, outbuf, 97 sizeof(outbuf), &outlen); 98 if (rc) 99 return rc; 100 if (outlen < sizeof(outbuf)) 101 return -EIO; 102 103 nic_data->vf_index = MCDI_DWORD(outbuf, GET_FUNCTION_INFO_OUT_VF); 104 return 0; 105 } 106 #endif 107 108 static int efx_ef10_init_datapath_caps(struct efx_nic *efx) 109 { 110 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CAPABILITIES_V7_OUT_LEN); 111 struct efx_ef10_nic_data *nic_data = efx->nic_data; 112 size_t outlen; 113 int rc; 114 115 BUILD_BUG_ON(MC_CMD_GET_CAPABILITIES_IN_LEN != 0); 116 117 rc = efx_mcdi_rpc(efx, MC_CMD_GET_CAPABILITIES, NULL, 0, 118 outbuf, sizeof(outbuf), &outlen); 119 if (rc) 120 return rc; 121 if (outlen < MC_CMD_GET_CAPABILITIES_OUT_LEN) { 122 netif_err(efx, drv, efx->net_dev, 123 "unable to read datapath firmware capabilities\n"); 124 return -EIO; 125 } 126 127 nic_data->datapath_caps = 128 MCDI_DWORD(outbuf, GET_CAPABILITIES_OUT_FLAGS1); 129 130 if (outlen >= MC_CMD_GET_CAPABILITIES_V2_OUT_LEN) { 131 nic_data->datapath_caps2 = MCDI_DWORD(outbuf, 132 GET_CAPABILITIES_V2_OUT_FLAGS2); 133 nic_data->piobuf_size = MCDI_WORD(outbuf, 134 GET_CAPABILITIES_V2_OUT_SIZE_PIO_BUFF); 135 } else { 136 nic_data->datapath_caps2 = 0; 137 nic_data->piobuf_size = ER_DZ_TX_PIOBUF_SIZE; 138 } 139 140 /* record the DPCPU firmware IDs to determine VEB vswitching support. 141 */ 142 nic_data->rx_dpcpu_fw_id = 143 MCDI_WORD(outbuf, GET_CAPABILITIES_OUT_RX_DPCPU_FW_ID); 144 nic_data->tx_dpcpu_fw_id = 145 MCDI_WORD(outbuf, GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID); 146 147 if (!(nic_data->datapath_caps & 148 (1 << MC_CMD_GET_CAPABILITIES_OUT_RX_PREFIX_LEN_14_LBN))) { 149 netif_err(efx, probe, efx->net_dev, 150 "current firmware does not support an RX prefix\n"); 151 return -ENODEV; 152 } 153 154 if (outlen >= MC_CMD_GET_CAPABILITIES_V3_OUT_LEN) { 155 u8 vi_window_mode = MCDI_BYTE(outbuf, 156 GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE); 157 158 rc = efx_mcdi_window_mode_to_stride(efx, vi_window_mode); 159 if (rc) 160 return rc; 161 } else { 162 /* keep default VI stride */ 163 netif_dbg(efx, probe, efx->net_dev, 164 "firmware did not report VI window mode, assuming vi_stride = %u\n", 165 efx->vi_stride); 166 } 167 168 if (outlen >= MC_CMD_GET_CAPABILITIES_V4_OUT_LEN) { 169 efx->num_mac_stats = MCDI_WORD(outbuf, 170 GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS); 171 netif_dbg(efx, probe, efx->net_dev, 172 "firmware reports num_mac_stats = %u\n", 173 efx->num_mac_stats); 174 } else { 175 /* leave num_mac_stats as the default value, MC_CMD_MAC_NSTATS */ 176 netif_dbg(efx, probe, efx->net_dev, 177 "firmware did not report num_mac_stats, assuming %u\n", 178 efx->num_mac_stats); 179 } 180 181 if (outlen < MC_CMD_GET_CAPABILITIES_V7_OUT_LEN) 182 nic_data->datapath_caps3 = 0; 183 else 184 nic_data->datapath_caps3 = MCDI_DWORD(outbuf, 185 GET_CAPABILITIES_V7_OUT_FLAGS3); 186 187 return 0; 188 } 189 190 static void efx_ef10_read_licensed_features(struct efx_nic *efx) 191 { 192 MCDI_DECLARE_BUF(inbuf, MC_CMD_LICENSING_V3_IN_LEN); 193 MCDI_DECLARE_BUF(outbuf, MC_CMD_LICENSING_V3_OUT_LEN); 194 struct efx_ef10_nic_data *nic_data = efx->nic_data; 195 size_t outlen; 196 int rc; 197 198 MCDI_SET_DWORD(inbuf, LICENSING_V3_IN_OP, 199 MC_CMD_LICENSING_V3_IN_OP_REPORT_LICENSE); 200 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_LICENSING_V3, inbuf, sizeof(inbuf), 201 outbuf, sizeof(outbuf), &outlen); 202 if (rc || (outlen < MC_CMD_LICENSING_V3_OUT_LEN)) 203 return; 204 205 nic_data->licensed_features = MCDI_QWORD(outbuf, 206 LICENSING_V3_OUT_LICENSED_FEATURES); 207 } 208 209 static int efx_ef10_get_sysclk_freq(struct efx_nic *efx) 210 { 211 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CLOCK_OUT_LEN); 212 int rc; 213 214 rc = efx_mcdi_rpc(efx, MC_CMD_GET_CLOCK, NULL, 0, 215 outbuf, sizeof(outbuf), NULL); 216 if (rc) 217 return rc; 218 rc = MCDI_DWORD(outbuf, GET_CLOCK_OUT_SYS_FREQ); 219 return rc > 0 ? rc : -ERANGE; 220 } 221 222 static int efx_ef10_get_timer_workarounds(struct efx_nic *efx) 223 { 224 struct efx_ef10_nic_data *nic_data = efx->nic_data; 225 unsigned int implemented; 226 unsigned int enabled; 227 int rc; 228 229 nic_data->workaround_35388 = false; 230 nic_data->workaround_61265 = false; 231 232 rc = efx_mcdi_get_workarounds(efx, &implemented, &enabled); 233 234 if (rc == -ENOSYS) { 235 /* Firmware without GET_WORKAROUNDS - not a problem. */ 236 rc = 0; 237 } else if (rc == 0) { 238 /* Bug61265 workaround is always enabled if implemented. */ 239 if (enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG61265) 240 nic_data->workaround_61265 = true; 241 242 if (enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG35388) { 243 nic_data->workaround_35388 = true; 244 } else if (implemented & MC_CMD_GET_WORKAROUNDS_OUT_BUG35388) { 245 /* Workaround is implemented but not enabled. 246 * Try to enable it. 247 */ 248 rc = efx_mcdi_set_workaround(efx, 249 MC_CMD_WORKAROUND_BUG35388, 250 true, NULL); 251 if (rc == 0) 252 nic_data->workaround_35388 = true; 253 /* If we failed to set the workaround just carry on. */ 254 rc = 0; 255 } 256 } 257 258 netif_dbg(efx, probe, efx->net_dev, 259 "workaround for bug 35388 is %sabled\n", 260 nic_data->workaround_35388 ? "en" : "dis"); 261 netif_dbg(efx, probe, efx->net_dev, 262 "workaround for bug 61265 is %sabled\n", 263 nic_data->workaround_61265 ? "en" : "dis"); 264 265 return rc; 266 } 267 268 static void efx_ef10_process_timer_config(struct efx_nic *efx, 269 const efx_dword_t *data) 270 { 271 unsigned int max_count; 272 273 if (EFX_EF10_WORKAROUND_61265(efx)) { 274 efx->timer_quantum_ns = MCDI_DWORD(data, 275 GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_STEP_NS); 276 efx->timer_max_ns = MCDI_DWORD(data, 277 GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_MAX_NS); 278 } else if (EFX_EF10_WORKAROUND_35388(efx)) { 279 efx->timer_quantum_ns = MCDI_DWORD(data, 280 GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_NS_PER_COUNT); 281 max_count = MCDI_DWORD(data, 282 GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_MAX_COUNT); 283 efx->timer_max_ns = max_count * efx->timer_quantum_ns; 284 } else { 285 efx->timer_quantum_ns = MCDI_DWORD(data, 286 GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_NS_PER_COUNT); 287 max_count = MCDI_DWORD(data, 288 GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_MAX_COUNT); 289 efx->timer_max_ns = max_count * efx->timer_quantum_ns; 290 } 291 292 netif_dbg(efx, probe, efx->net_dev, 293 "got timer properties from MC: quantum %u ns; max %u ns\n", 294 efx->timer_quantum_ns, efx->timer_max_ns); 295 } 296 297 static int efx_ef10_get_timer_config(struct efx_nic *efx) 298 { 299 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN); 300 int rc; 301 302 rc = efx_ef10_get_timer_workarounds(efx); 303 if (rc) 304 return rc; 305 306 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_GET_EVQ_TMR_PROPERTIES, NULL, 0, 307 outbuf, sizeof(outbuf), NULL); 308 309 if (rc == 0) { 310 efx_ef10_process_timer_config(efx, outbuf); 311 } else if (rc == -ENOSYS || rc == -EPERM) { 312 /* Not available - fall back to Huntington defaults. */ 313 unsigned int quantum; 314 315 rc = efx_ef10_get_sysclk_freq(efx); 316 if (rc < 0) 317 return rc; 318 319 quantum = 1536000 / rc; /* 1536 cycles */ 320 efx->timer_quantum_ns = quantum; 321 efx->timer_max_ns = efx->type->timer_period_max * quantum; 322 rc = 0; 323 } else { 324 efx_mcdi_display_error(efx, MC_CMD_GET_EVQ_TMR_PROPERTIES, 325 MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN, 326 NULL, 0, rc); 327 } 328 329 return rc; 330 } 331 332 static int efx_ef10_get_mac_address_pf(struct efx_nic *efx, u8 *mac_address) 333 { 334 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_MAC_ADDRESSES_OUT_LEN); 335 size_t outlen; 336 int rc; 337 338 BUILD_BUG_ON(MC_CMD_GET_MAC_ADDRESSES_IN_LEN != 0); 339 340 rc = efx_mcdi_rpc(efx, MC_CMD_GET_MAC_ADDRESSES, NULL, 0, 341 outbuf, sizeof(outbuf), &outlen); 342 if (rc) 343 return rc; 344 if (outlen < MC_CMD_GET_MAC_ADDRESSES_OUT_LEN) 345 return -EIO; 346 347 ether_addr_copy(mac_address, 348 MCDI_PTR(outbuf, GET_MAC_ADDRESSES_OUT_MAC_ADDR_BASE)); 349 return 0; 350 } 351 352 static int efx_ef10_get_mac_address_vf(struct efx_nic *efx, u8 *mac_address) 353 { 354 MCDI_DECLARE_BUF(inbuf, MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_LEN); 355 MCDI_DECLARE_BUF(outbuf, MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMAX); 356 size_t outlen; 357 int num_addrs, rc; 358 359 MCDI_SET_DWORD(inbuf, VPORT_GET_MAC_ADDRESSES_IN_VPORT_ID, 360 EVB_PORT_ID_ASSIGNED); 361 rc = efx_mcdi_rpc(efx, MC_CMD_VPORT_GET_MAC_ADDRESSES, inbuf, 362 sizeof(inbuf), outbuf, sizeof(outbuf), &outlen); 363 364 if (rc) 365 return rc; 366 if (outlen < MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMIN) 367 return -EIO; 368 369 num_addrs = MCDI_DWORD(outbuf, 370 VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_COUNT); 371 372 WARN_ON(num_addrs != 1); 373 374 ether_addr_copy(mac_address, 375 MCDI_PTR(outbuf, VPORT_GET_MAC_ADDRESSES_OUT_MACADDR)); 376 377 return 0; 378 } 379 380 static ssize_t link_control_flag_show(struct device *dev, 381 struct device_attribute *attr, 382 char *buf) 383 { 384 struct efx_nic *efx = dev_get_drvdata(dev); 385 386 return sprintf(buf, "%d\n", 387 ((efx->mcdi->fn_flags) & 388 (1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL)) 389 ? 1 : 0); 390 } 391 392 static ssize_t primary_flag_show(struct device *dev, 393 struct device_attribute *attr, 394 char *buf) 395 { 396 struct efx_nic *efx = dev_get_drvdata(dev); 397 398 return sprintf(buf, "%d\n", 399 ((efx->mcdi->fn_flags) & 400 (1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_PRIMARY)) 401 ? 1 : 0); 402 } 403 404 static struct efx_ef10_vlan *efx_ef10_find_vlan(struct efx_nic *efx, u16 vid) 405 { 406 struct efx_ef10_nic_data *nic_data = efx->nic_data; 407 struct efx_ef10_vlan *vlan; 408 409 WARN_ON(!mutex_is_locked(&nic_data->vlan_lock)); 410 411 list_for_each_entry(vlan, &nic_data->vlan_list, list) { 412 if (vlan->vid == vid) 413 return vlan; 414 } 415 416 return NULL; 417 } 418 419 static int efx_ef10_add_vlan(struct efx_nic *efx, u16 vid) 420 { 421 struct efx_ef10_nic_data *nic_data = efx->nic_data; 422 struct efx_ef10_vlan *vlan; 423 int rc; 424 425 mutex_lock(&nic_data->vlan_lock); 426 427 vlan = efx_ef10_find_vlan(efx, vid); 428 if (vlan) { 429 /* We add VID 0 on init. 8021q adds it on module init 430 * for all interfaces with VLAN filtring feature. 431 */ 432 if (vid == 0) 433 goto done_unlock; 434 netif_warn(efx, drv, efx->net_dev, 435 "VLAN %u already added\n", vid); 436 rc = -EALREADY; 437 goto fail_exist; 438 } 439 440 rc = -ENOMEM; 441 vlan = kzalloc_obj(*vlan); 442 if (!vlan) 443 goto fail_alloc; 444 445 vlan->vid = vid; 446 447 list_add_tail(&vlan->list, &nic_data->vlan_list); 448 449 if (efx->filter_state) { 450 mutex_lock(&efx->mac_lock); 451 down_write(&efx->filter_sem); 452 rc = efx_mcdi_filter_add_vlan(efx, vlan->vid); 453 up_write(&efx->filter_sem); 454 mutex_unlock(&efx->mac_lock); 455 if (rc) 456 goto fail_filter_add_vlan; 457 } 458 459 done_unlock: 460 mutex_unlock(&nic_data->vlan_lock); 461 return 0; 462 463 fail_filter_add_vlan: 464 list_del(&vlan->list); 465 kfree(vlan); 466 fail_alloc: 467 fail_exist: 468 mutex_unlock(&nic_data->vlan_lock); 469 return rc; 470 } 471 472 static void efx_ef10_del_vlan_internal(struct efx_nic *efx, 473 struct efx_ef10_vlan *vlan) 474 { 475 struct efx_ef10_nic_data *nic_data = efx->nic_data; 476 477 WARN_ON(!mutex_is_locked(&nic_data->vlan_lock)); 478 479 if (efx->filter_state) { 480 down_write(&efx->filter_sem); 481 efx_mcdi_filter_del_vlan(efx, vlan->vid); 482 up_write(&efx->filter_sem); 483 } 484 485 list_del(&vlan->list); 486 kfree(vlan); 487 } 488 489 static int efx_ef10_del_vlan(struct efx_nic *efx, u16 vid) 490 { 491 struct efx_ef10_nic_data *nic_data = efx->nic_data; 492 struct efx_ef10_vlan *vlan; 493 int rc = 0; 494 495 /* 8021q removes VID 0 on module unload for all interfaces 496 * with VLAN filtering feature. We need to keep it to receive 497 * untagged traffic. 498 */ 499 if (vid == 0) 500 return 0; 501 502 mutex_lock(&nic_data->vlan_lock); 503 504 vlan = efx_ef10_find_vlan(efx, vid); 505 if (!vlan) { 506 netif_err(efx, drv, efx->net_dev, 507 "VLAN %u to be deleted not found\n", vid); 508 rc = -ENOENT; 509 } else { 510 efx_ef10_del_vlan_internal(efx, vlan); 511 } 512 513 mutex_unlock(&nic_data->vlan_lock); 514 515 return rc; 516 } 517 518 static void efx_ef10_cleanup_vlans(struct efx_nic *efx) 519 { 520 struct efx_ef10_nic_data *nic_data = efx->nic_data; 521 struct efx_ef10_vlan *vlan, *next_vlan; 522 523 mutex_lock(&nic_data->vlan_lock); 524 list_for_each_entry_safe(vlan, next_vlan, &nic_data->vlan_list, list) 525 efx_ef10_del_vlan_internal(efx, vlan); 526 mutex_unlock(&nic_data->vlan_lock); 527 } 528 529 static DEVICE_ATTR_RO(link_control_flag); 530 static DEVICE_ATTR_RO(primary_flag); 531 532 static int efx_ef10_probe(struct efx_nic *efx) 533 { 534 struct efx_ef10_nic_data *nic_data; 535 int i, rc; 536 537 nic_data = kzalloc_obj(*nic_data); 538 if (!nic_data) 539 return -ENOMEM; 540 efx->nic_data = nic_data; 541 542 /* we assume later that we can copy from this buffer in dwords */ 543 BUILD_BUG_ON(MCDI_CTL_SDU_LEN_MAX_V2 % 4); 544 545 rc = efx_nic_alloc_buffer(efx, &nic_data->mcdi_buf, 546 8 + MCDI_CTL_SDU_LEN_MAX_V2, GFP_KERNEL); 547 if (rc) 548 goto fail1; 549 550 /* Get the MC's warm boot count. In case it's rebooting right 551 * now, be prepared to retry. 552 */ 553 i = 0; 554 for (;;) { 555 rc = efx_ef10_get_warm_boot_count(efx); 556 if (rc >= 0) 557 break; 558 if (++i == 5) 559 goto fail2; 560 ssleep(1); 561 } 562 nic_data->warm_boot_count = rc; 563 564 /* In case we're recovering from a crash (kexec), we want to 565 * cancel any outstanding request by the previous user of this 566 * function. We send a special message using the least 567 * significant bits of the 'high' (doorbell) register. 568 */ 569 _efx_writed(efx, cpu_to_le32(1), ER_DZ_MC_DB_HWRD); 570 571 rc = efx_mcdi_init(efx); 572 if (rc) 573 goto fail2; 574 575 mutex_init(&nic_data->udp_tunnels_lock); 576 for (i = 0; i < ARRAY_SIZE(nic_data->udp_tunnels); ++i) 577 nic_data->udp_tunnels[i].type = 578 TUNNEL_ENCAP_UDP_PORT_ENTRY_INVALID; 579 580 /* Reset (most) configuration for this function */ 581 rc = efx_mcdi_reset(efx, RESET_TYPE_ALL); 582 if (rc) 583 goto fail3; 584 585 /* Enable event logging */ 586 rc = efx_mcdi_log_ctrl(efx, true, false, 0); 587 if (rc) 588 goto fail3; 589 590 rc = device_create_file(&efx->pci_dev->dev, 591 &dev_attr_link_control_flag); 592 if (rc) 593 goto fail3; 594 595 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_primary_flag); 596 if (rc) 597 goto fail4; 598 599 rc = efx_get_pf_index(efx, &nic_data->pf_index); 600 if (rc) 601 goto fail5; 602 603 rc = efx_ef10_init_datapath_caps(efx); 604 if (rc < 0) 605 goto fail5; 606 607 efx_ef10_read_licensed_features(efx); 608 609 /* We can have one VI for each vi_stride-byte region. 610 * However, until we use TX option descriptors we need up to four 611 * TX queues per channel for different checksumming combinations. 612 */ 613 if (nic_data->datapath_caps & 614 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN)) 615 efx->tx_queues_per_channel = 4; 616 else 617 efx->tx_queues_per_channel = 2; 618 efx->max_vis = efx_ef10_mem_map_size(efx) / efx->vi_stride; 619 if (!efx->max_vis) { 620 netif_err(efx, drv, efx->net_dev, "error determining max VIs\n"); 621 rc = -EIO; 622 goto fail5; 623 } 624 efx->max_channels = min_t(unsigned int, EFX_MAX_CHANNELS, 625 efx->max_vis / efx->tx_queues_per_channel); 626 efx->max_tx_channels = efx->max_channels; 627 if (WARN_ON(efx->max_channels == 0)) { 628 rc = -EIO; 629 goto fail5; 630 } 631 632 efx->rx_packet_len_offset = 633 ES_DZ_RX_PREFIX_PKTLEN_OFST - ES_DZ_RX_PREFIX_SIZE; 634 635 if (nic_data->datapath_caps & 636 (1 << MC_CMD_GET_CAPABILITIES_OUT_RX_INCLUDE_FCS_LBN)) 637 efx->net_dev->hw_features |= NETIF_F_RXFCS; 638 639 rc = efx_mcdi_port_get_number(efx); 640 if (rc < 0) 641 goto fail5; 642 efx->port_num = rc; 643 644 rc = efx->type->get_mac_address(efx, efx->net_dev->perm_addr); 645 if (rc) 646 goto fail5; 647 648 rc = efx_ef10_get_timer_config(efx); 649 if (rc < 0) 650 goto fail5; 651 652 rc = efx_mcdi_mon_probe(efx); 653 if (rc && rc != -EPERM) 654 goto fail5; 655 656 efx_ptp_defer_probe_with_channel(efx); 657 658 #ifdef CONFIG_SFC_SRIOV 659 if ((efx->pci_dev->physfn) && (!efx->pci_dev->is_physfn)) { 660 struct pci_dev *pci_dev_pf = efx->pci_dev->physfn; 661 struct efx_nic *efx_pf = pci_get_drvdata(pci_dev_pf); 662 663 efx_pf->type->get_mac_address(efx_pf, nic_data->port_id); 664 } else 665 #endif 666 ether_addr_copy(nic_data->port_id, efx->net_dev->perm_addr); 667 668 INIT_LIST_HEAD(&nic_data->vlan_list); 669 mutex_init(&nic_data->vlan_lock); 670 671 /* Add unspecified VID to support VLAN filtering being disabled */ 672 rc = efx_ef10_add_vlan(efx, EFX_FILTER_VID_UNSPEC); 673 if (rc) 674 goto fail_add_vid_unspec; 675 676 /* If VLAN filtering is enabled, we need VID 0 to get untagged 677 * traffic. It is added automatically if 8021q module is loaded, 678 * but we can't rely on it since module may be not loaded. 679 */ 680 rc = efx_ef10_add_vlan(efx, 0); 681 if (rc) 682 goto fail_add_vid_0; 683 684 if (nic_data->datapath_caps & 685 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN) && 686 efx->mcdi->fn_flags & 687 (1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_TRUSTED)) 688 efx->net_dev->udp_tunnel_nic_info = &efx_ef10_udp_tunnels; 689 690 return 0; 691 692 fail_add_vid_0: 693 efx_ef10_cleanup_vlans(efx); 694 fail_add_vid_unspec: 695 mutex_destroy(&nic_data->vlan_lock); 696 efx_ptp_remove(efx); 697 efx_mcdi_mon_remove(efx); 698 fail5: 699 device_remove_file(&efx->pci_dev->dev, &dev_attr_primary_flag); 700 fail4: 701 device_remove_file(&efx->pci_dev->dev, &dev_attr_link_control_flag); 702 fail3: 703 efx_mcdi_detach(efx); 704 705 mutex_lock(&nic_data->udp_tunnels_lock); 706 memset(nic_data->udp_tunnels, 0, sizeof(nic_data->udp_tunnels)); 707 (void)efx_ef10_set_udp_tnl_ports(efx, true); 708 mutex_unlock(&nic_data->udp_tunnels_lock); 709 mutex_destroy(&nic_data->udp_tunnels_lock); 710 711 efx_mcdi_fini(efx); 712 fail2: 713 efx_nic_free_buffer(efx, &nic_data->mcdi_buf); 714 fail1: 715 kfree(nic_data); 716 efx->nic_data = NULL; 717 return rc; 718 } 719 720 #ifdef EFX_USE_PIO 721 722 static void efx_ef10_free_piobufs(struct efx_nic *efx) 723 { 724 struct efx_ef10_nic_data *nic_data = efx->nic_data; 725 MCDI_DECLARE_BUF(inbuf, MC_CMD_FREE_PIOBUF_IN_LEN); 726 unsigned int i; 727 int rc; 728 729 BUILD_BUG_ON(MC_CMD_FREE_PIOBUF_OUT_LEN != 0); 730 731 for (i = 0; i < nic_data->n_piobufs; i++) { 732 MCDI_SET_DWORD(inbuf, FREE_PIOBUF_IN_PIOBUF_HANDLE, 733 nic_data->piobuf_handle[i]); 734 rc = efx_mcdi_rpc(efx, MC_CMD_FREE_PIOBUF, inbuf, sizeof(inbuf), 735 NULL, 0, NULL); 736 WARN_ON(rc); 737 } 738 739 nic_data->n_piobufs = 0; 740 } 741 742 static int efx_ef10_alloc_piobufs(struct efx_nic *efx, unsigned int n) 743 { 744 struct efx_ef10_nic_data *nic_data = efx->nic_data; 745 MCDI_DECLARE_BUF(outbuf, MC_CMD_ALLOC_PIOBUF_OUT_LEN); 746 unsigned int i; 747 size_t outlen; 748 int rc = 0; 749 750 BUILD_BUG_ON(MC_CMD_ALLOC_PIOBUF_IN_LEN != 0); 751 752 for (i = 0; i < n; i++) { 753 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_ALLOC_PIOBUF, NULL, 0, 754 outbuf, sizeof(outbuf), &outlen); 755 if (rc) { 756 /* Don't display the MC error if we didn't have space 757 * for a VF. 758 */ 759 if (!(efx_ef10_is_vf(efx) && rc == -ENOSPC)) 760 efx_mcdi_display_error(efx, MC_CMD_ALLOC_PIOBUF, 761 0, outbuf, outlen, rc); 762 break; 763 } 764 if (outlen < MC_CMD_ALLOC_PIOBUF_OUT_LEN) { 765 rc = -EIO; 766 break; 767 } 768 nic_data->piobuf_handle[i] = 769 MCDI_DWORD(outbuf, ALLOC_PIOBUF_OUT_PIOBUF_HANDLE); 770 netif_dbg(efx, probe, efx->net_dev, 771 "allocated PIO buffer %u handle %x\n", i, 772 nic_data->piobuf_handle[i]); 773 } 774 775 nic_data->n_piobufs = i; 776 if (rc) 777 efx_ef10_free_piobufs(efx); 778 return rc; 779 } 780 781 static int efx_ef10_link_piobufs(struct efx_nic *efx) 782 { 783 struct efx_ef10_nic_data *nic_data = efx->nic_data; 784 MCDI_DECLARE_BUF(inbuf, MC_CMD_LINK_PIOBUF_IN_LEN); 785 struct efx_channel *channel; 786 struct efx_tx_queue *tx_queue; 787 unsigned int offset, index; 788 int rc; 789 790 BUILD_BUG_ON(MC_CMD_LINK_PIOBUF_OUT_LEN != 0); 791 BUILD_BUG_ON(MC_CMD_UNLINK_PIOBUF_OUT_LEN != 0); 792 793 /* Link a buffer to each VI in the write-combining mapping */ 794 for (index = 0; index < nic_data->n_piobufs; ++index) { 795 MCDI_SET_DWORD(inbuf, LINK_PIOBUF_IN_PIOBUF_HANDLE, 796 nic_data->piobuf_handle[index]); 797 MCDI_SET_DWORD(inbuf, LINK_PIOBUF_IN_TXQ_INSTANCE, 798 nic_data->pio_write_vi_base + index); 799 rc = efx_mcdi_rpc(efx, MC_CMD_LINK_PIOBUF, 800 inbuf, MC_CMD_LINK_PIOBUF_IN_LEN, 801 NULL, 0, NULL); 802 if (rc) { 803 netif_err(efx, drv, efx->net_dev, 804 "failed to link VI %u to PIO buffer %u (%d)\n", 805 nic_data->pio_write_vi_base + index, index, 806 rc); 807 goto fail; 808 } 809 netif_dbg(efx, probe, efx->net_dev, 810 "linked VI %u to PIO buffer %u\n", 811 nic_data->pio_write_vi_base + index, index); 812 } 813 814 /* Link a buffer to each TX queue */ 815 efx_for_each_channel(channel, efx) { 816 /* Extra channels, even those with TXQs (PTP), do not require 817 * PIO resources. 818 */ 819 if (!channel->type->want_pio || 820 channel->channel >= efx->xdp_channel_offset) 821 continue; 822 823 efx_for_each_channel_tx_queue(tx_queue, channel) { 824 /* We assign the PIO buffers to queues in 825 * reverse order to allow for the following 826 * special case. 827 */ 828 offset = ((efx->tx_channel_offset + efx->n_tx_channels - 829 tx_queue->channel->channel - 1) * 830 efx_piobuf_size); 831 index = offset / nic_data->piobuf_size; 832 offset = offset % nic_data->piobuf_size; 833 834 /* When the host page size is 4K, the first 835 * host page in the WC mapping may be within 836 * the same VI page as the last TX queue. We 837 * can only link one buffer to each VI. 838 */ 839 if (tx_queue->queue == nic_data->pio_write_vi_base) { 840 BUG_ON(index != 0); 841 rc = 0; 842 } else { 843 MCDI_SET_DWORD(inbuf, 844 LINK_PIOBUF_IN_PIOBUF_HANDLE, 845 nic_data->piobuf_handle[index]); 846 MCDI_SET_DWORD(inbuf, 847 LINK_PIOBUF_IN_TXQ_INSTANCE, 848 tx_queue->queue); 849 rc = efx_mcdi_rpc(efx, MC_CMD_LINK_PIOBUF, 850 inbuf, MC_CMD_LINK_PIOBUF_IN_LEN, 851 NULL, 0, NULL); 852 } 853 854 if (rc) { 855 /* This is non-fatal; the TX path just 856 * won't use PIO for this queue 857 */ 858 netif_err(efx, drv, efx->net_dev, 859 "failed to link VI %u to PIO buffer %u (%d)\n", 860 tx_queue->queue, index, rc); 861 tx_queue->piobuf = NULL; 862 } else { 863 tx_queue->piobuf = 864 nic_data->pio_write_base + 865 index * efx->vi_stride + offset; 866 tx_queue->piobuf_offset = offset; 867 netif_dbg(efx, probe, efx->net_dev, 868 "linked VI %u to PIO buffer %u offset %x addr %p\n", 869 tx_queue->queue, index, 870 tx_queue->piobuf_offset, 871 tx_queue->piobuf); 872 } 873 } 874 } 875 876 return 0; 877 878 fail: 879 /* inbuf was defined for MC_CMD_LINK_PIOBUF. We can use the same 880 * buffer for MC_CMD_UNLINK_PIOBUF because it's shorter. 881 */ 882 BUILD_BUG_ON(MC_CMD_LINK_PIOBUF_IN_LEN < MC_CMD_UNLINK_PIOBUF_IN_LEN); 883 while (index--) { 884 MCDI_SET_DWORD(inbuf, UNLINK_PIOBUF_IN_TXQ_INSTANCE, 885 nic_data->pio_write_vi_base + index); 886 efx_mcdi_rpc(efx, MC_CMD_UNLINK_PIOBUF, 887 inbuf, MC_CMD_UNLINK_PIOBUF_IN_LEN, 888 NULL, 0, NULL); 889 } 890 return rc; 891 } 892 893 static void efx_ef10_forget_old_piobufs(struct efx_nic *efx) 894 { 895 struct efx_channel *channel; 896 struct efx_tx_queue *tx_queue; 897 898 /* All our existing PIO buffers went away */ 899 efx_for_each_channel(channel, efx) 900 efx_for_each_channel_tx_queue(tx_queue, channel) 901 tx_queue->piobuf = NULL; 902 } 903 904 #else /* !EFX_USE_PIO */ 905 906 static int efx_ef10_alloc_piobufs(struct efx_nic *efx, unsigned int n) 907 { 908 return n == 0 ? 0 : -ENOBUFS; 909 } 910 911 static int efx_ef10_link_piobufs(struct efx_nic *efx) 912 { 913 return 0; 914 } 915 916 static void efx_ef10_free_piobufs(struct efx_nic *efx) 917 { 918 } 919 920 static void efx_ef10_forget_old_piobufs(struct efx_nic *efx) 921 { 922 } 923 924 #endif /* EFX_USE_PIO */ 925 926 static void efx_ef10_remove(struct efx_nic *efx) 927 { 928 struct efx_ef10_nic_data *nic_data = efx->nic_data; 929 int rc; 930 931 #ifdef CONFIG_SFC_SRIOV 932 struct efx_ef10_nic_data *nic_data_pf; 933 struct pci_dev *pci_dev_pf; 934 struct efx_nic *efx_pf; 935 struct ef10_vf *vf; 936 937 if (efx->pci_dev->is_virtfn) { 938 pci_dev_pf = efx->pci_dev->physfn; 939 if (pci_dev_pf) { 940 efx_pf = pci_get_drvdata(pci_dev_pf); 941 nic_data_pf = efx_pf->nic_data; 942 vf = nic_data_pf->vf + nic_data->vf_index; 943 vf->efx = NULL; 944 } else 945 netif_info(efx, drv, efx->net_dev, 946 "Could not get the PF id from VF\n"); 947 } 948 #endif 949 950 efx_ef10_cleanup_vlans(efx); 951 mutex_destroy(&nic_data->vlan_lock); 952 953 efx_ptp_remove(efx); 954 955 efx_mcdi_mon_remove(efx); 956 957 efx_mcdi_rx_free_indir_table(efx); 958 959 if (nic_data->wc_membase) 960 iounmap(nic_data->wc_membase); 961 962 rc = efx_mcdi_free_vis(efx); 963 WARN_ON(rc != 0); 964 965 if (!nic_data->must_restore_piobufs) 966 efx_ef10_free_piobufs(efx); 967 968 device_remove_file(&efx->pci_dev->dev, &dev_attr_primary_flag); 969 device_remove_file(&efx->pci_dev->dev, &dev_attr_link_control_flag); 970 971 efx_mcdi_detach(efx); 972 973 memset(nic_data->udp_tunnels, 0, sizeof(nic_data->udp_tunnels)); 974 mutex_lock(&nic_data->udp_tunnels_lock); 975 (void)efx_ef10_set_udp_tnl_ports(efx, true); 976 mutex_unlock(&nic_data->udp_tunnels_lock); 977 978 mutex_destroy(&nic_data->udp_tunnels_lock); 979 980 efx_mcdi_fini(efx); 981 efx_nic_free_buffer(efx, &nic_data->mcdi_buf); 982 kfree(nic_data); 983 } 984 985 static int efx_ef10_probe_pf(struct efx_nic *efx) 986 { 987 return efx_ef10_probe(efx); 988 } 989 990 int efx_ef10_vadaptor_query(struct efx_nic *efx, unsigned int port_id, 991 u32 *port_flags, u32 *vadaptor_flags, 992 unsigned int *vlan_tags) 993 { 994 struct efx_ef10_nic_data *nic_data = efx->nic_data; 995 MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_QUERY_IN_LEN); 996 MCDI_DECLARE_BUF(outbuf, MC_CMD_VADAPTOR_QUERY_OUT_LEN); 997 size_t outlen; 998 int rc; 999 1000 if (nic_data->datapath_caps & 1001 (1 << MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_QUERY_LBN)) { 1002 MCDI_SET_DWORD(inbuf, VADAPTOR_QUERY_IN_UPSTREAM_PORT_ID, 1003 port_id); 1004 1005 rc = efx_mcdi_rpc(efx, MC_CMD_VADAPTOR_QUERY, inbuf, sizeof(inbuf), 1006 outbuf, sizeof(outbuf), &outlen); 1007 if (rc) 1008 return rc; 1009 1010 if (outlen < sizeof(outbuf)) { 1011 rc = -EIO; 1012 return rc; 1013 } 1014 } 1015 1016 if (port_flags) 1017 *port_flags = MCDI_DWORD(outbuf, VADAPTOR_QUERY_OUT_PORT_FLAGS); 1018 if (vadaptor_flags) 1019 *vadaptor_flags = 1020 MCDI_DWORD(outbuf, VADAPTOR_QUERY_OUT_VADAPTOR_FLAGS); 1021 if (vlan_tags) 1022 *vlan_tags = 1023 MCDI_DWORD(outbuf, 1024 VADAPTOR_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS); 1025 1026 return 0; 1027 } 1028 1029 int efx_ef10_vadaptor_alloc(struct efx_nic *efx, unsigned int port_id) 1030 { 1031 MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_ALLOC_IN_LEN); 1032 1033 MCDI_SET_DWORD(inbuf, VADAPTOR_ALLOC_IN_UPSTREAM_PORT_ID, port_id); 1034 return efx_mcdi_rpc(efx, MC_CMD_VADAPTOR_ALLOC, inbuf, sizeof(inbuf), 1035 NULL, 0, NULL); 1036 } 1037 1038 int efx_ef10_vadaptor_free(struct efx_nic *efx, unsigned int port_id) 1039 { 1040 MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_FREE_IN_LEN); 1041 1042 MCDI_SET_DWORD(inbuf, VADAPTOR_FREE_IN_UPSTREAM_PORT_ID, port_id); 1043 return efx_mcdi_rpc(efx, MC_CMD_VADAPTOR_FREE, inbuf, sizeof(inbuf), 1044 NULL, 0, NULL); 1045 } 1046 1047 int efx_ef10_vport_add_mac(struct efx_nic *efx, 1048 unsigned int port_id, const u8 *mac) 1049 { 1050 MCDI_DECLARE_BUF(inbuf, MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_LEN); 1051 1052 MCDI_SET_DWORD(inbuf, VPORT_ADD_MAC_ADDRESS_IN_VPORT_ID, port_id); 1053 ether_addr_copy(MCDI_PTR(inbuf, VPORT_ADD_MAC_ADDRESS_IN_MACADDR), mac); 1054 1055 return efx_mcdi_rpc(efx, MC_CMD_VPORT_ADD_MAC_ADDRESS, inbuf, 1056 sizeof(inbuf), NULL, 0, NULL); 1057 } 1058 1059 int efx_ef10_vport_del_mac(struct efx_nic *efx, 1060 unsigned int port_id, const u8 *mac) 1061 { 1062 MCDI_DECLARE_BUF(inbuf, MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_LEN); 1063 1064 MCDI_SET_DWORD(inbuf, VPORT_DEL_MAC_ADDRESS_IN_VPORT_ID, port_id); 1065 ether_addr_copy(MCDI_PTR(inbuf, VPORT_DEL_MAC_ADDRESS_IN_MACADDR), mac); 1066 1067 return efx_mcdi_rpc(efx, MC_CMD_VPORT_DEL_MAC_ADDRESS, inbuf, 1068 sizeof(inbuf), NULL, 0, NULL); 1069 } 1070 1071 #ifdef CONFIG_SFC_SRIOV 1072 static int efx_ef10_probe_vf(struct efx_nic *efx) 1073 { 1074 int rc; 1075 struct pci_dev *pci_dev_pf; 1076 1077 /* If the parent PF has no VF data structure, it doesn't know about this 1078 * VF so fail probe. The VF needs to be re-created. This can happen 1079 * if the PF driver was unloaded while any VF was assigned to a guest 1080 * (using Xen, only). 1081 */ 1082 pci_dev_pf = efx->pci_dev->physfn; 1083 if (pci_dev_pf) { 1084 struct efx_nic *efx_pf = pci_get_drvdata(pci_dev_pf); 1085 struct efx_ef10_nic_data *nic_data_pf = efx_pf->nic_data; 1086 1087 if (!nic_data_pf->vf) { 1088 netif_info(efx, drv, efx->net_dev, 1089 "The VF cannot link to its parent PF; " 1090 "please destroy and re-create the VF\n"); 1091 return -EBUSY; 1092 } 1093 } 1094 1095 rc = efx_ef10_probe(efx); 1096 if (rc) 1097 return rc; 1098 1099 rc = efx_ef10_get_vf_index(efx); 1100 if (rc) 1101 goto fail; 1102 1103 if (efx->pci_dev->is_virtfn) { 1104 if (efx->pci_dev->physfn) { 1105 struct efx_nic *efx_pf = 1106 pci_get_drvdata(efx->pci_dev->physfn); 1107 struct efx_ef10_nic_data *nic_data_p = efx_pf->nic_data; 1108 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1109 1110 nic_data_p->vf[nic_data->vf_index].efx = efx; 1111 nic_data_p->vf[nic_data->vf_index].pci_dev = 1112 efx->pci_dev; 1113 } else 1114 netif_info(efx, drv, efx->net_dev, 1115 "Could not get the PF id from VF\n"); 1116 } 1117 1118 return 0; 1119 1120 fail: 1121 efx_ef10_remove(efx); 1122 return rc; 1123 } 1124 #else 1125 static int efx_ef10_probe_vf(struct efx_nic *efx __attribute__ ((unused))) 1126 { 1127 return 0; 1128 } 1129 #endif 1130 1131 static int efx_ef10_alloc_vis(struct efx_nic *efx, 1132 unsigned int min_vis, unsigned int max_vis) 1133 { 1134 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1135 1136 return efx_mcdi_alloc_vis(efx, min_vis, max_vis, &nic_data->vi_base, 1137 &nic_data->n_allocated_vis); 1138 } 1139 1140 /* Note that the failure path of this function does not free 1141 * resources, as this will be done by efx_ef10_remove(). 1142 */ 1143 static int efx_ef10_dimension_resources(struct efx_nic *efx) 1144 { 1145 unsigned int min_vis = max_t(unsigned int, efx->tx_queues_per_channel, 1146 efx_separate_tx_channels ? 2 : 1); 1147 unsigned int channel_vis, pio_write_vi_base, max_vis; 1148 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1149 unsigned int uc_mem_map_size, wc_mem_map_size; 1150 #ifdef CONFIG_SFC_CXL 1151 struct efx_probe_data *probe_data; 1152 #endif 1153 void __iomem *membase; 1154 int rc; 1155 1156 channel_vis = max(efx->n_channels, 1157 ((efx->n_tx_channels + efx->n_extra_tx_channels) * 1158 efx->tx_queues_per_channel) + 1159 efx->n_xdp_channels * efx->xdp_tx_per_channel); 1160 if (efx->max_vis && efx->max_vis < channel_vis) { 1161 netif_dbg(efx, drv, efx->net_dev, 1162 "Reducing channel VIs from %u to %u\n", 1163 channel_vis, efx->max_vis); 1164 channel_vis = efx->max_vis; 1165 } 1166 1167 #ifdef EFX_USE_PIO 1168 /* Try to allocate PIO buffers if wanted and if the full 1169 * number of PIO buffers would be sufficient to allocate one 1170 * copy-buffer per TX channel. Failure is non-fatal, as there 1171 * are only a small number of PIO buffers shared between all 1172 * functions of the controller. 1173 */ 1174 if (efx_piobuf_size != 0 && 1175 nic_data->piobuf_size / efx_piobuf_size * EF10_TX_PIOBUF_COUNT >= 1176 efx->n_tx_channels) { 1177 unsigned int n_piobufs = 1178 DIV_ROUND_UP(efx->n_tx_channels, 1179 nic_data->piobuf_size / efx_piobuf_size); 1180 1181 rc = efx_ef10_alloc_piobufs(efx, n_piobufs); 1182 if (rc == -ENOSPC) 1183 netif_dbg(efx, probe, efx->net_dev, 1184 "out of PIO buffers; cannot allocate more\n"); 1185 else if (rc == -EPERM) 1186 netif_dbg(efx, probe, efx->net_dev, 1187 "not permitted to allocate PIO buffers\n"); 1188 else if (rc) 1189 netif_err(efx, probe, efx->net_dev, 1190 "failed to allocate PIO buffers (%d)\n", rc); 1191 else 1192 netif_dbg(efx, probe, efx->net_dev, 1193 "allocated %u PIO buffers\n", n_piobufs); 1194 } 1195 #else 1196 nic_data->n_piobufs = 0; 1197 #endif 1198 1199 /* PIO buffers should be mapped with write-combining enabled, 1200 * and we want to make single UC and WC mappings rather than 1201 * several of each (in fact that's the only option if host 1202 * page size is >4K). So we may allocate some extra VIs just 1203 * for writing PIO buffers through. 1204 * 1205 * The UC mapping contains (channel_vis - 1) complete VIs and the 1206 * first 4K of the next VI. Then the WC mapping begins with 1207 * the remainder of this last VI. 1208 */ 1209 uc_mem_map_size = PAGE_ALIGN((channel_vis - 1) * efx->vi_stride + 1210 ER_DZ_TX_PIOBUF); 1211 if (nic_data->n_piobufs) { 1212 /* pio_write_vi_base rounds down to give the number of complete 1213 * VIs inside the UC mapping. 1214 */ 1215 pio_write_vi_base = uc_mem_map_size / efx->vi_stride; 1216 wc_mem_map_size = (PAGE_ALIGN((pio_write_vi_base + 1217 nic_data->n_piobufs) * 1218 efx->vi_stride) - 1219 uc_mem_map_size); 1220 max_vis = pio_write_vi_base + nic_data->n_piobufs; 1221 } else { 1222 pio_write_vi_base = 0; 1223 wc_mem_map_size = 0; 1224 max_vis = channel_vis; 1225 } 1226 1227 /* In case the last attached driver failed to free VIs, do it now */ 1228 rc = efx_mcdi_free_vis(efx); 1229 if (rc != 0) 1230 return rc; 1231 1232 rc = efx_ef10_alloc_vis(efx, min_vis, max_vis); 1233 if (rc != 0) 1234 return rc; 1235 1236 if (nic_data->n_allocated_vis < channel_vis) { 1237 netif_info(efx, drv, efx->net_dev, 1238 "Could not allocate enough VIs to satisfy RSS" 1239 " requirements. Performance may not be optimal.\n"); 1240 /* We didn't get the VIs to populate our channels. 1241 * We could keep what we got but then we'd have more 1242 * interrupts than we need. 1243 * Instead calculate new max_channels and restart 1244 */ 1245 efx->max_channels = nic_data->n_allocated_vis; 1246 efx->max_tx_channels = 1247 nic_data->n_allocated_vis / efx->tx_queues_per_channel; 1248 1249 efx_mcdi_free_vis(efx); 1250 return -EAGAIN; 1251 } 1252 1253 /* If we didn't get enough VIs to map all the PIO buffers, free the 1254 * PIO buffers 1255 */ 1256 if (nic_data->n_piobufs && 1257 nic_data->n_allocated_vis < 1258 pio_write_vi_base + nic_data->n_piobufs) { 1259 netif_dbg(efx, probe, efx->net_dev, 1260 "%u VIs are not sufficient to map %u PIO buffers\n", 1261 nic_data->n_allocated_vis, nic_data->n_piobufs); 1262 efx_ef10_free_piobufs(efx); 1263 } 1264 1265 /* Shrink the original UC mapping of the memory BAR */ 1266 membase = ioremap(efx->membase_phys, uc_mem_map_size); 1267 if (!membase) { 1268 netif_err(efx, probe, efx->net_dev, 1269 "could not shrink memory BAR to %x\n", 1270 uc_mem_map_size); 1271 return -ENOMEM; 1272 } 1273 iounmap(efx->membase); 1274 efx->membase = membase; 1275 1276 if (!wc_mem_map_size) 1277 goto skip_pio; 1278 1279 /* Set up the WC mapping */ 1280 1281 #ifdef CONFIG_SFC_CXL 1282 probe_data = container_of(efx, struct efx_probe_data, efx); 1283 if ((nic_data->datapath_caps3 & 1284 (1 << MC_CMD_GET_CAPABILITIES_V7_OUT_CXL_CONFIG_ENABLE_LBN)) && 1285 probe_data->cxl_pio_initialised) { 1286 /* Using PIO through CXL mapping */ 1287 nic_data->pio_write_base = probe_data->cxl->ctpio_cxl; 1288 nic_data->pio_write_vi_base = pio_write_vi_base; 1289 } else 1290 #endif 1291 { 1292 /* Using legacy PIO BAR mapping */ 1293 nic_data->wc_membase = ioremap_wc(efx->membase_phys + 1294 uc_mem_map_size, 1295 wc_mem_map_size); 1296 if (!nic_data->wc_membase) { 1297 netif_err(efx, probe, efx->net_dev, 1298 "could not allocate WC mapping of size %x\n", 1299 wc_mem_map_size); 1300 return -ENOMEM; 1301 } 1302 nic_data->pio_write_vi_base = pio_write_vi_base; 1303 nic_data->pio_write_base = 1304 nic_data->wc_membase + 1305 (pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF - 1306 uc_mem_map_size); 1307 } 1308 1309 rc = efx_ef10_link_piobufs(efx); 1310 if (rc) 1311 efx_ef10_free_piobufs(efx); 1312 1313 skip_pio: 1314 1315 netif_dbg(efx, probe, efx->net_dev, 1316 "memory BAR at %pa (virtual %p+%x UC, %p+%x WC)\n", 1317 &efx->membase_phys, efx->membase, uc_mem_map_size, 1318 nic_data->wc_membase, wc_mem_map_size); 1319 1320 return 0; 1321 } 1322 1323 static void efx_ef10_fini_nic(struct efx_nic *efx) 1324 { 1325 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1326 1327 spin_lock_bh(&efx->stats_lock); 1328 kfree(nic_data->mc_stats); 1329 nic_data->mc_stats = NULL; 1330 spin_unlock_bh(&efx->stats_lock); 1331 } 1332 1333 static int efx_ef10_init_nic(struct efx_nic *efx) 1334 { 1335 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1336 struct net_device *net_dev = efx->net_dev; 1337 netdev_features_t tun_feats, tso_feats; 1338 int rc; 1339 1340 if (nic_data->must_check_datapath_caps) { 1341 rc = efx_ef10_init_datapath_caps(efx); 1342 if (rc) 1343 return rc; 1344 nic_data->must_check_datapath_caps = false; 1345 } 1346 1347 if (efx->must_realloc_vis) { 1348 /* We cannot let the number of VIs change now */ 1349 rc = efx_ef10_alloc_vis(efx, nic_data->n_allocated_vis, 1350 nic_data->n_allocated_vis); 1351 if (rc) 1352 return rc; 1353 efx->must_realloc_vis = false; 1354 } 1355 1356 nic_data->mc_stats = kmalloc(efx->num_mac_stats * sizeof(__le64), 1357 GFP_KERNEL); 1358 if (!nic_data->mc_stats) 1359 return -ENOMEM; 1360 1361 if (nic_data->must_restore_piobufs && nic_data->n_piobufs) { 1362 rc = efx_ef10_alloc_piobufs(efx, nic_data->n_piobufs); 1363 if (rc == 0) { 1364 rc = efx_ef10_link_piobufs(efx); 1365 if (rc) 1366 efx_ef10_free_piobufs(efx); 1367 } 1368 1369 /* Log an error on failure, but this is non-fatal. 1370 * Permission errors are less important - we've presumably 1371 * had the PIO buffer licence removed. 1372 */ 1373 if (rc == -EPERM) 1374 netif_dbg(efx, drv, efx->net_dev, 1375 "not permitted to restore PIO buffers\n"); 1376 else if (rc) 1377 netif_err(efx, drv, efx->net_dev, 1378 "failed to restore PIO buffers (%d)\n", rc); 1379 nic_data->must_restore_piobufs = false; 1380 } 1381 1382 /* encap features might change during reset if fw variant changed */ 1383 if (efx_has_cap(efx, VXLAN_NVGRE) && !efx_ef10_is_vf(efx)) 1384 net_dev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 1385 else 1386 net_dev->hw_enc_features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); 1387 1388 tun_feats = NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_GRE | 1389 NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM; 1390 tso_feats = NETIF_F_TSO | NETIF_F_TSO6; 1391 1392 if (efx_has_cap(efx, TX_TSO_V2_ENCAP)) { 1393 /* If this is first nic_init, or if it is a reset and a new fw 1394 * variant has added new features, enable them by default. 1395 * If the features are not new, maintain their current value. 1396 */ 1397 if (!(net_dev->hw_features & tun_feats)) 1398 net_dev->features |= tun_feats; 1399 net_dev->hw_enc_features |= tun_feats | tso_feats; 1400 net_dev->hw_features |= tun_feats; 1401 } else { 1402 net_dev->hw_enc_features &= ~(tun_feats | tso_feats); 1403 net_dev->hw_features &= ~tun_feats; 1404 net_dev->features &= ~tun_feats; 1405 } 1406 1407 /* don't fail init if RSS setup doesn't work */ 1408 rc = efx->type->rx_push_rss_config(efx, false, 1409 efx->rss_context.rx_indir_table, NULL); 1410 1411 return 0; 1412 } 1413 1414 static void efx_ef10_table_reset_mc_allocations(struct efx_nic *efx) 1415 { 1416 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1417 #ifdef CONFIG_SFC_SRIOV 1418 unsigned int i; 1419 #endif 1420 1421 /* All our allocations have been reset */ 1422 efx->must_realloc_vis = true; 1423 efx_mcdi_filter_table_reset_mc_allocations(efx); 1424 nic_data->must_restore_piobufs = true; 1425 efx_ef10_forget_old_piobufs(efx); 1426 efx->rss_context.priv.context_id = EFX_MCDI_RSS_CONTEXT_INVALID; 1427 1428 /* Driver-created vswitches and vports must be re-created */ 1429 nic_data->must_probe_vswitching = true; 1430 efx->vport_id = EVB_PORT_ID_ASSIGNED; 1431 #ifdef CONFIG_SFC_SRIOV 1432 if (nic_data->vf) 1433 for (i = 0; i < efx->vf_count; i++) 1434 nic_data->vf[i].vport_id = 0; 1435 #endif 1436 } 1437 1438 static enum reset_type efx_ef10_map_reset_reason(enum reset_type reason) 1439 { 1440 if (reason == RESET_TYPE_MC_FAILURE) 1441 return RESET_TYPE_DATAPATH; 1442 1443 return efx_mcdi_map_reset_reason(reason); 1444 } 1445 1446 static int efx_ef10_map_reset_flags(u32 *flags) 1447 { 1448 enum { 1449 EF10_RESET_PORT = ((ETH_RESET_MAC | ETH_RESET_PHY) << 1450 ETH_RESET_SHARED_SHIFT), 1451 EF10_RESET_MC = ((ETH_RESET_DMA | ETH_RESET_FILTER | 1452 ETH_RESET_OFFLOAD | ETH_RESET_MAC | 1453 ETH_RESET_PHY | ETH_RESET_MGMT) << 1454 ETH_RESET_SHARED_SHIFT) 1455 }; 1456 1457 /* We assume for now that our PCI function is permitted to 1458 * reset everything. 1459 */ 1460 1461 if ((*flags & EF10_RESET_MC) == EF10_RESET_MC) { 1462 *flags &= ~EF10_RESET_MC; 1463 return RESET_TYPE_WORLD; 1464 } 1465 1466 if ((*flags & EF10_RESET_PORT) == EF10_RESET_PORT) { 1467 *flags &= ~EF10_RESET_PORT; 1468 return RESET_TYPE_ALL; 1469 } 1470 1471 /* no invisible reset implemented */ 1472 1473 return -EINVAL; 1474 } 1475 1476 static int efx_ef10_reset(struct efx_nic *efx, enum reset_type reset_type) 1477 { 1478 int rc = efx_mcdi_reset(efx, reset_type); 1479 1480 /* Unprivileged functions return -EPERM, but need to return success 1481 * here so that the datapath is brought back up. 1482 */ 1483 if (reset_type == RESET_TYPE_WORLD && rc == -EPERM) 1484 rc = 0; 1485 1486 /* If it was a port reset, trigger reallocation of MC resources. 1487 * Note that on an MC reset nothing needs to be done now because we'll 1488 * detect the MC reset later and handle it then. 1489 * For an FLR, we never get an MC reset event, but the MC has reset all 1490 * resources assigned to us, so we have to trigger reallocation now. 1491 */ 1492 if ((reset_type == RESET_TYPE_ALL || 1493 reset_type == RESET_TYPE_MCDI_TIMEOUT) && !rc) 1494 efx_ef10_table_reset_mc_allocations(efx); 1495 return rc; 1496 } 1497 1498 #define EF10_DMA_STAT(ext_name, mcdi_name) \ 1499 [EF10_STAT_ ## ext_name] = \ 1500 { #ext_name, 64, 8 * MC_CMD_MAC_ ## mcdi_name } 1501 #define EF10_DMA_INVIS_STAT(int_name, mcdi_name) \ 1502 [EF10_STAT_ ## int_name] = \ 1503 { NULL, 64, 8 * MC_CMD_MAC_ ## mcdi_name } 1504 #define EF10_OTHER_STAT(ext_name) \ 1505 [EF10_STAT_ ## ext_name] = { #ext_name, 0, 0 } 1506 1507 static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = { 1508 EF10_DMA_STAT(port_tx_bytes, TX_BYTES), 1509 EF10_DMA_STAT(port_tx_packets, TX_PKTS), 1510 EF10_DMA_STAT(port_tx_pause, TX_PAUSE_PKTS), 1511 EF10_DMA_STAT(port_tx_control, TX_CONTROL_PKTS), 1512 EF10_DMA_STAT(port_tx_unicast, TX_UNICAST_PKTS), 1513 EF10_DMA_STAT(port_tx_multicast, TX_MULTICAST_PKTS), 1514 EF10_DMA_STAT(port_tx_broadcast, TX_BROADCAST_PKTS), 1515 EF10_DMA_STAT(port_tx_lt64, TX_LT64_PKTS), 1516 EF10_DMA_STAT(port_tx_64, TX_64_PKTS), 1517 EF10_DMA_STAT(port_tx_65_to_127, TX_65_TO_127_PKTS), 1518 EF10_DMA_STAT(port_tx_128_to_255, TX_128_TO_255_PKTS), 1519 EF10_DMA_STAT(port_tx_256_to_511, TX_256_TO_511_PKTS), 1520 EF10_DMA_STAT(port_tx_512_to_1023, TX_512_TO_1023_PKTS), 1521 EF10_DMA_STAT(port_tx_1024_to_15xx, TX_1024_TO_15XX_PKTS), 1522 EF10_DMA_STAT(port_tx_15xx_to_jumbo, TX_15XX_TO_JUMBO_PKTS), 1523 EF10_DMA_STAT(port_rx_bytes, RX_BYTES), 1524 EF10_DMA_INVIS_STAT(port_rx_bytes_minus_good_bytes, RX_BAD_BYTES), 1525 EF10_OTHER_STAT(port_rx_good_bytes), 1526 EF10_OTHER_STAT(port_rx_bad_bytes), 1527 EF10_DMA_STAT(port_rx_packets, RX_PKTS), 1528 EF10_DMA_STAT(port_rx_good, RX_GOOD_PKTS), 1529 EF10_DMA_STAT(port_rx_bad, RX_BAD_FCS_PKTS), 1530 EF10_DMA_STAT(port_rx_pause, RX_PAUSE_PKTS), 1531 EF10_DMA_STAT(port_rx_control, RX_CONTROL_PKTS), 1532 EF10_DMA_STAT(port_rx_unicast, RX_UNICAST_PKTS), 1533 EF10_DMA_STAT(port_rx_multicast, RX_MULTICAST_PKTS), 1534 EF10_DMA_STAT(port_rx_broadcast, RX_BROADCAST_PKTS), 1535 EF10_DMA_STAT(port_rx_lt64, RX_UNDERSIZE_PKTS), 1536 EF10_DMA_STAT(port_rx_64, RX_64_PKTS), 1537 EF10_DMA_STAT(port_rx_65_to_127, RX_65_TO_127_PKTS), 1538 EF10_DMA_STAT(port_rx_128_to_255, RX_128_TO_255_PKTS), 1539 EF10_DMA_STAT(port_rx_256_to_511, RX_256_TO_511_PKTS), 1540 EF10_DMA_STAT(port_rx_512_to_1023, RX_512_TO_1023_PKTS), 1541 EF10_DMA_STAT(port_rx_1024_to_15xx, RX_1024_TO_15XX_PKTS), 1542 EF10_DMA_STAT(port_rx_15xx_to_jumbo, RX_15XX_TO_JUMBO_PKTS), 1543 EF10_DMA_STAT(port_rx_gtjumbo, RX_GTJUMBO_PKTS), 1544 EF10_DMA_STAT(port_rx_bad_gtjumbo, RX_JABBER_PKTS), 1545 EF10_DMA_STAT(port_rx_overflow, RX_OVERFLOW_PKTS), 1546 EF10_DMA_STAT(port_rx_align_error, RX_ALIGN_ERROR_PKTS), 1547 EF10_DMA_STAT(port_rx_length_error, RX_LENGTH_ERROR_PKTS), 1548 EF10_DMA_STAT(port_rx_nodesc_drops, RX_NODESC_DROPS), 1549 EFX_GENERIC_SW_STAT(rx_nodesc_trunc), 1550 EFX_GENERIC_SW_STAT(rx_noskb_drops), 1551 EF10_DMA_STAT(port_rx_pm_trunc_bb_overflow, PM_TRUNC_BB_OVERFLOW), 1552 EF10_DMA_STAT(port_rx_pm_discard_bb_overflow, PM_DISCARD_BB_OVERFLOW), 1553 EF10_DMA_STAT(port_rx_pm_trunc_vfifo_full, PM_TRUNC_VFIFO_FULL), 1554 EF10_DMA_STAT(port_rx_pm_discard_vfifo_full, PM_DISCARD_VFIFO_FULL), 1555 EF10_DMA_STAT(port_rx_pm_trunc_qbb, PM_TRUNC_QBB), 1556 EF10_DMA_STAT(port_rx_pm_discard_qbb, PM_DISCARD_QBB), 1557 EF10_DMA_STAT(port_rx_pm_discard_mapping, PM_DISCARD_MAPPING), 1558 EF10_DMA_STAT(port_rx_dp_q_disabled_packets, RXDP_Q_DISABLED_PKTS), 1559 EF10_DMA_STAT(port_rx_dp_di_dropped_packets, RXDP_DI_DROPPED_PKTS), 1560 EF10_DMA_STAT(port_rx_dp_streaming_packets, RXDP_STREAMING_PKTS), 1561 EF10_DMA_STAT(port_rx_dp_hlb_fetch, RXDP_HLB_FETCH_CONDITIONS), 1562 EF10_DMA_STAT(port_rx_dp_hlb_wait, RXDP_HLB_WAIT_CONDITIONS), 1563 EF10_DMA_STAT(rx_unicast, VADAPTER_RX_UNICAST_PACKETS), 1564 EF10_DMA_STAT(rx_unicast_bytes, VADAPTER_RX_UNICAST_BYTES), 1565 EF10_DMA_STAT(rx_multicast, VADAPTER_RX_MULTICAST_PACKETS), 1566 EF10_DMA_STAT(rx_multicast_bytes, VADAPTER_RX_MULTICAST_BYTES), 1567 EF10_DMA_STAT(rx_broadcast, VADAPTER_RX_BROADCAST_PACKETS), 1568 EF10_DMA_STAT(rx_broadcast_bytes, VADAPTER_RX_BROADCAST_BYTES), 1569 EF10_DMA_STAT(rx_bad, VADAPTER_RX_BAD_PACKETS), 1570 EF10_DMA_STAT(rx_bad_bytes, VADAPTER_RX_BAD_BYTES), 1571 EF10_DMA_STAT(rx_overflow, VADAPTER_RX_OVERFLOW), 1572 EF10_DMA_STAT(tx_unicast, VADAPTER_TX_UNICAST_PACKETS), 1573 EF10_DMA_STAT(tx_unicast_bytes, VADAPTER_TX_UNICAST_BYTES), 1574 EF10_DMA_STAT(tx_multicast, VADAPTER_TX_MULTICAST_PACKETS), 1575 EF10_DMA_STAT(tx_multicast_bytes, VADAPTER_TX_MULTICAST_BYTES), 1576 EF10_DMA_STAT(tx_broadcast, VADAPTER_TX_BROADCAST_PACKETS), 1577 EF10_DMA_STAT(tx_broadcast_bytes, VADAPTER_TX_BROADCAST_BYTES), 1578 EF10_DMA_STAT(tx_bad, VADAPTER_TX_BAD_PACKETS), 1579 EF10_DMA_STAT(tx_bad_bytes, VADAPTER_TX_BAD_BYTES), 1580 EF10_DMA_STAT(tx_overflow, VADAPTER_TX_OVERFLOW), 1581 EF10_DMA_STAT(fec_uncorrected_errors, FEC_UNCORRECTED_ERRORS), 1582 EF10_DMA_STAT(fec_corrected_errors, FEC_CORRECTED_ERRORS), 1583 EF10_DMA_STAT(fec_corrected_symbols_lane0, FEC_CORRECTED_SYMBOLS_LANE0), 1584 EF10_DMA_STAT(fec_corrected_symbols_lane1, FEC_CORRECTED_SYMBOLS_LANE1), 1585 EF10_DMA_STAT(fec_corrected_symbols_lane2, FEC_CORRECTED_SYMBOLS_LANE2), 1586 EF10_DMA_STAT(fec_corrected_symbols_lane3, FEC_CORRECTED_SYMBOLS_LANE3), 1587 EF10_DMA_STAT(ctpio_vi_busy_fallback, CTPIO_VI_BUSY_FALLBACK), 1588 EF10_DMA_STAT(ctpio_long_write_success, CTPIO_LONG_WRITE_SUCCESS), 1589 EF10_DMA_STAT(ctpio_missing_dbell_fail, CTPIO_MISSING_DBELL_FAIL), 1590 EF10_DMA_STAT(ctpio_overflow_fail, CTPIO_OVERFLOW_FAIL), 1591 EF10_DMA_STAT(ctpio_underflow_fail, CTPIO_UNDERFLOW_FAIL), 1592 EF10_DMA_STAT(ctpio_timeout_fail, CTPIO_TIMEOUT_FAIL), 1593 EF10_DMA_STAT(ctpio_noncontig_wr_fail, CTPIO_NONCONTIG_WR_FAIL), 1594 EF10_DMA_STAT(ctpio_frm_clobber_fail, CTPIO_FRM_CLOBBER_FAIL), 1595 EF10_DMA_STAT(ctpio_invalid_wr_fail, CTPIO_INVALID_WR_FAIL), 1596 EF10_DMA_STAT(ctpio_vi_clobber_fallback, CTPIO_VI_CLOBBER_FALLBACK), 1597 EF10_DMA_STAT(ctpio_unqualified_fallback, CTPIO_UNQUALIFIED_FALLBACK), 1598 EF10_DMA_STAT(ctpio_runt_fallback, CTPIO_RUNT_FALLBACK), 1599 EF10_DMA_STAT(ctpio_success, CTPIO_SUCCESS), 1600 EF10_DMA_STAT(ctpio_fallback, CTPIO_FALLBACK), 1601 EF10_DMA_STAT(ctpio_poison, CTPIO_POISON), 1602 EF10_DMA_STAT(ctpio_erase, CTPIO_ERASE), 1603 }; 1604 1605 #define HUNT_COMMON_STAT_MASK ((1ULL << EF10_STAT_port_tx_bytes) | \ 1606 (1ULL << EF10_STAT_port_tx_packets) | \ 1607 (1ULL << EF10_STAT_port_tx_pause) | \ 1608 (1ULL << EF10_STAT_port_tx_unicast) | \ 1609 (1ULL << EF10_STAT_port_tx_multicast) | \ 1610 (1ULL << EF10_STAT_port_tx_broadcast) | \ 1611 (1ULL << EF10_STAT_port_rx_bytes) | \ 1612 (1ULL << \ 1613 EF10_STAT_port_rx_bytes_minus_good_bytes) | \ 1614 (1ULL << EF10_STAT_port_rx_good_bytes) | \ 1615 (1ULL << EF10_STAT_port_rx_bad_bytes) | \ 1616 (1ULL << EF10_STAT_port_rx_packets) | \ 1617 (1ULL << EF10_STAT_port_rx_good) | \ 1618 (1ULL << EF10_STAT_port_rx_bad) | \ 1619 (1ULL << EF10_STAT_port_rx_pause) | \ 1620 (1ULL << EF10_STAT_port_rx_control) | \ 1621 (1ULL << EF10_STAT_port_rx_unicast) | \ 1622 (1ULL << EF10_STAT_port_rx_multicast) | \ 1623 (1ULL << EF10_STAT_port_rx_broadcast) | \ 1624 (1ULL << EF10_STAT_port_rx_lt64) | \ 1625 (1ULL << EF10_STAT_port_rx_64) | \ 1626 (1ULL << EF10_STAT_port_rx_65_to_127) | \ 1627 (1ULL << EF10_STAT_port_rx_128_to_255) | \ 1628 (1ULL << EF10_STAT_port_rx_256_to_511) | \ 1629 (1ULL << EF10_STAT_port_rx_512_to_1023) |\ 1630 (1ULL << EF10_STAT_port_rx_1024_to_15xx) |\ 1631 (1ULL << EF10_STAT_port_rx_15xx_to_jumbo) |\ 1632 (1ULL << EF10_STAT_port_rx_gtjumbo) | \ 1633 (1ULL << EF10_STAT_port_rx_bad_gtjumbo) |\ 1634 (1ULL << EF10_STAT_port_rx_overflow) | \ 1635 (1ULL << EF10_STAT_port_rx_nodesc_drops) |\ 1636 (1ULL << GENERIC_STAT_rx_nodesc_trunc) | \ 1637 (1ULL << GENERIC_STAT_rx_noskb_drops)) 1638 1639 /* On 7000 series NICs, these statistics are only provided by the 10G MAC. 1640 * For a 10G/40G switchable port we do not expose these because they might 1641 * not include all the packets they should. 1642 * On 8000 series NICs these statistics are always provided. 1643 */ 1644 #define HUNT_10G_ONLY_STAT_MASK ((1ULL << EF10_STAT_port_tx_control) | \ 1645 (1ULL << EF10_STAT_port_tx_lt64) | \ 1646 (1ULL << EF10_STAT_port_tx_64) | \ 1647 (1ULL << EF10_STAT_port_tx_65_to_127) |\ 1648 (1ULL << EF10_STAT_port_tx_128_to_255) |\ 1649 (1ULL << EF10_STAT_port_tx_256_to_511) |\ 1650 (1ULL << EF10_STAT_port_tx_512_to_1023) |\ 1651 (1ULL << EF10_STAT_port_tx_1024_to_15xx) |\ 1652 (1ULL << EF10_STAT_port_tx_15xx_to_jumbo)) 1653 1654 /* These statistics are only provided by the 40G MAC. For a 10G/40G 1655 * switchable port we do expose these because the errors will otherwise 1656 * be silent. 1657 */ 1658 #define HUNT_40G_EXTRA_STAT_MASK ((1ULL << EF10_STAT_port_rx_align_error) |\ 1659 (1ULL << EF10_STAT_port_rx_length_error)) 1660 1661 /* These statistics are only provided if the firmware supports the 1662 * capability PM_AND_RXDP_COUNTERS. 1663 */ 1664 #define HUNT_PM_AND_RXDP_STAT_MASK ( \ 1665 (1ULL << EF10_STAT_port_rx_pm_trunc_bb_overflow) | \ 1666 (1ULL << EF10_STAT_port_rx_pm_discard_bb_overflow) | \ 1667 (1ULL << EF10_STAT_port_rx_pm_trunc_vfifo_full) | \ 1668 (1ULL << EF10_STAT_port_rx_pm_discard_vfifo_full) | \ 1669 (1ULL << EF10_STAT_port_rx_pm_trunc_qbb) | \ 1670 (1ULL << EF10_STAT_port_rx_pm_discard_qbb) | \ 1671 (1ULL << EF10_STAT_port_rx_pm_discard_mapping) | \ 1672 (1ULL << EF10_STAT_port_rx_dp_q_disabled_packets) | \ 1673 (1ULL << EF10_STAT_port_rx_dp_di_dropped_packets) | \ 1674 (1ULL << EF10_STAT_port_rx_dp_streaming_packets) | \ 1675 (1ULL << EF10_STAT_port_rx_dp_hlb_fetch) | \ 1676 (1ULL << EF10_STAT_port_rx_dp_hlb_wait)) 1677 1678 /* These statistics are only provided if the NIC supports MC_CMD_MAC_STATS_V2, 1679 * indicated by returning a value >= MC_CMD_MAC_NSTATS_V2 in 1680 * MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS. 1681 * These bits are in the second u64 of the raw mask. 1682 */ 1683 #define EF10_FEC_STAT_MASK ( \ 1684 (1ULL << (EF10_STAT_fec_uncorrected_errors - 64)) | \ 1685 (1ULL << (EF10_STAT_fec_corrected_errors - 64)) | \ 1686 (1ULL << (EF10_STAT_fec_corrected_symbols_lane0 - 64)) | \ 1687 (1ULL << (EF10_STAT_fec_corrected_symbols_lane1 - 64)) | \ 1688 (1ULL << (EF10_STAT_fec_corrected_symbols_lane2 - 64)) | \ 1689 (1ULL << (EF10_STAT_fec_corrected_symbols_lane3 - 64))) 1690 1691 /* These statistics are only provided if the NIC supports MC_CMD_MAC_STATS_V3, 1692 * indicated by returning a value >= MC_CMD_MAC_NSTATS_V3 in 1693 * MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS. 1694 * These bits are in the second u64 of the raw mask. 1695 */ 1696 #define EF10_CTPIO_STAT_MASK ( \ 1697 (1ULL << (EF10_STAT_ctpio_vi_busy_fallback - 64)) | \ 1698 (1ULL << (EF10_STAT_ctpio_long_write_success - 64)) | \ 1699 (1ULL << (EF10_STAT_ctpio_missing_dbell_fail - 64)) | \ 1700 (1ULL << (EF10_STAT_ctpio_overflow_fail - 64)) | \ 1701 (1ULL << (EF10_STAT_ctpio_underflow_fail - 64)) | \ 1702 (1ULL << (EF10_STAT_ctpio_timeout_fail - 64)) | \ 1703 (1ULL << (EF10_STAT_ctpio_noncontig_wr_fail - 64)) | \ 1704 (1ULL << (EF10_STAT_ctpio_frm_clobber_fail - 64)) | \ 1705 (1ULL << (EF10_STAT_ctpio_invalid_wr_fail - 64)) | \ 1706 (1ULL << (EF10_STAT_ctpio_vi_clobber_fallback - 64)) | \ 1707 (1ULL << (EF10_STAT_ctpio_unqualified_fallback - 64)) | \ 1708 (1ULL << (EF10_STAT_ctpio_runt_fallback - 64)) | \ 1709 (1ULL << (EF10_STAT_ctpio_success - 64)) | \ 1710 (1ULL << (EF10_STAT_ctpio_fallback - 64)) | \ 1711 (1ULL << (EF10_STAT_ctpio_poison - 64)) | \ 1712 (1ULL << (EF10_STAT_ctpio_erase - 64))) 1713 1714 static u64 efx_ef10_raw_stat_mask(struct efx_nic *efx) 1715 { 1716 u64 raw_mask = HUNT_COMMON_STAT_MASK; 1717 u32 port_caps = efx_mcdi_phy_get_caps(efx); 1718 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1719 1720 if (!(efx->mcdi->fn_flags & 1721 1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL)) 1722 return 0; 1723 1724 if (port_caps & (1 << MC_CMD_PHY_CAP_40000FDX_LBN)) { 1725 raw_mask |= HUNT_40G_EXTRA_STAT_MASK; 1726 /* 8000 series have everything even at 40G */ 1727 if (nic_data->datapath_caps2 & 1728 (1 << MC_CMD_GET_CAPABILITIES_V2_OUT_MAC_STATS_40G_TX_SIZE_BINS_LBN)) 1729 raw_mask |= HUNT_10G_ONLY_STAT_MASK; 1730 } else { 1731 raw_mask |= HUNT_10G_ONLY_STAT_MASK; 1732 } 1733 1734 if (nic_data->datapath_caps & 1735 (1 << MC_CMD_GET_CAPABILITIES_OUT_PM_AND_RXDP_COUNTERS_LBN)) 1736 raw_mask |= HUNT_PM_AND_RXDP_STAT_MASK; 1737 1738 return raw_mask; 1739 } 1740 1741 static void efx_ef10_get_stat_mask(struct efx_nic *efx, unsigned long *mask) 1742 { 1743 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1744 u64 raw_mask[2]; 1745 1746 raw_mask[0] = efx_ef10_raw_stat_mask(efx); 1747 1748 /* Only show vadaptor stats when EVB capability is present */ 1749 if (nic_data->datapath_caps & 1750 (1 << MC_CMD_GET_CAPABILITIES_OUT_EVB_LBN)) { 1751 raw_mask[0] |= ~((1ULL << EF10_STAT_rx_unicast) - 1); 1752 raw_mask[1] = (1ULL << (EF10_STAT_V1_COUNT - 64)) - 1; 1753 } else { 1754 raw_mask[1] = 0; 1755 } 1756 /* Only show FEC stats when NIC supports MC_CMD_MAC_STATS_V2 */ 1757 if (efx->num_mac_stats >= MC_CMD_MAC_NSTATS_V2) 1758 raw_mask[1] |= EF10_FEC_STAT_MASK; 1759 1760 /* CTPIO stats appear in V3. Only show them on devices that actually 1761 * support CTPIO. Although this driver doesn't use CTPIO others might, 1762 * and we may be reporting the stats for the underlying port. 1763 */ 1764 if (efx->num_mac_stats >= MC_CMD_MAC_NSTATS_V3 && 1765 (nic_data->datapath_caps2 & 1766 (1 << MC_CMD_GET_CAPABILITIES_V4_OUT_CTPIO_LBN))) 1767 raw_mask[1] |= EF10_CTPIO_STAT_MASK; 1768 1769 #if BITS_PER_LONG == 64 1770 BUILD_BUG_ON(BITS_TO_LONGS(EF10_STAT_COUNT) != 2); 1771 mask[0] = raw_mask[0]; 1772 mask[1] = raw_mask[1]; 1773 #else 1774 BUILD_BUG_ON(BITS_TO_LONGS(EF10_STAT_COUNT) != 3); 1775 mask[0] = raw_mask[0] & 0xffffffff; 1776 mask[1] = raw_mask[0] >> 32; 1777 mask[2] = raw_mask[1] & 0xffffffff; 1778 #endif 1779 } 1780 1781 static size_t efx_ef10_describe_stats(struct efx_nic *efx, u8 **names) 1782 { 1783 DECLARE_BITMAP(mask, EF10_STAT_COUNT); 1784 1785 efx_ef10_get_stat_mask(efx, mask); 1786 return efx_nic_describe_stats(efx_ef10_stat_desc, EF10_STAT_COUNT, 1787 mask, names); 1788 } 1789 1790 static void efx_ef10_get_fec_stats(struct efx_nic *efx, 1791 struct ethtool_fec_stats *fec_stats) 1792 { 1793 DECLARE_BITMAP(mask, EF10_STAT_COUNT); 1794 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1795 u64 *stats = nic_data->stats; 1796 1797 efx_ef10_get_stat_mask(efx, mask); 1798 if (test_bit(EF10_STAT_fec_corrected_errors, mask)) 1799 fec_stats->corrected_blocks.total = 1800 stats[EF10_STAT_fec_corrected_errors]; 1801 if (test_bit(EF10_STAT_fec_uncorrected_errors, mask)) 1802 fec_stats->uncorrectable_blocks.total = 1803 stats[EF10_STAT_fec_uncorrected_errors]; 1804 } 1805 1806 static size_t efx_ef10_update_stats_common(struct efx_nic *efx, u64 *full_stats, 1807 struct rtnl_link_stats64 *core_stats) 1808 { 1809 DECLARE_BITMAP(mask, EF10_STAT_COUNT); 1810 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1811 u64 *stats = nic_data->stats; 1812 size_t stats_count = 0, index; 1813 1814 efx_ef10_get_stat_mask(efx, mask); 1815 1816 if (full_stats) { 1817 for_each_set_bit(index, mask, EF10_STAT_COUNT) { 1818 if (efx_ef10_stat_desc[index].name) { 1819 *full_stats++ = stats[index]; 1820 ++stats_count; 1821 } 1822 } 1823 } 1824 1825 if (!core_stats) 1826 return stats_count; 1827 1828 if (nic_data->datapath_caps & 1829 1 << MC_CMD_GET_CAPABILITIES_OUT_EVB_LBN) { 1830 /* Use vadaptor stats. */ 1831 core_stats->rx_packets = stats[EF10_STAT_rx_unicast] + 1832 stats[EF10_STAT_rx_multicast] + 1833 stats[EF10_STAT_rx_broadcast]; 1834 core_stats->tx_packets = stats[EF10_STAT_tx_unicast] + 1835 stats[EF10_STAT_tx_multicast] + 1836 stats[EF10_STAT_tx_broadcast]; 1837 core_stats->rx_bytes = stats[EF10_STAT_rx_unicast_bytes] + 1838 stats[EF10_STAT_rx_multicast_bytes] + 1839 stats[EF10_STAT_rx_broadcast_bytes]; 1840 core_stats->tx_bytes = stats[EF10_STAT_tx_unicast_bytes] + 1841 stats[EF10_STAT_tx_multicast_bytes] + 1842 stats[EF10_STAT_tx_broadcast_bytes]; 1843 core_stats->rx_dropped = stats[GENERIC_STAT_rx_nodesc_trunc] + 1844 stats[GENERIC_STAT_rx_noskb_drops]; 1845 core_stats->multicast = stats[EF10_STAT_rx_multicast]; 1846 core_stats->rx_crc_errors = stats[EF10_STAT_rx_bad]; 1847 core_stats->rx_fifo_errors = stats[EF10_STAT_rx_overflow]; 1848 core_stats->rx_errors = core_stats->rx_crc_errors; 1849 core_stats->tx_errors = stats[EF10_STAT_tx_bad]; 1850 } else { 1851 /* Use port stats. */ 1852 core_stats->rx_packets = stats[EF10_STAT_port_rx_packets]; 1853 core_stats->tx_packets = stats[EF10_STAT_port_tx_packets]; 1854 core_stats->rx_bytes = stats[EF10_STAT_port_rx_bytes]; 1855 core_stats->tx_bytes = stats[EF10_STAT_port_tx_bytes]; 1856 core_stats->rx_dropped = stats[EF10_STAT_port_rx_nodesc_drops] + 1857 stats[GENERIC_STAT_rx_nodesc_trunc] + 1858 stats[GENERIC_STAT_rx_noskb_drops]; 1859 core_stats->multicast = stats[EF10_STAT_port_rx_multicast]; 1860 core_stats->rx_length_errors = 1861 stats[EF10_STAT_port_rx_gtjumbo] + 1862 stats[EF10_STAT_port_rx_length_error]; 1863 core_stats->rx_crc_errors = stats[EF10_STAT_port_rx_bad]; 1864 core_stats->rx_frame_errors = 1865 stats[EF10_STAT_port_rx_align_error]; 1866 core_stats->rx_fifo_errors = stats[EF10_STAT_port_rx_overflow]; 1867 core_stats->rx_errors = (core_stats->rx_length_errors + 1868 core_stats->rx_crc_errors + 1869 core_stats->rx_frame_errors); 1870 } 1871 1872 return stats_count; 1873 } 1874 1875 static size_t efx_ef10_update_stats_pf(struct efx_nic *efx, u64 *full_stats, 1876 struct rtnl_link_stats64 *core_stats) 1877 { 1878 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1879 DECLARE_BITMAP(mask, EF10_STAT_COUNT); 1880 u64 *stats = nic_data->stats; 1881 1882 efx_ef10_get_stat_mask(efx, mask); 1883 1884 /* If NIC was fini'd (probably resetting), then we can't read 1885 * updated stats right now. 1886 */ 1887 if (nic_data->mc_stats) { 1888 efx_nic_copy_stats(efx, nic_data->mc_stats); 1889 efx_nic_update_stats(efx_ef10_stat_desc, EF10_STAT_COUNT, 1890 mask, stats, nic_data->mc_stats, false); 1891 } 1892 1893 /* Update derived statistics */ 1894 efx_nic_fix_nodesc_drop_stat(efx, 1895 &stats[EF10_STAT_port_rx_nodesc_drops]); 1896 /* MC Firmware reads RX_BYTES and RX_GOOD_BYTES from the MAC. 1897 * It then calculates RX_BAD_BYTES and DMAs it to us with RX_BYTES. 1898 * We report these as port_rx_ stats. We are not given RX_GOOD_BYTES. 1899 * Here we calculate port_rx_good_bytes. 1900 */ 1901 stats[EF10_STAT_port_rx_good_bytes] = 1902 stats[EF10_STAT_port_rx_bytes] - 1903 stats[EF10_STAT_port_rx_bytes_minus_good_bytes]; 1904 1905 /* The asynchronous reads used to calculate RX_BAD_BYTES in 1906 * MC Firmware are done such that we should not see an increase in 1907 * RX_BAD_BYTES when a good packet has arrived. Unfortunately this 1908 * does mean that the stat can decrease at times. Here we do not 1909 * update the stat unless it has increased or has gone to zero 1910 * (In the case of the NIC rebooting). 1911 * Please see Bug 33781 for a discussion of why things work this way. 1912 */ 1913 efx_update_diff_stat(&stats[EF10_STAT_port_rx_bad_bytes], 1914 stats[EF10_STAT_port_rx_bytes_minus_good_bytes]); 1915 efx_update_sw_stats(efx, stats); 1916 1917 return efx_ef10_update_stats_common(efx, full_stats, core_stats); 1918 } 1919 1920 static int efx_ef10_try_update_nic_stats_vf(struct efx_nic *efx) 1921 __must_hold(&efx->stats_lock) 1922 { 1923 MCDI_DECLARE_BUF(inbuf, MC_CMD_MAC_STATS_IN_LEN); 1924 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1925 DECLARE_BITMAP(mask, EF10_STAT_COUNT); 1926 __le64 generation_start, generation_end; 1927 u64 *stats = nic_data->stats; 1928 u32 dma_len = efx->num_mac_stats * sizeof(u64); 1929 struct efx_buffer stats_buf; 1930 __le64 *dma_stats; 1931 int rc; 1932 1933 spin_unlock_bh(&efx->stats_lock); 1934 1935 efx_ef10_get_stat_mask(efx, mask); 1936 1937 rc = efx_nic_alloc_buffer(efx, &stats_buf, dma_len, GFP_KERNEL); 1938 if (rc) { 1939 spin_lock_bh(&efx->stats_lock); 1940 return rc; 1941 } 1942 1943 dma_stats = stats_buf.addr; 1944 dma_stats[efx->num_mac_stats - 1] = EFX_MC_STATS_GENERATION_INVALID; 1945 1946 MCDI_SET_QWORD(inbuf, MAC_STATS_IN_DMA_ADDR, stats_buf.dma_addr); 1947 MCDI_POPULATE_DWORD_1(inbuf, MAC_STATS_IN_CMD, 1948 MAC_STATS_IN_DMA, 1); 1949 MCDI_SET_DWORD(inbuf, MAC_STATS_IN_DMA_LEN, dma_len); 1950 MCDI_SET_DWORD(inbuf, MAC_STATS_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); 1951 1952 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_MAC_STATS, inbuf, sizeof(inbuf), 1953 NULL, 0, NULL); 1954 spin_lock_bh(&efx->stats_lock); 1955 if (rc) { 1956 /* Expect ENOENT if DMA queues have not been set up */ 1957 if (rc != -ENOENT || atomic_read(&efx->active_queues)) 1958 efx_mcdi_display_error(efx, MC_CMD_MAC_STATS, 1959 sizeof(inbuf), NULL, 0, rc); 1960 goto out; 1961 } 1962 1963 generation_end = dma_stats[efx->num_mac_stats - 1]; 1964 if (generation_end == EFX_MC_STATS_GENERATION_INVALID) { 1965 WARN_ON_ONCE(1); 1966 goto out; 1967 } 1968 rmb(); 1969 efx_nic_update_stats(efx_ef10_stat_desc, EF10_STAT_COUNT, mask, 1970 stats, stats_buf.addr, false); 1971 rmb(); 1972 generation_start = dma_stats[MC_CMD_MAC_GENERATION_START]; 1973 if (generation_end != generation_start) { 1974 rc = -EAGAIN; 1975 goto out; 1976 } 1977 1978 efx_update_sw_stats(efx, stats); 1979 out: 1980 /* releasing a DMA coherent buffer with BH disabled can panic */ 1981 spin_unlock_bh(&efx->stats_lock); 1982 efx_nic_free_buffer(efx, &stats_buf); 1983 spin_lock_bh(&efx->stats_lock); 1984 return rc; 1985 } 1986 1987 static size_t efx_ef10_update_stats_vf(struct efx_nic *efx, u64 *full_stats, 1988 struct rtnl_link_stats64 *core_stats) 1989 { 1990 if (efx_ef10_try_update_nic_stats_vf(efx)) 1991 return 0; 1992 1993 return efx_ef10_update_stats_common(efx, full_stats, core_stats); 1994 } 1995 1996 static size_t efx_ef10_update_stats_atomic_vf(struct efx_nic *efx, u64 *full_stats, 1997 struct rtnl_link_stats64 *core_stats) 1998 { 1999 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2000 2001 /* In atomic context, cannot update HW stats. Just update the 2002 * software stats and return so the caller can continue. 2003 */ 2004 efx_update_sw_stats(efx, nic_data->stats); 2005 return efx_ef10_update_stats_common(efx, full_stats, core_stats); 2006 } 2007 2008 static void efx_ef10_push_irq_moderation(struct efx_channel *channel) 2009 { 2010 struct efx_nic *efx = channel->efx; 2011 unsigned int mode, usecs; 2012 efx_dword_t timer_cmd; 2013 2014 if (channel->irq_moderation_us) { 2015 mode = 3; 2016 usecs = channel->irq_moderation_us; 2017 } else { 2018 mode = 0; 2019 usecs = 0; 2020 } 2021 2022 if (EFX_EF10_WORKAROUND_61265(efx)) { 2023 MCDI_DECLARE_BUF(inbuf, MC_CMD_SET_EVQ_TMR_IN_LEN); 2024 unsigned int ns = usecs * 1000; 2025 2026 MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_INSTANCE, 2027 channel->channel); 2028 MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_LOAD_REQ_NS, ns); 2029 MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_RELOAD_REQ_NS, ns); 2030 MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_MODE, mode); 2031 2032 efx_mcdi_rpc_async(efx, MC_CMD_SET_EVQ_TMR, 2033 inbuf, sizeof(inbuf), 0, NULL, 0); 2034 } else if (EFX_EF10_WORKAROUND_35388(efx)) { 2035 unsigned int ticks = efx_usecs_to_ticks(efx, usecs); 2036 2037 EFX_POPULATE_DWORD_3(timer_cmd, ERF_DD_EVQ_IND_TIMER_FLAGS, 2038 EFE_DD_EVQ_IND_TIMER_FLAGS, 2039 ERF_DD_EVQ_IND_TIMER_MODE, mode, 2040 ERF_DD_EVQ_IND_TIMER_VAL, ticks); 2041 efx_writed_page(efx, &timer_cmd, ER_DD_EVQ_INDIRECT, 2042 channel->channel); 2043 } else { 2044 unsigned int ticks = efx_usecs_to_ticks(efx, usecs); 2045 2046 EFX_POPULATE_DWORD_3(timer_cmd, ERF_DZ_TC_TIMER_MODE, mode, 2047 ERF_DZ_TC_TIMER_VAL, ticks, 2048 ERF_FZ_TC_TMR_REL_VAL, ticks); 2049 efx_writed_page(efx, &timer_cmd, ER_DZ_EVQ_TMR, 2050 channel->channel); 2051 } 2052 } 2053 2054 static void efx_ef10_get_wol_vf(struct efx_nic *efx, 2055 struct ethtool_wolinfo *wol) {} 2056 2057 static int efx_ef10_set_wol_vf(struct efx_nic *efx, u32 type) 2058 { 2059 return -EOPNOTSUPP; 2060 } 2061 2062 static void efx_ef10_get_wol(struct efx_nic *efx, struct ethtool_wolinfo *wol) 2063 { 2064 wol->supported = 0; 2065 wol->wolopts = 0; 2066 memset(&wol->sopass, 0, sizeof(wol->sopass)); 2067 } 2068 2069 static int efx_ef10_set_wol(struct efx_nic *efx, u32 type) 2070 { 2071 if (type != 0) 2072 return -EINVAL; 2073 return 0; 2074 } 2075 2076 static void efx_ef10_mcdi_request(struct efx_nic *efx, 2077 const efx_dword_t *hdr, size_t hdr_len, 2078 const efx_dword_t *sdu, size_t sdu_len) 2079 { 2080 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2081 u8 *pdu = nic_data->mcdi_buf.addr; 2082 2083 memcpy(pdu, hdr, hdr_len); 2084 memcpy(pdu + hdr_len, sdu, sdu_len); 2085 wmb(); 2086 2087 /* The hardware provides 'low' and 'high' (doorbell) registers 2088 * for passing the 64-bit address of an MCDI request to 2089 * firmware. However the dwords are swapped by firmware. The 2090 * least significant bits of the doorbell are then 0 for all 2091 * MCDI requests due to alignment. 2092 */ 2093 _efx_writed(efx, cpu_to_le32((u64)nic_data->mcdi_buf.dma_addr >> 32), 2094 ER_DZ_MC_DB_LWRD); 2095 _efx_writed(efx, cpu_to_le32((u32)nic_data->mcdi_buf.dma_addr), 2096 ER_DZ_MC_DB_HWRD); 2097 } 2098 2099 static bool efx_ef10_mcdi_poll_response(struct efx_nic *efx) 2100 { 2101 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2102 const efx_dword_t hdr = *(const efx_dword_t *)nic_data->mcdi_buf.addr; 2103 2104 rmb(); 2105 return EFX_DWORD_FIELD(hdr, MCDI_HEADER_RESPONSE); 2106 } 2107 2108 static void 2109 efx_ef10_mcdi_read_response(struct efx_nic *efx, efx_dword_t *outbuf, 2110 size_t offset, size_t outlen) 2111 { 2112 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2113 const u8 *pdu = nic_data->mcdi_buf.addr; 2114 2115 memcpy(outbuf, pdu + offset, outlen); 2116 } 2117 2118 static void efx_ef10_mcdi_reboot_detected(struct efx_nic *efx) 2119 { 2120 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2121 2122 /* All our allocations have been reset */ 2123 efx_ef10_table_reset_mc_allocations(efx); 2124 2125 /* The datapath firmware might have been changed */ 2126 nic_data->must_check_datapath_caps = true; 2127 2128 /* MAC statistics have been cleared on the NIC; clear the local 2129 * statistic that we update with efx_update_diff_stat(). 2130 */ 2131 nic_data->stats[EF10_STAT_port_rx_bad_bytes] = 0; 2132 } 2133 2134 static int efx_ef10_mcdi_poll_reboot(struct efx_nic *efx) 2135 { 2136 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2137 int rc; 2138 2139 rc = efx_ef10_get_warm_boot_count(efx); 2140 if (rc < 0) { 2141 /* The firmware is presumably in the process of 2142 * rebooting. However, we are supposed to report each 2143 * reboot just once, so we must only do that once we 2144 * can read and store the updated warm boot count. 2145 */ 2146 return 0; 2147 } 2148 2149 if (rc == nic_data->warm_boot_count) 2150 return 0; 2151 2152 nic_data->warm_boot_count = rc; 2153 efx_ef10_mcdi_reboot_detected(efx); 2154 2155 return -EIO; 2156 } 2157 2158 /* Handle an MSI interrupt 2159 * 2160 * Handle an MSI hardware interrupt. This routine schedules event 2161 * queue processing. No interrupt acknowledgement cycle is necessary. 2162 * Also, we never need to check that the interrupt is for us, since 2163 * MSI interrupts cannot be shared. 2164 */ 2165 static irqreturn_t efx_ef10_msi_interrupt(int irq, void *dev_id) 2166 { 2167 struct efx_msi_context *context = dev_id; 2168 struct efx_nic *efx = context->efx; 2169 2170 netif_vdbg(efx, intr, efx->net_dev, 2171 "IRQ %d on CPU %d\n", irq, raw_smp_processor_id()); 2172 2173 if (likely(READ_ONCE(efx->irq_soft_enabled))) { 2174 /* Note test interrupts */ 2175 if (context->index == efx->irq_level) 2176 efx->last_irq_cpu = raw_smp_processor_id(); 2177 2178 /* Schedule processing of the channel */ 2179 efx_schedule_channel_irq(efx->channel[context->index]); 2180 } 2181 2182 return IRQ_HANDLED; 2183 } 2184 2185 static irqreturn_t efx_ef10_legacy_interrupt(int irq, void *dev_id) 2186 { 2187 struct efx_nic *efx = dev_id; 2188 bool soft_enabled = READ_ONCE(efx->irq_soft_enabled); 2189 struct efx_channel *channel; 2190 efx_dword_t reg; 2191 u32 queues; 2192 2193 /* Read the ISR which also ACKs the interrupts */ 2194 efx_readd(efx, ®, ER_DZ_BIU_INT_ISR); 2195 queues = EFX_DWORD_FIELD(reg, ERF_DZ_ISR_REG); 2196 2197 if (queues == 0) 2198 return IRQ_NONE; 2199 2200 if (likely(soft_enabled)) { 2201 /* Note test interrupts */ 2202 if (queues & (1U << efx->irq_level)) 2203 efx->last_irq_cpu = raw_smp_processor_id(); 2204 2205 efx_for_each_channel(channel, efx) { 2206 if (queues & 1) 2207 efx_schedule_channel_irq(channel); 2208 queues >>= 1; 2209 } 2210 } 2211 2212 netif_vdbg(efx, intr, efx->net_dev, 2213 "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n", 2214 irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg)); 2215 2216 return IRQ_HANDLED; 2217 } 2218 2219 static int efx_ef10_irq_test_generate(struct efx_nic *efx) 2220 { 2221 MCDI_DECLARE_BUF(inbuf, MC_CMD_TRIGGER_INTERRUPT_IN_LEN); 2222 2223 if (efx_mcdi_set_workaround(efx, MC_CMD_WORKAROUND_BUG41750, true, 2224 NULL) == 0) 2225 return -ENOTSUPP; 2226 2227 BUILD_BUG_ON(MC_CMD_TRIGGER_INTERRUPT_OUT_LEN != 0); 2228 2229 MCDI_SET_DWORD(inbuf, TRIGGER_INTERRUPT_IN_INTR_LEVEL, efx->irq_level); 2230 return efx_mcdi_rpc(efx, MC_CMD_TRIGGER_INTERRUPT, 2231 inbuf, sizeof(inbuf), NULL, 0, NULL); 2232 } 2233 2234 static int efx_ef10_tx_probe(struct efx_tx_queue *tx_queue) 2235 { 2236 /* low two bits of label are what we want for type */ 2237 BUILD_BUG_ON((EFX_TXQ_TYPE_OUTER_CSUM | EFX_TXQ_TYPE_INNER_CSUM) != 3); 2238 tx_queue->type = tx_queue->label & 3; 2239 return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd, 2240 (tx_queue->ptr_mask + 1) * 2241 sizeof(efx_qword_t), 2242 GFP_KERNEL); 2243 } 2244 2245 /* This writes to the TX_DESC_WPTR and also pushes data */ 2246 static inline void efx_ef10_push_tx_desc(struct efx_tx_queue *tx_queue, 2247 const efx_qword_t *txd) 2248 { 2249 unsigned int write_ptr; 2250 efx_oword_t reg; 2251 2252 write_ptr = tx_queue->write_count & tx_queue->ptr_mask; 2253 EFX_POPULATE_OWORD_1(reg, ERF_DZ_TX_DESC_WPTR, write_ptr); 2254 reg.qword[0] = *txd; 2255 efx_writeo_page(tx_queue->efx, ®, 2256 ER_DZ_TX_DESC_UPD, tx_queue->queue); 2257 } 2258 2259 /* Add Firmware-Assisted TSO v2 option descriptors to a queue. 2260 */ 2261 int efx_ef10_tx_tso_desc(struct efx_tx_queue *tx_queue, struct sk_buff *skb, 2262 bool *data_mapped) 2263 { 2264 struct efx_tx_buffer *buffer; 2265 u16 inner_ipv4_id = 0; 2266 u16 outer_ipv4_id = 0; 2267 struct tcphdr *tcp; 2268 struct iphdr *ip; 2269 u16 ip_tot_len; 2270 u32 seqnum; 2271 u32 mss; 2272 2273 EFX_WARN_ON_ONCE_PARANOID(tx_queue->tso_version != 2); 2274 2275 mss = skb_shinfo(skb)->gso_size; 2276 2277 if (unlikely(mss < 4)) { 2278 WARN_ONCE(1, "MSS of %u is too small for TSO v2\n", mss); 2279 return -EINVAL; 2280 } 2281 2282 if (skb->encapsulation) { 2283 if (!tx_queue->tso_encap) 2284 return -EINVAL; 2285 ip = ip_hdr(skb); 2286 if (ip->version == 4) 2287 outer_ipv4_id = ntohs(ip->id); 2288 2289 ip = inner_ip_hdr(skb); 2290 tcp = inner_tcp_hdr(skb); 2291 } else { 2292 ip = ip_hdr(skb); 2293 tcp = tcp_hdr(skb); 2294 } 2295 2296 /* 8000-series EF10 hardware requires that IP Total Length be 2297 * greater than or equal to the value it will have in each segment 2298 * (which is at most mss + 208 + TCP header length), but also less 2299 * than (0x10000 - inner_network_header). Otherwise the TCP 2300 * checksum calculation will be broken for encapsulated packets. 2301 * We fill in ip->tot_len with 0xff30, which should satisfy the 2302 * first requirement unless the MSS is ridiculously large (which 2303 * should be impossible as the driver max MTU is 9216); it is 2304 * guaranteed to satisfy the second as we only attempt TSO if 2305 * inner_network_header <= 208. 2306 */ 2307 ip_tot_len = 0x10000 - EFX_TSO2_MAX_HDRLEN; 2308 EFX_WARN_ON_ONCE_PARANOID(mss + EFX_TSO2_MAX_HDRLEN + 2309 (tcp->doff << 2u) > ip_tot_len); 2310 2311 if (ip->version == 4) { 2312 ip->tot_len = htons(ip_tot_len); 2313 ip->check = 0; 2314 inner_ipv4_id = ntohs(ip->id); 2315 } else { 2316 ((struct ipv6hdr *)ip)->payload_len = htons(ip_tot_len); 2317 } 2318 2319 seqnum = ntohl(tcp->seq); 2320 2321 buffer = efx_tx_queue_get_insert_buffer(tx_queue); 2322 2323 buffer->flags = EFX_TX_BUF_OPTION; 2324 buffer->len = 0; 2325 buffer->unmap_len = 0; 2326 EFX_POPULATE_QWORD_5(buffer->option, 2327 ESF_DZ_TX_DESC_IS_OPT, 1, 2328 ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_TSO, 2329 ESF_DZ_TX_TSO_OPTION_TYPE, 2330 ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A, 2331 ESF_DZ_TX_TSO_IP_ID, inner_ipv4_id, 2332 ESF_DZ_TX_TSO_TCP_SEQNO, seqnum 2333 ); 2334 ++tx_queue->insert_count; 2335 2336 buffer = efx_tx_queue_get_insert_buffer(tx_queue); 2337 2338 buffer->flags = EFX_TX_BUF_OPTION; 2339 buffer->len = 0; 2340 buffer->unmap_len = 0; 2341 EFX_POPULATE_QWORD_5(buffer->option, 2342 ESF_DZ_TX_DESC_IS_OPT, 1, 2343 ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_TSO, 2344 ESF_DZ_TX_TSO_OPTION_TYPE, 2345 ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B, 2346 ESF_DZ_TX_TSO_OUTER_IPID, outer_ipv4_id, 2347 ESF_DZ_TX_TSO_TCP_MSS, mss 2348 ); 2349 ++tx_queue->insert_count; 2350 2351 return 0; 2352 } 2353 2354 static u32 efx_ef10_tso_versions(struct efx_nic *efx) 2355 { 2356 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2357 u32 tso_versions = 0; 2358 2359 if (nic_data->datapath_caps & 2360 (1 << MC_CMD_GET_CAPABILITIES_OUT_TX_TSO_LBN)) 2361 tso_versions |= BIT(1); 2362 if (nic_data->datapath_caps2 & 2363 (1 << MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_LBN)) 2364 tso_versions |= BIT(2); 2365 return tso_versions; 2366 } 2367 2368 static void efx_ef10_tx_init(struct efx_tx_queue *tx_queue) 2369 { 2370 bool csum_offload = tx_queue->type & EFX_TXQ_TYPE_OUTER_CSUM; 2371 bool inner_csum = tx_queue->type & EFX_TXQ_TYPE_INNER_CSUM; 2372 struct efx_channel *channel = tx_queue->channel; 2373 struct efx_nic *efx = tx_queue->efx; 2374 struct efx_ef10_nic_data *nic_data; 2375 efx_qword_t *txd; 2376 int rc; 2377 2378 nic_data = efx->nic_data; 2379 2380 /* Only attempt to enable TX timestamping if we have the license for it, 2381 * otherwise TXQ init will fail 2382 */ 2383 if (!(nic_data->licensed_features & 2384 (1 << LICENSED_V3_FEATURES_TX_TIMESTAMPS_LBN))) { 2385 tx_queue->timestamping = false; 2386 /* Disable sync events on this channel. */ 2387 if (efx->type->ptp_set_ts_sync_events) 2388 efx->type->ptp_set_ts_sync_events(efx, false, false); 2389 } 2390 2391 /* TSOv2 is a limited resource that can only be configured on a limited 2392 * number of queues. TSO without checksum offload is not really a thing, 2393 * so we only enable it for those queues. 2394 * TSOv2 cannot be used with Hardware timestamping, and is never needed 2395 * for XDP tx. 2396 */ 2397 if (efx_has_cap(efx, TX_TSO_V2)) { 2398 if ((csum_offload || inner_csum) && 2399 !tx_queue->timestamping && !tx_queue->xdp_tx) { 2400 tx_queue->tso_version = 2; 2401 netif_dbg(efx, hw, efx->net_dev, "Using TSOv2 for channel %u\n", 2402 channel->channel); 2403 } 2404 } else if (efx_has_cap(efx, TX_TSO)) { 2405 tx_queue->tso_version = 1; 2406 } 2407 2408 rc = efx_mcdi_tx_init(tx_queue); 2409 if (rc) 2410 goto fail; 2411 2412 /* A previous user of this TX queue might have set us up the 2413 * bomb by writing a descriptor to the TX push collector but 2414 * not the doorbell. (Each collector belongs to a port, not a 2415 * queue or function, so cannot easily be reset.) We must 2416 * attempt to push a no-op descriptor in its place. 2417 */ 2418 tx_queue->buffer[0].flags = EFX_TX_BUF_OPTION; 2419 tx_queue->insert_count = 1; 2420 txd = efx_tx_desc(tx_queue, 0); 2421 EFX_POPULATE_QWORD_7(*txd, 2422 ESF_DZ_TX_DESC_IS_OPT, true, 2423 ESF_DZ_TX_OPTION_TYPE, 2424 ESE_DZ_TX_OPTION_DESC_CRC_CSUM, 2425 ESF_DZ_TX_OPTION_UDP_TCP_CSUM, csum_offload, 2426 ESF_DZ_TX_OPTION_IP_CSUM, csum_offload && tx_queue->tso_version != 2, 2427 ESF_DZ_TX_OPTION_INNER_UDP_TCP_CSUM, inner_csum, 2428 ESF_DZ_TX_OPTION_INNER_IP_CSUM, inner_csum && tx_queue->tso_version != 2, 2429 ESF_DZ_TX_TIMESTAMP, tx_queue->timestamping); 2430 tx_queue->write_count = 1; 2431 2432 if (tx_queue->tso_version == 2 && efx_has_cap(efx, TX_TSO_V2_ENCAP)) 2433 tx_queue->tso_encap = true; 2434 2435 wmb(); 2436 efx_ef10_push_tx_desc(tx_queue, txd); 2437 2438 return; 2439 2440 fail: 2441 netdev_WARN(efx->net_dev, "failed to initialise TXQ %d\n", 2442 tx_queue->queue); 2443 } 2444 2445 /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */ 2446 static inline void efx_ef10_notify_tx_desc(struct efx_tx_queue *tx_queue) 2447 { 2448 unsigned int write_ptr; 2449 efx_dword_t reg; 2450 2451 write_ptr = tx_queue->write_count & tx_queue->ptr_mask; 2452 EFX_POPULATE_DWORD_1(reg, ERF_DZ_TX_DESC_WPTR_DWORD, write_ptr); 2453 efx_writed_page(tx_queue->efx, ®, 2454 ER_DZ_TX_DESC_UPD_DWORD, tx_queue->queue); 2455 } 2456 2457 #define EFX_EF10_MAX_TX_DESCRIPTOR_LEN 0x3fff 2458 2459 static unsigned int efx_ef10_tx_limit_len(struct efx_tx_queue *tx_queue, 2460 dma_addr_t dma_addr, unsigned int len) 2461 { 2462 if (len > EFX_EF10_MAX_TX_DESCRIPTOR_LEN) { 2463 /* If we need to break across multiple descriptors we should 2464 * stop at a page boundary. This assumes the length limit is 2465 * greater than the page size. 2466 */ 2467 dma_addr_t end = dma_addr + EFX_EF10_MAX_TX_DESCRIPTOR_LEN; 2468 2469 BUILD_BUG_ON(EFX_EF10_MAX_TX_DESCRIPTOR_LEN < EFX_PAGE_SIZE); 2470 len = (end & (~(EFX_PAGE_SIZE - 1))) - dma_addr; 2471 } 2472 2473 return len; 2474 } 2475 2476 static void efx_ef10_tx_write(struct efx_tx_queue *tx_queue) 2477 { 2478 unsigned int old_write_count = tx_queue->write_count; 2479 struct efx_tx_buffer *buffer; 2480 unsigned int write_ptr; 2481 efx_qword_t *txd; 2482 2483 tx_queue->xmit_pending = false; 2484 if (unlikely(tx_queue->write_count == tx_queue->insert_count)) 2485 return; 2486 2487 do { 2488 write_ptr = tx_queue->write_count & tx_queue->ptr_mask; 2489 buffer = &tx_queue->buffer[write_ptr]; 2490 txd = efx_tx_desc(tx_queue, write_ptr); 2491 ++tx_queue->write_count; 2492 2493 /* Create TX descriptor ring entry */ 2494 if (buffer->flags & EFX_TX_BUF_OPTION) { 2495 *txd = buffer->option; 2496 if (EFX_QWORD_FIELD(*txd, ESF_DZ_TX_OPTION_TYPE) == 1) 2497 /* PIO descriptor */ 2498 tx_queue->packet_write_count = tx_queue->write_count; 2499 } else { 2500 tx_queue->packet_write_count = tx_queue->write_count; 2501 BUILD_BUG_ON(EFX_TX_BUF_CONT != 1); 2502 EFX_POPULATE_QWORD_3( 2503 *txd, 2504 ESF_DZ_TX_KER_CONT, 2505 buffer->flags & EFX_TX_BUF_CONT, 2506 ESF_DZ_TX_KER_BYTE_CNT, buffer->len, 2507 ESF_DZ_TX_KER_BUF_ADDR, buffer->dma_addr); 2508 } 2509 } while (tx_queue->write_count != tx_queue->insert_count); 2510 2511 wmb(); /* Ensure descriptors are written before they are fetched */ 2512 2513 if (efx_nic_may_push_tx_desc(tx_queue, old_write_count)) { 2514 txd = efx_tx_desc(tx_queue, 2515 old_write_count & tx_queue->ptr_mask); 2516 efx_ef10_push_tx_desc(tx_queue, txd); 2517 ++tx_queue->pushes; 2518 } else { 2519 efx_ef10_notify_tx_desc(tx_queue); 2520 } 2521 } 2522 2523 static int efx_ef10_probe_multicast_chaining(struct efx_nic *efx) 2524 { 2525 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2526 unsigned int enabled, implemented; 2527 bool want_workaround_26807; 2528 int rc; 2529 2530 rc = efx_mcdi_get_workarounds(efx, &implemented, &enabled); 2531 if (rc == -ENOSYS) { 2532 /* GET_WORKAROUNDS was implemented before this workaround, 2533 * thus it must be unavailable in this firmware. 2534 */ 2535 nic_data->workaround_26807 = false; 2536 return 0; 2537 } 2538 if (rc) 2539 return rc; 2540 want_workaround_26807 = 2541 implemented & MC_CMD_GET_WORKAROUNDS_OUT_BUG26807; 2542 nic_data->workaround_26807 = 2543 !!(enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG26807); 2544 2545 if (want_workaround_26807 && !nic_data->workaround_26807) { 2546 unsigned int flags; 2547 2548 rc = efx_mcdi_set_workaround(efx, 2549 MC_CMD_WORKAROUND_BUG26807, 2550 true, &flags); 2551 if (!rc) { 2552 if (flags & 2553 1 << MC_CMD_WORKAROUND_EXT_OUT_FLR_DONE_LBN) { 2554 netif_info(efx, drv, efx->net_dev, 2555 "other functions on NIC have been reset\n"); 2556 2557 /* With MCFW v4.6.x and earlier, the 2558 * boot count will have incremented, 2559 * so re-read the warm_boot_count 2560 * value now to ensure this function 2561 * doesn't think it has changed next 2562 * time it checks. 2563 */ 2564 rc = efx_ef10_get_warm_boot_count(efx); 2565 if (rc >= 0) { 2566 nic_data->warm_boot_count = rc; 2567 rc = 0; 2568 } 2569 } 2570 nic_data->workaround_26807 = true; 2571 } else if (rc == -EPERM) { 2572 rc = 0; 2573 } 2574 } 2575 return rc; 2576 } 2577 2578 static int efx_ef10_filter_table_probe(struct efx_nic *efx) 2579 { 2580 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2581 int rc = efx_ef10_probe_multicast_chaining(efx); 2582 struct efx_mcdi_filter_vlan *vlan; 2583 2584 if (rc) 2585 return rc; 2586 down_write(&efx->filter_sem); 2587 rc = efx_mcdi_filter_table_probe(efx, nic_data->workaround_26807); 2588 2589 if (rc) 2590 goto out_unlock; 2591 2592 list_for_each_entry(vlan, &nic_data->vlan_list, list) { 2593 rc = efx_mcdi_filter_add_vlan(efx, vlan->vid); 2594 if (rc) 2595 goto fail_add_vlan; 2596 } 2597 goto out_unlock; 2598 2599 fail_add_vlan: 2600 efx_mcdi_filter_table_remove(efx); 2601 out_unlock: 2602 up_write(&efx->filter_sem); 2603 return rc; 2604 } 2605 2606 static void efx_ef10_filter_table_remove(struct efx_nic *efx) 2607 { 2608 down_write(&efx->filter_sem); 2609 efx_mcdi_filter_table_remove(efx); 2610 up_write(&efx->filter_sem); 2611 } 2612 2613 /* This creates an entry in the RX descriptor queue */ 2614 static inline void 2615 efx_ef10_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index) 2616 { 2617 struct efx_rx_buffer *rx_buf; 2618 efx_qword_t *rxd; 2619 2620 rxd = efx_rx_desc(rx_queue, index); 2621 rx_buf = efx_rx_buffer(rx_queue, index); 2622 EFX_POPULATE_QWORD_2(*rxd, 2623 ESF_DZ_RX_KER_BYTE_CNT, rx_buf->len, 2624 ESF_DZ_RX_KER_BUF_ADDR, rx_buf->dma_addr); 2625 } 2626 2627 static void efx_ef10_rx_write(struct efx_rx_queue *rx_queue) 2628 { 2629 struct efx_nic *efx = rx_queue->efx; 2630 unsigned int write_count; 2631 efx_dword_t reg; 2632 2633 /* Firmware requires that RX_DESC_WPTR be a multiple of 8 */ 2634 write_count = rx_queue->added_count & ~7; 2635 if (rx_queue->notified_count == write_count) 2636 return; 2637 2638 do 2639 efx_ef10_build_rx_desc( 2640 rx_queue, 2641 rx_queue->notified_count & rx_queue->ptr_mask); 2642 while (++rx_queue->notified_count != write_count); 2643 2644 wmb(); 2645 EFX_POPULATE_DWORD_1(reg, ERF_DZ_RX_DESC_WPTR, 2646 write_count & rx_queue->ptr_mask); 2647 efx_writed_page(efx, ®, ER_DZ_RX_DESC_UPD, 2648 efx_rx_queue_index(rx_queue)); 2649 } 2650 2651 static efx_mcdi_async_completer efx_ef10_rx_defer_refill_complete; 2652 2653 static void efx_ef10_rx_defer_refill(struct efx_rx_queue *rx_queue) 2654 { 2655 struct efx_channel *channel = efx_rx_queue_channel(rx_queue); 2656 MCDI_DECLARE_BUF(inbuf, MC_CMD_DRIVER_EVENT_IN_LEN); 2657 efx_qword_t event; 2658 2659 EFX_POPULATE_QWORD_2(event, 2660 ESF_DZ_EV_CODE, EFX_EF10_DRVGEN_EV, 2661 ESF_DZ_EV_DATA, EFX_EF10_REFILL); 2662 2663 MCDI_SET_DWORD(inbuf, DRIVER_EVENT_IN_EVQ, channel->channel); 2664 2665 /* MCDI_SET_QWORD is not appropriate here since EFX_POPULATE_* has 2666 * already swapped the data to little-endian order. 2667 */ 2668 memcpy(MCDI_PTR(inbuf, DRIVER_EVENT_IN_DATA), &event.u64[0], 2669 sizeof(efx_qword_t)); 2670 2671 efx_mcdi_rpc_async(channel->efx, MC_CMD_DRIVER_EVENT, 2672 inbuf, sizeof(inbuf), 0, 2673 efx_ef10_rx_defer_refill_complete, 0); 2674 } 2675 2676 static void 2677 efx_ef10_rx_defer_refill_complete(struct efx_nic *efx, unsigned long cookie, 2678 int rc, efx_dword_t *outbuf, 2679 size_t outlen_actual) 2680 { 2681 /* nothing to do */ 2682 } 2683 2684 static int efx_ef10_ev_init(struct efx_channel *channel) 2685 { 2686 struct efx_nic *efx = channel->efx; 2687 struct efx_ef10_nic_data *nic_data; 2688 bool use_v2, cut_thru; 2689 2690 nic_data = efx->nic_data; 2691 use_v2 = nic_data->datapath_caps2 & 2692 1 << MC_CMD_GET_CAPABILITIES_V2_OUT_INIT_EVQ_V2_LBN; 2693 cut_thru = !(nic_data->datapath_caps & 2694 1 << MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_LBN); 2695 return efx_mcdi_ev_init(channel, cut_thru, use_v2); 2696 } 2697 2698 static void efx_ef10_handle_rx_wrong_queue(struct efx_rx_queue *rx_queue, 2699 unsigned int rx_queue_label) 2700 { 2701 struct efx_nic *efx = rx_queue->efx; 2702 2703 netif_info(efx, hw, efx->net_dev, 2704 "rx event arrived on queue %d labeled as queue %u\n", 2705 efx_rx_queue_index(rx_queue), rx_queue_label); 2706 2707 efx_schedule_reset(efx, RESET_TYPE_DISABLE); 2708 } 2709 2710 static void 2711 efx_ef10_handle_rx_bad_lbits(struct efx_rx_queue *rx_queue, 2712 unsigned int actual, unsigned int expected) 2713 { 2714 unsigned int dropped = (actual - expected) & rx_queue->ptr_mask; 2715 struct efx_nic *efx = rx_queue->efx; 2716 2717 netif_info(efx, hw, efx->net_dev, 2718 "dropped %d events (index=%d expected=%d)\n", 2719 dropped, actual, expected); 2720 2721 efx_schedule_reset(efx, RESET_TYPE_DISABLE); 2722 } 2723 2724 /* partially received RX was aborted. clean up. */ 2725 static void efx_ef10_handle_rx_abort(struct efx_rx_queue *rx_queue) 2726 { 2727 unsigned int rx_desc_ptr; 2728 2729 netif_dbg(rx_queue->efx, hw, rx_queue->efx->net_dev, 2730 "scattered RX aborted (dropping %u buffers)\n", 2731 rx_queue->scatter_n); 2732 2733 rx_desc_ptr = rx_queue->removed_count & rx_queue->ptr_mask; 2734 2735 efx_rx_packet(rx_queue, rx_desc_ptr, rx_queue->scatter_n, 2736 0, EFX_RX_PKT_DISCARD); 2737 2738 rx_queue->removed_count += rx_queue->scatter_n; 2739 rx_queue->scatter_n = 0; 2740 rx_queue->scatter_len = 0; 2741 ++efx_rx_queue_channel(rx_queue)->n_rx_nodesc_trunc; 2742 } 2743 2744 static u16 efx_ef10_handle_rx_event_errors(struct efx_channel *channel, 2745 unsigned int n_packets, 2746 unsigned int rx_encap_hdr, 2747 unsigned int rx_l3_class, 2748 unsigned int rx_l4_class, 2749 const efx_qword_t *event) 2750 { 2751 struct efx_nic *efx = channel->efx; 2752 bool handled = false; 2753 2754 if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_ECRC_ERR)) { 2755 if (!(efx->net_dev->features & NETIF_F_RXALL)) { 2756 if (!efx->loopback_selftest) 2757 channel->n_rx_eth_crc_err += n_packets; 2758 return EFX_RX_PKT_DISCARD; 2759 } 2760 handled = true; 2761 } 2762 if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_IPCKSUM_ERR)) { 2763 if (unlikely(rx_encap_hdr != ESE_EZ_ENCAP_HDR_VXLAN && 2764 rx_l3_class != ESE_DZ_L3_CLASS_IP4 && 2765 rx_l3_class != ESE_DZ_L3_CLASS_IP4_FRAG && 2766 rx_l3_class != ESE_DZ_L3_CLASS_IP6 && 2767 rx_l3_class != ESE_DZ_L3_CLASS_IP6_FRAG)) 2768 netdev_WARN(efx->net_dev, 2769 "invalid class for RX_IPCKSUM_ERR: event=" 2770 EFX_QWORD_FMT "\n", 2771 EFX_QWORD_VAL(*event)); 2772 if (!efx->loopback_selftest) 2773 *(rx_encap_hdr ? 2774 &channel->n_rx_outer_ip_hdr_chksum_err : 2775 &channel->n_rx_ip_hdr_chksum_err) += n_packets; 2776 return 0; 2777 } 2778 if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_TCPUDP_CKSUM_ERR)) { 2779 if (unlikely(rx_encap_hdr != ESE_EZ_ENCAP_HDR_VXLAN && 2780 ((rx_l3_class != ESE_DZ_L3_CLASS_IP4 && 2781 rx_l3_class != ESE_DZ_L3_CLASS_IP6) || 2782 (rx_l4_class != ESE_FZ_L4_CLASS_TCP && 2783 rx_l4_class != ESE_FZ_L4_CLASS_UDP)))) 2784 netdev_WARN(efx->net_dev, 2785 "invalid class for RX_TCPUDP_CKSUM_ERR: event=" 2786 EFX_QWORD_FMT "\n", 2787 EFX_QWORD_VAL(*event)); 2788 if (!efx->loopback_selftest) 2789 *(rx_encap_hdr ? 2790 &channel->n_rx_outer_tcp_udp_chksum_err : 2791 &channel->n_rx_tcp_udp_chksum_err) += n_packets; 2792 return 0; 2793 } 2794 if (EFX_QWORD_FIELD(*event, ESF_EZ_RX_IP_INNER_CHKSUM_ERR)) { 2795 if (unlikely(!rx_encap_hdr)) 2796 netdev_WARN(efx->net_dev, 2797 "invalid encapsulation type for RX_IP_INNER_CHKSUM_ERR: event=" 2798 EFX_QWORD_FMT "\n", 2799 EFX_QWORD_VAL(*event)); 2800 else if (unlikely(rx_l3_class != ESE_DZ_L3_CLASS_IP4 && 2801 rx_l3_class != ESE_DZ_L3_CLASS_IP4_FRAG && 2802 rx_l3_class != ESE_DZ_L3_CLASS_IP6 && 2803 rx_l3_class != ESE_DZ_L3_CLASS_IP6_FRAG)) 2804 netdev_WARN(efx->net_dev, 2805 "invalid class for RX_IP_INNER_CHKSUM_ERR: event=" 2806 EFX_QWORD_FMT "\n", 2807 EFX_QWORD_VAL(*event)); 2808 if (!efx->loopback_selftest) 2809 channel->n_rx_inner_ip_hdr_chksum_err += n_packets; 2810 return 0; 2811 } 2812 if (EFX_QWORD_FIELD(*event, ESF_EZ_RX_TCP_UDP_INNER_CHKSUM_ERR)) { 2813 if (unlikely(!rx_encap_hdr)) 2814 netdev_WARN(efx->net_dev, 2815 "invalid encapsulation type for RX_TCP_UDP_INNER_CHKSUM_ERR: event=" 2816 EFX_QWORD_FMT "\n", 2817 EFX_QWORD_VAL(*event)); 2818 else if (unlikely((rx_l3_class != ESE_DZ_L3_CLASS_IP4 && 2819 rx_l3_class != ESE_DZ_L3_CLASS_IP6) || 2820 (rx_l4_class != ESE_FZ_L4_CLASS_TCP && 2821 rx_l4_class != ESE_FZ_L4_CLASS_UDP))) 2822 netdev_WARN(efx->net_dev, 2823 "invalid class for RX_TCP_UDP_INNER_CHKSUM_ERR: event=" 2824 EFX_QWORD_FMT "\n", 2825 EFX_QWORD_VAL(*event)); 2826 if (!efx->loopback_selftest) 2827 channel->n_rx_inner_tcp_udp_chksum_err += n_packets; 2828 return 0; 2829 } 2830 2831 WARN_ON(!handled); /* No error bits were recognised */ 2832 return 0; 2833 } 2834 2835 static int efx_ef10_handle_rx_event(struct efx_channel *channel, 2836 const efx_qword_t *event) 2837 { 2838 unsigned int rx_bytes, next_ptr_lbits, rx_queue_label; 2839 unsigned int rx_l3_class, rx_l4_class, rx_encap_hdr; 2840 unsigned int n_descs, n_packets, i; 2841 struct efx_nic *efx = channel->efx; 2842 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2843 struct efx_rx_queue *rx_queue; 2844 efx_qword_t errors; 2845 bool rx_cont; 2846 u16 flags = 0; 2847 2848 if (unlikely(READ_ONCE(efx->reset_pending))) 2849 return 0; 2850 2851 /* Basic packet information */ 2852 rx_bytes = EFX_QWORD_FIELD(*event, ESF_DZ_RX_BYTES); 2853 next_ptr_lbits = EFX_QWORD_FIELD(*event, ESF_DZ_RX_DSC_PTR_LBITS); 2854 rx_queue_label = EFX_QWORD_FIELD(*event, ESF_DZ_RX_QLABEL); 2855 rx_l3_class = EFX_QWORD_FIELD(*event, ESF_DZ_RX_L3_CLASS); 2856 rx_l4_class = EFX_QWORD_FIELD(*event, ESF_FZ_RX_L4_CLASS); 2857 rx_cont = EFX_QWORD_FIELD(*event, ESF_DZ_RX_CONT); 2858 rx_encap_hdr = 2859 nic_data->datapath_caps & 2860 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN) ? 2861 EFX_QWORD_FIELD(*event, ESF_EZ_RX_ENCAP_HDR) : 2862 ESE_EZ_ENCAP_HDR_NONE; 2863 2864 if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_DROP_EVENT)) 2865 netdev_WARN(efx->net_dev, "saw RX_DROP_EVENT: event=" 2866 EFX_QWORD_FMT "\n", 2867 EFX_QWORD_VAL(*event)); 2868 2869 rx_queue = efx_channel_get_rx_queue(channel); 2870 2871 if (unlikely(rx_queue_label != efx_rx_queue_index(rx_queue))) 2872 efx_ef10_handle_rx_wrong_queue(rx_queue, rx_queue_label); 2873 2874 n_descs = ((next_ptr_lbits - rx_queue->removed_count) & 2875 ((1 << ESF_DZ_RX_DSC_PTR_LBITS_WIDTH) - 1)); 2876 2877 if (n_descs != rx_queue->scatter_n + 1) { 2878 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2879 2880 /* detect rx abort */ 2881 if (unlikely(n_descs == rx_queue->scatter_n)) { 2882 if (rx_queue->scatter_n == 0 || rx_bytes != 0) 2883 netdev_WARN(efx->net_dev, 2884 "invalid RX abort: scatter_n=%u event=" 2885 EFX_QWORD_FMT "\n", 2886 rx_queue->scatter_n, 2887 EFX_QWORD_VAL(*event)); 2888 efx_ef10_handle_rx_abort(rx_queue); 2889 return 0; 2890 } 2891 2892 /* Check that RX completion merging is valid, i.e. 2893 * the current firmware supports it and this is a 2894 * non-scattered packet. 2895 */ 2896 if (!(nic_data->datapath_caps & 2897 (1 << MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_LBN)) || 2898 rx_queue->scatter_n != 0 || rx_cont) { 2899 efx_ef10_handle_rx_bad_lbits( 2900 rx_queue, next_ptr_lbits, 2901 (rx_queue->removed_count + 2902 rx_queue->scatter_n + 1) & 2903 ((1 << ESF_DZ_RX_DSC_PTR_LBITS_WIDTH) - 1)); 2904 return 0; 2905 } 2906 2907 /* Merged completion for multiple non-scattered packets */ 2908 rx_queue->scatter_n = 1; 2909 rx_queue->scatter_len = 0; 2910 n_packets = n_descs; 2911 ++channel->n_rx_merge_events; 2912 channel->n_rx_merge_packets += n_packets; 2913 flags |= EFX_RX_PKT_PREFIX_LEN; 2914 } else { 2915 ++rx_queue->scatter_n; 2916 rx_queue->scatter_len += rx_bytes; 2917 if (rx_cont) 2918 return 0; 2919 n_packets = 1; 2920 } 2921 2922 EFX_POPULATE_QWORD_5(errors, ESF_DZ_RX_ECRC_ERR, 1, 2923 ESF_DZ_RX_IPCKSUM_ERR, 1, 2924 ESF_DZ_RX_TCPUDP_CKSUM_ERR, 1, 2925 ESF_EZ_RX_IP_INNER_CHKSUM_ERR, 1, 2926 ESF_EZ_RX_TCP_UDP_INNER_CHKSUM_ERR, 1); 2927 EFX_AND_QWORD(errors, *event, errors); 2928 if (unlikely(!EFX_QWORD_IS_ZERO(errors))) { 2929 flags |= efx_ef10_handle_rx_event_errors(channel, n_packets, 2930 rx_encap_hdr, 2931 rx_l3_class, rx_l4_class, 2932 event); 2933 } else { 2934 bool tcpudp = rx_l4_class == ESE_FZ_L4_CLASS_TCP || 2935 rx_l4_class == ESE_FZ_L4_CLASS_UDP; 2936 2937 switch (rx_encap_hdr) { 2938 case ESE_EZ_ENCAP_HDR_VXLAN: /* VxLAN or GENEVE */ 2939 flags |= EFX_RX_PKT_CSUMMED; /* outer UDP csum */ 2940 if (tcpudp) 2941 flags |= EFX_RX_PKT_CSUM_LEVEL; /* inner L4 */ 2942 break; 2943 case ESE_EZ_ENCAP_HDR_GRE: 2944 case ESE_EZ_ENCAP_HDR_NONE: 2945 if (tcpudp) 2946 flags |= EFX_RX_PKT_CSUMMED; 2947 break; 2948 default: 2949 netdev_WARN(efx->net_dev, 2950 "unknown encapsulation type: event=" 2951 EFX_QWORD_FMT "\n", 2952 EFX_QWORD_VAL(*event)); 2953 } 2954 } 2955 2956 if (rx_l4_class == ESE_FZ_L4_CLASS_TCP) 2957 flags |= EFX_RX_PKT_TCP; 2958 2959 channel->irq_mod_score += 2 * n_packets; 2960 2961 /* Handle received packet(s) */ 2962 for (i = 0; i < n_packets; i++) { 2963 efx_rx_packet(rx_queue, 2964 rx_queue->removed_count & rx_queue->ptr_mask, 2965 rx_queue->scatter_n, rx_queue->scatter_len, 2966 flags); 2967 rx_queue->removed_count += rx_queue->scatter_n; 2968 } 2969 2970 rx_queue->scatter_n = 0; 2971 rx_queue->scatter_len = 0; 2972 2973 return n_packets; 2974 } 2975 2976 static u32 efx_ef10_extract_event_ts(efx_qword_t *event) 2977 { 2978 u32 tstamp; 2979 2980 tstamp = EFX_QWORD_FIELD(*event, TX_TIMESTAMP_EVENT_TSTAMP_DATA_HI); 2981 tstamp <<= 16; 2982 tstamp |= EFX_QWORD_FIELD(*event, TX_TIMESTAMP_EVENT_TSTAMP_DATA_LO); 2983 2984 return tstamp; 2985 } 2986 2987 static int 2988 efx_ef10_handle_tx_event(struct efx_channel *channel, efx_qword_t *event) 2989 { 2990 struct efx_nic *efx = channel->efx; 2991 struct efx_tx_queue *tx_queue; 2992 unsigned int tx_ev_desc_ptr; 2993 unsigned int tx_ev_q_label; 2994 unsigned int tx_ev_type; 2995 int work_done; 2996 u64 ts_part; 2997 2998 if (unlikely(READ_ONCE(efx->reset_pending))) 2999 return 0; 3000 3001 if (unlikely(EFX_QWORD_FIELD(*event, ESF_DZ_TX_DROP_EVENT))) 3002 return 0; 3003 3004 /* Get the transmit queue */ 3005 tx_ev_q_label = EFX_QWORD_FIELD(*event, ESF_DZ_TX_QLABEL); 3006 tx_queue = channel->tx_queue + (tx_ev_q_label % EFX_MAX_TXQ_PER_CHANNEL); 3007 3008 if (!tx_queue->timestamping) { 3009 /* Transmit completion */ 3010 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, ESF_DZ_TX_DESCR_INDX); 3011 return efx_xmit_done(tx_queue, tx_ev_desc_ptr & tx_queue->ptr_mask); 3012 } 3013 3014 /* Transmit timestamps are only available for 8XXX series. They result 3015 * in up to three events per packet. These occur in order, and are: 3016 * - the normal completion event (may be omitted) 3017 * - the low part of the timestamp 3018 * - the high part of the timestamp 3019 * 3020 * It's possible for multiple completion events to appear before the 3021 * corresponding timestamps. So we can for example get: 3022 * COMP N 3023 * COMP N+1 3024 * TS_LO N 3025 * TS_HI N 3026 * TS_LO N+1 3027 * TS_HI N+1 3028 * 3029 * In addition it's also possible for the adjacent completions to be 3030 * merged, so we may not see COMP N above. As such, the completion 3031 * events are not very useful here. 3032 * 3033 * Each part of the timestamp is itself split across two 16 bit 3034 * fields in the event. 3035 */ 3036 tx_ev_type = EFX_QWORD_FIELD(*event, ESF_EZ_TX_SOFT1); 3037 work_done = 0; 3038 3039 switch (tx_ev_type) { 3040 case TX_TIMESTAMP_EVENT_TX_EV_COMPLETION: 3041 /* Ignore this event - see above. */ 3042 break; 3043 3044 case TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_LO: 3045 ts_part = efx_ef10_extract_event_ts(event); 3046 tx_queue->completed_timestamp_minor = ts_part; 3047 break; 3048 3049 case TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_HI: 3050 ts_part = efx_ef10_extract_event_ts(event); 3051 tx_queue->completed_timestamp_major = ts_part; 3052 3053 efx_xmit_done_single(tx_queue); 3054 work_done = 1; 3055 break; 3056 3057 default: 3058 netif_err(efx, hw, efx->net_dev, 3059 "channel %d unknown tx event type %d (data " 3060 EFX_QWORD_FMT ")\n", 3061 channel->channel, tx_ev_type, 3062 EFX_QWORD_VAL(*event)); 3063 break; 3064 } 3065 3066 return work_done; 3067 } 3068 3069 static void 3070 efx_ef10_handle_driver_event(struct efx_channel *channel, efx_qword_t *event) 3071 { 3072 struct efx_nic *efx = channel->efx; 3073 int subcode; 3074 3075 subcode = EFX_QWORD_FIELD(*event, ESF_DZ_DRV_SUB_CODE); 3076 3077 switch (subcode) { 3078 case ESE_DZ_DRV_TIMER_EV: 3079 case ESE_DZ_DRV_WAKE_UP_EV: 3080 break; 3081 case ESE_DZ_DRV_START_UP_EV: 3082 /* event queue init complete. ok. */ 3083 break; 3084 default: 3085 netif_err(efx, hw, efx->net_dev, 3086 "channel %d unknown driver event type %d" 3087 " (data " EFX_QWORD_FMT ")\n", 3088 channel->channel, subcode, 3089 EFX_QWORD_VAL(*event)); 3090 3091 } 3092 } 3093 3094 static void efx_ef10_handle_driver_generated_event(struct efx_channel *channel, 3095 efx_qword_t *event) 3096 { 3097 struct efx_nic *efx = channel->efx; 3098 u32 subcode; 3099 3100 subcode = EFX_QWORD_FIELD(*event, EFX_DWORD_0); 3101 3102 switch (subcode) { 3103 case EFX_EF10_TEST: 3104 channel->event_test_cpu = raw_smp_processor_id(); 3105 break; 3106 case EFX_EF10_REFILL: 3107 /* The queue must be empty, so we won't receive any rx 3108 * events, so efx_process_channel() won't refill the 3109 * queue. Refill it here 3110 */ 3111 efx_fast_push_rx_descriptors(&channel->rx_queue, true); 3112 break; 3113 default: 3114 netif_err(efx, hw, efx->net_dev, 3115 "channel %d unknown driver event type %u" 3116 " (data " EFX_QWORD_FMT ")\n", 3117 channel->channel, (unsigned) subcode, 3118 EFX_QWORD_VAL(*event)); 3119 } 3120 } 3121 3122 #define EFX_NAPI_MAX_TX 512 3123 3124 static int efx_ef10_ev_process(struct efx_channel *channel, int quota) 3125 { 3126 struct efx_nic *efx = channel->efx; 3127 efx_qword_t event, *p_event; 3128 unsigned int read_ptr; 3129 int spent_tx = 0; 3130 int spent = 0; 3131 int ev_code; 3132 3133 if (quota <= 0) 3134 return spent; 3135 3136 read_ptr = channel->eventq_read_ptr; 3137 3138 for (;;) { 3139 p_event = efx_event(channel, read_ptr); 3140 event = *p_event; 3141 3142 if (!efx_event_present(&event)) 3143 break; 3144 3145 EFX_SET_QWORD(*p_event); 3146 3147 ++read_ptr; 3148 3149 ev_code = EFX_QWORD_FIELD(event, ESF_DZ_EV_CODE); 3150 3151 netif_vdbg(efx, drv, efx->net_dev, 3152 "processing event on %d " EFX_QWORD_FMT "\n", 3153 channel->channel, EFX_QWORD_VAL(event)); 3154 3155 switch (ev_code) { 3156 case ESE_DZ_EV_CODE_MCDI_EV: 3157 efx_mcdi_process_event(channel, &event); 3158 break; 3159 case ESE_DZ_EV_CODE_RX_EV: 3160 spent += efx_ef10_handle_rx_event(channel, &event); 3161 if (spent >= quota) { 3162 /* XXX can we split a merged event to 3163 * avoid going over-quota? 3164 */ 3165 spent = quota; 3166 goto out; 3167 } 3168 break; 3169 case ESE_DZ_EV_CODE_TX_EV: 3170 spent_tx += efx_ef10_handle_tx_event(channel, &event); 3171 if (spent_tx >= EFX_NAPI_MAX_TX) { 3172 spent = quota; 3173 goto out; 3174 } 3175 break; 3176 case ESE_DZ_EV_CODE_DRIVER_EV: 3177 efx_ef10_handle_driver_event(channel, &event); 3178 if (++spent == quota) 3179 goto out; 3180 break; 3181 case EFX_EF10_DRVGEN_EV: 3182 efx_ef10_handle_driver_generated_event(channel, &event); 3183 break; 3184 default: 3185 netif_err(efx, hw, efx->net_dev, 3186 "channel %d unknown event type %d" 3187 " (data " EFX_QWORD_FMT ")\n", 3188 channel->channel, ev_code, 3189 EFX_QWORD_VAL(event)); 3190 } 3191 } 3192 3193 out: 3194 channel->eventq_read_ptr = read_ptr; 3195 return spent; 3196 } 3197 3198 static void efx_ef10_ev_read_ack(struct efx_channel *channel) 3199 { 3200 struct efx_nic *efx = channel->efx; 3201 efx_dword_t rptr; 3202 3203 if (EFX_EF10_WORKAROUND_35388(efx)) { 3204 BUILD_BUG_ON(EFX_MIN_EVQ_SIZE < 3205 (1 << ERF_DD_EVQ_IND_RPTR_WIDTH)); 3206 BUILD_BUG_ON(EFX_MAX_EVQ_SIZE > 3207 (1 << 2 * ERF_DD_EVQ_IND_RPTR_WIDTH)); 3208 3209 EFX_POPULATE_DWORD_2(rptr, ERF_DD_EVQ_IND_RPTR_FLAGS, 3210 EFE_DD_EVQ_IND_RPTR_FLAGS_HIGH, 3211 ERF_DD_EVQ_IND_RPTR, 3212 (channel->eventq_read_ptr & 3213 channel->eventq_mask) >> 3214 ERF_DD_EVQ_IND_RPTR_WIDTH); 3215 efx_writed_page(efx, &rptr, ER_DD_EVQ_INDIRECT, 3216 channel->channel); 3217 EFX_POPULATE_DWORD_2(rptr, ERF_DD_EVQ_IND_RPTR_FLAGS, 3218 EFE_DD_EVQ_IND_RPTR_FLAGS_LOW, 3219 ERF_DD_EVQ_IND_RPTR, 3220 channel->eventq_read_ptr & 3221 ((1 << ERF_DD_EVQ_IND_RPTR_WIDTH) - 1)); 3222 efx_writed_page(efx, &rptr, ER_DD_EVQ_INDIRECT, 3223 channel->channel); 3224 } else { 3225 EFX_POPULATE_DWORD_1(rptr, ERF_DZ_EVQ_RPTR, 3226 channel->eventq_read_ptr & 3227 channel->eventq_mask); 3228 efx_writed_page(efx, &rptr, ER_DZ_EVQ_RPTR, channel->channel); 3229 } 3230 } 3231 3232 static void efx_ef10_ev_test_generate(struct efx_channel *channel) 3233 { 3234 MCDI_DECLARE_BUF(inbuf, MC_CMD_DRIVER_EVENT_IN_LEN); 3235 struct efx_nic *efx = channel->efx; 3236 efx_qword_t event; 3237 int rc; 3238 3239 EFX_POPULATE_QWORD_2(event, 3240 ESF_DZ_EV_CODE, EFX_EF10_DRVGEN_EV, 3241 ESF_DZ_EV_DATA, EFX_EF10_TEST); 3242 3243 MCDI_SET_DWORD(inbuf, DRIVER_EVENT_IN_EVQ, channel->channel); 3244 3245 /* MCDI_SET_QWORD is not appropriate here since EFX_POPULATE_* has 3246 * already swapped the data to little-endian order. 3247 */ 3248 memcpy(MCDI_PTR(inbuf, DRIVER_EVENT_IN_DATA), &event.u64[0], 3249 sizeof(efx_qword_t)); 3250 3251 rc = efx_mcdi_rpc(efx, MC_CMD_DRIVER_EVENT, inbuf, sizeof(inbuf), 3252 NULL, 0, NULL); 3253 if (rc != 0) 3254 goto fail; 3255 3256 return; 3257 3258 fail: 3259 WARN_ON(true); 3260 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc); 3261 } 3262 3263 static void efx_ef10_prepare_flr(struct efx_nic *efx) 3264 { 3265 atomic_set(&efx->active_queues, 0); 3266 } 3267 3268 static int efx_ef10_vport_set_mac_address(struct efx_nic *efx) 3269 { 3270 struct efx_ef10_nic_data *nic_data = efx->nic_data; 3271 u8 mac_old[ETH_ALEN]; 3272 int rc, rc2; 3273 3274 /* Only reconfigure a PF-created vport */ 3275 if (is_zero_ether_addr(nic_data->vport_mac)) 3276 return 0; 3277 3278 efx_device_detach_sync(efx); 3279 efx_net_stop(efx->net_dev); 3280 efx_ef10_filter_table_remove(efx); 3281 3282 rc = efx_ef10_vadaptor_free(efx, efx->vport_id); 3283 if (rc) 3284 goto restore_filters; 3285 3286 ether_addr_copy(mac_old, nic_data->vport_mac); 3287 rc = efx_ef10_vport_del_mac(efx, efx->vport_id, 3288 nic_data->vport_mac); 3289 if (rc) 3290 goto restore_vadaptor; 3291 3292 rc = efx_ef10_vport_add_mac(efx, efx->vport_id, 3293 efx->net_dev->dev_addr); 3294 if (!rc) { 3295 ether_addr_copy(nic_data->vport_mac, efx->net_dev->dev_addr); 3296 } else { 3297 rc2 = efx_ef10_vport_add_mac(efx, efx->vport_id, mac_old); 3298 if (rc2) { 3299 /* Failed to add original MAC, so clear vport_mac */ 3300 eth_zero_addr(nic_data->vport_mac); 3301 goto reset_nic; 3302 } 3303 } 3304 3305 restore_vadaptor: 3306 rc2 = efx_ef10_vadaptor_alloc(efx, efx->vport_id); 3307 if (rc2) 3308 goto reset_nic; 3309 restore_filters: 3310 rc2 = efx_ef10_filter_table_probe(efx); 3311 if (rc2) 3312 goto reset_nic; 3313 3314 rc2 = efx_net_open(efx->net_dev); 3315 if (rc2) 3316 goto reset_nic; 3317 3318 efx_device_attach_if_not_resetting(efx); 3319 3320 return rc; 3321 3322 reset_nic: 3323 netif_err(efx, drv, efx->net_dev, 3324 "Failed to restore when changing MAC address - scheduling reset\n"); 3325 efx_schedule_reset(efx, RESET_TYPE_DATAPATH); 3326 3327 return rc ? rc : rc2; 3328 } 3329 3330 static int efx_ef10_set_mac_address(struct efx_nic *efx) 3331 { 3332 MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_SET_MAC_IN_LEN); 3333 bool was_enabled = efx->port_enabled; 3334 int rc; 3335 3336 #ifdef CONFIG_SFC_SRIOV 3337 /* If this function is a VF and we have access to the parent PF, 3338 * then use the PF control path to attempt to change the VF MAC address. 3339 */ 3340 if (efx->pci_dev->is_virtfn && efx->pci_dev->physfn) { 3341 struct efx_nic *efx_pf = pci_get_drvdata(efx->pci_dev->physfn); 3342 struct efx_ef10_nic_data *nic_data = efx->nic_data; 3343 u8 mac[ETH_ALEN]; 3344 3345 /* net_dev->dev_addr can be zeroed by efx_net_stop in 3346 * efx_ef10_sriov_set_vf_mac, so pass in a copy. 3347 */ 3348 ether_addr_copy(mac, efx->net_dev->dev_addr); 3349 3350 rc = efx_ef10_sriov_set_vf_mac(efx_pf, nic_data->vf_index, mac); 3351 if (!rc) 3352 return 0; 3353 3354 netif_dbg(efx, drv, efx->net_dev, 3355 "Updating VF mac via PF failed (%d), setting directly\n", 3356 rc); 3357 } 3358 #endif 3359 3360 efx_device_detach_sync(efx); 3361 efx_net_stop(efx->net_dev); 3362 3363 mutex_lock(&efx->mac_lock); 3364 efx_ef10_filter_table_remove(efx); 3365 3366 ether_addr_copy(MCDI_PTR(inbuf, VADAPTOR_SET_MAC_IN_MACADDR), 3367 efx->net_dev->dev_addr); 3368 MCDI_SET_DWORD(inbuf, VADAPTOR_SET_MAC_IN_UPSTREAM_PORT_ID, 3369 efx->vport_id); 3370 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_VADAPTOR_SET_MAC, inbuf, 3371 sizeof(inbuf), NULL, 0, NULL); 3372 3373 efx_ef10_filter_table_probe(efx); 3374 mutex_unlock(&efx->mac_lock); 3375 3376 if (was_enabled) 3377 efx_net_open(efx->net_dev); 3378 efx_device_attach_if_not_resetting(efx); 3379 3380 if (rc == -EPERM) { 3381 netif_err(efx, drv, efx->net_dev, 3382 "Cannot change MAC address; use sfboot to enable" 3383 " mac-spoofing on this interface\n"); 3384 } else if (rc == -ENOSYS && !efx_ef10_is_vf(efx)) { 3385 /* If the active MCFW does not support MC_CMD_VADAPTOR_SET_MAC 3386 * fall-back to the method of changing the MAC address on the 3387 * vport. This only applies to PFs because such versions of 3388 * MCFW do not support VFs. 3389 */ 3390 rc = efx_ef10_vport_set_mac_address(efx); 3391 } else if (rc) { 3392 efx_mcdi_display_error(efx, MC_CMD_VADAPTOR_SET_MAC, 3393 sizeof(inbuf), NULL, 0, rc); 3394 } 3395 3396 return rc; 3397 } 3398 3399 static int efx_ef10_mac_reconfigure(struct efx_nic *efx, bool mtu_only) 3400 { 3401 WARN_ON(!mutex_is_locked(&efx->mac_lock)); 3402 3403 efx_mcdi_filter_sync_rx_mode(efx); 3404 3405 if (mtu_only && efx_has_cap(efx, SET_MAC_ENHANCED)) 3406 return efx_mcdi_set_mtu(efx); 3407 return efx_mcdi_set_mac(efx); 3408 } 3409 3410 static int efx_ef10_start_bist(struct efx_nic *efx, u32 bist_type) 3411 { 3412 MCDI_DECLARE_BUF(inbuf, MC_CMD_START_BIST_IN_LEN); 3413 3414 MCDI_SET_DWORD(inbuf, START_BIST_IN_TYPE, bist_type); 3415 return efx_mcdi_rpc(efx, MC_CMD_START_BIST, inbuf, sizeof(inbuf), 3416 NULL, 0, NULL); 3417 } 3418 3419 /* MC BISTs follow a different poll mechanism to phy BISTs. 3420 * The BIST is done in the poll handler on the MC, and the MCDI command 3421 * will block until the BIST is done. 3422 */ 3423 static int efx_ef10_poll_bist(struct efx_nic *efx) 3424 { 3425 int rc; 3426 MCDI_DECLARE_BUF(outbuf, MC_CMD_POLL_BIST_OUT_LEN); 3427 size_t outlen; 3428 u32 result; 3429 3430 rc = efx_mcdi_rpc(efx, MC_CMD_POLL_BIST, NULL, 0, 3431 outbuf, sizeof(outbuf), &outlen); 3432 if (rc != 0) 3433 return rc; 3434 3435 if (outlen < MC_CMD_POLL_BIST_OUT_LEN) 3436 return -EIO; 3437 3438 result = MCDI_DWORD(outbuf, POLL_BIST_OUT_RESULT); 3439 switch (result) { 3440 case MC_CMD_POLL_BIST_PASSED: 3441 netif_dbg(efx, hw, efx->net_dev, "BIST passed.\n"); 3442 return 0; 3443 case MC_CMD_POLL_BIST_TIMEOUT: 3444 netif_err(efx, hw, efx->net_dev, "BIST timed out\n"); 3445 return -EIO; 3446 case MC_CMD_POLL_BIST_FAILED: 3447 netif_err(efx, hw, efx->net_dev, "BIST failed.\n"); 3448 return -EIO; 3449 default: 3450 netif_err(efx, hw, efx->net_dev, 3451 "BIST returned unknown result %u", result); 3452 return -EIO; 3453 } 3454 } 3455 3456 static int efx_ef10_run_bist(struct efx_nic *efx, u32 bist_type) 3457 { 3458 int rc; 3459 3460 netif_dbg(efx, drv, efx->net_dev, "starting BIST type %u\n", bist_type); 3461 3462 rc = efx_ef10_start_bist(efx, bist_type); 3463 if (rc != 0) 3464 return rc; 3465 3466 return efx_ef10_poll_bist(efx); 3467 } 3468 3469 static int 3470 efx_ef10_test_chip(struct efx_nic *efx, struct efx_self_tests *tests) 3471 { 3472 int rc, rc2; 3473 3474 efx_reset_down(efx, RESET_TYPE_WORLD); 3475 3476 rc = efx_mcdi_rpc(efx, MC_CMD_ENABLE_OFFLINE_BIST, 3477 NULL, 0, NULL, 0, NULL); 3478 if (rc != 0) 3479 goto out; 3480 3481 tests->memory = efx_ef10_run_bist(efx, MC_CMD_MC_MEM_BIST) ? -1 : 1; 3482 tests->registers = efx_ef10_run_bist(efx, MC_CMD_REG_BIST) ? -1 : 1; 3483 3484 rc = efx_mcdi_reset(efx, RESET_TYPE_WORLD); 3485 3486 out: 3487 if (rc == -EPERM) 3488 rc = 0; 3489 rc2 = efx_reset_up(efx, RESET_TYPE_WORLD, rc == 0); 3490 return rc ? rc : rc2; 3491 } 3492 3493 #ifdef CONFIG_SFC_MTD 3494 3495 struct efx_ef10_nvram_type_info { 3496 u16 type, type_mask; 3497 u8 port; 3498 const char *name; 3499 }; 3500 3501 static const struct efx_ef10_nvram_type_info efx_ef10_nvram_types[] = { 3502 { NVRAM_PARTITION_TYPE_MC_FIRMWARE, 0, 0, "sfc_mcfw" }, 3503 { NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 0, 0, "sfc_mcfw_backup" }, 3504 { NVRAM_PARTITION_TYPE_EXPANSION_ROM, 0, 0, "sfc_exp_rom" }, 3505 { NVRAM_PARTITION_TYPE_STATIC_CONFIG, 0, 0, "sfc_static_cfg" }, 3506 { NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 0, 0, "sfc_dynamic_cfg" }, 3507 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0, 0, 0, "sfc_exp_rom_cfg" }, 3508 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT1, 0, 1, "sfc_exp_rom_cfg" }, 3509 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT2, 0, 2, "sfc_exp_rom_cfg" }, 3510 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT3, 0, 3, "sfc_exp_rom_cfg" }, 3511 { NVRAM_PARTITION_TYPE_LICENSE, 0, 0, "sfc_license" }, 3512 { NVRAM_PARTITION_TYPE_PHY_MIN, 0xff, 0, "sfc_phy_fw" }, 3513 { NVRAM_PARTITION_TYPE_MUM_FIRMWARE, 0, 0, "sfc_mumfw" }, 3514 { NVRAM_PARTITION_TYPE_EXPANSION_UEFI, 0, 0, "sfc_uefi" }, 3515 { NVRAM_PARTITION_TYPE_DYNCONFIG_DEFAULTS, 0, 0, "sfc_dynamic_cfg_dflt" }, 3516 { NVRAM_PARTITION_TYPE_ROMCONFIG_DEFAULTS, 0, 0, "sfc_exp_rom_cfg_dflt" }, 3517 { NVRAM_PARTITION_TYPE_STATUS, 0, 0, "sfc_status" }, 3518 { NVRAM_PARTITION_TYPE_BUNDLE, 0, 0, "sfc_bundle" }, 3519 { NVRAM_PARTITION_TYPE_BUNDLE_METADATA, 0, 0, "sfc_bundle_metadata" }, 3520 }; 3521 #define EF10_NVRAM_PARTITION_COUNT ARRAY_SIZE(efx_ef10_nvram_types) 3522 3523 static int efx_ef10_mtd_probe_partition(struct efx_nic *efx, 3524 struct efx_mcdi_mtd_partition *part, 3525 unsigned int type, 3526 unsigned long *found) 3527 { 3528 MCDI_DECLARE_BUF(inbuf, MC_CMD_NVRAM_METADATA_IN_LEN); 3529 MCDI_DECLARE_BUF(outbuf, MC_CMD_NVRAM_METADATA_OUT_LENMAX); 3530 const struct efx_ef10_nvram_type_info *info; 3531 size_t size, erase_size, write_size, outlen; 3532 int type_idx = 0; 3533 bool protected; 3534 int rc; 3535 3536 for (type_idx = 0; ; type_idx++) { 3537 if (type_idx == EF10_NVRAM_PARTITION_COUNT) 3538 return -ENODEV; 3539 info = efx_ef10_nvram_types + type_idx; 3540 if ((type & ~info->type_mask) == info->type) 3541 break; 3542 } 3543 if (info->port != efx_port_num(efx)) 3544 return -ENODEV; 3545 3546 rc = efx_mcdi_nvram_info(efx, type, &size, &erase_size, &write_size, 3547 &protected); 3548 if (rc) 3549 return rc; 3550 if (protected && 3551 (type != NVRAM_PARTITION_TYPE_DYNCONFIG_DEFAULTS && 3552 type != NVRAM_PARTITION_TYPE_ROMCONFIG_DEFAULTS)) 3553 /* Hide protected partitions that don't provide defaults. */ 3554 return -ENODEV; 3555 3556 if (protected) 3557 /* Protected partitions are read only. */ 3558 erase_size = 0; 3559 3560 /* If we've already exposed a partition of this type, hide this 3561 * duplicate. All operations on MTDs are keyed by the type anyway, 3562 * so we can't act on the duplicate. 3563 */ 3564 if (__test_and_set_bit(type_idx, found)) 3565 return -EEXIST; 3566 3567 part->nvram_type = type; 3568 3569 MCDI_SET_DWORD(inbuf, NVRAM_METADATA_IN_TYPE, type); 3570 rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_METADATA, inbuf, sizeof(inbuf), 3571 outbuf, sizeof(outbuf), &outlen); 3572 if (rc) 3573 return rc; 3574 if (outlen < MC_CMD_NVRAM_METADATA_OUT_LENMIN) 3575 return -EIO; 3576 if (MCDI_DWORD(outbuf, NVRAM_METADATA_OUT_FLAGS) & 3577 (1 << MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_VALID_LBN)) 3578 part->fw_subtype = MCDI_DWORD(outbuf, 3579 NVRAM_METADATA_OUT_SUBTYPE); 3580 3581 part->common.dev_type_name = "EF10 NVRAM manager"; 3582 part->common.type_name = info->name; 3583 3584 part->common.mtd.type = MTD_NORFLASH; 3585 part->common.mtd.flags = MTD_CAP_NORFLASH; 3586 part->common.mtd.size = size; 3587 part->common.mtd.erasesize = erase_size; 3588 /* sfc_status is read-only */ 3589 if (!erase_size) 3590 part->common.mtd.flags |= MTD_NO_ERASE; 3591 3592 part->common.mtd.writesize = write_size; 3593 3594 return 0; 3595 } 3596 3597 static int efx_ef10_mtd_probe(struct efx_nic *efx) 3598 { 3599 MCDI_DECLARE_BUF(outbuf, MC_CMD_NVRAM_PARTITIONS_OUT_LENMAX); 3600 DECLARE_BITMAP(found, EF10_NVRAM_PARTITION_COUNT) = { 0 }; 3601 struct efx_mcdi_mtd_partition *parts; 3602 size_t outlen, n_parts_total, i, n_parts; 3603 unsigned int type; 3604 int rc; 3605 3606 ASSERT_RTNL(); 3607 3608 BUILD_BUG_ON(MC_CMD_NVRAM_PARTITIONS_IN_LEN != 0); 3609 rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_PARTITIONS, NULL, 0, 3610 outbuf, sizeof(outbuf), &outlen); 3611 if (rc) 3612 return rc; 3613 if (outlen < MC_CMD_NVRAM_PARTITIONS_OUT_LENMIN) 3614 return -EIO; 3615 3616 n_parts_total = MCDI_DWORD(outbuf, NVRAM_PARTITIONS_OUT_NUM_PARTITIONS); 3617 if (n_parts_total > 3618 MCDI_VAR_ARRAY_LEN(outlen, NVRAM_PARTITIONS_OUT_TYPE_ID)) 3619 return -EIO; 3620 3621 parts = kzalloc_objs(*parts, n_parts_total); 3622 if (!parts) 3623 return -ENOMEM; 3624 3625 n_parts = 0; 3626 for (i = 0; i < n_parts_total; i++) { 3627 type = MCDI_ARRAY_DWORD(outbuf, NVRAM_PARTITIONS_OUT_TYPE_ID, 3628 i); 3629 rc = efx_ef10_mtd_probe_partition(efx, &parts[n_parts], type, 3630 found); 3631 if (rc == -EEXIST || rc == -ENODEV) 3632 continue; 3633 if (rc) 3634 goto fail; 3635 n_parts++; 3636 } 3637 3638 if (!n_parts) { 3639 kfree(parts); 3640 return 0; 3641 } 3642 3643 rc = efx_mtd_add(efx, &parts[0].common, n_parts, sizeof(*parts)); 3644 fail: 3645 if (rc) 3646 kfree(parts); 3647 return rc; 3648 } 3649 3650 #endif /* CONFIG_SFC_MTD */ 3651 3652 static void efx_ef10_ptp_write_host_time(struct efx_nic *efx, u32 host_time) 3653 { 3654 _efx_writed(efx, cpu_to_le32(host_time), ER_DZ_MC_DB_LWRD); 3655 } 3656 3657 static void efx_ef10_ptp_write_host_time_vf(struct efx_nic *efx, 3658 u32 host_time) {} 3659 3660 static int efx_ef10_rx_enable_timestamping(struct efx_channel *channel, 3661 bool temp) 3662 { 3663 MCDI_DECLARE_BUF(inbuf, MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_LEN); 3664 int rc; 3665 3666 if (channel->sync_events_state == SYNC_EVENTS_REQUESTED || 3667 channel->sync_events_state == SYNC_EVENTS_VALID || 3668 (temp && channel->sync_events_state == SYNC_EVENTS_DISABLED)) 3669 return 0; 3670 channel->sync_events_state = SYNC_EVENTS_REQUESTED; 3671 3672 MCDI_SET_DWORD(inbuf, PTP_IN_OP, MC_CMD_PTP_OP_TIME_EVENT_SUBSCRIBE); 3673 MCDI_SET_DWORD(inbuf, PTP_IN_PERIPH_ID, 0); 3674 MCDI_SET_DWORD(inbuf, PTP_IN_TIME_EVENT_SUBSCRIBE_QUEUE, 3675 channel->channel); 3676 3677 rc = efx_mcdi_rpc(channel->efx, MC_CMD_PTP, 3678 inbuf, sizeof(inbuf), NULL, 0, NULL); 3679 3680 if (rc != 0) 3681 channel->sync_events_state = temp ? SYNC_EVENTS_QUIESCENT : 3682 SYNC_EVENTS_DISABLED; 3683 3684 return rc; 3685 } 3686 3687 static int efx_ef10_rx_disable_timestamping(struct efx_channel *channel, 3688 bool temp) 3689 { 3690 MCDI_DECLARE_BUF(inbuf, MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_LEN); 3691 int rc; 3692 3693 if (channel->sync_events_state == SYNC_EVENTS_DISABLED || 3694 (temp && channel->sync_events_state == SYNC_EVENTS_QUIESCENT)) 3695 return 0; 3696 if (channel->sync_events_state == SYNC_EVENTS_QUIESCENT) { 3697 channel->sync_events_state = SYNC_EVENTS_DISABLED; 3698 return 0; 3699 } 3700 channel->sync_events_state = temp ? SYNC_EVENTS_QUIESCENT : 3701 SYNC_EVENTS_DISABLED; 3702 3703 MCDI_SET_DWORD(inbuf, PTP_IN_OP, MC_CMD_PTP_OP_TIME_EVENT_UNSUBSCRIBE); 3704 MCDI_SET_DWORD(inbuf, PTP_IN_PERIPH_ID, 0); 3705 MCDI_SET_DWORD(inbuf, PTP_IN_TIME_EVENT_UNSUBSCRIBE_CONTROL, 3706 MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_SINGLE); 3707 MCDI_SET_DWORD(inbuf, PTP_IN_TIME_EVENT_UNSUBSCRIBE_QUEUE, 3708 channel->channel); 3709 3710 rc = efx_mcdi_rpc(channel->efx, MC_CMD_PTP, 3711 inbuf, sizeof(inbuf), NULL, 0, NULL); 3712 3713 return rc; 3714 } 3715 3716 static int efx_ef10_ptp_set_ts_sync_events(struct efx_nic *efx, bool en, 3717 bool temp) 3718 { 3719 int (*set)(struct efx_channel *channel, bool temp); 3720 struct efx_channel *channel; 3721 3722 set = en ? 3723 efx_ef10_rx_enable_timestamping : 3724 efx_ef10_rx_disable_timestamping; 3725 3726 channel = efx_ptp_channel(efx); 3727 if (channel) { 3728 int rc = set(channel, temp); 3729 if (en && rc != 0) { 3730 efx_ef10_ptp_set_ts_sync_events(efx, false, temp); 3731 return rc; 3732 } 3733 } 3734 3735 return 0; 3736 } 3737 3738 static int efx_ef10_ptp_set_ts_config_vf(struct efx_nic *efx, 3739 struct kernel_hwtstamp_config *init) 3740 { 3741 return -EOPNOTSUPP; 3742 } 3743 3744 static int efx_ef10_ptp_set_ts_config(struct efx_nic *efx, 3745 struct kernel_hwtstamp_config *init) 3746 { 3747 int rc; 3748 3749 switch (init->rx_filter) { 3750 case HWTSTAMP_FILTER_NONE: 3751 efx_ef10_ptp_set_ts_sync_events(efx, false, false); 3752 /* if TX timestamping is still requested then leave PTP on */ 3753 return efx_ptp_change_mode(efx, 3754 init->tx_type != HWTSTAMP_TX_OFF, 0); 3755 case HWTSTAMP_FILTER_ALL: 3756 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 3757 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 3758 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 3759 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 3760 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 3761 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 3762 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 3763 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 3764 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 3765 case HWTSTAMP_FILTER_PTP_V2_EVENT: 3766 case HWTSTAMP_FILTER_PTP_V2_SYNC: 3767 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 3768 case HWTSTAMP_FILTER_NTP_ALL: 3769 init->rx_filter = HWTSTAMP_FILTER_ALL; 3770 rc = efx_ptp_change_mode(efx, true, 0); 3771 if (!rc) 3772 rc = efx_ef10_ptp_set_ts_sync_events(efx, true, false); 3773 if (rc) 3774 efx_ptp_change_mode(efx, false, 0); 3775 return rc; 3776 default: 3777 return -ERANGE; 3778 } 3779 } 3780 3781 static int efx_ef10_get_phys_port_id(struct efx_nic *efx, 3782 struct netdev_phys_item_id *ppid) 3783 { 3784 struct efx_ef10_nic_data *nic_data = efx->nic_data; 3785 3786 if (!is_valid_ether_addr(nic_data->port_id)) 3787 return -EOPNOTSUPP; 3788 3789 ppid->id_len = ETH_ALEN; 3790 memcpy(ppid->id, nic_data->port_id, ppid->id_len); 3791 3792 return 0; 3793 } 3794 3795 static int efx_ef10_vlan_rx_add_vid(struct efx_nic *efx, __be16 proto, u16 vid) 3796 { 3797 if (proto != htons(ETH_P_8021Q)) 3798 return -EINVAL; 3799 3800 return efx_ef10_add_vlan(efx, vid); 3801 } 3802 3803 static int efx_ef10_vlan_rx_kill_vid(struct efx_nic *efx, __be16 proto, u16 vid) 3804 { 3805 if (proto != htons(ETH_P_8021Q)) 3806 return -EINVAL; 3807 3808 return efx_ef10_del_vlan(efx, vid); 3809 } 3810 3811 /* We rely on the MCDI wiping out our TX rings if it made any changes to the 3812 * ports table, ensuring that any TSO descriptors that were made on a now- 3813 * removed tunnel port will be blown away and won't break things when we try 3814 * to transmit them using the new ports table. 3815 */ 3816 static int efx_ef10_set_udp_tnl_ports(struct efx_nic *efx, bool unloading) 3817 { 3818 struct efx_ef10_nic_data *nic_data = efx->nic_data; 3819 MCDI_DECLARE_BUF(inbuf, MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LENMAX); 3820 MCDI_DECLARE_BUF(outbuf, MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_LEN); 3821 bool will_reset = false; 3822 size_t num_entries = 0; 3823 size_t inlen, outlen; 3824 size_t i; 3825 int rc; 3826 efx_dword_t flags_and_num_entries; 3827 3828 WARN_ON(!mutex_is_locked(&nic_data->udp_tunnels_lock)); 3829 3830 nic_data->udp_tunnels_dirty = false; 3831 3832 if (!(nic_data->datapath_caps & 3833 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN))) { 3834 efx_device_attach_if_not_resetting(efx); 3835 return 0; 3836 } 3837 3838 BUILD_BUG_ON(ARRAY_SIZE(nic_data->udp_tunnels) > 3839 MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_MAXNUM); 3840 3841 for (i = 0; i < ARRAY_SIZE(nic_data->udp_tunnels); ++i) { 3842 if (nic_data->udp_tunnels[i].type != 3843 TUNNEL_ENCAP_UDP_PORT_ENTRY_INVALID) { 3844 efx_dword_t entry; 3845 3846 EFX_POPULATE_DWORD_2(entry, 3847 TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT, 3848 ntohs(nic_data->udp_tunnels[i].port), 3849 TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL, 3850 nic_data->udp_tunnels[i].type); 3851 *_MCDI_ARRAY_DWORD(inbuf, 3852 SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES, 3853 num_entries++) = entry; 3854 } 3855 } 3856 3857 BUILD_BUG_ON((MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_NUM_ENTRIES_OFST - 3858 MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_FLAGS_OFST) * 8 != 3859 EFX_WORD_1_LBN); 3860 BUILD_BUG_ON(MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_NUM_ENTRIES_LEN * 8 != 3861 EFX_WORD_1_WIDTH); 3862 EFX_POPULATE_DWORD_2(flags_and_num_entries, 3863 MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_UNLOADING, 3864 !!unloading, 3865 EFX_WORD_1, num_entries); 3866 *_MCDI_DWORD(inbuf, SET_TUNNEL_ENCAP_UDP_PORTS_IN_FLAGS) = 3867 flags_and_num_entries; 3868 3869 inlen = MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LEN(num_entries); 3870 3871 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS, 3872 inbuf, inlen, outbuf, sizeof(outbuf), &outlen); 3873 if (rc == -EIO) { 3874 /* Most likely the MC rebooted due to another function also 3875 * setting its tunnel port list. Mark the tunnel port list as 3876 * dirty, so it will be pushed upon coming up from the reboot. 3877 */ 3878 nic_data->udp_tunnels_dirty = true; 3879 return 0; 3880 } 3881 3882 if (rc) { 3883 /* expected not available on unprivileged functions */ 3884 if (rc != -EPERM) 3885 netif_warn(efx, drv, efx->net_dev, 3886 "Unable to set UDP tunnel ports; rc=%d.\n", rc); 3887 } else if (MCDI_DWORD(outbuf, SET_TUNNEL_ENCAP_UDP_PORTS_OUT_FLAGS) & 3888 (1 << MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_RESETTING_LBN)) { 3889 netif_info(efx, drv, efx->net_dev, 3890 "Rebooting MC due to UDP tunnel port list change\n"); 3891 will_reset = true; 3892 if (unloading) 3893 /* Delay for the MC reset to complete. This will make 3894 * unloading other functions a bit smoother. This is a 3895 * race, but the other unload will work whichever way 3896 * it goes, this just avoids an unnecessary error 3897 * message. 3898 */ 3899 msleep(100); 3900 } 3901 if (!will_reset && !unloading) { 3902 /* The caller will have detached, relying on the MC reset to 3903 * trigger a re-attach. Since there won't be an MC reset, we 3904 * have to do the attach ourselves. 3905 */ 3906 efx_device_attach_if_not_resetting(efx); 3907 } 3908 3909 return rc; 3910 } 3911 3912 static int efx_ef10_udp_tnl_push_ports(struct efx_nic *efx) 3913 { 3914 struct efx_ef10_nic_data *nic_data = efx->nic_data; 3915 int rc = 0; 3916 3917 mutex_lock(&nic_data->udp_tunnels_lock); 3918 if (nic_data->udp_tunnels_dirty) { 3919 /* Make sure all TX are stopped while we modify the table, else 3920 * we might race against an efx_features_check(). 3921 */ 3922 efx_device_detach_sync(efx); 3923 rc = efx_ef10_set_udp_tnl_ports(efx, false); 3924 } 3925 mutex_unlock(&nic_data->udp_tunnels_lock); 3926 return rc; 3927 } 3928 3929 static int efx_ef10_udp_tnl_set_port(struct net_device *dev, 3930 unsigned int table, unsigned int entry, 3931 struct udp_tunnel_info *ti) 3932 { 3933 struct efx_nic *efx = efx_netdev_priv(dev); 3934 struct efx_ef10_nic_data *nic_data; 3935 int efx_tunnel_type, rc; 3936 3937 if (ti->type == UDP_TUNNEL_TYPE_VXLAN) 3938 efx_tunnel_type = TUNNEL_ENCAP_UDP_PORT_ENTRY_VXLAN; 3939 else 3940 efx_tunnel_type = TUNNEL_ENCAP_UDP_PORT_ENTRY_GENEVE; 3941 3942 nic_data = efx->nic_data; 3943 if (!(nic_data->datapath_caps & 3944 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN))) 3945 return -EOPNOTSUPP; 3946 3947 mutex_lock(&nic_data->udp_tunnels_lock); 3948 /* Make sure all TX are stopped while we add to the table, else we 3949 * might race against an efx_features_check(). 3950 */ 3951 efx_device_detach_sync(efx); 3952 nic_data->udp_tunnels[entry].type = efx_tunnel_type; 3953 nic_data->udp_tunnels[entry].port = ti->port; 3954 rc = efx_ef10_set_udp_tnl_ports(efx, false); 3955 mutex_unlock(&nic_data->udp_tunnels_lock); 3956 3957 return rc; 3958 } 3959 3960 /* Called under the TX lock with the TX queue running, hence no-one can be 3961 * in the middle of updating the UDP tunnels table. However, they could 3962 * have tried and failed the MCDI, in which case they'll have set the dirty 3963 * flag before dropping their locks. 3964 */ 3965 static bool efx_ef10_udp_tnl_has_port(struct efx_nic *efx, __be16 port) 3966 { 3967 struct efx_ef10_nic_data *nic_data = efx->nic_data; 3968 size_t i; 3969 3970 if (!(nic_data->datapath_caps & 3971 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN))) 3972 return false; 3973 3974 if (nic_data->udp_tunnels_dirty) 3975 /* SW table may not match HW state, so just assume we can't 3976 * use any UDP tunnel offloads. 3977 */ 3978 return false; 3979 3980 for (i = 0; i < ARRAY_SIZE(nic_data->udp_tunnels); ++i) 3981 if (nic_data->udp_tunnels[i].type != 3982 TUNNEL_ENCAP_UDP_PORT_ENTRY_INVALID && 3983 nic_data->udp_tunnels[i].port == port) 3984 return true; 3985 3986 return false; 3987 } 3988 3989 static int efx_ef10_udp_tnl_unset_port(struct net_device *dev, 3990 unsigned int table, unsigned int entry, 3991 struct udp_tunnel_info *ti) 3992 { 3993 struct efx_nic *efx = efx_netdev_priv(dev); 3994 struct efx_ef10_nic_data *nic_data; 3995 int rc; 3996 3997 nic_data = efx->nic_data; 3998 3999 mutex_lock(&nic_data->udp_tunnels_lock); 4000 /* Make sure all TX are stopped while we remove from the table, else we 4001 * might race against an efx_features_check(). 4002 */ 4003 efx_device_detach_sync(efx); 4004 nic_data->udp_tunnels[entry].type = TUNNEL_ENCAP_UDP_PORT_ENTRY_INVALID; 4005 nic_data->udp_tunnels[entry].port = 0; 4006 rc = efx_ef10_set_udp_tnl_ports(efx, false); 4007 mutex_unlock(&nic_data->udp_tunnels_lock); 4008 4009 return rc; 4010 } 4011 4012 static const struct udp_tunnel_nic_info efx_ef10_udp_tunnels = { 4013 .set_port = efx_ef10_udp_tnl_set_port, 4014 .unset_port = efx_ef10_udp_tnl_unset_port, 4015 .tables = { 4016 { 4017 .n_entries = 16, 4018 .tunnel_types = UDP_TUNNEL_TYPE_VXLAN | 4019 UDP_TUNNEL_TYPE_GENEVE, 4020 }, 4021 }, 4022 }; 4023 4024 /* EF10 may have multiple datapath firmware variants within a 4025 * single version. Report which variants are running. 4026 */ 4027 static size_t efx_ef10_print_additional_fwver(struct efx_nic *efx, char *buf, 4028 size_t len) 4029 { 4030 struct efx_ef10_nic_data *nic_data = efx->nic_data; 4031 4032 return scnprintf(buf, len, " rx%x tx%x", 4033 nic_data->rx_dpcpu_fw_id, 4034 nic_data->tx_dpcpu_fw_id); 4035 } 4036 4037 static unsigned int ef10_check_caps(const struct efx_nic *efx, 4038 u8 flag, 4039 u32 offset) 4040 { 4041 const struct efx_ef10_nic_data *nic_data = efx->nic_data; 4042 4043 switch (offset) { 4044 case(MC_CMD_GET_CAPABILITIES_V4_OUT_FLAGS1_OFST): 4045 return nic_data->datapath_caps & BIT_ULL(flag); 4046 case(MC_CMD_GET_CAPABILITIES_V4_OUT_FLAGS2_OFST): 4047 return nic_data->datapath_caps2 & BIT_ULL(flag); 4048 default: 4049 return 0; 4050 } 4051 } 4052 4053 static unsigned int efx_ef10_recycle_ring_size(const struct efx_nic *efx) 4054 { 4055 unsigned int ret = EFX_RECYCLE_RING_SIZE_10G; 4056 4057 /* There is no difference between PFs and VFs. The side is based on 4058 * the maximum link speed of a given NIC. 4059 */ 4060 switch (efx->pci_dev->device & 0xfff) { 4061 case 0x0903: /* Farmingdale can do up to 10G */ 4062 break; 4063 case 0x0923: /* Greenport can do up to 40G */ 4064 case 0x0a03: /* Medford can do up to 40G */ 4065 ret *= 4; 4066 break; 4067 default: /* Medford2 can do up to 100G */ 4068 ret *= 10; 4069 } 4070 4071 if (IS_ENABLED(CONFIG_PPC64)) 4072 ret *= 4; 4073 4074 return ret; 4075 } 4076 4077 #define EF10_OFFLOAD_FEATURES \ 4078 (NETIF_F_IP_CSUM | \ 4079 NETIF_F_HW_VLAN_CTAG_FILTER | \ 4080 NETIF_F_IPV6_CSUM | \ 4081 NETIF_F_RXHASH | \ 4082 NETIF_F_NTUPLE | \ 4083 NETIF_F_SG | \ 4084 NETIF_F_RXCSUM | \ 4085 NETIF_F_RXALL) 4086 4087 const struct efx_nic_type efx_hunt_a0_vf_nic_type = { 4088 .is_vf = true, 4089 .mem_bar = efx_ef10_vf_mem_bar, 4090 .mem_map_size = efx_ef10_mem_map_size, 4091 .probe = efx_ef10_probe_vf, 4092 .remove = efx_ef10_remove, 4093 .dimension_resources = efx_ef10_dimension_resources, 4094 .init = efx_ef10_init_nic, 4095 .fini = efx_ef10_fini_nic, 4096 .map_reset_reason = efx_ef10_map_reset_reason, 4097 .map_reset_flags = efx_ef10_map_reset_flags, 4098 .reset = efx_ef10_reset, 4099 .probe_port = efx_mcdi_port_probe, 4100 .remove_port = efx_mcdi_port_remove, 4101 .fini_dmaq = efx_fini_dmaq, 4102 .prepare_flr = efx_ef10_prepare_flr, 4103 .finish_flr = efx_port_dummy_op_void, 4104 .describe_stats = efx_ef10_describe_stats, 4105 .update_stats = efx_ef10_update_stats_vf, 4106 .update_stats_atomic = efx_ef10_update_stats_atomic_vf, 4107 .start_stats = efx_port_dummy_op_void, 4108 .pull_stats = efx_port_dummy_op_void, 4109 .stop_stats = efx_port_dummy_op_void, 4110 .push_irq_moderation = efx_ef10_push_irq_moderation, 4111 .reconfigure_mac = efx_ef10_mac_reconfigure, 4112 .check_mac_fault = efx_mcdi_mac_check_fault, 4113 .reconfigure_port = efx_mcdi_port_reconfigure, 4114 .get_wol = efx_ef10_get_wol_vf, 4115 .set_wol = efx_ef10_set_wol_vf, 4116 .resume_wol = efx_port_dummy_op_void, 4117 .mcdi_request = efx_ef10_mcdi_request, 4118 .mcdi_poll_response = efx_ef10_mcdi_poll_response, 4119 .mcdi_read_response = efx_ef10_mcdi_read_response, 4120 .mcdi_poll_reboot = efx_ef10_mcdi_poll_reboot, 4121 .mcdi_reboot_detected = efx_ef10_mcdi_reboot_detected, 4122 .irq_enable_master = efx_port_dummy_op_void, 4123 .irq_test_generate = efx_ef10_irq_test_generate, 4124 .irq_disable_non_ev = efx_port_dummy_op_void, 4125 .irq_handle_msi = efx_ef10_msi_interrupt, 4126 .irq_handle_legacy = efx_ef10_legacy_interrupt, 4127 .tx_probe = efx_ef10_tx_probe, 4128 .tx_init = efx_ef10_tx_init, 4129 .tx_remove = efx_mcdi_tx_remove, 4130 .tx_write = efx_ef10_tx_write, 4131 .tx_limit_len = efx_ef10_tx_limit_len, 4132 .tx_enqueue = __efx_enqueue_skb, 4133 .rx_push_rss_config = efx_mcdi_vf_rx_push_rss_config, 4134 .rx_pull_rss_config = efx_mcdi_rx_pull_rss_config, 4135 .rx_probe = efx_mcdi_rx_probe, 4136 .rx_init = efx_mcdi_rx_init, 4137 .rx_remove = efx_mcdi_rx_remove, 4138 .rx_write = efx_ef10_rx_write, 4139 .rx_defer_refill = efx_ef10_rx_defer_refill, 4140 .rx_packet = __efx_rx_packet, 4141 .ev_probe = efx_mcdi_ev_probe, 4142 .ev_init = efx_ef10_ev_init, 4143 .ev_fini = efx_mcdi_ev_fini, 4144 .ev_remove = efx_mcdi_ev_remove, 4145 .ev_process = efx_ef10_ev_process, 4146 .ev_read_ack = efx_ef10_ev_read_ack, 4147 .ev_test_generate = efx_ef10_ev_test_generate, 4148 .filter_table_probe = efx_ef10_filter_table_probe, 4149 .filter_table_restore = efx_mcdi_filter_table_restore, 4150 .filter_table_remove = efx_ef10_filter_table_remove, 4151 .filter_update_rx_scatter = efx_mcdi_update_rx_scatter, 4152 .filter_insert = efx_mcdi_filter_insert, 4153 .filter_remove_safe = efx_mcdi_filter_remove_safe, 4154 .filter_get_safe = efx_mcdi_filter_get_safe, 4155 .filter_clear_rx = efx_mcdi_filter_clear_rx, 4156 .filter_count_rx_used = efx_mcdi_filter_count_rx_used, 4157 .filter_get_rx_id_limit = efx_mcdi_filter_get_rx_id_limit, 4158 .filter_get_rx_ids = efx_mcdi_filter_get_rx_ids, 4159 #ifdef CONFIG_RFS_ACCEL 4160 .filter_rfs_expire_one = efx_mcdi_filter_rfs_expire_one, 4161 #endif 4162 #ifdef CONFIG_SFC_MTD 4163 .mtd_probe = efx_port_dummy_op_int, 4164 #endif 4165 .ptp_write_host_time = efx_ef10_ptp_write_host_time_vf, 4166 .ptp_set_ts_config = efx_ef10_ptp_set_ts_config_vf, 4167 .vlan_rx_add_vid = efx_ef10_vlan_rx_add_vid, 4168 .vlan_rx_kill_vid = efx_ef10_vlan_rx_kill_vid, 4169 #ifdef CONFIG_SFC_SRIOV 4170 .vswitching_probe = efx_ef10_vswitching_probe_vf, 4171 .vswitching_restore = efx_ef10_vswitching_restore_vf, 4172 .vswitching_remove = efx_ef10_vswitching_remove_vf, 4173 #endif 4174 .get_mac_address = efx_ef10_get_mac_address_vf, 4175 .set_mac_address = efx_ef10_set_mac_address, 4176 4177 .get_phys_port_id = efx_ef10_get_phys_port_id, 4178 .revision = EFX_REV_HUNT_A0, 4179 .max_dma_mask = DMA_BIT_MASK(ESF_DZ_TX_KER_BUF_ADDR_WIDTH), 4180 .rx_prefix_size = ES_DZ_RX_PREFIX_SIZE, 4181 .rx_hash_offset = ES_DZ_RX_PREFIX_HASH_OFST, 4182 .rx_ts_offset = ES_DZ_RX_PREFIX_TSTAMP_OFST, 4183 .can_rx_scatter = true, 4184 .always_rx_scatter = true, 4185 .min_interrupt_mode = EFX_INT_MODE_MSIX, 4186 .timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH, 4187 .offload_features = EF10_OFFLOAD_FEATURES, 4188 .mcdi_max_ver = 2, 4189 .max_rx_ip_filters = EFX_MCDI_FILTER_TBL_ROWS, 4190 .hwtstamp_filters = 1 << HWTSTAMP_FILTER_NONE | 4191 1 << HWTSTAMP_FILTER_ALL, 4192 .rx_hash_key_size = 40, 4193 .check_caps = ef10_check_caps, 4194 .print_additional_fwver = efx_ef10_print_additional_fwver, 4195 .sensor_event = efx_mcdi_sensor_event, 4196 .rx_recycle_ring_size = efx_ef10_recycle_ring_size, 4197 }; 4198 4199 const struct efx_nic_type efx_hunt_a0_nic_type = { 4200 .is_vf = false, 4201 .mem_bar = efx_ef10_pf_mem_bar, 4202 .mem_map_size = efx_ef10_mem_map_size, 4203 .probe = efx_ef10_probe_pf, 4204 .remove = efx_ef10_remove, 4205 .dimension_resources = efx_ef10_dimension_resources, 4206 .init = efx_ef10_init_nic, 4207 .fini = efx_ef10_fini_nic, 4208 .map_reset_reason = efx_ef10_map_reset_reason, 4209 .map_reset_flags = efx_ef10_map_reset_flags, 4210 .reset = efx_ef10_reset, 4211 .probe_port = efx_mcdi_port_probe, 4212 .remove_port = efx_mcdi_port_remove, 4213 .fini_dmaq = efx_fini_dmaq, 4214 .prepare_flr = efx_ef10_prepare_flr, 4215 .finish_flr = efx_port_dummy_op_void, 4216 .describe_stats = efx_ef10_describe_stats, 4217 .update_stats = efx_ef10_update_stats_pf, 4218 .start_stats = efx_mcdi_mac_start_stats, 4219 .pull_stats = efx_mcdi_mac_pull_stats, 4220 .stop_stats = efx_mcdi_mac_stop_stats, 4221 .push_irq_moderation = efx_ef10_push_irq_moderation, 4222 .reconfigure_mac = efx_ef10_mac_reconfigure, 4223 .check_mac_fault = efx_mcdi_mac_check_fault, 4224 .reconfigure_port = efx_mcdi_port_reconfigure, 4225 .get_wol = efx_ef10_get_wol, 4226 .set_wol = efx_ef10_set_wol, 4227 .resume_wol = efx_port_dummy_op_void, 4228 .get_fec_stats = efx_ef10_get_fec_stats, 4229 .test_chip = efx_ef10_test_chip, 4230 .test_nvram = efx_mcdi_nvram_test_all, 4231 .mcdi_request = efx_ef10_mcdi_request, 4232 .mcdi_poll_response = efx_ef10_mcdi_poll_response, 4233 .mcdi_read_response = efx_ef10_mcdi_read_response, 4234 .mcdi_poll_reboot = efx_ef10_mcdi_poll_reboot, 4235 .mcdi_reboot_detected = efx_ef10_mcdi_reboot_detected, 4236 .irq_enable_master = efx_port_dummy_op_void, 4237 .irq_test_generate = efx_ef10_irq_test_generate, 4238 .irq_disable_non_ev = efx_port_dummy_op_void, 4239 .irq_handle_msi = efx_ef10_msi_interrupt, 4240 .irq_handle_legacy = efx_ef10_legacy_interrupt, 4241 .tx_probe = efx_ef10_tx_probe, 4242 .tx_init = efx_ef10_tx_init, 4243 .tx_remove = efx_mcdi_tx_remove, 4244 .tx_write = efx_ef10_tx_write, 4245 .tx_limit_len = efx_ef10_tx_limit_len, 4246 .tx_enqueue = __efx_enqueue_skb, 4247 .rx_push_rss_config = efx_mcdi_pf_rx_push_rss_config, 4248 .rx_pull_rss_config = efx_mcdi_rx_pull_rss_config, 4249 .rx_push_rss_context_config = efx_mcdi_rx_push_rss_context_config, 4250 .rx_pull_rss_context_config = efx_mcdi_rx_pull_rss_context_config, 4251 .rx_restore_rss_contexts = efx_mcdi_rx_restore_rss_contexts, 4252 .rx_probe = efx_mcdi_rx_probe, 4253 .rx_init = efx_mcdi_rx_init, 4254 .rx_remove = efx_mcdi_rx_remove, 4255 .rx_write = efx_ef10_rx_write, 4256 .rx_defer_refill = efx_ef10_rx_defer_refill, 4257 .rx_packet = __efx_rx_packet, 4258 .ev_probe = efx_mcdi_ev_probe, 4259 .ev_init = efx_ef10_ev_init, 4260 .ev_fini = efx_mcdi_ev_fini, 4261 .ev_remove = efx_mcdi_ev_remove, 4262 .ev_process = efx_ef10_ev_process, 4263 .ev_read_ack = efx_ef10_ev_read_ack, 4264 .ev_test_generate = efx_ef10_ev_test_generate, 4265 .filter_table_probe = efx_ef10_filter_table_probe, 4266 .filter_table_restore = efx_mcdi_filter_table_restore, 4267 .filter_table_remove = efx_ef10_filter_table_remove, 4268 .filter_update_rx_scatter = efx_mcdi_update_rx_scatter, 4269 .filter_insert = efx_mcdi_filter_insert, 4270 .filter_remove_safe = efx_mcdi_filter_remove_safe, 4271 .filter_get_safe = efx_mcdi_filter_get_safe, 4272 .filter_clear_rx = efx_mcdi_filter_clear_rx, 4273 .filter_count_rx_used = efx_mcdi_filter_count_rx_used, 4274 .filter_get_rx_id_limit = efx_mcdi_filter_get_rx_id_limit, 4275 .filter_get_rx_ids = efx_mcdi_filter_get_rx_ids, 4276 #ifdef CONFIG_RFS_ACCEL 4277 .filter_rfs_expire_one = efx_mcdi_filter_rfs_expire_one, 4278 #endif 4279 #ifdef CONFIG_SFC_MTD 4280 .mtd_probe = efx_ef10_mtd_probe, 4281 .mtd_rename = efx_mcdi_mtd_rename, 4282 .mtd_read = efx_mcdi_mtd_read, 4283 .mtd_erase = efx_mcdi_mtd_erase, 4284 .mtd_write = efx_mcdi_mtd_write, 4285 .mtd_sync = efx_mcdi_mtd_sync, 4286 #endif 4287 .ptp_write_host_time = efx_ef10_ptp_write_host_time, 4288 .ptp_set_ts_sync_events = efx_ef10_ptp_set_ts_sync_events, 4289 .ptp_set_ts_config = efx_ef10_ptp_set_ts_config, 4290 .vlan_rx_add_vid = efx_ef10_vlan_rx_add_vid, 4291 .vlan_rx_kill_vid = efx_ef10_vlan_rx_kill_vid, 4292 .udp_tnl_push_ports = efx_ef10_udp_tnl_push_ports, 4293 .udp_tnl_has_port = efx_ef10_udp_tnl_has_port, 4294 #ifdef CONFIG_SFC_SRIOV 4295 .sriov_configure = efx_ef10_sriov_configure, 4296 .sriov_init = efx_ef10_sriov_init, 4297 .sriov_fini = efx_ef10_sriov_fini, 4298 .sriov_wanted = efx_ef10_sriov_wanted, 4299 .sriov_set_vf_mac = efx_ef10_sriov_set_vf_mac, 4300 .sriov_set_vf_vlan = efx_ef10_sriov_set_vf_vlan, 4301 .sriov_set_vf_spoofchk = efx_ef10_sriov_set_vf_spoofchk, 4302 .sriov_get_vf_config = efx_ef10_sriov_get_vf_config, 4303 .sriov_set_vf_link_state = efx_ef10_sriov_set_vf_link_state, 4304 .vswitching_probe = efx_ef10_vswitching_probe_pf, 4305 .vswitching_restore = efx_ef10_vswitching_restore_pf, 4306 .vswitching_remove = efx_ef10_vswitching_remove_pf, 4307 #endif 4308 .get_mac_address = efx_ef10_get_mac_address_pf, 4309 .set_mac_address = efx_ef10_set_mac_address, 4310 .tso_versions = efx_ef10_tso_versions, 4311 4312 .get_phys_port_id = efx_ef10_get_phys_port_id, 4313 .revision = EFX_REV_HUNT_A0, 4314 .max_dma_mask = DMA_BIT_MASK(ESF_DZ_TX_KER_BUF_ADDR_WIDTH), 4315 .rx_prefix_size = ES_DZ_RX_PREFIX_SIZE, 4316 .rx_hash_offset = ES_DZ_RX_PREFIX_HASH_OFST, 4317 .rx_ts_offset = ES_DZ_RX_PREFIX_TSTAMP_OFST, 4318 .can_rx_scatter = true, 4319 .always_rx_scatter = true, 4320 .option_descriptors = true, 4321 .min_interrupt_mode = EFX_INT_MODE_LEGACY, 4322 .timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH, 4323 .offload_features = EF10_OFFLOAD_FEATURES, 4324 .mcdi_max_ver = 2, 4325 .max_rx_ip_filters = EFX_MCDI_FILTER_TBL_ROWS, 4326 .hwtstamp_filters = 1 << HWTSTAMP_FILTER_NONE | 4327 1 << HWTSTAMP_FILTER_ALL, 4328 .rx_hash_key_size = 40, 4329 .check_caps = ef10_check_caps, 4330 .print_additional_fwver = efx_ef10_print_additional_fwver, 4331 .sensor_event = efx_mcdi_sensor_event, 4332 .rx_recycle_ring_size = efx_ef10_recycle_ring_size, 4333 }; 4334 4335 const struct efx_nic_type efx_x4_nic_type = { 4336 .is_vf = false, 4337 .mem_bar = efx_ef10_pf_mem_bar, 4338 .mem_map_size = efx_ef10_mem_map_size, 4339 .probe = efx_ef10_probe_pf, 4340 .remove = efx_ef10_remove, 4341 .dimension_resources = efx_ef10_dimension_resources, 4342 .init = efx_ef10_init_nic, 4343 .fini = efx_ef10_fini_nic, 4344 .map_reset_reason = efx_ef10_map_reset_reason, 4345 .map_reset_flags = efx_ef10_map_reset_flags, 4346 .reset = efx_ef10_reset, 4347 .probe_port = efx_mcdi_port_probe, 4348 .remove_port = efx_mcdi_port_remove, 4349 .fini_dmaq = efx_fini_dmaq, 4350 .prepare_flr = efx_ef10_prepare_flr, 4351 .finish_flr = efx_port_dummy_op_void, 4352 .describe_stats = efx_ef10_describe_stats, 4353 .update_stats = efx_ef10_update_stats_pf, 4354 .start_stats = efx_mcdi_mac_start_stats, 4355 .pull_stats = efx_mcdi_mac_pull_stats, 4356 .stop_stats = efx_mcdi_mac_stop_stats, 4357 .push_irq_moderation = efx_ef10_push_irq_moderation, 4358 .reconfigure_mac = efx_ef10_mac_reconfigure, 4359 .check_mac_fault = efx_mcdi_mac_check_fault, 4360 .reconfigure_port = efx_mcdi_port_reconfigure, 4361 .get_wol = efx_ef10_get_wol, 4362 .set_wol = efx_ef10_set_wol, 4363 .resume_wol = efx_port_dummy_op_void, 4364 .get_fec_stats = efx_ef10_get_fec_stats, 4365 .test_chip = efx_ef10_test_chip, 4366 .test_nvram = efx_mcdi_nvram_test_all, 4367 .mcdi_request = efx_ef10_mcdi_request, 4368 .mcdi_poll_response = efx_ef10_mcdi_poll_response, 4369 .mcdi_read_response = efx_ef10_mcdi_read_response, 4370 .mcdi_poll_reboot = efx_ef10_mcdi_poll_reboot, 4371 .mcdi_reboot_detected = efx_ef10_mcdi_reboot_detected, 4372 .irq_enable_master = efx_port_dummy_op_void, 4373 .irq_test_generate = efx_ef10_irq_test_generate, 4374 .irq_disable_non_ev = efx_port_dummy_op_void, 4375 .irq_handle_msi = efx_ef10_msi_interrupt, 4376 .tx_probe = efx_ef10_tx_probe, 4377 .tx_init = efx_ef10_tx_init, 4378 .tx_write = efx_ef10_tx_write, 4379 .tx_limit_len = efx_ef10_tx_limit_len, 4380 .tx_enqueue = __efx_enqueue_skb, 4381 .rx_push_rss_config = efx_mcdi_pf_rx_push_rss_config, 4382 .rx_pull_rss_config = efx_mcdi_rx_pull_rss_config, 4383 .rx_push_rss_context_config = efx_mcdi_rx_push_rss_context_config, 4384 .rx_pull_rss_context_config = efx_mcdi_rx_pull_rss_context_config, 4385 .rx_restore_rss_contexts = efx_mcdi_rx_restore_rss_contexts, 4386 .rx_probe = efx_mcdi_rx_probe, 4387 .rx_init = efx_mcdi_rx_init, 4388 .rx_remove = efx_mcdi_rx_remove, 4389 .rx_write = efx_ef10_rx_write, 4390 .rx_defer_refill = efx_ef10_rx_defer_refill, 4391 .rx_packet = __efx_rx_packet, 4392 .ev_probe = efx_mcdi_ev_probe, 4393 .ev_init = efx_ef10_ev_init, 4394 .ev_fini = efx_mcdi_ev_fini, 4395 .ev_remove = efx_mcdi_ev_remove, 4396 .ev_process = efx_ef10_ev_process, 4397 .ev_read_ack = efx_ef10_ev_read_ack, 4398 .ev_test_generate = efx_ef10_ev_test_generate, 4399 .filter_table_probe = efx_ef10_filter_table_probe, 4400 .filter_table_restore = efx_mcdi_filter_table_restore, 4401 .filter_table_remove = efx_ef10_filter_table_remove, 4402 .filter_insert = efx_mcdi_filter_insert, 4403 .filter_remove_safe = efx_mcdi_filter_remove_safe, 4404 .filter_get_safe = efx_mcdi_filter_get_safe, 4405 .filter_clear_rx = efx_mcdi_filter_clear_rx, 4406 .filter_count_rx_used = efx_mcdi_filter_count_rx_used, 4407 .filter_get_rx_id_limit = efx_mcdi_filter_get_rx_id_limit, 4408 .filter_get_rx_ids = efx_mcdi_filter_get_rx_ids, 4409 #ifdef CONFIG_RFS_ACCEL 4410 .filter_rfs_expire_one = efx_mcdi_filter_rfs_expire_one, 4411 #endif 4412 #ifdef CONFIG_SFC_MTD 4413 .mtd_probe = efx_ef10_mtd_probe, 4414 .mtd_rename = efx_mcdi_mtd_rename, 4415 .mtd_read = efx_mcdi_mtd_read, 4416 .mtd_erase = efx_mcdi_mtd_erase, 4417 .mtd_write = efx_mcdi_mtd_write, 4418 .mtd_sync = efx_mcdi_mtd_sync, 4419 #endif 4420 .ptp_write_host_time = efx_ef10_ptp_write_host_time, 4421 .ptp_set_ts_sync_events = efx_ef10_ptp_set_ts_sync_events, 4422 .ptp_set_ts_config = efx_ef10_ptp_set_ts_config, 4423 .vlan_rx_add_vid = efx_ef10_vlan_rx_add_vid, 4424 .vlan_rx_kill_vid = efx_ef10_vlan_rx_kill_vid, 4425 .udp_tnl_push_ports = efx_ef10_udp_tnl_push_ports, 4426 .udp_tnl_has_port = efx_ef10_udp_tnl_has_port, 4427 #ifdef CONFIG_SFC_SRIOV 4428 /* currently set to the VF versions of these functions 4429 * because SRIOV will be reimplemented later. 4430 */ 4431 .vswitching_probe = efx_ef10_vswitching_probe_vf, 4432 .vswitching_restore = efx_ef10_vswitching_restore_vf, 4433 .vswitching_remove = efx_ef10_vswitching_remove_vf, 4434 #endif 4435 .get_mac_address = efx_ef10_get_mac_address_pf, 4436 .set_mac_address = efx_ef10_set_mac_address, 4437 .tso_versions = efx_ef10_tso_versions, 4438 4439 .get_phys_port_id = efx_ef10_get_phys_port_id, 4440 .revision = EFX_REV_X4, 4441 .max_dma_mask = DMA_BIT_MASK(ESF_DZ_TX_KER_BUF_ADDR_WIDTH), 4442 .rx_prefix_size = ES_DZ_RX_PREFIX_SIZE, 4443 .rx_hash_offset = ES_DZ_RX_PREFIX_HASH_OFST, 4444 .rx_ts_offset = ES_DZ_RX_PREFIX_TSTAMP_OFST, 4445 .can_rx_scatter = true, 4446 .always_rx_scatter = true, 4447 .option_descriptors = true, 4448 .flash_auto_partition = true, 4449 .min_interrupt_mode = EFX_INT_MODE_MSIX, 4450 .timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH, 4451 .offload_features = EF10_OFFLOAD_FEATURES, 4452 .mcdi_max_ver = 2, 4453 .max_rx_ip_filters = EFX_MCDI_FILTER_TBL_ROWS, 4454 .hwtstamp_filters = 1 << HWTSTAMP_FILTER_NONE | 4455 1 << HWTSTAMP_FILTER_ALL, 4456 .check_caps = ef10_check_caps, 4457 .print_additional_fwver = efx_ef10_print_additional_fwver, 4458 .sensor_event = efx_mcdi_sensor_event, 4459 .rx_recycle_ring_size = efx_ef10_recycle_ring_size, 4460 }; 4461 4462