1 /**************************************************************************** 2 * Driver for Solarflare network controllers and boards 3 * Copyright 2012-2013 Solarflare Communications Inc. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 as published 7 * by the Free Software Foundation, incorporated herein by reference. 8 */ 9 10 #include "net_driver.h" 11 #include "ef10_regs.h" 12 #include "io.h" 13 #include "mcdi.h" 14 #include "mcdi_pcol.h" 15 #include "nic.h" 16 #include "workarounds.h" 17 #include "selftest.h" 18 #include "ef10_sriov.h" 19 #include <linux/in.h> 20 #include <linux/jhash.h> 21 #include <linux/wait.h> 22 #include <linux/workqueue.h> 23 24 /* Hardware control for EF10 architecture including 'Huntington'. */ 25 26 #define EFX_EF10_DRVGEN_EV 7 27 enum { 28 EFX_EF10_TEST = 1, 29 EFX_EF10_REFILL, 30 }; 31 /* The maximum size of a shared RSS context */ 32 /* TODO: this should really be from the mcdi protocol export */ 33 #define EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE 64UL 34 35 /* The filter table(s) are managed by firmware and we have write-only 36 * access. When removing filters we must identify them to the 37 * firmware by a 64-bit handle, but this is too wide for Linux kernel 38 * interfaces (32-bit for RX NFC, 16-bit for RFS). Also, we need to 39 * be able to tell in advance whether a requested insertion will 40 * replace an existing filter. Therefore we maintain a software hash 41 * table, which should be at least as large as the hardware hash 42 * table. 43 * 44 * Huntington has a single 8K filter table shared between all filter 45 * types and both ports. 46 */ 47 #define HUNT_FILTER_TBL_ROWS 8192 48 49 #define EFX_EF10_FILTER_ID_INVALID 0xffff 50 51 #define EFX_EF10_FILTER_DEV_UC_MAX 32 52 #define EFX_EF10_FILTER_DEV_MC_MAX 256 53 54 /* VLAN list entry */ 55 struct efx_ef10_vlan { 56 struct list_head list; 57 u16 vid; 58 }; 59 60 enum efx_ef10_default_filters { 61 EFX_EF10_BCAST, 62 EFX_EF10_UCDEF, 63 EFX_EF10_MCDEF, 64 EFX_EF10_VXLAN4_UCDEF, 65 EFX_EF10_VXLAN4_MCDEF, 66 EFX_EF10_VXLAN6_UCDEF, 67 EFX_EF10_VXLAN6_MCDEF, 68 EFX_EF10_NVGRE4_UCDEF, 69 EFX_EF10_NVGRE4_MCDEF, 70 EFX_EF10_NVGRE6_UCDEF, 71 EFX_EF10_NVGRE6_MCDEF, 72 EFX_EF10_GENEVE4_UCDEF, 73 EFX_EF10_GENEVE4_MCDEF, 74 EFX_EF10_GENEVE6_UCDEF, 75 EFX_EF10_GENEVE6_MCDEF, 76 77 EFX_EF10_NUM_DEFAULT_FILTERS 78 }; 79 80 /* Per-VLAN filters information */ 81 struct efx_ef10_filter_vlan { 82 struct list_head list; 83 u16 vid; 84 u16 uc[EFX_EF10_FILTER_DEV_UC_MAX]; 85 u16 mc[EFX_EF10_FILTER_DEV_MC_MAX]; 86 u16 default_filters[EFX_EF10_NUM_DEFAULT_FILTERS]; 87 }; 88 89 struct efx_ef10_dev_addr { 90 u8 addr[ETH_ALEN]; 91 }; 92 93 struct efx_ef10_filter_table { 94 /* The MCDI match masks supported by this fw & hw, in order of priority */ 95 u32 rx_match_mcdi_flags[ 96 MC_CMD_GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES_MAXNUM * 2]; 97 unsigned int rx_match_count; 98 99 struct rw_semaphore lock; /* Protects entries */ 100 struct { 101 unsigned long spec; /* pointer to spec plus flag bits */ 102 /* AUTO_OLD is used to mark and sweep MAC filters for the device address lists. */ 103 /* unused flag 1UL */ 104 #define EFX_EF10_FILTER_FLAG_AUTO_OLD 2UL 105 #define EFX_EF10_FILTER_FLAGS 3UL 106 u64 handle; /* firmware handle */ 107 } *entry; 108 /* Shadow of net_device address lists, guarded by mac_lock */ 109 struct efx_ef10_dev_addr dev_uc_list[EFX_EF10_FILTER_DEV_UC_MAX]; 110 struct efx_ef10_dev_addr dev_mc_list[EFX_EF10_FILTER_DEV_MC_MAX]; 111 int dev_uc_count; 112 int dev_mc_count; 113 bool uc_promisc; 114 bool mc_promisc; 115 /* Whether in multicast promiscuous mode when last changed */ 116 bool mc_promisc_last; 117 bool mc_overflow; /* Too many MC addrs; should always imply mc_promisc */ 118 bool vlan_filter; 119 struct list_head vlan_list; 120 }; 121 122 /* An arbitrary search limit for the software hash table */ 123 #define EFX_EF10_FILTER_SEARCH_LIMIT 200 124 125 static void efx_ef10_rx_free_indir_table(struct efx_nic *efx); 126 static void efx_ef10_filter_table_remove(struct efx_nic *efx); 127 static int efx_ef10_filter_add_vlan(struct efx_nic *efx, u16 vid); 128 static void efx_ef10_filter_del_vlan_internal(struct efx_nic *efx, 129 struct efx_ef10_filter_vlan *vlan); 130 static void efx_ef10_filter_del_vlan(struct efx_nic *efx, u16 vid); 131 static int efx_ef10_set_udp_tnl_ports(struct efx_nic *efx, bool unloading); 132 133 static u32 efx_ef10_filter_get_unsafe_id(u32 filter_id) 134 { 135 WARN_ON_ONCE(filter_id == EFX_EF10_FILTER_ID_INVALID); 136 return filter_id & (HUNT_FILTER_TBL_ROWS - 1); 137 } 138 139 static unsigned int efx_ef10_filter_get_unsafe_pri(u32 filter_id) 140 { 141 return filter_id / (HUNT_FILTER_TBL_ROWS * 2); 142 } 143 144 static u32 efx_ef10_make_filter_id(unsigned int pri, u16 idx) 145 { 146 return pri * HUNT_FILTER_TBL_ROWS * 2 + idx; 147 } 148 149 static int efx_ef10_get_warm_boot_count(struct efx_nic *efx) 150 { 151 efx_dword_t reg; 152 153 efx_readd(efx, ®, ER_DZ_BIU_MC_SFT_STATUS); 154 return EFX_DWORD_FIELD(reg, EFX_WORD_1) == 0xb007 ? 155 EFX_DWORD_FIELD(reg, EFX_WORD_0) : -EIO; 156 } 157 158 /* On all EF10s up to and including SFC9220 (Medford1), all PFs use BAR 0 for 159 * I/O space and BAR 2(&3) for memory. On SFC9250 (Medford2), there is no I/O 160 * bar; PFs use BAR 0/1 for memory. 161 */ 162 static unsigned int efx_ef10_pf_mem_bar(struct efx_nic *efx) 163 { 164 switch (efx->pci_dev->device) { 165 case 0x0b03: /* SFC9250 PF */ 166 return 0; 167 default: 168 return 2; 169 } 170 } 171 172 /* All VFs use BAR 0/1 for memory */ 173 static unsigned int efx_ef10_vf_mem_bar(struct efx_nic *efx) 174 { 175 return 0; 176 } 177 178 static unsigned int efx_ef10_mem_map_size(struct efx_nic *efx) 179 { 180 int bar; 181 182 bar = efx->type->mem_bar(efx); 183 return resource_size(&efx->pci_dev->resource[bar]); 184 } 185 186 static bool efx_ef10_is_vf(struct efx_nic *efx) 187 { 188 return efx->type->is_vf; 189 } 190 191 static int efx_ef10_get_pf_index(struct efx_nic *efx) 192 { 193 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_FUNCTION_INFO_OUT_LEN); 194 struct efx_ef10_nic_data *nic_data = efx->nic_data; 195 size_t outlen; 196 int rc; 197 198 rc = efx_mcdi_rpc(efx, MC_CMD_GET_FUNCTION_INFO, NULL, 0, outbuf, 199 sizeof(outbuf), &outlen); 200 if (rc) 201 return rc; 202 if (outlen < sizeof(outbuf)) 203 return -EIO; 204 205 nic_data->pf_index = MCDI_DWORD(outbuf, GET_FUNCTION_INFO_OUT_PF); 206 return 0; 207 } 208 209 #ifdef CONFIG_SFC_SRIOV 210 static int efx_ef10_get_vf_index(struct efx_nic *efx) 211 { 212 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_FUNCTION_INFO_OUT_LEN); 213 struct efx_ef10_nic_data *nic_data = efx->nic_data; 214 size_t outlen; 215 int rc; 216 217 rc = efx_mcdi_rpc(efx, MC_CMD_GET_FUNCTION_INFO, NULL, 0, outbuf, 218 sizeof(outbuf), &outlen); 219 if (rc) 220 return rc; 221 if (outlen < sizeof(outbuf)) 222 return -EIO; 223 224 nic_data->vf_index = MCDI_DWORD(outbuf, GET_FUNCTION_INFO_OUT_VF); 225 return 0; 226 } 227 #endif 228 229 static int efx_ef10_init_datapath_caps(struct efx_nic *efx) 230 { 231 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CAPABILITIES_V4_OUT_LEN); 232 struct efx_ef10_nic_data *nic_data = efx->nic_data; 233 size_t outlen; 234 int rc; 235 236 BUILD_BUG_ON(MC_CMD_GET_CAPABILITIES_IN_LEN != 0); 237 238 rc = efx_mcdi_rpc(efx, MC_CMD_GET_CAPABILITIES, NULL, 0, 239 outbuf, sizeof(outbuf), &outlen); 240 if (rc) 241 return rc; 242 if (outlen < MC_CMD_GET_CAPABILITIES_OUT_LEN) { 243 netif_err(efx, drv, efx->net_dev, 244 "unable to read datapath firmware capabilities\n"); 245 return -EIO; 246 } 247 248 nic_data->datapath_caps = 249 MCDI_DWORD(outbuf, GET_CAPABILITIES_OUT_FLAGS1); 250 251 if (outlen >= MC_CMD_GET_CAPABILITIES_V2_OUT_LEN) { 252 nic_data->datapath_caps2 = MCDI_DWORD(outbuf, 253 GET_CAPABILITIES_V2_OUT_FLAGS2); 254 nic_data->piobuf_size = MCDI_WORD(outbuf, 255 GET_CAPABILITIES_V2_OUT_SIZE_PIO_BUFF); 256 } else { 257 nic_data->datapath_caps2 = 0; 258 nic_data->piobuf_size = ER_DZ_TX_PIOBUF_SIZE; 259 } 260 261 /* record the DPCPU firmware IDs to determine VEB vswitching support. 262 */ 263 nic_data->rx_dpcpu_fw_id = 264 MCDI_WORD(outbuf, GET_CAPABILITIES_OUT_RX_DPCPU_FW_ID); 265 nic_data->tx_dpcpu_fw_id = 266 MCDI_WORD(outbuf, GET_CAPABILITIES_OUT_TX_DPCPU_FW_ID); 267 268 if (!(nic_data->datapath_caps & 269 (1 << MC_CMD_GET_CAPABILITIES_OUT_RX_PREFIX_LEN_14_LBN))) { 270 netif_err(efx, probe, efx->net_dev, 271 "current firmware does not support an RX prefix\n"); 272 return -ENODEV; 273 } 274 275 if (outlen >= MC_CMD_GET_CAPABILITIES_V3_OUT_LEN) { 276 u8 vi_window_mode = MCDI_BYTE(outbuf, 277 GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE); 278 279 switch (vi_window_mode) { 280 case MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_8K: 281 efx->vi_stride = 8192; 282 break; 283 case MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_16K: 284 efx->vi_stride = 16384; 285 break; 286 case MC_CMD_GET_CAPABILITIES_V3_OUT_VI_WINDOW_MODE_64K: 287 efx->vi_stride = 65536; 288 break; 289 default: 290 netif_err(efx, probe, efx->net_dev, 291 "Unrecognised VI window mode %d\n", 292 vi_window_mode); 293 return -EIO; 294 } 295 netif_dbg(efx, probe, efx->net_dev, "vi_stride = %u\n", 296 efx->vi_stride); 297 } else { 298 /* keep default VI stride */ 299 netif_dbg(efx, probe, efx->net_dev, 300 "firmware did not report VI window mode, assuming vi_stride = %u\n", 301 efx->vi_stride); 302 } 303 304 if (outlen >= MC_CMD_GET_CAPABILITIES_V4_OUT_LEN) { 305 efx->num_mac_stats = MCDI_WORD(outbuf, 306 GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS); 307 netif_dbg(efx, probe, efx->net_dev, 308 "firmware reports num_mac_stats = %u\n", 309 efx->num_mac_stats); 310 } else { 311 /* leave num_mac_stats as the default value, MC_CMD_MAC_NSTATS */ 312 netif_dbg(efx, probe, efx->net_dev, 313 "firmware did not report num_mac_stats, assuming %u\n", 314 efx->num_mac_stats); 315 } 316 317 return 0; 318 } 319 320 static void efx_ef10_read_licensed_features(struct efx_nic *efx) 321 { 322 MCDI_DECLARE_BUF(inbuf, MC_CMD_LICENSING_V3_IN_LEN); 323 MCDI_DECLARE_BUF(outbuf, MC_CMD_LICENSING_V3_OUT_LEN); 324 struct efx_ef10_nic_data *nic_data = efx->nic_data; 325 size_t outlen; 326 int rc; 327 328 MCDI_SET_DWORD(inbuf, LICENSING_V3_IN_OP, 329 MC_CMD_LICENSING_V3_IN_OP_REPORT_LICENSE); 330 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_LICENSING_V3, inbuf, sizeof(inbuf), 331 outbuf, sizeof(outbuf), &outlen); 332 if (rc || (outlen < MC_CMD_LICENSING_V3_OUT_LEN)) 333 return; 334 335 nic_data->licensed_features = MCDI_QWORD(outbuf, 336 LICENSING_V3_OUT_LICENSED_FEATURES); 337 } 338 339 static int efx_ef10_get_sysclk_freq(struct efx_nic *efx) 340 { 341 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_CLOCK_OUT_LEN); 342 int rc; 343 344 rc = efx_mcdi_rpc(efx, MC_CMD_GET_CLOCK, NULL, 0, 345 outbuf, sizeof(outbuf), NULL); 346 if (rc) 347 return rc; 348 rc = MCDI_DWORD(outbuf, GET_CLOCK_OUT_SYS_FREQ); 349 return rc > 0 ? rc : -ERANGE; 350 } 351 352 static int efx_ef10_get_timer_workarounds(struct efx_nic *efx) 353 { 354 struct efx_ef10_nic_data *nic_data = efx->nic_data; 355 unsigned int implemented; 356 unsigned int enabled; 357 int rc; 358 359 nic_data->workaround_35388 = false; 360 nic_data->workaround_61265 = false; 361 362 rc = efx_mcdi_get_workarounds(efx, &implemented, &enabled); 363 364 if (rc == -ENOSYS) { 365 /* Firmware without GET_WORKAROUNDS - not a problem. */ 366 rc = 0; 367 } else if (rc == 0) { 368 /* Bug61265 workaround is always enabled if implemented. */ 369 if (enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG61265) 370 nic_data->workaround_61265 = true; 371 372 if (enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG35388) { 373 nic_data->workaround_35388 = true; 374 } else if (implemented & MC_CMD_GET_WORKAROUNDS_OUT_BUG35388) { 375 /* Workaround is implemented but not enabled. 376 * Try to enable it. 377 */ 378 rc = efx_mcdi_set_workaround(efx, 379 MC_CMD_WORKAROUND_BUG35388, 380 true, NULL); 381 if (rc == 0) 382 nic_data->workaround_35388 = true; 383 /* If we failed to set the workaround just carry on. */ 384 rc = 0; 385 } 386 } 387 388 netif_dbg(efx, probe, efx->net_dev, 389 "workaround for bug 35388 is %sabled\n", 390 nic_data->workaround_35388 ? "en" : "dis"); 391 netif_dbg(efx, probe, efx->net_dev, 392 "workaround for bug 61265 is %sabled\n", 393 nic_data->workaround_61265 ? "en" : "dis"); 394 395 return rc; 396 } 397 398 static void efx_ef10_process_timer_config(struct efx_nic *efx, 399 const efx_dword_t *data) 400 { 401 unsigned int max_count; 402 403 if (EFX_EF10_WORKAROUND_61265(efx)) { 404 efx->timer_quantum_ns = MCDI_DWORD(data, 405 GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_STEP_NS); 406 efx->timer_max_ns = MCDI_DWORD(data, 407 GET_EVQ_TMR_PROPERTIES_OUT_MCDI_TMR_MAX_NS); 408 } else if (EFX_EF10_WORKAROUND_35388(efx)) { 409 efx->timer_quantum_ns = MCDI_DWORD(data, 410 GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_NS_PER_COUNT); 411 max_count = MCDI_DWORD(data, 412 GET_EVQ_TMR_PROPERTIES_OUT_BUG35388_TMR_MAX_COUNT); 413 efx->timer_max_ns = max_count * efx->timer_quantum_ns; 414 } else { 415 efx->timer_quantum_ns = MCDI_DWORD(data, 416 GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_NS_PER_COUNT); 417 max_count = MCDI_DWORD(data, 418 GET_EVQ_TMR_PROPERTIES_OUT_TMR_REG_MAX_COUNT); 419 efx->timer_max_ns = max_count * efx->timer_quantum_ns; 420 } 421 422 netif_dbg(efx, probe, efx->net_dev, 423 "got timer properties from MC: quantum %u ns; max %u ns\n", 424 efx->timer_quantum_ns, efx->timer_max_ns); 425 } 426 427 static int efx_ef10_get_timer_config(struct efx_nic *efx) 428 { 429 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN); 430 int rc; 431 432 rc = efx_ef10_get_timer_workarounds(efx); 433 if (rc) 434 return rc; 435 436 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_GET_EVQ_TMR_PROPERTIES, NULL, 0, 437 outbuf, sizeof(outbuf), NULL); 438 439 if (rc == 0) { 440 efx_ef10_process_timer_config(efx, outbuf); 441 } else if (rc == -ENOSYS || rc == -EPERM) { 442 /* Not available - fall back to Huntington defaults. */ 443 unsigned int quantum; 444 445 rc = efx_ef10_get_sysclk_freq(efx); 446 if (rc < 0) 447 return rc; 448 449 quantum = 1536000 / rc; /* 1536 cycles */ 450 efx->timer_quantum_ns = quantum; 451 efx->timer_max_ns = efx->type->timer_period_max * quantum; 452 rc = 0; 453 } else { 454 efx_mcdi_display_error(efx, MC_CMD_GET_EVQ_TMR_PROPERTIES, 455 MC_CMD_GET_EVQ_TMR_PROPERTIES_OUT_LEN, 456 NULL, 0, rc); 457 } 458 459 return rc; 460 } 461 462 static int efx_ef10_get_mac_address_pf(struct efx_nic *efx, u8 *mac_address) 463 { 464 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_MAC_ADDRESSES_OUT_LEN); 465 size_t outlen; 466 int rc; 467 468 BUILD_BUG_ON(MC_CMD_GET_MAC_ADDRESSES_IN_LEN != 0); 469 470 rc = efx_mcdi_rpc(efx, MC_CMD_GET_MAC_ADDRESSES, NULL, 0, 471 outbuf, sizeof(outbuf), &outlen); 472 if (rc) 473 return rc; 474 if (outlen < MC_CMD_GET_MAC_ADDRESSES_OUT_LEN) 475 return -EIO; 476 477 ether_addr_copy(mac_address, 478 MCDI_PTR(outbuf, GET_MAC_ADDRESSES_OUT_MAC_ADDR_BASE)); 479 return 0; 480 } 481 482 static int efx_ef10_get_mac_address_vf(struct efx_nic *efx, u8 *mac_address) 483 { 484 MCDI_DECLARE_BUF(inbuf, MC_CMD_VPORT_GET_MAC_ADDRESSES_IN_LEN); 485 MCDI_DECLARE_BUF(outbuf, MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMAX); 486 size_t outlen; 487 int num_addrs, rc; 488 489 MCDI_SET_DWORD(inbuf, VPORT_GET_MAC_ADDRESSES_IN_VPORT_ID, 490 EVB_PORT_ID_ASSIGNED); 491 rc = efx_mcdi_rpc(efx, MC_CMD_VPORT_GET_MAC_ADDRESSES, inbuf, 492 sizeof(inbuf), outbuf, sizeof(outbuf), &outlen); 493 494 if (rc) 495 return rc; 496 if (outlen < MC_CMD_VPORT_GET_MAC_ADDRESSES_OUT_LENMIN) 497 return -EIO; 498 499 num_addrs = MCDI_DWORD(outbuf, 500 VPORT_GET_MAC_ADDRESSES_OUT_MACADDR_COUNT); 501 502 WARN_ON(num_addrs != 1); 503 504 ether_addr_copy(mac_address, 505 MCDI_PTR(outbuf, VPORT_GET_MAC_ADDRESSES_OUT_MACADDR)); 506 507 return 0; 508 } 509 510 static ssize_t efx_ef10_show_link_control_flag(struct device *dev, 511 struct device_attribute *attr, 512 char *buf) 513 { 514 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 515 516 return sprintf(buf, "%d\n", 517 ((efx->mcdi->fn_flags) & 518 (1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL)) 519 ? 1 : 0); 520 } 521 522 static ssize_t efx_ef10_show_primary_flag(struct device *dev, 523 struct device_attribute *attr, 524 char *buf) 525 { 526 struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev)); 527 528 return sprintf(buf, "%d\n", 529 ((efx->mcdi->fn_flags) & 530 (1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_PRIMARY)) 531 ? 1 : 0); 532 } 533 534 static struct efx_ef10_vlan *efx_ef10_find_vlan(struct efx_nic *efx, u16 vid) 535 { 536 struct efx_ef10_nic_data *nic_data = efx->nic_data; 537 struct efx_ef10_vlan *vlan; 538 539 WARN_ON(!mutex_is_locked(&nic_data->vlan_lock)); 540 541 list_for_each_entry(vlan, &nic_data->vlan_list, list) { 542 if (vlan->vid == vid) 543 return vlan; 544 } 545 546 return NULL; 547 } 548 549 static int efx_ef10_add_vlan(struct efx_nic *efx, u16 vid) 550 { 551 struct efx_ef10_nic_data *nic_data = efx->nic_data; 552 struct efx_ef10_vlan *vlan; 553 int rc; 554 555 mutex_lock(&nic_data->vlan_lock); 556 557 vlan = efx_ef10_find_vlan(efx, vid); 558 if (vlan) { 559 /* We add VID 0 on init. 8021q adds it on module init 560 * for all interfaces with VLAN filtring feature. 561 */ 562 if (vid == 0) 563 goto done_unlock; 564 netif_warn(efx, drv, efx->net_dev, 565 "VLAN %u already added\n", vid); 566 rc = -EALREADY; 567 goto fail_exist; 568 } 569 570 rc = -ENOMEM; 571 vlan = kzalloc(sizeof(*vlan), GFP_KERNEL); 572 if (!vlan) 573 goto fail_alloc; 574 575 vlan->vid = vid; 576 577 list_add_tail(&vlan->list, &nic_data->vlan_list); 578 579 if (efx->filter_state) { 580 mutex_lock(&efx->mac_lock); 581 down_write(&efx->filter_sem); 582 rc = efx_ef10_filter_add_vlan(efx, vlan->vid); 583 up_write(&efx->filter_sem); 584 mutex_unlock(&efx->mac_lock); 585 if (rc) 586 goto fail_filter_add_vlan; 587 } 588 589 done_unlock: 590 mutex_unlock(&nic_data->vlan_lock); 591 return 0; 592 593 fail_filter_add_vlan: 594 list_del(&vlan->list); 595 kfree(vlan); 596 fail_alloc: 597 fail_exist: 598 mutex_unlock(&nic_data->vlan_lock); 599 return rc; 600 } 601 602 static void efx_ef10_del_vlan_internal(struct efx_nic *efx, 603 struct efx_ef10_vlan *vlan) 604 { 605 struct efx_ef10_nic_data *nic_data = efx->nic_data; 606 607 WARN_ON(!mutex_is_locked(&nic_data->vlan_lock)); 608 609 if (efx->filter_state) { 610 down_write(&efx->filter_sem); 611 efx_ef10_filter_del_vlan(efx, vlan->vid); 612 up_write(&efx->filter_sem); 613 } 614 615 list_del(&vlan->list); 616 kfree(vlan); 617 } 618 619 static int efx_ef10_del_vlan(struct efx_nic *efx, u16 vid) 620 { 621 struct efx_ef10_nic_data *nic_data = efx->nic_data; 622 struct efx_ef10_vlan *vlan; 623 int rc = 0; 624 625 /* 8021q removes VID 0 on module unload for all interfaces 626 * with VLAN filtering feature. We need to keep it to receive 627 * untagged traffic. 628 */ 629 if (vid == 0) 630 return 0; 631 632 mutex_lock(&nic_data->vlan_lock); 633 634 vlan = efx_ef10_find_vlan(efx, vid); 635 if (!vlan) { 636 netif_err(efx, drv, efx->net_dev, 637 "VLAN %u to be deleted not found\n", vid); 638 rc = -ENOENT; 639 } else { 640 efx_ef10_del_vlan_internal(efx, vlan); 641 } 642 643 mutex_unlock(&nic_data->vlan_lock); 644 645 return rc; 646 } 647 648 static void efx_ef10_cleanup_vlans(struct efx_nic *efx) 649 { 650 struct efx_ef10_nic_data *nic_data = efx->nic_data; 651 struct efx_ef10_vlan *vlan, *next_vlan; 652 653 mutex_lock(&nic_data->vlan_lock); 654 list_for_each_entry_safe(vlan, next_vlan, &nic_data->vlan_list, list) 655 efx_ef10_del_vlan_internal(efx, vlan); 656 mutex_unlock(&nic_data->vlan_lock); 657 } 658 659 static DEVICE_ATTR(link_control_flag, 0444, efx_ef10_show_link_control_flag, 660 NULL); 661 static DEVICE_ATTR(primary_flag, 0444, efx_ef10_show_primary_flag, NULL); 662 663 static int efx_ef10_probe(struct efx_nic *efx) 664 { 665 struct efx_ef10_nic_data *nic_data; 666 int i, rc; 667 668 nic_data = kzalloc(sizeof(*nic_data), GFP_KERNEL); 669 if (!nic_data) 670 return -ENOMEM; 671 efx->nic_data = nic_data; 672 673 /* we assume later that we can copy from this buffer in dwords */ 674 BUILD_BUG_ON(MCDI_CTL_SDU_LEN_MAX_V2 % 4); 675 676 rc = efx_nic_alloc_buffer(efx, &nic_data->mcdi_buf, 677 8 + MCDI_CTL_SDU_LEN_MAX_V2, GFP_KERNEL); 678 if (rc) 679 goto fail1; 680 681 /* Get the MC's warm boot count. In case it's rebooting right 682 * now, be prepared to retry. 683 */ 684 i = 0; 685 for (;;) { 686 rc = efx_ef10_get_warm_boot_count(efx); 687 if (rc >= 0) 688 break; 689 if (++i == 5) 690 goto fail2; 691 ssleep(1); 692 } 693 nic_data->warm_boot_count = rc; 694 695 efx->rss_context.context_id = EFX_EF10_RSS_CONTEXT_INVALID; 696 697 nic_data->vport_id = EVB_PORT_ID_ASSIGNED; 698 699 /* In case we're recovering from a crash (kexec), we want to 700 * cancel any outstanding request by the previous user of this 701 * function. We send a special message using the least 702 * significant bits of the 'high' (doorbell) register. 703 */ 704 _efx_writed(efx, cpu_to_le32(1), ER_DZ_MC_DB_HWRD); 705 706 rc = efx_mcdi_init(efx); 707 if (rc) 708 goto fail2; 709 710 mutex_init(&nic_data->udp_tunnels_lock); 711 712 /* Reset (most) configuration for this function */ 713 rc = efx_mcdi_reset(efx, RESET_TYPE_ALL); 714 if (rc) 715 goto fail3; 716 717 /* Enable event logging */ 718 rc = efx_mcdi_log_ctrl(efx, true, false, 0); 719 if (rc) 720 goto fail3; 721 722 rc = device_create_file(&efx->pci_dev->dev, 723 &dev_attr_link_control_flag); 724 if (rc) 725 goto fail3; 726 727 rc = device_create_file(&efx->pci_dev->dev, &dev_attr_primary_flag); 728 if (rc) 729 goto fail4; 730 731 rc = efx_ef10_get_pf_index(efx); 732 if (rc) 733 goto fail5; 734 735 rc = efx_ef10_init_datapath_caps(efx); 736 if (rc < 0) 737 goto fail5; 738 739 efx_ef10_read_licensed_features(efx); 740 741 /* We can have one VI for each vi_stride-byte region. 742 * However, until we use TX option descriptors we need two TX queues 743 * per channel. 744 */ 745 efx->max_channels = min_t(unsigned int, 746 EFX_MAX_CHANNELS, 747 efx_ef10_mem_map_size(efx) / 748 (efx->vi_stride * EFX_TXQ_TYPES)); 749 efx->max_tx_channels = efx->max_channels; 750 if (WARN_ON(efx->max_channels == 0)) { 751 rc = -EIO; 752 goto fail5; 753 } 754 755 efx->rx_packet_len_offset = 756 ES_DZ_RX_PREFIX_PKTLEN_OFST - ES_DZ_RX_PREFIX_SIZE; 757 758 if (nic_data->datapath_caps & 759 (1 << MC_CMD_GET_CAPABILITIES_OUT_RX_INCLUDE_FCS_LBN)) 760 efx->net_dev->hw_features |= NETIF_F_RXFCS; 761 762 rc = efx_mcdi_port_get_number(efx); 763 if (rc < 0) 764 goto fail5; 765 efx->port_num = rc; 766 767 rc = efx->type->get_mac_address(efx, efx->net_dev->perm_addr); 768 if (rc) 769 goto fail5; 770 771 rc = efx_ef10_get_timer_config(efx); 772 if (rc < 0) 773 goto fail5; 774 775 rc = efx_mcdi_mon_probe(efx); 776 if (rc && rc != -EPERM) 777 goto fail5; 778 779 efx_ptp_defer_probe_with_channel(efx); 780 781 #ifdef CONFIG_SFC_SRIOV 782 if ((efx->pci_dev->physfn) && (!efx->pci_dev->is_physfn)) { 783 struct pci_dev *pci_dev_pf = efx->pci_dev->physfn; 784 struct efx_nic *efx_pf = pci_get_drvdata(pci_dev_pf); 785 786 efx_pf->type->get_mac_address(efx_pf, nic_data->port_id); 787 } else 788 #endif 789 ether_addr_copy(nic_data->port_id, efx->net_dev->perm_addr); 790 791 INIT_LIST_HEAD(&nic_data->vlan_list); 792 mutex_init(&nic_data->vlan_lock); 793 794 /* Add unspecified VID to support VLAN filtering being disabled */ 795 rc = efx_ef10_add_vlan(efx, EFX_FILTER_VID_UNSPEC); 796 if (rc) 797 goto fail_add_vid_unspec; 798 799 /* If VLAN filtering is enabled, we need VID 0 to get untagged 800 * traffic. It is added automatically if 8021q module is loaded, 801 * but we can't rely on it since module may be not loaded. 802 */ 803 rc = efx_ef10_add_vlan(efx, 0); 804 if (rc) 805 goto fail_add_vid_0; 806 807 return 0; 808 809 fail_add_vid_0: 810 efx_ef10_cleanup_vlans(efx); 811 fail_add_vid_unspec: 812 mutex_destroy(&nic_data->vlan_lock); 813 efx_ptp_remove(efx); 814 efx_mcdi_mon_remove(efx); 815 fail5: 816 device_remove_file(&efx->pci_dev->dev, &dev_attr_primary_flag); 817 fail4: 818 device_remove_file(&efx->pci_dev->dev, &dev_attr_link_control_flag); 819 fail3: 820 efx_mcdi_detach(efx); 821 822 mutex_lock(&nic_data->udp_tunnels_lock); 823 memset(nic_data->udp_tunnels, 0, sizeof(nic_data->udp_tunnels)); 824 (void)efx_ef10_set_udp_tnl_ports(efx, true); 825 mutex_unlock(&nic_data->udp_tunnels_lock); 826 mutex_destroy(&nic_data->udp_tunnels_lock); 827 828 efx_mcdi_fini(efx); 829 fail2: 830 efx_nic_free_buffer(efx, &nic_data->mcdi_buf); 831 fail1: 832 kfree(nic_data); 833 efx->nic_data = NULL; 834 return rc; 835 } 836 837 static int efx_ef10_free_vis(struct efx_nic *efx) 838 { 839 MCDI_DECLARE_BUF_ERR(outbuf); 840 size_t outlen; 841 int rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FREE_VIS, NULL, 0, 842 outbuf, sizeof(outbuf), &outlen); 843 844 /* -EALREADY means nothing to free, so ignore */ 845 if (rc == -EALREADY) 846 rc = 0; 847 if (rc) 848 efx_mcdi_display_error(efx, MC_CMD_FREE_VIS, 0, outbuf, outlen, 849 rc); 850 return rc; 851 } 852 853 #ifdef EFX_USE_PIO 854 855 static void efx_ef10_free_piobufs(struct efx_nic *efx) 856 { 857 struct efx_ef10_nic_data *nic_data = efx->nic_data; 858 MCDI_DECLARE_BUF(inbuf, MC_CMD_FREE_PIOBUF_IN_LEN); 859 unsigned int i; 860 int rc; 861 862 BUILD_BUG_ON(MC_CMD_FREE_PIOBUF_OUT_LEN != 0); 863 864 for (i = 0; i < nic_data->n_piobufs; i++) { 865 MCDI_SET_DWORD(inbuf, FREE_PIOBUF_IN_PIOBUF_HANDLE, 866 nic_data->piobuf_handle[i]); 867 rc = efx_mcdi_rpc(efx, MC_CMD_FREE_PIOBUF, inbuf, sizeof(inbuf), 868 NULL, 0, NULL); 869 WARN_ON(rc); 870 } 871 872 nic_data->n_piobufs = 0; 873 } 874 875 static int efx_ef10_alloc_piobufs(struct efx_nic *efx, unsigned int n) 876 { 877 struct efx_ef10_nic_data *nic_data = efx->nic_data; 878 MCDI_DECLARE_BUF(outbuf, MC_CMD_ALLOC_PIOBUF_OUT_LEN); 879 unsigned int i; 880 size_t outlen; 881 int rc = 0; 882 883 BUILD_BUG_ON(MC_CMD_ALLOC_PIOBUF_IN_LEN != 0); 884 885 for (i = 0; i < n; i++) { 886 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_ALLOC_PIOBUF, NULL, 0, 887 outbuf, sizeof(outbuf), &outlen); 888 if (rc) { 889 /* Don't display the MC error if we didn't have space 890 * for a VF. 891 */ 892 if (!(efx_ef10_is_vf(efx) && rc == -ENOSPC)) 893 efx_mcdi_display_error(efx, MC_CMD_ALLOC_PIOBUF, 894 0, outbuf, outlen, rc); 895 break; 896 } 897 if (outlen < MC_CMD_ALLOC_PIOBUF_OUT_LEN) { 898 rc = -EIO; 899 break; 900 } 901 nic_data->piobuf_handle[i] = 902 MCDI_DWORD(outbuf, ALLOC_PIOBUF_OUT_PIOBUF_HANDLE); 903 netif_dbg(efx, probe, efx->net_dev, 904 "allocated PIO buffer %u handle %x\n", i, 905 nic_data->piobuf_handle[i]); 906 } 907 908 nic_data->n_piobufs = i; 909 if (rc) 910 efx_ef10_free_piobufs(efx); 911 return rc; 912 } 913 914 static int efx_ef10_link_piobufs(struct efx_nic *efx) 915 { 916 struct efx_ef10_nic_data *nic_data = efx->nic_data; 917 MCDI_DECLARE_BUF(inbuf, MC_CMD_LINK_PIOBUF_IN_LEN); 918 struct efx_channel *channel; 919 struct efx_tx_queue *tx_queue; 920 unsigned int offset, index; 921 int rc; 922 923 BUILD_BUG_ON(MC_CMD_LINK_PIOBUF_OUT_LEN != 0); 924 BUILD_BUG_ON(MC_CMD_UNLINK_PIOBUF_OUT_LEN != 0); 925 926 /* Link a buffer to each VI in the write-combining mapping */ 927 for (index = 0; index < nic_data->n_piobufs; ++index) { 928 MCDI_SET_DWORD(inbuf, LINK_PIOBUF_IN_PIOBUF_HANDLE, 929 nic_data->piobuf_handle[index]); 930 MCDI_SET_DWORD(inbuf, LINK_PIOBUF_IN_TXQ_INSTANCE, 931 nic_data->pio_write_vi_base + index); 932 rc = efx_mcdi_rpc(efx, MC_CMD_LINK_PIOBUF, 933 inbuf, MC_CMD_LINK_PIOBUF_IN_LEN, 934 NULL, 0, NULL); 935 if (rc) { 936 netif_err(efx, drv, efx->net_dev, 937 "failed to link VI %u to PIO buffer %u (%d)\n", 938 nic_data->pio_write_vi_base + index, index, 939 rc); 940 goto fail; 941 } 942 netif_dbg(efx, probe, efx->net_dev, 943 "linked VI %u to PIO buffer %u\n", 944 nic_data->pio_write_vi_base + index, index); 945 } 946 947 /* Link a buffer to each TX queue */ 948 efx_for_each_channel(channel, efx) { 949 /* Extra channels, even those with TXQs (PTP), do not require 950 * PIO resources. 951 */ 952 if (!channel->type->want_pio) 953 continue; 954 efx_for_each_channel_tx_queue(tx_queue, channel) { 955 /* We assign the PIO buffers to queues in 956 * reverse order to allow for the following 957 * special case. 958 */ 959 offset = ((efx->tx_channel_offset + efx->n_tx_channels - 960 tx_queue->channel->channel - 1) * 961 efx_piobuf_size); 962 index = offset / nic_data->piobuf_size; 963 offset = offset % nic_data->piobuf_size; 964 965 /* When the host page size is 4K, the first 966 * host page in the WC mapping may be within 967 * the same VI page as the last TX queue. We 968 * can only link one buffer to each VI. 969 */ 970 if (tx_queue->queue == nic_data->pio_write_vi_base) { 971 BUG_ON(index != 0); 972 rc = 0; 973 } else { 974 MCDI_SET_DWORD(inbuf, 975 LINK_PIOBUF_IN_PIOBUF_HANDLE, 976 nic_data->piobuf_handle[index]); 977 MCDI_SET_DWORD(inbuf, 978 LINK_PIOBUF_IN_TXQ_INSTANCE, 979 tx_queue->queue); 980 rc = efx_mcdi_rpc(efx, MC_CMD_LINK_PIOBUF, 981 inbuf, MC_CMD_LINK_PIOBUF_IN_LEN, 982 NULL, 0, NULL); 983 } 984 985 if (rc) { 986 /* This is non-fatal; the TX path just 987 * won't use PIO for this queue 988 */ 989 netif_err(efx, drv, efx->net_dev, 990 "failed to link VI %u to PIO buffer %u (%d)\n", 991 tx_queue->queue, index, rc); 992 tx_queue->piobuf = NULL; 993 } else { 994 tx_queue->piobuf = 995 nic_data->pio_write_base + 996 index * efx->vi_stride + offset; 997 tx_queue->piobuf_offset = offset; 998 netif_dbg(efx, probe, efx->net_dev, 999 "linked VI %u to PIO buffer %u offset %x addr %p\n", 1000 tx_queue->queue, index, 1001 tx_queue->piobuf_offset, 1002 tx_queue->piobuf); 1003 } 1004 } 1005 } 1006 1007 return 0; 1008 1009 fail: 1010 /* inbuf was defined for MC_CMD_LINK_PIOBUF. We can use the same 1011 * buffer for MC_CMD_UNLINK_PIOBUF because it's shorter. 1012 */ 1013 BUILD_BUG_ON(MC_CMD_LINK_PIOBUF_IN_LEN < MC_CMD_UNLINK_PIOBUF_IN_LEN); 1014 while (index--) { 1015 MCDI_SET_DWORD(inbuf, UNLINK_PIOBUF_IN_TXQ_INSTANCE, 1016 nic_data->pio_write_vi_base + index); 1017 efx_mcdi_rpc(efx, MC_CMD_UNLINK_PIOBUF, 1018 inbuf, MC_CMD_UNLINK_PIOBUF_IN_LEN, 1019 NULL, 0, NULL); 1020 } 1021 return rc; 1022 } 1023 1024 static void efx_ef10_forget_old_piobufs(struct efx_nic *efx) 1025 { 1026 struct efx_channel *channel; 1027 struct efx_tx_queue *tx_queue; 1028 1029 /* All our existing PIO buffers went away */ 1030 efx_for_each_channel(channel, efx) 1031 efx_for_each_channel_tx_queue(tx_queue, channel) 1032 tx_queue->piobuf = NULL; 1033 } 1034 1035 #else /* !EFX_USE_PIO */ 1036 1037 static int efx_ef10_alloc_piobufs(struct efx_nic *efx, unsigned int n) 1038 { 1039 return n == 0 ? 0 : -ENOBUFS; 1040 } 1041 1042 static int efx_ef10_link_piobufs(struct efx_nic *efx) 1043 { 1044 return 0; 1045 } 1046 1047 static void efx_ef10_free_piobufs(struct efx_nic *efx) 1048 { 1049 } 1050 1051 static void efx_ef10_forget_old_piobufs(struct efx_nic *efx) 1052 { 1053 } 1054 1055 #endif /* EFX_USE_PIO */ 1056 1057 static void efx_ef10_remove(struct efx_nic *efx) 1058 { 1059 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1060 int rc; 1061 1062 #ifdef CONFIG_SFC_SRIOV 1063 struct efx_ef10_nic_data *nic_data_pf; 1064 struct pci_dev *pci_dev_pf; 1065 struct efx_nic *efx_pf; 1066 struct ef10_vf *vf; 1067 1068 if (efx->pci_dev->is_virtfn) { 1069 pci_dev_pf = efx->pci_dev->physfn; 1070 if (pci_dev_pf) { 1071 efx_pf = pci_get_drvdata(pci_dev_pf); 1072 nic_data_pf = efx_pf->nic_data; 1073 vf = nic_data_pf->vf + nic_data->vf_index; 1074 vf->efx = NULL; 1075 } else 1076 netif_info(efx, drv, efx->net_dev, 1077 "Could not get the PF id from VF\n"); 1078 } 1079 #endif 1080 1081 efx_ef10_cleanup_vlans(efx); 1082 mutex_destroy(&nic_data->vlan_lock); 1083 1084 efx_ptp_remove(efx); 1085 1086 efx_mcdi_mon_remove(efx); 1087 1088 efx_ef10_rx_free_indir_table(efx); 1089 1090 if (nic_data->wc_membase) 1091 iounmap(nic_data->wc_membase); 1092 1093 rc = efx_ef10_free_vis(efx); 1094 WARN_ON(rc != 0); 1095 1096 if (!nic_data->must_restore_piobufs) 1097 efx_ef10_free_piobufs(efx); 1098 1099 device_remove_file(&efx->pci_dev->dev, &dev_attr_primary_flag); 1100 device_remove_file(&efx->pci_dev->dev, &dev_attr_link_control_flag); 1101 1102 efx_mcdi_detach(efx); 1103 1104 memset(nic_data->udp_tunnels, 0, sizeof(nic_data->udp_tunnels)); 1105 mutex_lock(&nic_data->udp_tunnels_lock); 1106 (void)efx_ef10_set_udp_tnl_ports(efx, true); 1107 mutex_unlock(&nic_data->udp_tunnels_lock); 1108 1109 mutex_destroy(&nic_data->udp_tunnels_lock); 1110 1111 efx_mcdi_fini(efx); 1112 efx_nic_free_buffer(efx, &nic_data->mcdi_buf); 1113 kfree(nic_data); 1114 } 1115 1116 static int efx_ef10_probe_pf(struct efx_nic *efx) 1117 { 1118 return efx_ef10_probe(efx); 1119 } 1120 1121 int efx_ef10_vadaptor_query(struct efx_nic *efx, unsigned int port_id, 1122 u32 *port_flags, u32 *vadaptor_flags, 1123 unsigned int *vlan_tags) 1124 { 1125 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1126 MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_QUERY_IN_LEN); 1127 MCDI_DECLARE_BUF(outbuf, MC_CMD_VADAPTOR_QUERY_OUT_LEN); 1128 size_t outlen; 1129 int rc; 1130 1131 if (nic_data->datapath_caps & 1132 (1 << MC_CMD_GET_CAPABILITIES_OUT_VADAPTOR_QUERY_LBN)) { 1133 MCDI_SET_DWORD(inbuf, VADAPTOR_QUERY_IN_UPSTREAM_PORT_ID, 1134 port_id); 1135 1136 rc = efx_mcdi_rpc(efx, MC_CMD_VADAPTOR_QUERY, inbuf, sizeof(inbuf), 1137 outbuf, sizeof(outbuf), &outlen); 1138 if (rc) 1139 return rc; 1140 1141 if (outlen < sizeof(outbuf)) { 1142 rc = -EIO; 1143 return rc; 1144 } 1145 } 1146 1147 if (port_flags) 1148 *port_flags = MCDI_DWORD(outbuf, VADAPTOR_QUERY_OUT_PORT_FLAGS); 1149 if (vadaptor_flags) 1150 *vadaptor_flags = 1151 MCDI_DWORD(outbuf, VADAPTOR_QUERY_OUT_VADAPTOR_FLAGS); 1152 if (vlan_tags) 1153 *vlan_tags = 1154 MCDI_DWORD(outbuf, 1155 VADAPTOR_QUERY_OUT_NUM_AVAILABLE_VLAN_TAGS); 1156 1157 return 0; 1158 } 1159 1160 int efx_ef10_vadaptor_alloc(struct efx_nic *efx, unsigned int port_id) 1161 { 1162 MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_ALLOC_IN_LEN); 1163 1164 MCDI_SET_DWORD(inbuf, VADAPTOR_ALLOC_IN_UPSTREAM_PORT_ID, port_id); 1165 return efx_mcdi_rpc(efx, MC_CMD_VADAPTOR_ALLOC, inbuf, sizeof(inbuf), 1166 NULL, 0, NULL); 1167 } 1168 1169 int efx_ef10_vadaptor_free(struct efx_nic *efx, unsigned int port_id) 1170 { 1171 MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_FREE_IN_LEN); 1172 1173 MCDI_SET_DWORD(inbuf, VADAPTOR_FREE_IN_UPSTREAM_PORT_ID, port_id); 1174 return efx_mcdi_rpc(efx, MC_CMD_VADAPTOR_FREE, inbuf, sizeof(inbuf), 1175 NULL, 0, NULL); 1176 } 1177 1178 int efx_ef10_vport_add_mac(struct efx_nic *efx, 1179 unsigned int port_id, u8 *mac) 1180 { 1181 MCDI_DECLARE_BUF(inbuf, MC_CMD_VPORT_ADD_MAC_ADDRESS_IN_LEN); 1182 1183 MCDI_SET_DWORD(inbuf, VPORT_ADD_MAC_ADDRESS_IN_VPORT_ID, port_id); 1184 ether_addr_copy(MCDI_PTR(inbuf, VPORT_ADD_MAC_ADDRESS_IN_MACADDR), mac); 1185 1186 return efx_mcdi_rpc(efx, MC_CMD_VPORT_ADD_MAC_ADDRESS, inbuf, 1187 sizeof(inbuf), NULL, 0, NULL); 1188 } 1189 1190 int efx_ef10_vport_del_mac(struct efx_nic *efx, 1191 unsigned int port_id, u8 *mac) 1192 { 1193 MCDI_DECLARE_BUF(inbuf, MC_CMD_VPORT_DEL_MAC_ADDRESS_IN_LEN); 1194 1195 MCDI_SET_DWORD(inbuf, VPORT_DEL_MAC_ADDRESS_IN_VPORT_ID, port_id); 1196 ether_addr_copy(MCDI_PTR(inbuf, VPORT_DEL_MAC_ADDRESS_IN_MACADDR), mac); 1197 1198 return efx_mcdi_rpc(efx, MC_CMD_VPORT_DEL_MAC_ADDRESS, inbuf, 1199 sizeof(inbuf), NULL, 0, NULL); 1200 } 1201 1202 #ifdef CONFIG_SFC_SRIOV 1203 static int efx_ef10_probe_vf(struct efx_nic *efx) 1204 { 1205 int rc; 1206 struct pci_dev *pci_dev_pf; 1207 1208 /* If the parent PF has no VF data structure, it doesn't know about this 1209 * VF so fail probe. The VF needs to be re-created. This can happen 1210 * if the PF driver is unloaded while the VF is assigned to a guest. 1211 */ 1212 pci_dev_pf = efx->pci_dev->physfn; 1213 if (pci_dev_pf) { 1214 struct efx_nic *efx_pf = pci_get_drvdata(pci_dev_pf); 1215 struct efx_ef10_nic_data *nic_data_pf = efx_pf->nic_data; 1216 1217 if (!nic_data_pf->vf) { 1218 netif_info(efx, drv, efx->net_dev, 1219 "The VF cannot link to its parent PF; " 1220 "please destroy and re-create the VF\n"); 1221 return -EBUSY; 1222 } 1223 } 1224 1225 rc = efx_ef10_probe(efx); 1226 if (rc) 1227 return rc; 1228 1229 rc = efx_ef10_get_vf_index(efx); 1230 if (rc) 1231 goto fail; 1232 1233 if (efx->pci_dev->is_virtfn) { 1234 if (efx->pci_dev->physfn) { 1235 struct efx_nic *efx_pf = 1236 pci_get_drvdata(efx->pci_dev->physfn); 1237 struct efx_ef10_nic_data *nic_data_p = efx_pf->nic_data; 1238 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1239 1240 nic_data_p->vf[nic_data->vf_index].efx = efx; 1241 nic_data_p->vf[nic_data->vf_index].pci_dev = 1242 efx->pci_dev; 1243 } else 1244 netif_info(efx, drv, efx->net_dev, 1245 "Could not get the PF id from VF\n"); 1246 } 1247 1248 return 0; 1249 1250 fail: 1251 efx_ef10_remove(efx); 1252 return rc; 1253 } 1254 #else 1255 static int efx_ef10_probe_vf(struct efx_nic *efx __attribute__ ((unused))) 1256 { 1257 return 0; 1258 } 1259 #endif 1260 1261 static int efx_ef10_alloc_vis(struct efx_nic *efx, 1262 unsigned int min_vis, unsigned int max_vis) 1263 { 1264 MCDI_DECLARE_BUF(inbuf, MC_CMD_ALLOC_VIS_IN_LEN); 1265 MCDI_DECLARE_BUF(outbuf, MC_CMD_ALLOC_VIS_OUT_LEN); 1266 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1267 size_t outlen; 1268 int rc; 1269 1270 MCDI_SET_DWORD(inbuf, ALLOC_VIS_IN_MIN_VI_COUNT, min_vis); 1271 MCDI_SET_DWORD(inbuf, ALLOC_VIS_IN_MAX_VI_COUNT, max_vis); 1272 rc = efx_mcdi_rpc(efx, MC_CMD_ALLOC_VIS, inbuf, sizeof(inbuf), 1273 outbuf, sizeof(outbuf), &outlen); 1274 if (rc != 0) 1275 return rc; 1276 1277 if (outlen < MC_CMD_ALLOC_VIS_OUT_LEN) 1278 return -EIO; 1279 1280 netif_dbg(efx, drv, efx->net_dev, "base VI is A0x%03x\n", 1281 MCDI_DWORD(outbuf, ALLOC_VIS_OUT_VI_BASE)); 1282 1283 nic_data->vi_base = MCDI_DWORD(outbuf, ALLOC_VIS_OUT_VI_BASE); 1284 nic_data->n_allocated_vis = MCDI_DWORD(outbuf, ALLOC_VIS_OUT_VI_COUNT); 1285 return 0; 1286 } 1287 1288 /* Note that the failure path of this function does not free 1289 * resources, as this will be done by efx_ef10_remove(). 1290 */ 1291 static int efx_ef10_dimension_resources(struct efx_nic *efx) 1292 { 1293 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1294 unsigned int uc_mem_map_size, wc_mem_map_size; 1295 unsigned int min_vis = max(EFX_TXQ_TYPES, 1296 efx_separate_tx_channels ? 2 : 1); 1297 unsigned int channel_vis, pio_write_vi_base, max_vis; 1298 void __iomem *membase; 1299 int rc; 1300 1301 channel_vis = max(efx->n_channels, 1302 (efx->n_tx_channels + efx->n_extra_tx_channels) * 1303 EFX_TXQ_TYPES); 1304 1305 #ifdef EFX_USE_PIO 1306 /* Try to allocate PIO buffers if wanted and if the full 1307 * number of PIO buffers would be sufficient to allocate one 1308 * copy-buffer per TX channel. Failure is non-fatal, as there 1309 * are only a small number of PIO buffers shared between all 1310 * functions of the controller. 1311 */ 1312 if (efx_piobuf_size != 0 && 1313 nic_data->piobuf_size / efx_piobuf_size * EF10_TX_PIOBUF_COUNT >= 1314 efx->n_tx_channels) { 1315 unsigned int n_piobufs = 1316 DIV_ROUND_UP(efx->n_tx_channels, 1317 nic_data->piobuf_size / efx_piobuf_size); 1318 1319 rc = efx_ef10_alloc_piobufs(efx, n_piobufs); 1320 if (rc == -ENOSPC) 1321 netif_dbg(efx, probe, efx->net_dev, 1322 "out of PIO buffers; cannot allocate more\n"); 1323 else if (rc == -EPERM) 1324 netif_dbg(efx, probe, efx->net_dev, 1325 "not permitted to allocate PIO buffers\n"); 1326 else if (rc) 1327 netif_err(efx, probe, efx->net_dev, 1328 "failed to allocate PIO buffers (%d)\n", rc); 1329 else 1330 netif_dbg(efx, probe, efx->net_dev, 1331 "allocated %u PIO buffers\n", n_piobufs); 1332 } 1333 #else 1334 nic_data->n_piobufs = 0; 1335 #endif 1336 1337 /* PIO buffers should be mapped with write-combining enabled, 1338 * and we want to make single UC and WC mappings rather than 1339 * several of each (in fact that's the only option if host 1340 * page size is >4K). So we may allocate some extra VIs just 1341 * for writing PIO buffers through. 1342 * 1343 * The UC mapping contains (channel_vis - 1) complete VIs and the 1344 * first 4K of the next VI. Then the WC mapping begins with 1345 * the remainder of this last VI. 1346 */ 1347 uc_mem_map_size = PAGE_ALIGN((channel_vis - 1) * efx->vi_stride + 1348 ER_DZ_TX_PIOBUF); 1349 if (nic_data->n_piobufs) { 1350 /* pio_write_vi_base rounds down to give the number of complete 1351 * VIs inside the UC mapping. 1352 */ 1353 pio_write_vi_base = uc_mem_map_size / efx->vi_stride; 1354 wc_mem_map_size = (PAGE_ALIGN((pio_write_vi_base + 1355 nic_data->n_piobufs) * 1356 efx->vi_stride) - 1357 uc_mem_map_size); 1358 max_vis = pio_write_vi_base + nic_data->n_piobufs; 1359 } else { 1360 pio_write_vi_base = 0; 1361 wc_mem_map_size = 0; 1362 max_vis = channel_vis; 1363 } 1364 1365 /* In case the last attached driver failed to free VIs, do it now */ 1366 rc = efx_ef10_free_vis(efx); 1367 if (rc != 0) 1368 return rc; 1369 1370 rc = efx_ef10_alloc_vis(efx, min_vis, max_vis); 1371 if (rc != 0) 1372 return rc; 1373 1374 if (nic_data->n_allocated_vis < channel_vis) { 1375 netif_info(efx, drv, efx->net_dev, 1376 "Could not allocate enough VIs to satisfy RSS" 1377 " requirements. Performance may not be optimal.\n"); 1378 /* We didn't get the VIs to populate our channels. 1379 * We could keep what we got but then we'd have more 1380 * interrupts than we need. 1381 * Instead calculate new max_channels and restart 1382 */ 1383 efx->max_channels = nic_data->n_allocated_vis; 1384 efx->max_tx_channels = 1385 nic_data->n_allocated_vis / EFX_TXQ_TYPES; 1386 1387 efx_ef10_free_vis(efx); 1388 return -EAGAIN; 1389 } 1390 1391 /* If we didn't get enough VIs to map all the PIO buffers, free the 1392 * PIO buffers 1393 */ 1394 if (nic_data->n_piobufs && 1395 nic_data->n_allocated_vis < 1396 pio_write_vi_base + nic_data->n_piobufs) { 1397 netif_dbg(efx, probe, efx->net_dev, 1398 "%u VIs are not sufficient to map %u PIO buffers\n", 1399 nic_data->n_allocated_vis, nic_data->n_piobufs); 1400 efx_ef10_free_piobufs(efx); 1401 } 1402 1403 /* Shrink the original UC mapping of the memory BAR */ 1404 membase = ioremap_nocache(efx->membase_phys, uc_mem_map_size); 1405 if (!membase) { 1406 netif_err(efx, probe, efx->net_dev, 1407 "could not shrink memory BAR to %x\n", 1408 uc_mem_map_size); 1409 return -ENOMEM; 1410 } 1411 iounmap(efx->membase); 1412 efx->membase = membase; 1413 1414 /* Set up the WC mapping if needed */ 1415 if (wc_mem_map_size) { 1416 nic_data->wc_membase = ioremap_wc(efx->membase_phys + 1417 uc_mem_map_size, 1418 wc_mem_map_size); 1419 if (!nic_data->wc_membase) { 1420 netif_err(efx, probe, efx->net_dev, 1421 "could not allocate WC mapping of size %x\n", 1422 wc_mem_map_size); 1423 return -ENOMEM; 1424 } 1425 nic_data->pio_write_vi_base = pio_write_vi_base; 1426 nic_data->pio_write_base = 1427 nic_data->wc_membase + 1428 (pio_write_vi_base * efx->vi_stride + ER_DZ_TX_PIOBUF - 1429 uc_mem_map_size); 1430 1431 rc = efx_ef10_link_piobufs(efx); 1432 if (rc) 1433 efx_ef10_free_piobufs(efx); 1434 } 1435 1436 netif_dbg(efx, probe, efx->net_dev, 1437 "memory BAR at %pa (virtual %p+%x UC, %p+%x WC)\n", 1438 &efx->membase_phys, efx->membase, uc_mem_map_size, 1439 nic_data->wc_membase, wc_mem_map_size); 1440 1441 return 0; 1442 } 1443 1444 static int efx_ef10_init_nic(struct efx_nic *efx) 1445 { 1446 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1447 int rc; 1448 1449 if (nic_data->must_check_datapath_caps) { 1450 rc = efx_ef10_init_datapath_caps(efx); 1451 if (rc) 1452 return rc; 1453 nic_data->must_check_datapath_caps = false; 1454 } 1455 1456 if (nic_data->must_realloc_vis) { 1457 /* We cannot let the number of VIs change now */ 1458 rc = efx_ef10_alloc_vis(efx, nic_data->n_allocated_vis, 1459 nic_data->n_allocated_vis); 1460 if (rc) 1461 return rc; 1462 nic_data->must_realloc_vis = false; 1463 } 1464 1465 if (nic_data->must_restore_piobufs && nic_data->n_piobufs) { 1466 rc = efx_ef10_alloc_piobufs(efx, nic_data->n_piobufs); 1467 if (rc == 0) { 1468 rc = efx_ef10_link_piobufs(efx); 1469 if (rc) 1470 efx_ef10_free_piobufs(efx); 1471 } 1472 1473 /* Log an error on failure, but this is non-fatal. 1474 * Permission errors are less important - we've presumably 1475 * had the PIO buffer licence removed. 1476 */ 1477 if (rc == -EPERM) 1478 netif_dbg(efx, drv, efx->net_dev, 1479 "not permitted to restore PIO buffers\n"); 1480 else if (rc) 1481 netif_err(efx, drv, efx->net_dev, 1482 "failed to restore PIO buffers (%d)\n", rc); 1483 nic_data->must_restore_piobufs = false; 1484 } 1485 1486 /* don't fail init if RSS setup doesn't work */ 1487 rc = efx->type->rx_push_rss_config(efx, false, 1488 efx->rss_context.rx_indir_table, NULL); 1489 1490 return 0; 1491 } 1492 1493 static void efx_ef10_reset_mc_allocations(struct efx_nic *efx) 1494 { 1495 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1496 #ifdef CONFIG_SFC_SRIOV 1497 unsigned int i; 1498 #endif 1499 1500 /* All our allocations have been reset */ 1501 nic_data->must_realloc_vis = true; 1502 nic_data->must_restore_rss_contexts = true; 1503 nic_data->must_restore_filters = true; 1504 nic_data->must_restore_piobufs = true; 1505 efx_ef10_forget_old_piobufs(efx); 1506 efx->rss_context.context_id = EFX_EF10_RSS_CONTEXT_INVALID; 1507 1508 /* Driver-created vswitches and vports must be re-created */ 1509 nic_data->must_probe_vswitching = true; 1510 nic_data->vport_id = EVB_PORT_ID_ASSIGNED; 1511 #ifdef CONFIG_SFC_SRIOV 1512 if (nic_data->vf) 1513 for (i = 0; i < efx->vf_count; i++) 1514 nic_data->vf[i].vport_id = 0; 1515 #endif 1516 } 1517 1518 static enum reset_type efx_ef10_map_reset_reason(enum reset_type reason) 1519 { 1520 if (reason == RESET_TYPE_MC_FAILURE) 1521 return RESET_TYPE_DATAPATH; 1522 1523 return efx_mcdi_map_reset_reason(reason); 1524 } 1525 1526 static int efx_ef10_map_reset_flags(u32 *flags) 1527 { 1528 enum { 1529 EF10_RESET_PORT = ((ETH_RESET_MAC | ETH_RESET_PHY) << 1530 ETH_RESET_SHARED_SHIFT), 1531 EF10_RESET_MC = ((ETH_RESET_DMA | ETH_RESET_FILTER | 1532 ETH_RESET_OFFLOAD | ETH_RESET_MAC | 1533 ETH_RESET_PHY | ETH_RESET_MGMT) << 1534 ETH_RESET_SHARED_SHIFT) 1535 }; 1536 1537 /* We assume for now that our PCI function is permitted to 1538 * reset everything. 1539 */ 1540 1541 if ((*flags & EF10_RESET_MC) == EF10_RESET_MC) { 1542 *flags &= ~EF10_RESET_MC; 1543 return RESET_TYPE_WORLD; 1544 } 1545 1546 if ((*flags & EF10_RESET_PORT) == EF10_RESET_PORT) { 1547 *flags &= ~EF10_RESET_PORT; 1548 return RESET_TYPE_ALL; 1549 } 1550 1551 /* no invisible reset implemented */ 1552 1553 return -EINVAL; 1554 } 1555 1556 static int efx_ef10_reset(struct efx_nic *efx, enum reset_type reset_type) 1557 { 1558 int rc = efx_mcdi_reset(efx, reset_type); 1559 1560 /* Unprivileged functions return -EPERM, but need to return success 1561 * here so that the datapath is brought back up. 1562 */ 1563 if (reset_type == RESET_TYPE_WORLD && rc == -EPERM) 1564 rc = 0; 1565 1566 /* If it was a port reset, trigger reallocation of MC resources. 1567 * Note that on an MC reset nothing needs to be done now because we'll 1568 * detect the MC reset later and handle it then. 1569 * For an FLR, we never get an MC reset event, but the MC has reset all 1570 * resources assigned to us, so we have to trigger reallocation now. 1571 */ 1572 if ((reset_type == RESET_TYPE_ALL || 1573 reset_type == RESET_TYPE_MCDI_TIMEOUT) && !rc) 1574 efx_ef10_reset_mc_allocations(efx); 1575 return rc; 1576 } 1577 1578 #define EF10_DMA_STAT(ext_name, mcdi_name) \ 1579 [EF10_STAT_ ## ext_name] = \ 1580 { #ext_name, 64, 8 * MC_CMD_MAC_ ## mcdi_name } 1581 #define EF10_DMA_INVIS_STAT(int_name, mcdi_name) \ 1582 [EF10_STAT_ ## int_name] = \ 1583 { NULL, 64, 8 * MC_CMD_MAC_ ## mcdi_name } 1584 #define EF10_OTHER_STAT(ext_name) \ 1585 [EF10_STAT_ ## ext_name] = { #ext_name, 0, 0 } 1586 #define GENERIC_SW_STAT(ext_name) \ 1587 [GENERIC_STAT_ ## ext_name] = { #ext_name, 0, 0 } 1588 1589 static const struct efx_hw_stat_desc efx_ef10_stat_desc[EF10_STAT_COUNT] = { 1590 EF10_DMA_STAT(port_tx_bytes, TX_BYTES), 1591 EF10_DMA_STAT(port_tx_packets, TX_PKTS), 1592 EF10_DMA_STAT(port_tx_pause, TX_PAUSE_PKTS), 1593 EF10_DMA_STAT(port_tx_control, TX_CONTROL_PKTS), 1594 EF10_DMA_STAT(port_tx_unicast, TX_UNICAST_PKTS), 1595 EF10_DMA_STAT(port_tx_multicast, TX_MULTICAST_PKTS), 1596 EF10_DMA_STAT(port_tx_broadcast, TX_BROADCAST_PKTS), 1597 EF10_DMA_STAT(port_tx_lt64, TX_LT64_PKTS), 1598 EF10_DMA_STAT(port_tx_64, TX_64_PKTS), 1599 EF10_DMA_STAT(port_tx_65_to_127, TX_65_TO_127_PKTS), 1600 EF10_DMA_STAT(port_tx_128_to_255, TX_128_TO_255_PKTS), 1601 EF10_DMA_STAT(port_tx_256_to_511, TX_256_TO_511_PKTS), 1602 EF10_DMA_STAT(port_tx_512_to_1023, TX_512_TO_1023_PKTS), 1603 EF10_DMA_STAT(port_tx_1024_to_15xx, TX_1024_TO_15XX_PKTS), 1604 EF10_DMA_STAT(port_tx_15xx_to_jumbo, TX_15XX_TO_JUMBO_PKTS), 1605 EF10_DMA_STAT(port_rx_bytes, RX_BYTES), 1606 EF10_DMA_INVIS_STAT(port_rx_bytes_minus_good_bytes, RX_BAD_BYTES), 1607 EF10_OTHER_STAT(port_rx_good_bytes), 1608 EF10_OTHER_STAT(port_rx_bad_bytes), 1609 EF10_DMA_STAT(port_rx_packets, RX_PKTS), 1610 EF10_DMA_STAT(port_rx_good, RX_GOOD_PKTS), 1611 EF10_DMA_STAT(port_rx_bad, RX_BAD_FCS_PKTS), 1612 EF10_DMA_STAT(port_rx_pause, RX_PAUSE_PKTS), 1613 EF10_DMA_STAT(port_rx_control, RX_CONTROL_PKTS), 1614 EF10_DMA_STAT(port_rx_unicast, RX_UNICAST_PKTS), 1615 EF10_DMA_STAT(port_rx_multicast, RX_MULTICAST_PKTS), 1616 EF10_DMA_STAT(port_rx_broadcast, RX_BROADCAST_PKTS), 1617 EF10_DMA_STAT(port_rx_lt64, RX_UNDERSIZE_PKTS), 1618 EF10_DMA_STAT(port_rx_64, RX_64_PKTS), 1619 EF10_DMA_STAT(port_rx_65_to_127, RX_65_TO_127_PKTS), 1620 EF10_DMA_STAT(port_rx_128_to_255, RX_128_TO_255_PKTS), 1621 EF10_DMA_STAT(port_rx_256_to_511, RX_256_TO_511_PKTS), 1622 EF10_DMA_STAT(port_rx_512_to_1023, RX_512_TO_1023_PKTS), 1623 EF10_DMA_STAT(port_rx_1024_to_15xx, RX_1024_TO_15XX_PKTS), 1624 EF10_DMA_STAT(port_rx_15xx_to_jumbo, RX_15XX_TO_JUMBO_PKTS), 1625 EF10_DMA_STAT(port_rx_gtjumbo, RX_GTJUMBO_PKTS), 1626 EF10_DMA_STAT(port_rx_bad_gtjumbo, RX_JABBER_PKTS), 1627 EF10_DMA_STAT(port_rx_overflow, RX_OVERFLOW_PKTS), 1628 EF10_DMA_STAT(port_rx_align_error, RX_ALIGN_ERROR_PKTS), 1629 EF10_DMA_STAT(port_rx_length_error, RX_LENGTH_ERROR_PKTS), 1630 EF10_DMA_STAT(port_rx_nodesc_drops, RX_NODESC_DROPS), 1631 GENERIC_SW_STAT(rx_nodesc_trunc), 1632 GENERIC_SW_STAT(rx_noskb_drops), 1633 EF10_DMA_STAT(port_rx_pm_trunc_bb_overflow, PM_TRUNC_BB_OVERFLOW), 1634 EF10_DMA_STAT(port_rx_pm_discard_bb_overflow, PM_DISCARD_BB_OVERFLOW), 1635 EF10_DMA_STAT(port_rx_pm_trunc_vfifo_full, PM_TRUNC_VFIFO_FULL), 1636 EF10_DMA_STAT(port_rx_pm_discard_vfifo_full, PM_DISCARD_VFIFO_FULL), 1637 EF10_DMA_STAT(port_rx_pm_trunc_qbb, PM_TRUNC_QBB), 1638 EF10_DMA_STAT(port_rx_pm_discard_qbb, PM_DISCARD_QBB), 1639 EF10_DMA_STAT(port_rx_pm_discard_mapping, PM_DISCARD_MAPPING), 1640 EF10_DMA_STAT(port_rx_dp_q_disabled_packets, RXDP_Q_DISABLED_PKTS), 1641 EF10_DMA_STAT(port_rx_dp_di_dropped_packets, RXDP_DI_DROPPED_PKTS), 1642 EF10_DMA_STAT(port_rx_dp_streaming_packets, RXDP_STREAMING_PKTS), 1643 EF10_DMA_STAT(port_rx_dp_hlb_fetch, RXDP_HLB_FETCH_CONDITIONS), 1644 EF10_DMA_STAT(port_rx_dp_hlb_wait, RXDP_HLB_WAIT_CONDITIONS), 1645 EF10_DMA_STAT(rx_unicast, VADAPTER_RX_UNICAST_PACKETS), 1646 EF10_DMA_STAT(rx_unicast_bytes, VADAPTER_RX_UNICAST_BYTES), 1647 EF10_DMA_STAT(rx_multicast, VADAPTER_RX_MULTICAST_PACKETS), 1648 EF10_DMA_STAT(rx_multicast_bytes, VADAPTER_RX_MULTICAST_BYTES), 1649 EF10_DMA_STAT(rx_broadcast, VADAPTER_RX_BROADCAST_PACKETS), 1650 EF10_DMA_STAT(rx_broadcast_bytes, VADAPTER_RX_BROADCAST_BYTES), 1651 EF10_DMA_STAT(rx_bad, VADAPTER_RX_BAD_PACKETS), 1652 EF10_DMA_STAT(rx_bad_bytes, VADAPTER_RX_BAD_BYTES), 1653 EF10_DMA_STAT(rx_overflow, VADAPTER_RX_OVERFLOW), 1654 EF10_DMA_STAT(tx_unicast, VADAPTER_TX_UNICAST_PACKETS), 1655 EF10_DMA_STAT(tx_unicast_bytes, VADAPTER_TX_UNICAST_BYTES), 1656 EF10_DMA_STAT(tx_multicast, VADAPTER_TX_MULTICAST_PACKETS), 1657 EF10_DMA_STAT(tx_multicast_bytes, VADAPTER_TX_MULTICAST_BYTES), 1658 EF10_DMA_STAT(tx_broadcast, VADAPTER_TX_BROADCAST_PACKETS), 1659 EF10_DMA_STAT(tx_broadcast_bytes, VADAPTER_TX_BROADCAST_BYTES), 1660 EF10_DMA_STAT(tx_bad, VADAPTER_TX_BAD_PACKETS), 1661 EF10_DMA_STAT(tx_bad_bytes, VADAPTER_TX_BAD_BYTES), 1662 EF10_DMA_STAT(tx_overflow, VADAPTER_TX_OVERFLOW), 1663 EF10_DMA_STAT(fec_uncorrected_errors, FEC_UNCORRECTED_ERRORS), 1664 EF10_DMA_STAT(fec_corrected_errors, FEC_CORRECTED_ERRORS), 1665 EF10_DMA_STAT(fec_corrected_symbols_lane0, FEC_CORRECTED_SYMBOLS_LANE0), 1666 EF10_DMA_STAT(fec_corrected_symbols_lane1, FEC_CORRECTED_SYMBOLS_LANE1), 1667 EF10_DMA_STAT(fec_corrected_symbols_lane2, FEC_CORRECTED_SYMBOLS_LANE2), 1668 EF10_DMA_STAT(fec_corrected_symbols_lane3, FEC_CORRECTED_SYMBOLS_LANE3), 1669 EF10_DMA_STAT(ctpio_dmabuf_start, CTPIO_DMABUF_START), 1670 EF10_DMA_STAT(ctpio_vi_busy_fallback, CTPIO_VI_BUSY_FALLBACK), 1671 EF10_DMA_STAT(ctpio_long_write_success, CTPIO_LONG_WRITE_SUCCESS), 1672 EF10_DMA_STAT(ctpio_missing_dbell_fail, CTPIO_MISSING_DBELL_FAIL), 1673 EF10_DMA_STAT(ctpio_overflow_fail, CTPIO_OVERFLOW_FAIL), 1674 EF10_DMA_STAT(ctpio_underflow_fail, CTPIO_UNDERFLOW_FAIL), 1675 EF10_DMA_STAT(ctpio_timeout_fail, CTPIO_TIMEOUT_FAIL), 1676 EF10_DMA_STAT(ctpio_noncontig_wr_fail, CTPIO_NONCONTIG_WR_FAIL), 1677 EF10_DMA_STAT(ctpio_frm_clobber_fail, CTPIO_FRM_CLOBBER_FAIL), 1678 EF10_DMA_STAT(ctpio_invalid_wr_fail, CTPIO_INVALID_WR_FAIL), 1679 EF10_DMA_STAT(ctpio_vi_clobber_fallback, CTPIO_VI_CLOBBER_FALLBACK), 1680 EF10_DMA_STAT(ctpio_unqualified_fallback, CTPIO_UNQUALIFIED_FALLBACK), 1681 EF10_DMA_STAT(ctpio_runt_fallback, CTPIO_RUNT_FALLBACK), 1682 EF10_DMA_STAT(ctpio_success, CTPIO_SUCCESS), 1683 EF10_DMA_STAT(ctpio_fallback, CTPIO_FALLBACK), 1684 EF10_DMA_STAT(ctpio_poison, CTPIO_POISON), 1685 EF10_DMA_STAT(ctpio_erase, CTPIO_ERASE), 1686 }; 1687 1688 #define HUNT_COMMON_STAT_MASK ((1ULL << EF10_STAT_port_tx_bytes) | \ 1689 (1ULL << EF10_STAT_port_tx_packets) | \ 1690 (1ULL << EF10_STAT_port_tx_pause) | \ 1691 (1ULL << EF10_STAT_port_tx_unicast) | \ 1692 (1ULL << EF10_STAT_port_tx_multicast) | \ 1693 (1ULL << EF10_STAT_port_tx_broadcast) | \ 1694 (1ULL << EF10_STAT_port_rx_bytes) | \ 1695 (1ULL << \ 1696 EF10_STAT_port_rx_bytes_minus_good_bytes) | \ 1697 (1ULL << EF10_STAT_port_rx_good_bytes) | \ 1698 (1ULL << EF10_STAT_port_rx_bad_bytes) | \ 1699 (1ULL << EF10_STAT_port_rx_packets) | \ 1700 (1ULL << EF10_STAT_port_rx_good) | \ 1701 (1ULL << EF10_STAT_port_rx_bad) | \ 1702 (1ULL << EF10_STAT_port_rx_pause) | \ 1703 (1ULL << EF10_STAT_port_rx_control) | \ 1704 (1ULL << EF10_STAT_port_rx_unicast) | \ 1705 (1ULL << EF10_STAT_port_rx_multicast) | \ 1706 (1ULL << EF10_STAT_port_rx_broadcast) | \ 1707 (1ULL << EF10_STAT_port_rx_lt64) | \ 1708 (1ULL << EF10_STAT_port_rx_64) | \ 1709 (1ULL << EF10_STAT_port_rx_65_to_127) | \ 1710 (1ULL << EF10_STAT_port_rx_128_to_255) | \ 1711 (1ULL << EF10_STAT_port_rx_256_to_511) | \ 1712 (1ULL << EF10_STAT_port_rx_512_to_1023) |\ 1713 (1ULL << EF10_STAT_port_rx_1024_to_15xx) |\ 1714 (1ULL << EF10_STAT_port_rx_15xx_to_jumbo) |\ 1715 (1ULL << EF10_STAT_port_rx_gtjumbo) | \ 1716 (1ULL << EF10_STAT_port_rx_bad_gtjumbo) |\ 1717 (1ULL << EF10_STAT_port_rx_overflow) | \ 1718 (1ULL << EF10_STAT_port_rx_nodesc_drops) |\ 1719 (1ULL << GENERIC_STAT_rx_nodesc_trunc) | \ 1720 (1ULL << GENERIC_STAT_rx_noskb_drops)) 1721 1722 /* On 7000 series NICs, these statistics are only provided by the 10G MAC. 1723 * For a 10G/40G switchable port we do not expose these because they might 1724 * not include all the packets they should. 1725 * On 8000 series NICs these statistics are always provided. 1726 */ 1727 #define HUNT_10G_ONLY_STAT_MASK ((1ULL << EF10_STAT_port_tx_control) | \ 1728 (1ULL << EF10_STAT_port_tx_lt64) | \ 1729 (1ULL << EF10_STAT_port_tx_64) | \ 1730 (1ULL << EF10_STAT_port_tx_65_to_127) |\ 1731 (1ULL << EF10_STAT_port_tx_128_to_255) |\ 1732 (1ULL << EF10_STAT_port_tx_256_to_511) |\ 1733 (1ULL << EF10_STAT_port_tx_512_to_1023) |\ 1734 (1ULL << EF10_STAT_port_tx_1024_to_15xx) |\ 1735 (1ULL << EF10_STAT_port_tx_15xx_to_jumbo)) 1736 1737 /* These statistics are only provided by the 40G MAC. For a 10G/40G 1738 * switchable port we do expose these because the errors will otherwise 1739 * be silent. 1740 */ 1741 #define HUNT_40G_EXTRA_STAT_MASK ((1ULL << EF10_STAT_port_rx_align_error) |\ 1742 (1ULL << EF10_STAT_port_rx_length_error)) 1743 1744 /* These statistics are only provided if the firmware supports the 1745 * capability PM_AND_RXDP_COUNTERS. 1746 */ 1747 #define HUNT_PM_AND_RXDP_STAT_MASK ( \ 1748 (1ULL << EF10_STAT_port_rx_pm_trunc_bb_overflow) | \ 1749 (1ULL << EF10_STAT_port_rx_pm_discard_bb_overflow) | \ 1750 (1ULL << EF10_STAT_port_rx_pm_trunc_vfifo_full) | \ 1751 (1ULL << EF10_STAT_port_rx_pm_discard_vfifo_full) | \ 1752 (1ULL << EF10_STAT_port_rx_pm_trunc_qbb) | \ 1753 (1ULL << EF10_STAT_port_rx_pm_discard_qbb) | \ 1754 (1ULL << EF10_STAT_port_rx_pm_discard_mapping) | \ 1755 (1ULL << EF10_STAT_port_rx_dp_q_disabled_packets) | \ 1756 (1ULL << EF10_STAT_port_rx_dp_di_dropped_packets) | \ 1757 (1ULL << EF10_STAT_port_rx_dp_streaming_packets) | \ 1758 (1ULL << EF10_STAT_port_rx_dp_hlb_fetch) | \ 1759 (1ULL << EF10_STAT_port_rx_dp_hlb_wait)) 1760 1761 /* These statistics are only provided if the NIC supports MC_CMD_MAC_STATS_V2, 1762 * indicated by returning a value >= MC_CMD_MAC_NSTATS_V2 in 1763 * MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS. 1764 * These bits are in the second u64 of the raw mask. 1765 */ 1766 #define EF10_FEC_STAT_MASK ( \ 1767 (1ULL << (EF10_STAT_fec_uncorrected_errors - 64)) | \ 1768 (1ULL << (EF10_STAT_fec_corrected_errors - 64)) | \ 1769 (1ULL << (EF10_STAT_fec_corrected_symbols_lane0 - 64)) | \ 1770 (1ULL << (EF10_STAT_fec_corrected_symbols_lane1 - 64)) | \ 1771 (1ULL << (EF10_STAT_fec_corrected_symbols_lane2 - 64)) | \ 1772 (1ULL << (EF10_STAT_fec_corrected_symbols_lane3 - 64))) 1773 1774 /* These statistics are only provided if the NIC supports MC_CMD_MAC_STATS_V3, 1775 * indicated by returning a value >= MC_CMD_MAC_NSTATS_V3 in 1776 * MC_CMD_GET_CAPABILITIES_V4_OUT_MAC_STATS_NUM_STATS. 1777 * These bits are in the second u64 of the raw mask. 1778 */ 1779 #define EF10_CTPIO_STAT_MASK ( \ 1780 (1ULL << (EF10_STAT_ctpio_dmabuf_start - 64)) | \ 1781 (1ULL << (EF10_STAT_ctpio_vi_busy_fallback - 64)) | \ 1782 (1ULL << (EF10_STAT_ctpio_long_write_success - 64)) | \ 1783 (1ULL << (EF10_STAT_ctpio_missing_dbell_fail - 64)) | \ 1784 (1ULL << (EF10_STAT_ctpio_overflow_fail - 64)) | \ 1785 (1ULL << (EF10_STAT_ctpio_underflow_fail - 64)) | \ 1786 (1ULL << (EF10_STAT_ctpio_timeout_fail - 64)) | \ 1787 (1ULL << (EF10_STAT_ctpio_noncontig_wr_fail - 64)) | \ 1788 (1ULL << (EF10_STAT_ctpio_frm_clobber_fail - 64)) | \ 1789 (1ULL << (EF10_STAT_ctpio_invalid_wr_fail - 64)) | \ 1790 (1ULL << (EF10_STAT_ctpio_vi_clobber_fallback - 64)) | \ 1791 (1ULL << (EF10_STAT_ctpio_unqualified_fallback - 64)) | \ 1792 (1ULL << (EF10_STAT_ctpio_runt_fallback - 64)) | \ 1793 (1ULL << (EF10_STAT_ctpio_success - 64)) | \ 1794 (1ULL << (EF10_STAT_ctpio_fallback - 64)) | \ 1795 (1ULL << (EF10_STAT_ctpio_poison - 64)) | \ 1796 (1ULL << (EF10_STAT_ctpio_erase - 64))) 1797 1798 static u64 efx_ef10_raw_stat_mask(struct efx_nic *efx) 1799 { 1800 u64 raw_mask = HUNT_COMMON_STAT_MASK; 1801 u32 port_caps = efx_mcdi_phy_get_caps(efx); 1802 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1803 1804 if (!(efx->mcdi->fn_flags & 1805 1 << MC_CMD_DRV_ATTACH_EXT_OUT_FLAG_LINKCTRL)) 1806 return 0; 1807 1808 if (port_caps & (1 << MC_CMD_PHY_CAP_40000FDX_LBN)) { 1809 raw_mask |= HUNT_40G_EXTRA_STAT_MASK; 1810 /* 8000 series have everything even at 40G */ 1811 if (nic_data->datapath_caps2 & 1812 (1 << MC_CMD_GET_CAPABILITIES_V2_OUT_MAC_STATS_40G_TX_SIZE_BINS_LBN)) 1813 raw_mask |= HUNT_10G_ONLY_STAT_MASK; 1814 } else { 1815 raw_mask |= HUNT_10G_ONLY_STAT_MASK; 1816 } 1817 1818 if (nic_data->datapath_caps & 1819 (1 << MC_CMD_GET_CAPABILITIES_OUT_PM_AND_RXDP_COUNTERS_LBN)) 1820 raw_mask |= HUNT_PM_AND_RXDP_STAT_MASK; 1821 1822 return raw_mask; 1823 } 1824 1825 static void efx_ef10_get_stat_mask(struct efx_nic *efx, unsigned long *mask) 1826 { 1827 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1828 u64 raw_mask[2]; 1829 1830 raw_mask[0] = efx_ef10_raw_stat_mask(efx); 1831 1832 /* Only show vadaptor stats when EVB capability is present */ 1833 if (nic_data->datapath_caps & 1834 (1 << MC_CMD_GET_CAPABILITIES_OUT_EVB_LBN)) { 1835 raw_mask[0] |= ~((1ULL << EF10_STAT_rx_unicast) - 1); 1836 raw_mask[1] = (1ULL << (EF10_STAT_V1_COUNT - 64)) - 1; 1837 } else { 1838 raw_mask[1] = 0; 1839 } 1840 /* Only show FEC stats when NIC supports MC_CMD_MAC_STATS_V2 */ 1841 if (efx->num_mac_stats >= MC_CMD_MAC_NSTATS_V2) 1842 raw_mask[1] |= EF10_FEC_STAT_MASK; 1843 1844 /* CTPIO stats appear in V3. Only show them on devices that actually 1845 * support CTPIO. Although this driver doesn't use CTPIO others might, 1846 * and we may be reporting the stats for the underlying port. 1847 */ 1848 if (efx->num_mac_stats >= MC_CMD_MAC_NSTATS_V3 && 1849 (nic_data->datapath_caps2 & 1850 (1 << MC_CMD_GET_CAPABILITIES_V4_OUT_CTPIO_LBN))) 1851 raw_mask[1] |= EF10_CTPIO_STAT_MASK; 1852 1853 #if BITS_PER_LONG == 64 1854 BUILD_BUG_ON(BITS_TO_LONGS(EF10_STAT_COUNT) != 2); 1855 mask[0] = raw_mask[0]; 1856 mask[1] = raw_mask[1]; 1857 #else 1858 BUILD_BUG_ON(BITS_TO_LONGS(EF10_STAT_COUNT) != 3); 1859 mask[0] = raw_mask[0] & 0xffffffff; 1860 mask[1] = raw_mask[0] >> 32; 1861 mask[2] = raw_mask[1] & 0xffffffff; 1862 #endif 1863 } 1864 1865 static size_t efx_ef10_describe_stats(struct efx_nic *efx, u8 *names) 1866 { 1867 DECLARE_BITMAP(mask, EF10_STAT_COUNT); 1868 1869 efx_ef10_get_stat_mask(efx, mask); 1870 return efx_nic_describe_stats(efx_ef10_stat_desc, EF10_STAT_COUNT, 1871 mask, names); 1872 } 1873 1874 static size_t efx_ef10_update_stats_common(struct efx_nic *efx, u64 *full_stats, 1875 struct rtnl_link_stats64 *core_stats) 1876 { 1877 DECLARE_BITMAP(mask, EF10_STAT_COUNT); 1878 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1879 u64 *stats = nic_data->stats; 1880 size_t stats_count = 0, index; 1881 1882 efx_ef10_get_stat_mask(efx, mask); 1883 1884 if (full_stats) { 1885 for_each_set_bit(index, mask, EF10_STAT_COUNT) { 1886 if (efx_ef10_stat_desc[index].name) { 1887 *full_stats++ = stats[index]; 1888 ++stats_count; 1889 } 1890 } 1891 } 1892 1893 if (!core_stats) 1894 return stats_count; 1895 1896 if (nic_data->datapath_caps & 1897 1 << MC_CMD_GET_CAPABILITIES_OUT_EVB_LBN) { 1898 /* Use vadaptor stats. */ 1899 core_stats->rx_packets = stats[EF10_STAT_rx_unicast] + 1900 stats[EF10_STAT_rx_multicast] + 1901 stats[EF10_STAT_rx_broadcast]; 1902 core_stats->tx_packets = stats[EF10_STAT_tx_unicast] + 1903 stats[EF10_STAT_tx_multicast] + 1904 stats[EF10_STAT_tx_broadcast]; 1905 core_stats->rx_bytes = stats[EF10_STAT_rx_unicast_bytes] + 1906 stats[EF10_STAT_rx_multicast_bytes] + 1907 stats[EF10_STAT_rx_broadcast_bytes]; 1908 core_stats->tx_bytes = stats[EF10_STAT_tx_unicast_bytes] + 1909 stats[EF10_STAT_tx_multicast_bytes] + 1910 stats[EF10_STAT_tx_broadcast_bytes]; 1911 core_stats->rx_dropped = stats[GENERIC_STAT_rx_nodesc_trunc] + 1912 stats[GENERIC_STAT_rx_noskb_drops]; 1913 core_stats->multicast = stats[EF10_STAT_rx_multicast]; 1914 core_stats->rx_crc_errors = stats[EF10_STAT_rx_bad]; 1915 core_stats->rx_fifo_errors = stats[EF10_STAT_rx_overflow]; 1916 core_stats->rx_errors = core_stats->rx_crc_errors; 1917 core_stats->tx_errors = stats[EF10_STAT_tx_bad]; 1918 } else { 1919 /* Use port stats. */ 1920 core_stats->rx_packets = stats[EF10_STAT_port_rx_packets]; 1921 core_stats->tx_packets = stats[EF10_STAT_port_tx_packets]; 1922 core_stats->rx_bytes = stats[EF10_STAT_port_rx_bytes]; 1923 core_stats->tx_bytes = stats[EF10_STAT_port_tx_bytes]; 1924 core_stats->rx_dropped = stats[EF10_STAT_port_rx_nodesc_drops] + 1925 stats[GENERIC_STAT_rx_nodesc_trunc] + 1926 stats[GENERIC_STAT_rx_noskb_drops]; 1927 core_stats->multicast = stats[EF10_STAT_port_rx_multicast]; 1928 core_stats->rx_length_errors = 1929 stats[EF10_STAT_port_rx_gtjumbo] + 1930 stats[EF10_STAT_port_rx_length_error]; 1931 core_stats->rx_crc_errors = stats[EF10_STAT_port_rx_bad]; 1932 core_stats->rx_frame_errors = 1933 stats[EF10_STAT_port_rx_align_error]; 1934 core_stats->rx_fifo_errors = stats[EF10_STAT_port_rx_overflow]; 1935 core_stats->rx_errors = (core_stats->rx_length_errors + 1936 core_stats->rx_crc_errors + 1937 core_stats->rx_frame_errors); 1938 } 1939 1940 return stats_count; 1941 } 1942 1943 static int efx_ef10_try_update_nic_stats_pf(struct efx_nic *efx) 1944 { 1945 struct efx_ef10_nic_data *nic_data = efx->nic_data; 1946 DECLARE_BITMAP(mask, EF10_STAT_COUNT); 1947 __le64 generation_start, generation_end; 1948 u64 *stats = nic_data->stats; 1949 __le64 *dma_stats; 1950 1951 efx_ef10_get_stat_mask(efx, mask); 1952 1953 dma_stats = efx->stats_buffer.addr; 1954 1955 generation_end = dma_stats[efx->num_mac_stats - 1]; 1956 if (generation_end == EFX_MC_STATS_GENERATION_INVALID) 1957 return 0; 1958 rmb(); 1959 efx_nic_update_stats(efx_ef10_stat_desc, EF10_STAT_COUNT, mask, 1960 stats, efx->stats_buffer.addr, false); 1961 rmb(); 1962 generation_start = dma_stats[MC_CMD_MAC_GENERATION_START]; 1963 if (generation_end != generation_start) 1964 return -EAGAIN; 1965 1966 /* Update derived statistics */ 1967 efx_nic_fix_nodesc_drop_stat(efx, 1968 &stats[EF10_STAT_port_rx_nodesc_drops]); 1969 stats[EF10_STAT_port_rx_good_bytes] = 1970 stats[EF10_STAT_port_rx_bytes] - 1971 stats[EF10_STAT_port_rx_bytes_minus_good_bytes]; 1972 efx_update_diff_stat(&stats[EF10_STAT_port_rx_bad_bytes], 1973 stats[EF10_STAT_port_rx_bytes_minus_good_bytes]); 1974 efx_update_sw_stats(efx, stats); 1975 return 0; 1976 } 1977 1978 1979 static size_t efx_ef10_update_stats_pf(struct efx_nic *efx, u64 *full_stats, 1980 struct rtnl_link_stats64 *core_stats) 1981 { 1982 int retry; 1983 1984 /* If we're unlucky enough to read statistics during the DMA, wait 1985 * up to 10ms for it to finish (typically takes <500us) 1986 */ 1987 for (retry = 0; retry < 100; ++retry) { 1988 if (efx_ef10_try_update_nic_stats_pf(efx) == 0) 1989 break; 1990 udelay(100); 1991 } 1992 1993 return efx_ef10_update_stats_common(efx, full_stats, core_stats); 1994 } 1995 1996 static int efx_ef10_try_update_nic_stats_vf(struct efx_nic *efx) 1997 { 1998 MCDI_DECLARE_BUF(inbuf, MC_CMD_MAC_STATS_IN_LEN); 1999 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2000 DECLARE_BITMAP(mask, EF10_STAT_COUNT); 2001 __le64 generation_start, generation_end; 2002 u64 *stats = nic_data->stats; 2003 u32 dma_len = efx->num_mac_stats * sizeof(u64); 2004 struct efx_buffer stats_buf; 2005 __le64 *dma_stats; 2006 int rc; 2007 2008 spin_unlock_bh(&efx->stats_lock); 2009 2010 if (in_interrupt()) { 2011 /* If in atomic context, cannot update stats. Just update the 2012 * software stats and return so the caller can continue. 2013 */ 2014 spin_lock_bh(&efx->stats_lock); 2015 efx_update_sw_stats(efx, stats); 2016 return 0; 2017 } 2018 2019 efx_ef10_get_stat_mask(efx, mask); 2020 2021 rc = efx_nic_alloc_buffer(efx, &stats_buf, dma_len, GFP_ATOMIC); 2022 if (rc) { 2023 spin_lock_bh(&efx->stats_lock); 2024 return rc; 2025 } 2026 2027 dma_stats = stats_buf.addr; 2028 dma_stats[efx->num_mac_stats - 1] = EFX_MC_STATS_GENERATION_INVALID; 2029 2030 MCDI_SET_QWORD(inbuf, MAC_STATS_IN_DMA_ADDR, stats_buf.dma_addr); 2031 MCDI_POPULATE_DWORD_1(inbuf, MAC_STATS_IN_CMD, 2032 MAC_STATS_IN_DMA, 1); 2033 MCDI_SET_DWORD(inbuf, MAC_STATS_IN_DMA_LEN, dma_len); 2034 MCDI_SET_DWORD(inbuf, MAC_STATS_IN_PORT_ID, EVB_PORT_ID_ASSIGNED); 2035 2036 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_MAC_STATS, inbuf, sizeof(inbuf), 2037 NULL, 0, NULL); 2038 spin_lock_bh(&efx->stats_lock); 2039 if (rc) { 2040 /* Expect ENOENT if DMA queues have not been set up */ 2041 if (rc != -ENOENT || atomic_read(&efx->active_queues)) 2042 efx_mcdi_display_error(efx, MC_CMD_MAC_STATS, 2043 sizeof(inbuf), NULL, 0, rc); 2044 goto out; 2045 } 2046 2047 generation_end = dma_stats[efx->num_mac_stats - 1]; 2048 if (generation_end == EFX_MC_STATS_GENERATION_INVALID) { 2049 WARN_ON_ONCE(1); 2050 goto out; 2051 } 2052 rmb(); 2053 efx_nic_update_stats(efx_ef10_stat_desc, EF10_STAT_COUNT, mask, 2054 stats, stats_buf.addr, false); 2055 rmb(); 2056 generation_start = dma_stats[MC_CMD_MAC_GENERATION_START]; 2057 if (generation_end != generation_start) { 2058 rc = -EAGAIN; 2059 goto out; 2060 } 2061 2062 efx_update_sw_stats(efx, stats); 2063 out: 2064 efx_nic_free_buffer(efx, &stats_buf); 2065 return rc; 2066 } 2067 2068 static size_t efx_ef10_update_stats_vf(struct efx_nic *efx, u64 *full_stats, 2069 struct rtnl_link_stats64 *core_stats) 2070 { 2071 if (efx_ef10_try_update_nic_stats_vf(efx)) 2072 return 0; 2073 2074 return efx_ef10_update_stats_common(efx, full_stats, core_stats); 2075 } 2076 2077 static void efx_ef10_push_irq_moderation(struct efx_channel *channel) 2078 { 2079 struct efx_nic *efx = channel->efx; 2080 unsigned int mode, usecs; 2081 efx_dword_t timer_cmd; 2082 2083 if (channel->irq_moderation_us) { 2084 mode = 3; 2085 usecs = channel->irq_moderation_us; 2086 } else { 2087 mode = 0; 2088 usecs = 0; 2089 } 2090 2091 if (EFX_EF10_WORKAROUND_61265(efx)) { 2092 MCDI_DECLARE_BUF(inbuf, MC_CMD_SET_EVQ_TMR_IN_LEN); 2093 unsigned int ns = usecs * 1000; 2094 2095 MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_INSTANCE, 2096 channel->channel); 2097 MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_LOAD_REQ_NS, ns); 2098 MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_RELOAD_REQ_NS, ns); 2099 MCDI_SET_DWORD(inbuf, SET_EVQ_TMR_IN_TMR_MODE, mode); 2100 2101 efx_mcdi_rpc_async(efx, MC_CMD_SET_EVQ_TMR, 2102 inbuf, sizeof(inbuf), 0, NULL, 0); 2103 } else if (EFX_EF10_WORKAROUND_35388(efx)) { 2104 unsigned int ticks = efx_usecs_to_ticks(efx, usecs); 2105 2106 EFX_POPULATE_DWORD_3(timer_cmd, ERF_DD_EVQ_IND_TIMER_FLAGS, 2107 EFE_DD_EVQ_IND_TIMER_FLAGS, 2108 ERF_DD_EVQ_IND_TIMER_MODE, mode, 2109 ERF_DD_EVQ_IND_TIMER_VAL, ticks); 2110 efx_writed_page(efx, &timer_cmd, ER_DD_EVQ_INDIRECT, 2111 channel->channel); 2112 } else { 2113 unsigned int ticks = efx_usecs_to_ticks(efx, usecs); 2114 2115 EFX_POPULATE_DWORD_3(timer_cmd, ERF_DZ_TC_TIMER_MODE, mode, 2116 ERF_DZ_TC_TIMER_VAL, ticks, 2117 ERF_FZ_TC_TMR_REL_VAL, ticks); 2118 efx_writed_page(efx, &timer_cmd, ER_DZ_EVQ_TMR, 2119 channel->channel); 2120 } 2121 } 2122 2123 static void efx_ef10_get_wol_vf(struct efx_nic *efx, 2124 struct ethtool_wolinfo *wol) {} 2125 2126 static int efx_ef10_set_wol_vf(struct efx_nic *efx, u32 type) 2127 { 2128 return -EOPNOTSUPP; 2129 } 2130 2131 static void efx_ef10_get_wol(struct efx_nic *efx, struct ethtool_wolinfo *wol) 2132 { 2133 wol->supported = 0; 2134 wol->wolopts = 0; 2135 memset(&wol->sopass, 0, sizeof(wol->sopass)); 2136 } 2137 2138 static int efx_ef10_set_wol(struct efx_nic *efx, u32 type) 2139 { 2140 if (type != 0) 2141 return -EINVAL; 2142 return 0; 2143 } 2144 2145 static void efx_ef10_mcdi_request(struct efx_nic *efx, 2146 const efx_dword_t *hdr, size_t hdr_len, 2147 const efx_dword_t *sdu, size_t sdu_len) 2148 { 2149 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2150 u8 *pdu = nic_data->mcdi_buf.addr; 2151 2152 memcpy(pdu, hdr, hdr_len); 2153 memcpy(pdu + hdr_len, sdu, sdu_len); 2154 wmb(); 2155 2156 /* The hardware provides 'low' and 'high' (doorbell) registers 2157 * for passing the 64-bit address of an MCDI request to 2158 * firmware. However the dwords are swapped by firmware. The 2159 * least significant bits of the doorbell are then 0 for all 2160 * MCDI requests due to alignment. 2161 */ 2162 _efx_writed(efx, cpu_to_le32((u64)nic_data->mcdi_buf.dma_addr >> 32), 2163 ER_DZ_MC_DB_LWRD); 2164 _efx_writed(efx, cpu_to_le32((u32)nic_data->mcdi_buf.dma_addr), 2165 ER_DZ_MC_DB_HWRD); 2166 } 2167 2168 static bool efx_ef10_mcdi_poll_response(struct efx_nic *efx) 2169 { 2170 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2171 const efx_dword_t hdr = *(const efx_dword_t *)nic_data->mcdi_buf.addr; 2172 2173 rmb(); 2174 return EFX_DWORD_FIELD(hdr, MCDI_HEADER_RESPONSE); 2175 } 2176 2177 static void 2178 efx_ef10_mcdi_read_response(struct efx_nic *efx, efx_dword_t *outbuf, 2179 size_t offset, size_t outlen) 2180 { 2181 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2182 const u8 *pdu = nic_data->mcdi_buf.addr; 2183 2184 memcpy(outbuf, pdu + offset, outlen); 2185 } 2186 2187 static void efx_ef10_mcdi_reboot_detected(struct efx_nic *efx) 2188 { 2189 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2190 2191 /* All our allocations have been reset */ 2192 efx_ef10_reset_mc_allocations(efx); 2193 2194 /* The datapath firmware might have been changed */ 2195 nic_data->must_check_datapath_caps = true; 2196 2197 /* MAC statistics have been cleared on the NIC; clear the local 2198 * statistic that we update with efx_update_diff_stat(). 2199 */ 2200 nic_data->stats[EF10_STAT_port_rx_bad_bytes] = 0; 2201 } 2202 2203 static int efx_ef10_mcdi_poll_reboot(struct efx_nic *efx) 2204 { 2205 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2206 int rc; 2207 2208 rc = efx_ef10_get_warm_boot_count(efx); 2209 if (rc < 0) { 2210 /* The firmware is presumably in the process of 2211 * rebooting. However, we are supposed to report each 2212 * reboot just once, so we must only do that once we 2213 * can read and store the updated warm boot count. 2214 */ 2215 return 0; 2216 } 2217 2218 if (rc == nic_data->warm_boot_count) 2219 return 0; 2220 2221 nic_data->warm_boot_count = rc; 2222 efx_ef10_mcdi_reboot_detected(efx); 2223 2224 return -EIO; 2225 } 2226 2227 /* Handle an MSI interrupt 2228 * 2229 * Handle an MSI hardware interrupt. This routine schedules event 2230 * queue processing. No interrupt acknowledgement cycle is necessary. 2231 * Also, we never need to check that the interrupt is for us, since 2232 * MSI interrupts cannot be shared. 2233 */ 2234 static irqreturn_t efx_ef10_msi_interrupt(int irq, void *dev_id) 2235 { 2236 struct efx_msi_context *context = dev_id; 2237 struct efx_nic *efx = context->efx; 2238 2239 netif_vdbg(efx, intr, efx->net_dev, 2240 "IRQ %d on CPU %d\n", irq, raw_smp_processor_id()); 2241 2242 if (likely(READ_ONCE(efx->irq_soft_enabled))) { 2243 /* Note test interrupts */ 2244 if (context->index == efx->irq_level) 2245 efx->last_irq_cpu = raw_smp_processor_id(); 2246 2247 /* Schedule processing of the channel */ 2248 efx_schedule_channel_irq(efx->channel[context->index]); 2249 } 2250 2251 return IRQ_HANDLED; 2252 } 2253 2254 static irqreturn_t efx_ef10_legacy_interrupt(int irq, void *dev_id) 2255 { 2256 struct efx_nic *efx = dev_id; 2257 bool soft_enabled = READ_ONCE(efx->irq_soft_enabled); 2258 struct efx_channel *channel; 2259 efx_dword_t reg; 2260 u32 queues; 2261 2262 /* Read the ISR which also ACKs the interrupts */ 2263 efx_readd(efx, ®, ER_DZ_BIU_INT_ISR); 2264 queues = EFX_DWORD_FIELD(reg, ERF_DZ_ISR_REG); 2265 2266 if (queues == 0) 2267 return IRQ_NONE; 2268 2269 if (likely(soft_enabled)) { 2270 /* Note test interrupts */ 2271 if (queues & (1U << efx->irq_level)) 2272 efx->last_irq_cpu = raw_smp_processor_id(); 2273 2274 efx_for_each_channel(channel, efx) { 2275 if (queues & 1) 2276 efx_schedule_channel_irq(channel); 2277 queues >>= 1; 2278 } 2279 } 2280 2281 netif_vdbg(efx, intr, efx->net_dev, 2282 "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n", 2283 irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg)); 2284 2285 return IRQ_HANDLED; 2286 } 2287 2288 static int efx_ef10_irq_test_generate(struct efx_nic *efx) 2289 { 2290 MCDI_DECLARE_BUF(inbuf, MC_CMD_TRIGGER_INTERRUPT_IN_LEN); 2291 2292 if (efx_mcdi_set_workaround(efx, MC_CMD_WORKAROUND_BUG41750, true, 2293 NULL) == 0) 2294 return -ENOTSUPP; 2295 2296 BUILD_BUG_ON(MC_CMD_TRIGGER_INTERRUPT_OUT_LEN != 0); 2297 2298 MCDI_SET_DWORD(inbuf, TRIGGER_INTERRUPT_IN_INTR_LEVEL, efx->irq_level); 2299 return efx_mcdi_rpc(efx, MC_CMD_TRIGGER_INTERRUPT, 2300 inbuf, sizeof(inbuf), NULL, 0, NULL); 2301 } 2302 2303 static int efx_ef10_tx_probe(struct efx_tx_queue *tx_queue) 2304 { 2305 return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd.buf, 2306 (tx_queue->ptr_mask + 1) * 2307 sizeof(efx_qword_t), 2308 GFP_KERNEL); 2309 } 2310 2311 /* This writes to the TX_DESC_WPTR and also pushes data */ 2312 static inline void efx_ef10_push_tx_desc(struct efx_tx_queue *tx_queue, 2313 const efx_qword_t *txd) 2314 { 2315 unsigned int write_ptr; 2316 efx_oword_t reg; 2317 2318 write_ptr = tx_queue->write_count & tx_queue->ptr_mask; 2319 EFX_POPULATE_OWORD_1(reg, ERF_DZ_TX_DESC_WPTR, write_ptr); 2320 reg.qword[0] = *txd; 2321 efx_writeo_page(tx_queue->efx, ®, 2322 ER_DZ_TX_DESC_UPD, tx_queue->queue); 2323 } 2324 2325 /* Add Firmware-Assisted TSO v2 option descriptors to a queue. 2326 */ 2327 static int efx_ef10_tx_tso_desc(struct efx_tx_queue *tx_queue, 2328 struct sk_buff *skb, 2329 bool *data_mapped) 2330 { 2331 struct efx_tx_buffer *buffer; 2332 struct tcphdr *tcp; 2333 struct iphdr *ip; 2334 2335 u16 ipv4_id; 2336 u32 seqnum; 2337 u32 mss; 2338 2339 EFX_WARN_ON_ONCE_PARANOID(tx_queue->tso_version != 2); 2340 2341 mss = skb_shinfo(skb)->gso_size; 2342 2343 if (unlikely(mss < 4)) { 2344 WARN_ONCE(1, "MSS of %u is too small for TSO v2\n", mss); 2345 return -EINVAL; 2346 } 2347 2348 ip = ip_hdr(skb); 2349 if (ip->version == 4) { 2350 /* Modify IPv4 header if needed. */ 2351 ip->tot_len = 0; 2352 ip->check = 0; 2353 ipv4_id = ntohs(ip->id); 2354 } else { 2355 /* Modify IPv6 header if needed. */ 2356 struct ipv6hdr *ipv6 = ipv6_hdr(skb); 2357 2358 ipv6->payload_len = 0; 2359 ipv4_id = 0; 2360 } 2361 2362 tcp = tcp_hdr(skb); 2363 seqnum = ntohl(tcp->seq); 2364 2365 buffer = efx_tx_queue_get_insert_buffer(tx_queue); 2366 2367 buffer->flags = EFX_TX_BUF_OPTION; 2368 buffer->len = 0; 2369 buffer->unmap_len = 0; 2370 EFX_POPULATE_QWORD_5(buffer->option, 2371 ESF_DZ_TX_DESC_IS_OPT, 1, 2372 ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_TSO, 2373 ESF_DZ_TX_TSO_OPTION_TYPE, 2374 ESE_DZ_TX_TSO_OPTION_DESC_FATSO2A, 2375 ESF_DZ_TX_TSO_IP_ID, ipv4_id, 2376 ESF_DZ_TX_TSO_TCP_SEQNO, seqnum 2377 ); 2378 ++tx_queue->insert_count; 2379 2380 buffer = efx_tx_queue_get_insert_buffer(tx_queue); 2381 2382 buffer->flags = EFX_TX_BUF_OPTION; 2383 buffer->len = 0; 2384 buffer->unmap_len = 0; 2385 EFX_POPULATE_QWORD_4(buffer->option, 2386 ESF_DZ_TX_DESC_IS_OPT, 1, 2387 ESF_DZ_TX_OPTION_TYPE, ESE_DZ_TX_OPTION_DESC_TSO, 2388 ESF_DZ_TX_TSO_OPTION_TYPE, 2389 ESE_DZ_TX_TSO_OPTION_DESC_FATSO2B, 2390 ESF_DZ_TX_TSO_TCP_MSS, mss 2391 ); 2392 ++tx_queue->insert_count; 2393 2394 return 0; 2395 } 2396 2397 static u32 efx_ef10_tso_versions(struct efx_nic *efx) 2398 { 2399 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2400 u32 tso_versions = 0; 2401 2402 if (nic_data->datapath_caps & 2403 (1 << MC_CMD_GET_CAPABILITIES_OUT_TX_TSO_LBN)) 2404 tso_versions |= BIT(1); 2405 if (nic_data->datapath_caps2 & 2406 (1 << MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_LBN)) 2407 tso_versions |= BIT(2); 2408 return tso_versions; 2409 } 2410 2411 static void efx_ef10_tx_init(struct efx_tx_queue *tx_queue) 2412 { 2413 MCDI_DECLARE_BUF(inbuf, MC_CMD_INIT_TXQ_IN_LEN(EFX_MAX_DMAQ_SIZE * 8 / 2414 EFX_BUF_SIZE)); 2415 bool csum_offload = tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD; 2416 size_t entries = tx_queue->txd.buf.len / EFX_BUF_SIZE; 2417 struct efx_channel *channel = tx_queue->channel; 2418 struct efx_nic *efx = tx_queue->efx; 2419 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2420 bool tso_v2 = false; 2421 size_t inlen; 2422 dma_addr_t dma_addr; 2423 efx_qword_t *txd; 2424 int rc; 2425 int i; 2426 BUILD_BUG_ON(MC_CMD_INIT_TXQ_OUT_LEN != 0); 2427 2428 /* Only attempt to enable TX timestamping if we have the license for it, 2429 * otherwise TXQ init will fail 2430 */ 2431 if (!(nic_data->licensed_features & 2432 (1 << LICENSED_V3_FEATURES_TX_TIMESTAMPS_LBN))) { 2433 tx_queue->timestamping = false; 2434 /* Disable sync events on this channel. */ 2435 if (efx->type->ptp_set_ts_sync_events) 2436 efx->type->ptp_set_ts_sync_events(efx, false, false); 2437 } 2438 2439 /* TSOv2 is a limited resource that can only be configured on a limited 2440 * number of queues. TSO without checksum offload is not really a thing, 2441 * so we only enable it for those queues. 2442 * TSOv2 cannot be used with Hardware timestamping. 2443 */ 2444 if (csum_offload && (nic_data->datapath_caps2 & 2445 (1 << MC_CMD_GET_CAPABILITIES_V2_OUT_TX_TSO_V2_LBN)) && 2446 !tx_queue->timestamping) { 2447 tso_v2 = true; 2448 netif_dbg(efx, hw, efx->net_dev, "Using TSOv2 for channel %u\n", 2449 channel->channel); 2450 } 2451 2452 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_SIZE, tx_queue->ptr_mask + 1); 2453 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_TARGET_EVQ, channel->channel); 2454 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_LABEL, tx_queue->queue); 2455 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_INSTANCE, tx_queue->queue); 2456 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_OWNER_ID, 0); 2457 MCDI_SET_DWORD(inbuf, INIT_TXQ_IN_PORT_ID, nic_data->vport_id); 2458 2459 dma_addr = tx_queue->txd.buf.dma_addr; 2460 2461 netif_dbg(efx, hw, efx->net_dev, "pushing TXQ %d. %zu entries (%llx)\n", 2462 tx_queue->queue, entries, (u64)dma_addr); 2463 2464 for (i = 0; i < entries; ++i) { 2465 MCDI_SET_ARRAY_QWORD(inbuf, INIT_TXQ_IN_DMA_ADDR, i, dma_addr); 2466 dma_addr += EFX_BUF_SIZE; 2467 } 2468 2469 inlen = MC_CMD_INIT_TXQ_IN_LEN(entries); 2470 2471 do { 2472 MCDI_POPULATE_DWORD_4(inbuf, INIT_TXQ_IN_FLAGS, 2473 /* This flag was removed from mcdi_pcol.h for 2474 * the non-_EXT version of INIT_TXQ. However, 2475 * firmware still honours it. 2476 */ 2477 INIT_TXQ_EXT_IN_FLAG_TSOV2_EN, tso_v2, 2478 INIT_TXQ_IN_FLAG_IP_CSUM_DIS, !csum_offload, 2479 INIT_TXQ_IN_FLAG_TCP_CSUM_DIS, !csum_offload, 2480 INIT_TXQ_EXT_IN_FLAG_TIMESTAMP, 2481 tx_queue->timestamping); 2482 2483 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_INIT_TXQ, inbuf, inlen, 2484 NULL, 0, NULL); 2485 if (rc == -ENOSPC && tso_v2) { 2486 /* Retry without TSOv2 if we're short on contexts. */ 2487 tso_v2 = false; 2488 netif_warn(efx, probe, efx->net_dev, 2489 "TSOv2 context not available to segment in hardware. TCP performance may be reduced.\n"); 2490 } else if (rc) { 2491 efx_mcdi_display_error(efx, MC_CMD_INIT_TXQ, 2492 MC_CMD_INIT_TXQ_EXT_IN_LEN, 2493 NULL, 0, rc); 2494 goto fail; 2495 } 2496 } while (rc); 2497 2498 /* A previous user of this TX queue might have set us up the 2499 * bomb by writing a descriptor to the TX push collector but 2500 * not the doorbell. (Each collector belongs to a port, not a 2501 * queue or function, so cannot easily be reset.) We must 2502 * attempt to push a no-op descriptor in its place. 2503 */ 2504 tx_queue->buffer[0].flags = EFX_TX_BUF_OPTION; 2505 tx_queue->insert_count = 1; 2506 txd = efx_tx_desc(tx_queue, 0); 2507 EFX_POPULATE_QWORD_5(*txd, 2508 ESF_DZ_TX_DESC_IS_OPT, true, 2509 ESF_DZ_TX_OPTION_TYPE, 2510 ESE_DZ_TX_OPTION_DESC_CRC_CSUM, 2511 ESF_DZ_TX_OPTION_UDP_TCP_CSUM, csum_offload, 2512 ESF_DZ_TX_OPTION_IP_CSUM, csum_offload, 2513 ESF_DZ_TX_TIMESTAMP, tx_queue->timestamping); 2514 tx_queue->write_count = 1; 2515 2516 if (tso_v2) { 2517 tx_queue->handle_tso = efx_ef10_tx_tso_desc; 2518 tx_queue->tso_version = 2; 2519 } else if (nic_data->datapath_caps & 2520 (1 << MC_CMD_GET_CAPABILITIES_OUT_TX_TSO_LBN)) { 2521 tx_queue->tso_version = 1; 2522 } 2523 2524 wmb(); 2525 efx_ef10_push_tx_desc(tx_queue, txd); 2526 2527 return; 2528 2529 fail: 2530 netdev_WARN(efx->net_dev, "failed to initialise TXQ %d\n", 2531 tx_queue->queue); 2532 } 2533 2534 static void efx_ef10_tx_fini(struct efx_tx_queue *tx_queue) 2535 { 2536 MCDI_DECLARE_BUF(inbuf, MC_CMD_FINI_TXQ_IN_LEN); 2537 MCDI_DECLARE_BUF_ERR(outbuf); 2538 struct efx_nic *efx = tx_queue->efx; 2539 size_t outlen; 2540 int rc; 2541 2542 MCDI_SET_DWORD(inbuf, FINI_TXQ_IN_INSTANCE, 2543 tx_queue->queue); 2544 2545 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FINI_TXQ, inbuf, sizeof(inbuf), 2546 outbuf, sizeof(outbuf), &outlen); 2547 2548 if (rc && rc != -EALREADY) 2549 goto fail; 2550 2551 return; 2552 2553 fail: 2554 efx_mcdi_display_error(efx, MC_CMD_FINI_TXQ, MC_CMD_FINI_TXQ_IN_LEN, 2555 outbuf, outlen, rc); 2556 } 2557 2558 static void efx_ef10_tx_remove(struct efx_tx_queue *tx_queue) 2559 { 2560 efx_nic_free_buffer(tx_queue->efx, &tx_queue->txd.buf); 2561 } 2562 2563 /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */ 2564 static inline void efx_ef10_notify_tx_desc(struct efx_tx_queue *tx_queue) 2565 { 2566 unsigned int write_ptr; 2567 efx_dword_t reg; 2568 2569 write_ptr = tx_queue->write_count & tx_queue->ptr_mask; 2570 EFX_POPULATE_DWORD_1(reg, ERF_DZ_TX_DESC_WPTR_DWORD, write_ptr); 2571 efx_writed_page(tx_queue->efx, ®, 2572 ER_DZ_TX_DESC_UPD_DWORD, tx_queue->queue); 2573 } 2574 2575 #define EFX_EF10_MAX_TX_DESCRIPTOR_LEN 0x3fff 2576 2577 static unsigned int efx_ef10_tx_limit_len(struct efx_tx_queue *tx_queue, 2578 dma_addr_t dma_addr, unsigned int len) 2579 { 2580 if (len > EFX_EF10_MAX_TX_DESCRIPTOR_LEN) { 2581 /* If we need to break across multiple descriptors we should 2582 * stop at a page boundary. This assumes the length limit is 2583 * greater than the page size. 2584 */ 2585 dma_addr_t end = dma_addr + EFX_EF10_MAX_TX_DESCRIPTOR_LEN; 2586 2587 BUILD_BUG_ON(EFX_EF10_MAX_TX_DESCRIPTOR_LEN < EFX_PAGE_SIZE); 2588 len = (end & (~(EFX_PAGE_SIZE - 1))) - dma_addr; 2589 } 2590 2591 return len; 2592 } 2593 2594 static void efx_ef10_tx_write(struct efx_tx_queue *tx_queue) 2595 { 2596 unsigned int old_write_count = tx_queue->write_count; 2597 struct efx_tx_buffer *buffer; 2598 unsigned int write_ptr; 2599 efx_qword_t *txd; 2600 2601 tx_queue->xmit_more_available = false; 2602 if (unlikely(tx_queue->write_count == tx_queue->insert_count)) 2603 return; 2604 2605 do { 2606 write_ptr = tx_queue->write_count & tx_queue->ptr_mask; 2607 buffer = &tx_queue->buffer[write_ptr]; 2608 txd = efx_tx_desc(tx_queue, write_ptr); 2609 ++tx_queue->write_count; 2610 2611 /* Create TX descriptor ring entry */ 2612 if (buffer->flags & EFX_TX_BUF_OPTION) { 2613 *txd = buffer->option; 2614 if (EFX_QWORD_FIELD(*txd, ESF_DZ_TX_OPTION_TYPE) == 1) 2615 /* PIO descriptor */ 2616 tx_queue->packet_write_count = tx_queue->write_count; 2617 } else { 2618 tx_queue->packet_write_count = tx_queue->write_count; 2619 BUILD_BUG_ON(EFX_TX_BUF_CONT != 1); 2620 EFX_POPULATE_QWORD_3( 2621 *txd, 2622 ESF_DZ_TX_KER_CONT, 2623 buffer->flags & EFX_TX_BUF_CONT, 2624 ESF_DZ_TX_KER_BYTE_CNT, buffer->len, 2625 ESF_DZ_TX_KER_BUF_ADDR, buffer->dma_addr); 2626 } 2627 } while (tx_queue->write_count != tx_queue->insert_count); 2628 2629 wmb(); /* Ensure descriptors are written before they are fetched */ 2630 2631 if (efx_nic_may_push_tx_desc(tx_queue, old_write_count)) { 2632 txd = efx_tx_desc(tx_queue, 2633 old_write_count & tx_queue->ptr_mask); 2634 efx_ef10_push_tx_desc(tx_queue, txd); 2635 ++tx_queue->pushes; 2636 } else { 2637 efx_ef10_notify_tx_desc(tx_queue); 2638 } 2639 } 2640 2641 #define RSS_MODE_HASH_ADDRS (1 << RSS_MODE_HASH_SRC_ADDR_LBN |\ 2642 1 << RSS_MODE_HASH_DST_ADDR_LBN) 2643 #define RSS_MODE_HASH_PORTS (1 << RSS_MODE_HASH_SRC_PORT_LBN |\ 2644 1 << RSS_MODE_HASH_DST_PORT_LBN) 2645 #define RSS_CONTEXT_FLAGS_DEFAULT (1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN |\ 2646 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN |\ 2647 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_LBN |\ 2648 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_LBN |\ 2649 (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_LBN |\ 2650 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN |\ 2651 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_LBN |\ 2652 (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_LBN |\ 2653 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN |\ 2654 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_LBN) 2655 2656 static int efx_ef10_get_rss_flags(struct efx_nic *efx, u32 context, u32 *flags) 2657 { 2658 /* Firmware had a bug (sfc bug 61952) where it would not actually 2659 * fill in the flags field in the response to MC_CMD_RSS_CONTEXT_GET_FLAGS. 2660 * This meant that it would always contain whatever was previously 2661 * in the MCDI buffer. Fortunately, all firmware versions with 2662 * this bug have the same default flags value for a newly-allocated 2663 * RSS context, and the only time we want to get the flags is just 2664 * after allocating. Moreover, the response has a 32-bit hole 2665 * where the context ID would be in the request, so we can use an 2666 * overlength buffer in the request and pre-fill the flags field 2667 * with what we believe the default to be. Thus if the firmware 2668 * has the bug, it will leave our pre-filled value in the flags 2669 * field of the response, and we will get the right answer. 2670 * 2671 * However, this does mean that this function should NOT be used if 2672 * the RSS context flags might not be their defaults - it is ONLY 2673 * reliably correct for a newly-allocated RSS context. 2674 */ 2675 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN); 2676 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN); 2677 size_t outlen; 2678 int rc; 2679 2680 /* Check we have a hole for the context ID */ 2681 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN != MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST); 2682 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID, context); 2683 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS, 2684 RSS_CONTEXT_FLAGS_DEFAULT); 2685 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_FLAGS, inbuf, 2686 sizeof(inbuf), outbuf, sizeof(outbuf), &outlen); 2687 if (rc == 0) { 2688 if (outlen < MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN) 2689 rc = -EIO; 2690 else 2691 *flags = MCDI_DWORD(outbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS); 2692 } 2693 return rc; 2694 } 2695 2696 /* Attempt to enable 4-tuple UDP hashing on the specified RSS context. 2697 * If we fail, we just leave the RSS context at its default hash settings, 2698 * which is safe but may slightly reduce performance. 2699 * Defaults are 4-tuple for TCP and 2-tuple for UDP and other-IP, so we 2700 * just need to set the UDP ports flags (for both IP versions). 2701 */ 2702 static void efx_ef10_set_rss_flags(struct efx_nic *efx, 2703 struct efx_rss_context *ctx) 2704 { 2705 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN); 2706 u32 flags; 2707 2708 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN != 0); 2709 2710 if (efx_ef10_get_rss_flags(efx, ctx->context_id, &flags) != 0) 2711 return; 2712 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 2713 ctx->context_id); 2714 flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN; 2715 flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN; 2716 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, flags); 2717 if (!efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_FLAGS, inbuf, sizeof(inbuf), 2718 NULL, 0, NULL)) 2719 /* Succeeded, so UDP 4-tuple is now enabled */ 2720 ctx->rx_hash_udp_4tuple = true; 2721 } 2722 2723 static int efx_ef10_alloc_rss_context(struct efx_nic *efx, bool exclusive, 2724 struct efx_rss_context *ctx, 2725 unsigned *context_size) 2726 { 2727 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN); 2728 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN); 2729 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2730 size_t outlen; 2731 int rc; 2732 u32 alloc_type = exclusive ? 2733 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE : 2734 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 2735 unsigned rss_spread = exclusive ? 2736 efx->rss_spread : 2737 min(rounddown_pow_of_two(efx->rss_spread), 2738 EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE); 2739 2740 if (!exclusive && rss_spread == 1) { 2741 ctx->context_id = EFX_EF10_RSS_CONTEXT_INVALID; 2742 if (context_size) 2743 *context_size = 1; 2744 return 0; 2745 } 2746 2747 if (nic_data->datapath_caps & 2748 1 << MC_CMD_GET_CAPABILITIES_OUT_RX_RSS_LIMITED_LBN) 2749 return -EOPNOTSUPP; 2750 2751 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 2752 nic_data->vport_id); 2753 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_TYPE, alloc_type); 2754 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, rss_spread); 2755 2756 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_ALLOC, inbuf, sizeof(inbuf), 2757 outbuf, sizeof(outbuf), &outlen); 2758 if (rc != 0) 2759 return rc; 2760 2761 if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) 2762 return -EIO; 2763 2764 ctx->context_id = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 2765 2766 if (context_size) 2767 *context_size = rss_spread; 2768 2769 if (nic_data->datapath_caps & 2770 1 << MC_CMD_GET_CAPABILITIES_OUT_ADDITIONAL_RSS_MODES_LBN) 2771 efx_ef10_set_rss_flags(efx, ctx); 2772 2773 return 0; 2774 } 2775 2776 static int efx_ef10_free_rss_context(struct efx_nic *efx, u32 context) 2777 { 2778 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN); 2779 2780 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, 2781 context); 2782 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf), 2783 NULL, 0, NULL); 2784 } 2785 2786 static int efx_ef10_populate_rss_table(struct efx_nic *efx, u32 context, 2787 const u32 *rx_indir_table, const u8 *key) 2788 { 2789 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN); 2790 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN); 2791 int i, rc; 2792 2793 MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 2794 context); 2795 BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) != 2796 MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN); 2797 2798 /* This iterates over the length of efx->rss_context.rx_indir_table, but 2799 * copies bytes from rx_indir_table. That's because the latter is a 2800 * pointer rather than an array, but should have the same length. 2801 * The efx->rss_context.rx_hash_key loop below is similar. 2802 */ 2803 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); ++i) 2804 MCDI_PTR(tablebuf, 2805 RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] = 2806 (u8) rx_indir_table[i]; 2807 2808 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_TABLE, tablebuf, 2809 sizeof(tablebuf), NULL, 0, NULL); 2810 if (rc != 0) 2811 return rc; 2812 2813 MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 2814 context); 2815 BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_hash_key) != 2816 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 2817 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_hash_key); ++i) 2818 MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] = key[i]; 2819 2820 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, keybuf, 2821 sizeof(keybuf), NULL, 0, NULL); 2822 } 2823 2824 static void efx_ef10_rx_free_indir_table(struct efx_nic *efx) 2825 { 2826 int rc; 2827 2828 if (efx->rss_context.context_id != EFX_EF10_RSS_CONTEXT_INVALID) { 2829 rc = efx_ef10_free_rss_context(efx, efx->rss_context.context_id); 2830 WARN_ON(rc != 0); 2831 } 2832 efx->rss_context.context_id = EFX_EF10_RSS_CONTEXT_INVALID; 2833 } 2834 2835 static int efx_ef10_rx_push_shared_rss_config(struct efx_nic *efx, 2836 unsigned *context_size) 2837 { 2838 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2839 int rc = efx_ef10_alloc_rss_context(efx, false, &efx->rss_context, 2840 context_size); 2841 2842 if (rc != 0) 2843 return rc; 2844 2845 nic_data->rx_rss_context_exclusive = false; 2846 efx_set_default_rx_indir_table(efx, &efx->rss_context); 2847 return 0; 2848 } 2849 2850 static int efx_ef10_rx_push_exclusive_rss_config(struct efx_nic *efx, 2851 const u32 *rx_indir_table, 2852 const u8 *key) 2853 { 2854 u32 old_rx_rss_context = efx->rss_context.context_id; 2855 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2856 int rc; 2857 2858 if (efx->rss_context.context_id == EFX_EF10_RSS_CONTEXT_INVALID || 2859 !nic_data->rx_rss_context_exclusive) { 2860 rc = efx_ef10_alloc_rss_context(efx, true, &efx->rss_context, 2861 NULL); 2862 if (rc == -EOPNOTSUPP) 2863 return rc; 2864 else if (rc != 0) 2865 goto fail1; 2866 } 2867 2868 rc = efx_ef10_populate_rss_table(efx, efx->rss_context.context_id, 2869 rx_indir_table, key); 2870 if (rc != 0) 2871 goto fail2; 2872 2873 if (efx->rss_context.context_id != old_rx_rss_context && 2874 old_rx_rss_context != EFX_EF10_RSS_CONTEXT_INVALID) 2875 WARN_ON(efx_ef10_free_rss_context(efx, old_rx_rss_context) != 0); 2876 nic_data->rx_rss_context_exclusive = true; 2877 if (rx_indir_table != efx->rss_context.rx_indir_table) 2878 memcpy(efx->rss_context.rx_indir_table, rx_indir_table, 2879 sizeof(efx->rss_context.rx_indir_table)); 2880 if (key != efx->rss_context.rx_hash_key) 2881 memcpy(efx->rss_context.rx_hash_key, key, 2882 efx->type->rx_hash_key_size); 2883 2884 return 0; 2885 2886 fail2: 2887 if (old_rx_rss_context != efx->rss_context.context_id) { 2888 WARN_ON(efx_ef10_free_rss_context(efx, efx->rss_context.context_id) != 0); 2889 efx->rss_context.context_id = old_rx_rss_context; 2890 } 2891 fail1: 2892 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc); 2893 return rc; 2894 } 2895 2896 static int efx_ef10_rx_push_rss_context_config(struct efx_nic *efx, 2897 struct efx_rss_context *ctx, 2898 const u32 *rx_indir_table, 2899 const u8 *key) 2900 { 2901 int rc; 2902 2903 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 2904 2905 if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) { 2906 rc = efx_ef10_alloc_rss_context(efx, true, ctx, NULL); 2907 if (rc) 2908 return rc; 2909 } 2910 2911 if (!rx_indir_table) /* Delete this context */ 2912 return efx_ef10_free_rss_context(efx, ctx->context_id); 2913 2914 rc = efx_ef10_populate_rss_table(efx, ctx->context_id, 2915 rx_indir_table, key); 2916 if (rc) 2917 return rc; 2918 2919 memcpy(ctx->rx_indir_table, rx_indir_table, 2920 sizeof(efx->rss_context.rx_indir_table)); 2921 memcpy(ctx->rx_hash_key, key, efx->type->rx_hash_key_size); 2922 2923 return 0; 2924 } 2925 2926 static int efx_ef10_rx_pull_rss_context_config(struct efx_nic *efx, 2927 struct efx_rss_context *ctx) 2928 { 2929 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN); 2930 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN); 2931 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN); 2932 size_t outlen; 2933 int rc, i; 2934 2935 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 2936 2937 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN != 2938 MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN); 2939 2940 if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) 2941 return -ENOENT; 2942 2943 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID, 2944 ctx->context_id); 2945 BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_indir_table) != 2946 MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN); 2947 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_TABLE, inbuf, sizeof(inbuf), 2948 tablebuf, sizeof(tablebuf), &outlen); 2949 if (rc != 0) 2950 return rc; 2951 2952 if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN)) 2953 return -EIO; 2954 2955 for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) 2956 ctx->rx_indir_table[i] = MCDI_PTR(tablebuf, 2957 RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE)[i]; 2958 2959 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID, 2960 ctx->context_id); 2961 BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_hash_key) != 2962 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 2963 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_KEY, inbuf, sizeof(inbuf), 2964 keybuf, sizeof(keybuf), &outlen); 2965 if (rc != 0) 2966 return rc; 2967 2968 if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN)) 2969 return -EIO; 2970 2971 for (i = 0; i < ARRAY_SIZE(ctx->rx_hash_key); ++i) 2972 ctx->rx_hash_key[i] = MCDI_PTR( 2973 keybuf, RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY)[i]; 2974 2975 return 0; 2976 } 2977 2978 static int efx_ef10_rx_pull_rss_config(struct efx_nic *efx) 2979 { 2980 int rc; 2981 2982 mutex_lock(&efx->rss_lock); 2983 rc = efx_ef10_rx_pull_rss_context_config(efx, &efx->rss_context); 2984 mutex_unlock(&efx->rss_lock); 2985 return rc; 2986 } 2987 2988 static void efx_ef10_rx_restore_rss_contexts(struct efx_nic *efx) 2989 { 2990 struct efx_ef10_nic_data *nic_data = efx->nic_data; 2991 struct efx_rss_context *ctx; 2992 int rc; 2993 2994 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 2995 2996 if (!nic_data->must_restore_rss_contexts) 2997 return; 2998 2999 list_for_each_entry(ctx, &efx->rss_context.list, list) { 3000 /* previous NIC RSS context is gone */ 3001 ctx->context_id = EFX_EF10_RSS_CONTEXT_INVALID; 3002 /* so try to allocate a new one */ 3003 rc = efx_ef10_rx_push_rss_context_config(efx, ctx, 3004 ctx->rx_indir_table, 3005 ctx->rx_hash_key); 3006 if (rc) 3007 netif_warn(efx, probe, efx->net_dev, 3008 "failed to restore RSS context %u, rc=%d" 3009 "; RSS filters may fail to be applied\n", 3010 ctx->user_id, rc); 3011 } 3012 nic_data->must_restore_rss_contexts = false; 3013 } 3014 3015 static int efx_ef10_pf_rx_push_rss_config(struct efx_nic *efx, bool user, 3016 const u32 *rx_indir_table, 3017 const u8 *key) 3018 { 3019 int rc; 3020 3021 if (efx->rss_spread == 1) 3022 return 0; 3023 3024 if (!key) 3025 key = efx->rss_context.rx_hash_key; 3026 3027 rc = efx_ef10_rx_push_exclusive_rss_config(efx, rx_indir_table, key); 3028 3029 if (rc == -ENOBUFS && !user) { 3030 unsigned context_size; 3031 bool mismatch = false; 3032 size_t i; 3033 3034 for (i = 0; 3035 i < ARRAY_SIZE(efx->rss_context.rx_indir_table) && !mismatch; 3036 i++) 3037 mismatch = rx_indir_table[i] != 3038 ethtool_rxfh_indir_default(i, efx->rss_spread); 3039 3040 rc = efx_ef10_rx_push_shared_rss_config(efx, &context_size); 3041 if (rc == 0) { 3042 if (context_size != efx->rss_spread) 3043 netif_warn(efx, probe, efx->net_dev, 3044 "Could not allocate an exclusive RSS" 3045 " context; allocated a shared one of" 3046 " different size." 3047 " Wanted %u, got %u.\n", 3048 efx->rss_spread, context_size); 3049 else if (mismatch) 3050 netif_warn(efx, probe, efx->net_dev, 3051 "Could not allocate an exclusive RSS" 3052 " context; allocated a shared one but" 3053 " could not apply custom" 3054 " indirection.\n"); 3055 else 3056 netif_info(efx, probe, efx->net_dev, 3057 "Could not allocate an exclusive RSS" 3058 " context; allocated a shared one.\n"); 3059 } 3060 } 3061 return rc; 3062 } 3063 3064 static int efx_ef10_vf_rx_push_rss_config(struct efx_nic *efx, bool user, 3065 const u32 *rx_indir_table 3066 __attribute__ ((unused)), 3067 const u8 *key 3068 __attribute__ ((unused))) 3069 { 3070 if (user) 3071 return -EOPNOTSUPP; 3072 if (efx->rss_context.context_id != EFX_EF10_RSS_CONTEXT_INVALID) 3073 return 0; 3074 return efx_ef10_rx_push_shared_rss_config(efx, NULL); 3075 } 3076 3077 static int efx_ef10_rx_probe(struct efx_rx_queue *rx_queue) 3078 { 3079 return efx_nic_alloc_buffer(rx_queue->efx, &rx_queue->rxd.buf, 3080 (rx_queue->ptr_mask + 1) * 3081 sizeof(efx_qword_t), 3082 GFP_KERNEL); 3083 } 3084 3085 static void efx_ef10_rx_init(struct efx_rx_queue *rx_queue) 3086 { 3087 MCDI_DECLARE_BUF(inbuf, 3088 MC_CMD_INIT_RXQ_IN_LEN(EFX_MAX_DMAQ_SIZE * 8 / 3089 EFX_BUF_SIZE)); 3090 struct efx_channel *channel = efx_rx_queue_channel(rx_queue); 3091 size_t entries = rx_queue->rxd.buf.len / EFX_BUF_SIZE; 3092 struct efx_nic *efx = rx_queue->efx; 3093 struct efx_ef10_nic_data *nic_data = efx->nic_data; 3094 size_t inlen; 3095 dma_addr_t dma_addr; 3096 int rc; 3097 int i; 3098 BUILD_BUG_ON(MC_CMD_INIT_RXQ_OUT_LEN != 0); 3099 3100 rx_queue->scatter_n = 0; 3101 rx_queue->scatter_len = 0; 3102 3103 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_SIZE, rx_queue->ptr_mask + 1); 3104 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_TARGET_EVQ, channel->channel); 3105 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_LABEL, efx_rx_queue_index(rx_queue)); 3106 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_INSTANCE, 3107 efx_rx_queue_index(rx_queue)); 3108 MCDI_POPULATE_DWORD_2(inbuf, INIT_RXQ_IN_FLAGS, 3109 INIT_RXQ_IN_FLAG_PREFIX, 1, 3110 INIT_RXQ_IN_FLAG_TIMESTAMP, 1); 3111 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_OWNER_ID, 0); 3112 MCDI_SET_DWORD(inbuf, INIT_RXQ_IN_PORT_ID, nic_data->vport_id); 3113 3114 dma_addr = rx_queue->rxd.buf.dma_addr; 3115 3116 netif_dbg(efx, hw, efx->net_dev, "pushing RXQ %d. %zu entries (%llx)\n", 3117 efx_rx_queue_index(rx_queue), entries, (u64)dma_addr); 3118 3119 for (i = 0; i < entries; ++i) { 3120 MCDI_SET_ARRAY_QWORD(inbuf, INIT_RXQ_IN_DMA_ADDR, i, dma_addr); 3121 dma_addr += EFX_BUF_SIZE; 3122 } 3123 3124 inlen = MC_CMD_INIT_RXQ_IN_LEN(entries); 3125 3126 rc = efx_mcdi_rpc(efx, MC_CMD_INIT_RXQ, inbuf, inlen, 3127 NULL, 0, NULL); 3128 if (rc) 3129 netdev_WARN(efx->net_dev, "failed to initialise RXQ %d\n", 3130 efx_rx_queue_index(rx_queue)); 3131 } 3132 3133 static void efx_ef10_rx_fini(struct efx_rx_queue *rx_queue) 3134 { 3135 MCDI_DECLARE_BUF(inbuf, MC_CMD_FINI_RXQ_IN_LEN); 3136 MCDI_DECLARE_BUF_ERR(outbuf); 3137 struct efx_nic *efx = rx_queue->efx; 3138 size_t outlen; 3139 int rc; 3140 3141 MCDI_SET_DWORD(inbuf, FINI_RXQ_IN_INSTANCE, 3142 efx_rx_queue_index(rx_queue)); 3143 3144 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FINI_RXQ, inbuf, sizeof(inbuf), 3145 outbuf, sizeof(outbuf), &outlen); 3146 3147 if (rc && rc != -EALREADY) 3148 goto fail; 3149 3150 return; 3151 3152 fail: 3153 efx_mcdi_display_error(efx, MC_CMD_FINI_RXQ, MC_CMD_FINI_RXQ_IN_LEN, 3154 outbuf, outlen, rc); 3155 } 3156 3157 static void efx_ef10_rx_remove(struct efx_rx_queue *rx_queue) 3158 { 3159 efx_nic_free_buffer(rx_queue->efx, &rx_queue->rxd.buf); 3160 } 3161 3162 /* This creates an entry in the RX descriptor queue */ 3163 static inline void 3164 efx_ef10_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index) 3165 { 3166 struct efx_rx_buffer *rx_buf; 3167 efx_qword_t *rxd; 3168 3169 rxd = efx_rx_desc(rx_queue, index); 3170 rx_buf = efx_rx_buffer(rx_queue, index); 3171 EFX_POPULATE_QWORD_2(*rxd, 3172 ESF_DZ_RX_KER_BYTE_CNT, rx_buf->len, 3173 ESF_DZ_RX_KER_BUF_ADDR, rx_buf->dma_addr); 3174 } 3175 3176 static void efx_ef10_rx_write(struct efx_rx_queue *rx_queue) 3177 { 3178 struct efx_nic *efx = rx_queue->efx; 3179 unsigned int write_count; 3180 efx_dword_t reg; 3181 3182 /* Firmware requires that RX_DESC_WPTR be a multiple of 8 */ 3183 write_count = rx_queue->added_count & ~7; 3184 if (rx_queue->notified_count == write_count) 3185 return; 3186 3187 do 3188 efx_ef10_build_rx_desc( 3189 rx_queue, 3190 rx_queue->notified_count & rx_queue->ptr_mask); 3191 while (++rx_queue->notified_count != write_count); 3192 3193 wmb(); 3194 EFX_POPULATE_DWORD_1(reg, ERF_DZ_RX_DESC_WPTR, 3195 write_count & rx_queue->ptr_mask); 3196 efx_writed_page(efx, ®, ER_DZ_RX_DESC_UPD, 3197 efx_rx_queue_index(rx_queue)); 3198 } 3199 3200 static efx_mcdi_async_completer efx_ef10_rx_defer_refill_complete; 3201 3202 static void efx_ef10_rx_defer_refill(struct efx_rx_queue *rx_queue) 3203 { 3204 struct efx_channel *channel = efx_rx_queue_channel(rx_queue); 3205 MCDI_DECLARE_BUF(inbuf, MC_CMD_DRIVER_EVENT_IN_LEN); 3206 efx_qword_t event; 3207 3208 EFX_POPULATE_QWORD_2(event, 3209 ESF_DZ_EV_CODE, EFX_EF10_DRVGEN_EV, 3210 ESF_DZ_EV_DATA, EFX_EF10_REFILL); 3211 3212 MCDI_SET_DWORD(inbuf, DRIVER_EVENT_IN_EVQ, channel->channel); 3213 3214 /* MCDI_SET_QWORD is not appropriate here since EFX_POPULATE_* has 3215 * already swapped the data to little-endian order. 3216 */ 3217 memcpy(MCDI_PTR(inbuf, DRIVER_EVENT_IN_DATA), &event.u64[0], 3218 sizeof(efx_qword_t)); 3219 3220 efx_mcdi_rpc_async(channel->efx, MC_CMD_DRIVER_EVENT, 3221 inbuf, sizeof(inbuf), 0, 3222 efx_ef10_rx_defer_refill_complete, 0); 3223 } 3224 3225 static void 3226 efx_ef10_rx_defer_refill_complete(struct efx_nic *efx, unsigned long cookie, 3227 int rc, efx_dword_t *outbuf, 3228 size_t outlen_actual) 3229 { 3230 /* nothing to do */ 3231 } 3232 3233 static int efx_ef10_ev_probe(struct efx_channel *channel) 3234 { 3235 return efx_nic_alloc_buffer(channel->efx, &channel->eventq.buf, 3236 (channel->eventq_mask + 1) * 3237 sizeof(efx_qword_t), 3238 GFP_KERNEL); 3239 } 3240 3241 static void efx_ef10_ev_fini(struct efx_channel *channel) 3242 { 3243 MCDI_DECLARE_BUF(inbuf, MC_CMD_FINI_EVQ_IN_LEN); 3244 MCDI_DECLARE_BUF_ERR(outbuf); 3245 struct efx_nic *efx = channel->efx; 3246 size_t outlen; 3247 int rc; 3248 3249 MCDI_SET_DWORD(inbuf, FINI_EVQ_IN_INSTANCE, channel->channel); 3250 3251 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FINI_EVQ, inbuf, sizeof(inbuf), 3252 outbuf, sizeof(outbuf), &outlen); 3253 3254 if (rc && rc != -EALREADY) 3255 goto fail; 3256 3257 return; 3258 3259 fail: 3260 efx_mcdi_display_error(efx, MC_CMD_FINI_EVQ, MC_CMD_FINI_EVQ_IN_LEN, 3261 outbuf, outlen, rc); 3262 } 3263 3264 static int efx_ef10_ev_init(struct efx_channel *channel) 3265 { 3266 MCDI_DECLARE_BUF(inbuf, 3267 MC_CMD_INIT_EVQ_V2_IN_LEN(EFX_MAX_EVQ_SIZE * 8 / 3268 EFX_BUF_SIZE)); 3269 MCDI_DECLARE_BUF(outbuf, MC_CMD_INIT_EVQ_V2_OUT_LEN); 3270 size_t entries = channel->eventq.buf.len / EFX_BUF_SIZE; 3271 struct efx_nic *efx = channel->efx; 3272 struct efx_ef10_nic_data *nic_data; 3273 size_t inlen, outlen; 3274 unsigned int enabled, implemented; 3275 dma_addr_t dma_addr; 3276 int rc; 3277 int i; 3278 3279 nic_data = efx->nic_data; 3280 3281 /* Fill event queue with all ones (i.e. empty events) */ 3282 memset(channel->eventq.buf.addr, 0xff, channel->eventq.buf.len); 3283 3284 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_SIZE, channel->eventq_mask + 1); 3285 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_INSTANCE, channel->channel); 3286 /* INIT_EVQ expects index in vector table, not absolute */ 3287 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_IRQ_NUM, channel->channel); 3288 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_TMR_MODE, 3289 MC_CMD_INIT_EVQ_IN_TMR_MODE_DIS); 3290 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_TMR_LOAD, 0); 3291 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_TMR_RELOAD, 0); 3292 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_COUNT_MODE, 3293 MC_CMD_INIT_EVQ_IN_COUNT_MODE_DIS); 3294 MCDI_SET_DWORD(inbuf, INIT_EVQ_IN_COUNT_THRSHLD, 0); 3295 3296 if (nic_data->datapath_caps2 & 3297 1 << MC_CMD_GET_CAPABILITIES_V2_OUT_INIT_EVQ_V2_LBN) { 3298 /* Use the new generic approach to specifying event queue 3299 * configuration, requesting lower latency or higher throughput. 3300 * The options that actually get used appear in the output. 3301 */ 3302 MCDI_POPULATE_DWORD_2(inbuf, INIT_EVQ_V2_IN_FLAGS, 3303 INIT_EVQ_V2_IN_FLAG_INTERRUPTING, 1, 3304 INIT_EVQ_V2_IN_FLAG_TYPE, 3305 MC_CMD_INIT_EVQ_V2_IN_FLAG_TYPE_AUTO); 3306 } else { 3307 bool cut_thru = !(nic_data->datapath_caps & 3308 1 << MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_LBN); 3309 3310 MCDI_POPULATE_DWORD_4(inbuf, INIT_EVQ_IN_FLAGS, 3311 INIT_EVQ_IN_FLAG_INTERRUPTING, 1, 3312 INIT_EVQ_IN_FLAG_RX_MERGE, 1, 3313 INIT_EVQ_IN_FLAG_TX_MERGE, 1, 3314 INIT_EVQ_IN_FLAG_CUT_THRU, cut_thru); 3315 } 3316 3317 dma_addr = channel->eventq.buf.dma_addr; 3318 for (i = 0; i < entries; ++i) { 3319 MCDI_SET_ARRAY_QWORD(inbuf, INIT_EVQ_IN_DMA_ADDR, i, dma_addr); 3320 dma_addr += EFX_BUF_SIZE; 3321 } 3322 3323 inlen = MC_CMD_INIT_EVQ_IN_LEN(entries); 3324 3325 rc = efx_mcdi_rpc(efx, MC_CMD_INIT_EVQ, inbuf, inlen, 3326 outbuf, sizeof(outbuf), &outlen); 3327 3328 if (outlen >= MC_CMD_INIT_EVQ_V2_OUT_LEN) 3329 netif_dbg(efx, drv, efx->net_dev, 3330 "Channel %d using event queue flags %08x\n", 3331 channel->channel, 3332 MCDI_DWORD(outbuf, INIT_EVQ_V2_OUT_FLAGS)); 3333 3334 /* IRQ return is ignored */ 3335 if (channel->channel || rc) 3336 return rc; 3337 3338 /* Successfully created event queue on channel 0 */ 3339 rc = efx_mcdi_get_workarounds(efx, &implemented, &enabled); 3340 if (rc == -ENOSYS) { 3341 /* GET_WORKAROUNDS was implemented before this workaround, 3342 * thus it must be unavailable in this firmware. 3343 */ 3344 nic_data->workaround_26807 = false; 3345 rc = 0; 3346 } else if (rc) { 3347 goto fail; 3348 } else { 3349 nic_data->workaround_26807 = 3350 !!(enabled & MC_CMD_GET_WORKAROUNDS_OUT_BUG26807); 3351 3352 if (implemented & MC_CMD_GET_WORKAROUNDS_OUT_BUG26807 && 3353 !nic_data->workaround_26807) { 3354 unsigned int flags; 3355 3356 rc = efx_mcdi_set_workaround(efx, 3357 MC_CMD_WORKAROUND_BUG26807, 3358 true, &flags); 3359 3360 if (!rc) { 3361 if (flags & 3362 1 << MC_CMD_WORKAROUND_EXT_OUT_FLR_DONE_LBN) { 3363 netif_info(efx, drv, efx->net_dev, 3364 "other functions on NIC have been reset\n"); 3365 3366 /* With MCFW v4.6.x and earlier, the 3367 * boot count will have incremented, 3368 * so re-read the warm_boot_count 3369 * value now to ensure this function 3370 * doesn't think it has changed next 3371 * time it checks. 3372 */ 3373 rc = efx_ef10_get_warm_boot_count(efx); 3374 if (rc >= 0) { 3375 nic_data->warm_boot_count = rc; 3376 rc = 0; 3377 } 3378 } 3379 nic_data->workaround_26807 = true; 3380 } else if (rc == -EPERM) { 3381 rc = 0; 3382 } 3383 } 3384 } 3385 3386 if (!rc) 3387 return 0; 3388 3389 fail: 3390 efx_ef10_ev_fini(channel); 3391 return rc; 3392 } 3393 3394 static void efx_ef10_ev_remove(struct efx_channel *channel) 3395 { 3396 efx_nic_free_buffer(channel->efx, &channel->eventq.buf); 3397 } 3398 3399 static void efx_ef10_handle_rx_wrong_queue(struct efx_rx_queue *rx_queue, 3400 unsigned int rx_queue_label) 3401 { 3402 struct efx_nic *efx = rx_queue->efx; 3403 3404 netif_info(efx, hw, efx->net_dev, 3405 "rx event arrived on queue %d labeled as queue %u\n", 3406 efx_rx_queue_index(rx_queue), rx_queue_label); 3407 3408 efx_schedule_reset(efx, RESET_TYPE_DISABLE); 3409 } 3410 3411 static void 3412 efx_ef10_handle_rx_bad_lbits(struct efx_rx_queue *rx_queue, 3413 unsigned int actual, unsigned int expected) 3414 { 3415 unsigned int dropped = (actual - expected) & rx_queue->ptr_mask; 3416 struct efx_nic *efx = rx_queue->efx; 3417 3418 netif_info(efx, hw, efx->net_dev, 3419 "dropped %d events (index=%d expected=%d)\n", 3420 dropped, actual, expected); 3421 3422 efx_schedule_reset(efx, RESET_TYPE_DISABLE); 3423 } 3424 3425 /* partially received RX was aborted. clean up. */ 3426 static void efx_ef10_handle_rx_abort(struct efx_rx_queue *rx_queue) 3427 { 3428 unsigned int rx_desc_ptr; 3429 3430 netif_dbg(rx_queue->efx, hw, rx_queue->efx->net_dev, 3431 "scattered RX aborted (dropping %u buffers)\n", 3432 rx_queue->scatter_n); 3433 3434 rx_desc_ptr = rx_queue->removed_count & rx_queue->ptr_mask; 3435 3436 efx_rx_packet(rx_queue, rx_desc_ptr, rx_queue->scatter_n, 3437 0, EFX_RX_PKT_DISCARD); 3438 3439 rx_queue->removed_count += rx_queue->scatter_n; 3440 rx_queue->scatter_n = 0; 3441 rx_queue->scatter_len = 0; 3442 ++efx_rx_queue_channel(rx_queue)->n_rx_nodesc_trunc; 3443 } 3444 3445 static u16 efx_ef10_handle_rx_event_errors(struct efx_channel *channel, 3446 unsigned int n_packets, 3447 unsigned int rx_encap_hdr, 3448 unsigned int rx_l3_class, 3449 unsigned int rx_l4_class, 3450 const efx_qword_t *event) 3451 { 3452 struct efx_nic *efx = channel->efx; 3453 bool handled = false; 3454 3455 if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_ECRC_ERR)) { 3456 if (!(efx->net_dev->features & NETIF_F_RXALL)) { 3457 if (!efx->loopback_selftest) 3458 channel->n_rx_eth_crc_err += n_packets; 3459 return EFX_RX_PKT_DISCARD; 3460 } 3461 handled = true; 3462 } 3463 if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_IPCKSUM_ERR)) { 3464 if (unlikely(rx_encap_hdr != ESE_EZ_ENCAP_HDR_VXLAN && 3465 rx_l3_class != ESE_DZ_L3_CLASS_IP4 && 3466 rx_l3_class != ESE_DZ_L3_CLASS_IP4_FRAG && 3467 rx_l3_class != ESE_DZ_L3_CLASS_IP6 && 3468 rx_l3_class != ESE_DZ_L3_CLASS_IP6_FRAG)) 3469 netdev_WARN(efx->net_dev, 3470 "invalid class for RX_IPCKSUM_ERR: event=" 3471 EFX_QWORD_FMT "\n", 3472 EFX_QWORD_VAL(*event)); 3473 if (!efx->loopback_selftest) 3474 *(rx_encap_hdr ? 3475 &channel->n_rx_outer_ip_hdr_chksum_err : 3476 &channel->n_rx_ip_hdr_chksum_err) += n_packets; 3477 return 0; 3478 } 3479 if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_TCPUDP_CKSUM_ERR)) { 3480 if (unlikely(rx_encap_hdr != ESE_EZ_ENCAP_HDR_VXLAN && 3481 ((rx_l3_class != ESE_DZ_L3_CLASS_IP4 && 3482 rx_l3_class != ESE_DZ_L3_CLASS_IP6) || 3483 (rx_l4_class != ESE_FZ_L4_CLASS_TCP && 3484 rx_l4_class != ESE_FZ_L4_CLASS_UDP)))) 3485 netdev_WARN(efx->net_dev, 3486 "invalid class for RX_TCPUDP_CKSUM_ERR: event=" 3487 EFX_QWORD_FMT "\n", 3488 EFX_QWORD_VAL(*event)); 3489 if (!efx->loopback_selftest) 3490 *(rx_encap_hdr ? 3491 &channel->n_rx_outer_tcp_udp_chksum_err : 3492 &channel->n_rx_tcp_udp_chksum_err) += n_packets; 3493 return 0; 3494 } 3495 if (EFX_QWORD_FIELD(*event, ESF_EZ_RX_IP_INNER_CHKSUM_ERR)) { 3496 if (unlikely(!rx_encap_hdr)) 3497 netdev_WARN(efx->net_dev, 3498 "invalid encapsulation type for RX_IP_INNER_CHKSUM_ERR: event=" 3499 EFX_QWORD_FMT "\n", 3500 EFX_QWORD_VAL(*event)); 3501 else if (unlikely(rx_l3_class != ESE_DZ_L3_CLASS_IP4 && 3502 rx_l3_class != ESE_DZ_L3_CLASS_IP4_FRAG && 3503 rx_l3_class != ESE_DZ_L3_CLASS_IP6 && 3504 rx_l3_class != ESE_DZ_L3_CLASS_IP6_FRAG)) 3505 netdev_WARN(efx->net_dev, 3506 "invalid class for RX_IP_INNER_CHKSUM_ERR: event=" 3507 EFX_QWORD_FMT "\n", 3508 EFX_QWORD_VAL(*event)); 3509 if (!efx->loopback_selftest) 3510 channel->n_rx_inner_ip_hdr_chksum_err += n_packets; 3511 return 0; 3512 } 3513 if (EFX_QWORD_FIELD(*event, ESF_EZ_RX_TCP_UDP_INNER_CHKSUM_ERR)) { 3514 if (unlikely(!rx_encap_hdr)) 3515 netdev_WARN(efx->net_dev, 3516 "invalid encapsulation type for RX_TCP_UDP_INNER_CHKSUM_ERR: event=" 3517 EFX_QWORD_FMT "\n", 3518 EFX_QWORD_VAL(*event)); 3519 else if (unlikely((rx_l3_class != ESE_DZ_L3_CLASS_IP4 && 3520 rx_l3_class != ESE_DZ_L3_CLASS_IP6) || 3521 (rx_l4_class != ESE_FZ_L4_CLASS_TCP && 3522 rx_l4_class != ESE_FZ_L4_CLASS_UDP))) 3523 netdev_WARN(efx->net_dev, 3524 "invalid class for RX_TCP_UDP_INNER_CHKSUM_ERR: event=" 3525 EFX_QWORD_FMT "\n", 3526 EFX_QWORD_VAL(*event)); 3527 if (!efx->loopback_selftest) 3528 channel->n_rx_inner_tcp_udp_chksum_err += n_packets; 3529 return 0; 3530 } 3531 3532 WARN_ON(!handled); /* No error bits were recognised */ 3533 return 0; 3534 } 3535 3536 static int efx_ef10_handle_rx_event(struct efx_channel *channel, 3537 const efx_qword_t *event) 3538 { 3539 unsigned int rx_bytes, next_ptr_lbits, rx_queue_label; 3540 unsigned int rx_l3_class, rx_l4_class, rx_encap_hdr; 3541 unsigned int n_descs, n_packets, i; 3542 struct efx_nic *efx = channel->efx; 3543 struct efx_ef10_nic_data *nic_data = efx->nic_data; 3544 struct efx_rx_queue *rx_queue; 3545 efx_qword_t errors; 3546 bool rx_cont; 3547 u16 flags = 0; 3548 3549 if (unlikely(READ_ONCE(efx->reset_pending))) 3550 return 0; 3551 3552 /* Basic packet information */ 3553 rx_bytes = EFX_QWORD_FIELD(*event, ESF_DZ_RX_BYTES); 3554 next_ptr_lbits = EFX_QWORD_FIELD(*event, ESF_DZ_RX_DSC_PTR_LBITS); 3555 rx_queue_label = EFX_QWORD_FIELD(*event, ESF_DZ_RX_QLABEL); 3556 rx_l3_class = EFX_QWORD_FIELD(*event, ESF_DZ_RX_L3_CLASS); 3557 rx_l4_class = EFX_QWORD_FIELD(*event, ESF_FZ_RX_L4_CLASS); 3558 rx_cont = EFX_QWORD_FIELD(*event, ESF_DZ_RX_CONT); 3559 rx_encap_hdr = 3560 nic_data->datapath_caps & 3561 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN) ? 3562 EFX_QWORD_FIELD(*event, ESF_EZ_RX_ENCAP_HDR) : 3563 ESE_EZ_ENCAP_HDR_NONE; 3564 3565 if (EFX_QWORD_FIELD(*event, ESF_DZ_RX_DROP_EVENT)) 3566 netdev_WARN(efx->net_dev, "saw RX_DROP_EVENT: event=" 3567 EFX_QWORD_FMT "\n", 3568 EFX_QWORD_VAL(*event)); 3569 3570 rx_queue = efx_channel_get_rx_queue(channel); 3571 3572 if (unlikely(rx_queue_label != efx_rx_queue_index(rx_queue))) 3573 efx_ef10_handle_rx_wrong_queue(rx_queue, rx_queue_label); 3574 3575 n_descs = ((next_ptr_lbits - rx_queue->removed_count) & 3576 ((1 << ESF_DZ_RX_DSC_PTR_LBITS_WIDTH) - 1)); 3577 3578 if (n_descs != rx_queue->scatter_n + 1) { 3579 struct efx_ef10_nic_data *nic_data = efx->nic_data; 3580 3581 /* detect rx abort */ 3582 if (unlikely(n_descs == rx_queue->scatter_n)) { 3583 if (rx_queue->scatter_n == 0 || rx_bytes != 0) 3584 netdev_WARN(efx->net_dev, 3585 "invalid RX abort: scatter_n=%u event=" 3586 EFX_QWORD_FMT "\n", 3587 rx_queue->scatter_n, 3588 EFX_QWORD_VAL(*event)); 3589 efx_ef10_handle_rx_abort(rx_queue); 3590 return 0; 3591 } 3592 3593 /* Check that RX completion merging is valid, i.e. 3594 * the current firmware supports it and this is a 3595 * non-scattered packet. 3596 */ 3597 if (!(nic_data->datapath_caps & 3598 (1 << MC_CMD_GET_CAPABILITIES_OUT_RX_BATCHING_LBN)) || 3599 rx_queue->scatter_n != 0 || rx_cont) { 3600 efx_ef10_handle_rx_bad_lbits( 3601 rx_queue, next_ptr_lbits, 3602 (rx_queue->removed_count + 3603 rx_queue->scatter_n + 1) & 3604 ((1 << ESF_DZ_RX_DSC_PTR_LBITS_WIDTH) - 1)); 3605 return 0; 3606 } 3607 3608 /* Merged completion for multiple non-scattered packets */ 3609 rx_queue->scatter_n = 1; 3610 rx_queue->scatter_len = 0; 3611 n_packets = n_descs; 3612 ++channel->n_rx_merge_events; 3613 channel->n_rx_merge_packets += n_packets; 3614 flags |= EFX_RX_PKT_PREFIX_LEN; 3615 } else { 3616 ++rx_queue->scatter_n; 3617 rx_queue->scatter_len += rx_bytes; 3618 if (rx_cont) 3619 return 0; 3620 n_packets = 1; 3621 } 3622 3623 EFX_POPULATE_QWORD_5(errors, ESF_DZ_RX_ECRC_ERR, 1, 3624 ESF_DZ_RX_IPCKSUM_ERR, 1, 3625 ESF_DZ_RX_TCPUDP_CKSUM_ERR, 1, 3626 ESF_EZ_RX_IP_INNER_CHKSUM_ERR, 1, 3627 ESF_EZ_RX_TCP_UDP_INNER_CHKSUM_ERR, 1); 3628 EFX_AND_QWORD(errors, *event, errors); 3629 if (unlikely(!EFX_QWORD_IS_ZERO(errors))) { 3630 flags |= efx_ef10_handle_rx_event_errors(channel, n_packets, 3631 rx_encap_hdr, 3632 rx_l3_class, rx_l4_class, 3633 event); 3634 } else { 3635 bool tcpudp = rx_l4_class == ESE_FZ_L4_CLASS_TCP || 3636 rx_l4_class == ESE_FZ_L4_CLASS_UDP; 3637 3638 switch (rx_encap_hdr) { 3639 case ESE_EZ_ENCAP_HDR_VXLAN: /* VxLAN or GENEVE */ 3640 flags |= EFX_RX_PKT_CSUMMED; /* outer UDP csum */ 3641 if (tcpudp) 3642 flags |= EFX_RX_PKT_CSUM_LEVEL; /* inner L4 */ 3643 break; 3644 case ESE_EZ_ENCAP_HDR_GRE: 3645 case ESE_EZ_ENCAP_HDR_NONE: 3646 if (tcpudp) 3647 flags |= EFX_RX_PKT_CSUMMED; 3648 break; 3649 default: 3650 netdev_WARN(efx->net_dev, 3651 "unknown encapsulation type: event=" 3652 EFX_QWORD_FMT "\n", 3653 EFX_QWORD_VAL(*event)); 3654 } 3655 } 3656 3657 if (rx_l4_class == ESE_FZ_L4_CLASS_TCP) 3658 flags |= EFX_RX_PKT_TCP; 3659 3660 channel->irq_mod_score += 2 * n_packets; 3661 3662 /* Handle received packet(s) */ 3663 for (i = 0; i < n_packets; i++) { 3664 efx_rx_packet(rx_queue, 3665 rx_queue->removed_count & rx_queue->ptr_mask, 3666 rx_queue->scatter_n, rx_queue->scatter_len, 3667 flags); 3668 rx_queue->removed_count += rx_queue->scatter_n; 3669 } 3670 3671 rx_queue->scatter_n = 0; 3672 rx_queue->scatter_len = 0; 3673 3674 return n_packets; 3675 } 3676 3677 static u32 efx_ef10_extract_event_ts(efx_qword_t *event) 3678 { 3679 u32 tstamp; 3680 3681 tstamp = EFX_QWORD_FIELD(*event, TX_TIMESTAMP_EVENT_TSTAMP_DATA_HI); 3682 tstamp <<= 16; 3683 tstamp |= EFX_QWORD_FIELD(*event, TX_TIMESTAMP_EVENT_TSTAMP_DATA_LO); 3684 3685 return tstamp; 3686 } 3687 3688 static void 3689 efx_ef10_handle_tx_event(struct efx_channel *channel, efx_qword_t *event) 3690 { 3691 struct efx_nic *efx = channel->efx; 3692 struct efx_tx_queue *tx_queue; 3693 unsigned int tx_ev_desc_ptr; 3694 unsigned int tx_ev_q_label; 3695 unsigned int tx_ev_type; 3696 u64 ts_part; 3697 3698 if (unlikely(READ_ONCE(efx->reset_pending))) 3699 return; 3700 3701 if (unlikely(EFX_QWORD_FIELD(*event, ESF_DZ_TX_DROP_EVENT))) 3702 return; 3703 3704 /* Get the transmit queue */ 3705 tx_ev_q_label = EFX_QWORD_FIELD(*event, ESF_DZ_TX_QLABEL); 3706 tx_queue = efx_channel_get_tx_queue(channel, 3707 tx_ev_q_label % EFX_TXQ_TYPES); 3708 3709 if (!tx_queue->timestamping) { 3710 /* Transmit completion */ 3711 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, ESF_DZ_TX_DESCR_INDX); 3712 efx_xmit_done(tx_queue, tx_ev_desc_ptr & tx_queue->ptr_mask); 3713 return; 3714 } 3715 3716 /* Transmit timestamps are only available for 8XXX series. They result 3717 * in three events per packet. These occur in order, and are: 3718 * - the normal completion event 3719 * - the low part of the timestamp 3720 * - the high part of the timestamp 3721 * 3722 * Each part of the timestamp is itself split across two 16 bit 3723 * fields in the event. 3724 */ 3725 tx_ev_type = EFX_QWORD_FIELD(*event, ESF_EZ_TX_SOFT1); 3726 3727 switch (tx_ev_type) { 3728 case TX_TIMESTAMP_EVENT_TX_EV_COMPLETION: 3729 /* In case of Queue flush or FLR, we might have received 3730 * the previous TX completion event but not the Timestamp 3731 * events. 3732 */ 3733 if (tx_queue->completed_desc_ptr != tx_queue->ptr_mask) 3734 efx_xmit_done(tx_queue, tx_queue->completed_desc_ptr); 3735 3736 tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, 3737 ESF_DZ_TX_DESCR_INDX); 3738 tx_queue->completed_desc_ptr = 3739 tx_ev_desc_ptr & tx_queue->ptr_mask; 3740 break; 3741 3742 case TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_LO: 3743 ts_part = efx_ef10_extract_event_ts(event); 3744 tx_queue->completed_timestamp_minor = ts_part; 3745 break; 3746 3747 case TX_TIMESTAMP_EVENT_TX_EV_TSTAMP_HI: 3748 ts_part = efx_ef10_extract_event_ts(event); 3749 tx_queue->completed_timestamp_major = ts_part; 3750 3751 efx_xmit_done(tx_queue, tx_queue->completed_desc_ptr); 3752 tx_queue->completed_desc_ptr = tx_queue->ptr_mask; 3753 break; 3754 3755 default: 3756 netif_err(efx, hw, efx->net_dev, 3757 "channel %d unknown tx event type %d (data " 3758 EFX_QWORD_FMT ")\n", 3759 channel->channel, tx_ev_type, 3760 EFX_QWORD_VAL(*event)); 3761 break; 3762 } 3763 } 3764 3765 static void 3766 efx_ef10_handle_driver_event(struct efx_channel *channel, efx_qword_t *event) 3767 { 3768 struct efx_nic *efx = channel->efx; 3769 int subcode; 3770 3771 subcode = EFX_QWORD_FIELD(*event, ESF_DZ_DRV_SUB_CODE); 3772 3773 switch (subcode) { 3774 case ESE_DZ_DRV_TIMER_EV: 3775 case ESE_DZ_DRV_WAKE_UP_EV: 3776 break; 3777 case ESE_DZ_DRV_START_UP_EV: 3778 /* event queue init complete. ok. */ 3779 break; 3780 default: 3781 netif_err(efx, hw, efx->net_dev, 3782 "channel %d unknown driver event type %d" 3783 " (data " EFX_QWORD_FMT ")\n", 3784 channel->channel, subcode, 3785 EFX_QWORD_VAL(*event)); 3786 3787 } 3788 } 3789 3790 static void efx_ef10_handle_driver_generated_event(struct efx_channel *channel, 3791 efx_qword_t *event) 3792 { 3793 struct efx_nic *efx = channel->efx; 3794 u32 subcode; 3795 3796 subcode = EFX_QWORD_FIELD(*event, EFX_DWORD_0); 3797 3798 switch (subcode) { 3799 case EFX_EF10_TEST: 3800 channel->event_test_cpu = raw_smp_processor_id(); 3801 break; 3802 case EFX_EF10_REFILL: 3803 /* The queue must be empty, so we won't receive any rx 3804 * events, so efx_process_channel() won't refill the 3805 * queue. Refill it here 3806 */ 3807 efx_fast_push_rx_descriptors(&channel->rx_queue, true); 3808 break; 3809 default: 3810 netif_err(efx, hw, efx->net_dev, 3811 "channel %d unknown driver event type %u" 3812 " (data " EFX_QWORD_FMT ")\n", 3813 channel->channel, (unsigned) subcode, 3814 EFX_QWORD_VAL(*event)); 3815 } 3816 } 3817 3818 static int efx_ef10_ev_process(struct efx_channel *channel, int quota) 3819 { 3820 struct efx_nic *efx = channel->efx; 3821 efx_qword_t event, *p_event; 3822 unsigned int read_ptr; 3823 int ev_code; 3824 int spent = 0; 3825 3826 if (quota <= 0) 3827 return spent; 3828 3829 read_ptr = channel->eventq_read_ptr; 3830 3831 for (;;) { 3832 p_event = efx_event(channel, read_ptr); 3833 event = *p_event; 3834 3835 if (!efx_event_present(&event)) 3836 break; 3837 3838 EFX_SET_QWORD(*p_event); 3839 3840 ++read_ptr; 3841 3842 ev_code = EFX_QWORD_FIELD(event, ESF_DZ_EV_CODE); 3843 3844 netif_vdbg(efx, drv, efx->net_dev, 3845 "processing event on %d " EFX_QWORD_FMT "\n", 3846 channel->channel, EFX_QWORD_VAL(event)); 3847 3848 switch (ev_code) { 3849 case ESE_DZ_EV_CODE_MCDI_EV: 3850 efx_mcdi_process_event(channel, &event); 3851 break; 3852 case ESE_DZ_EV_CODE_RX_EV: 3853 spent += efx_ef10_handle_rx_event(channel, &event); 3854 if (spent >= quota) { 3855 /* XXX can we split a merged event to 3856 * avoid going over-quota? 3857 */ 3858 spent = quota; 3859 goto out; 3860 } 3861 break; 3862 case ESE_DZ_EV_CODE_TX_EV: 3863 efx_ef10_handle_tx_event(channel, &event); 3864 break; 3865 case ESE_DZ_EV_CODE_DRIVER_EV: 3866 efx_ef10_handle_driver_event(channel, &event); 3867 if (++spent == quota) 3868 goto out; 3869 break; 3870 case EFX_EF10_DRVGEN_EV: 3871 efx_ef10_handle_driver_generated_event(channel, &event); 3872 break; 3873 default: 3874 netif_err(efx, hw, efx->net_dev, 3875 "channel %d unknown event type %d" 3876 " (data " EFX_QWORD_FMT ")\n", 3877 channel->channel, ev_code, 3878 EFX_QWORD_VAL(event)); 3879 } 3880 } 3881 3882 out: 3883 channel->eventq_read_ptr = read_ptr; 3884 return spent; 3885 } 3886 3887 static void efx_ef10_ev_read_ack(struct efx_channel *channel) 3888 { 3889 struct efx_nic *efx = channel->efx; 3890 efx_dword_t rptr; 3891 3892 if (EFX_EF10_WORKAROUND_35388(efx)) { 3893 BUILD_BUG_ON(EFX_MIN_EVQ_SIZE < 3894 (1 << ERF_DD_EVQ_IND_RPTR_WIDTH)); 3895 BUILD_BUG_ON(EFX_MAX_EVQ_SIZE > 3896 (1 << 2 * ERF_DD_EVQ_IND_RPTR_WIDTH)); 3897 3898 EFX_POPULATE_DWORD_2(rptr, ERF_DD_EVQ_IND_RPTR_FLAGS, 3899 EFE_DD_EVQ_IND_RPTR_FLAGS_HIGH, 3900 ERF_DD_EVQ_IND_RPTR, 3901 (channel->eventq_read_ptr & 3902 channel->eventq_mask) >> 3903 ERF_DD_EVQ_IND_RPTR_WIDTH); 3904 efx_writed_page(efx, &rptr, ER_DD_EVQ_INDIRECT, 3905 channel->channel); 3906 EFX_POPULATE_DWORD_2(rptr, ERF_DD_EVQ_IND_RPTR_FLAGS, 3907 EFE_DD_EVQ_IND_RPTR_FLAGS_LOW, 3908 ERF_DD_EVQ_IND_RPTR, 3909 channel->eventq_read_ptr & 3910 ((1 << ERF_DD_EVQ_IND_RPTR_WIDTH) - 1)); 3911 efx_writed_page(efx, &rptr, ER_DD_EVQ_INDIRECT, 3912 channel->channel); 3913 } else { 3914 EFX_POPULATE_DWORD_1(rptr, ERF_DZ_EVQ_RPTR, 3915 channel->eventq_read_ptr & 3916 channel->eventq_mask); 3917 efx_writed_page(efx, &rptr, ER_DZ_EVQ_RPTR, channel->channel); 3918 } 3919 } 3920 3921 static void efx_ef10_ev_test_generate(struct efx_channel *channel) 3922 { 3923 MCDI_DECLARE_BUF(inbuf, MC_CMD_DRIVER_EVENT_IN_LEN); 3924 struct efx_nic *efx = channel->efx; 3925 efx_qword_t event; 3926 int rc; 3927 3928 EFX_POPULATE_QWORD_2(event, 3929 ESF_DZ_EV_CODE, EFX_EF10_DRVGEN_EV, 3930 ESF_DZ_EV_DATA, EFX_EF10_TEST); 3931 3932 MCDI_SET_DWORD(inbuf, DRIVER_EVENT_IN_EVQ, channel->channel); 3933 3934 /* MCDI_SET_QWORD is not appropriate here since EFX_POPULATE_* has 3935 * already swapped the data to little-endian order. 3936 */ 3937 memcpy(MCDI_PTR(inbuf, DRIVER_EVENT_IN_DATA), &event.u64[0], 3938 sizeof(efx_qword_t)); 3939 3940 rc = efx_mcdi_rpc(efx, MC_CMD_DRIVER_EVENT, inbuf, sizeof(inbuf), 3941 NULL, 0, NULL); 3942 if (rc != 0) 3943 goto fail; 3944 3945 return; 3946 3947 fail: 3948 WARN_ON(true); 3949 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc); 3950 } 3951 3952 void efx_ef10_handle_drain_event(struct efx_nic *efx) 3953 { 3954 if (atomic_dec_and_test(&efx->active_queues)) 3955 wake_up(&efx->flush_wq); 3956 3957 WARN_ON(atomic_read(&efx->active_queues) < 0); 3958 } 3959 3960 static int efx_ef10_fini_dmaq(struct efx_nic *efx) 3961 { 3962 struct efx_ef10_nic_data *nic_data = efx->nic_data; 3963 struct efx_channel *channel; 3964 struct efx_tx_queue *tx_queue; 3965 struct efx_rx_queue *rx_queue; 3966 int pending; 3967 3968 /* If the MC has just rebooted, the TX/RX queues will have already been 3969 * torn down, but efx->active_queues needs to be set to zero. 3970 */ 3971 if (nic_data->must_realloc_vis) { 3972 atomic_set(&efx->active_queues, 0); 3973 return 0; 3974 } 3975 3976 /* Do not attempt to write to the NIC during EEH recovery */ 3977 if (efx->state != STATE_RECOVERY) { 3978 efx_for_each_channel(channel, efx) { 3979 efx_for_each_channel_rx_queue(rx_queue, channel) 3980 efx_ef10_rx_fini(rx_queue); 3981 efx_for_each_channel_tx_queue(tx_queue, channel) 3982 efx_ef10_tx_fini(tx_queue); 3983 } 3984 3985 wait_event_timeout(efx->flush_wq, 3986 atomic_read(&efx->active_queues) == 0, 3987 msecs_to_jiffies(EFX_MAX_FLUSH_TIME)); 3988 pending = atomic_read(&efx->active_queues); 3989 if (pending) { 3990 netif_err(efx, hw, efx->net_dev, "failed to flush %d queues\n", 3991 pending); 3992 return -ETIMEDOUT; 3993 } 3994 } 3995 3996 return 0; 3997 } 3998 3999 static void efx_ef10_prepare_flr(struct efx_nic *efx) 4000 { 4001 atomic_set(&efx->active_queues, 0); 4002 } 4003 4004 static bool efx_ef10_filter_equal(const struct efx_filter_spec *left, 4005 const struct efx_filter_spec *right) 4006 { 4007 if ((left->match_flags ^ right->match_flags) | 4008 ((left->flags ^ right->flags) & 4009 (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX))) 4010 return false; 4011 4012 return memcmp(&left->outer_vid, &right->outer_vid, 4013 sizeof(struct efx_filter_spec) - 4014 offsetof(struct efx_filter_spec, outer_vid)) == 0; 4015 } 4016 4017 static unsigned int efx_ef10_filter_hash(const struct efx_filter_spec *spec) 4018 { 4019 BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3); 4020 return jhash2((const u32 *)&spec->outer_vid, 4021 (sizeof(struct efx_filter_spec) - 4022 offsetof(struct efx_filter_spec, outer_vid)) / 4, 4023 0); 4024 /* XXX should we randomise the initval? */ 4025 } 4026 4027 /* Decide whether a filter should be exclusive or else should allow 4028 * delivery to additional recipients. Currently we decide that 4029 * filters for specific local unicast MAC and IP addresses are 4030 * exclusive. 4031 */ 4032 static bool efx_ef10_filter_is_exclusive(const struct efx_filter_spec *spec) 4033 { 4034 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC && 4035 !is_multicast_ether_addr(spec->loc_mac)) 4036 return true; 4037 4038 if ((spec->match_flags & 4039 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) == 4040 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) { 4041 if (spec->ether_type == htons(ETH_P_IP) && 4042 !ipv4_is_multicast(spec->loc_host[0])) 4043 return true; 4044 if (spec->ether_type == htons(ETH_P_IPV6) && 4045 ((const u8 *)spec->loc_host)[0] != 0xff) 4046 return true; 4047 } 4048 4049 return false; 4050 } 4051 4052 static struct efx_filter_spec * 4053 efx_ef10_filter_entry_spec(const struct efx_ef10_filter_table *table, 4054 unsigned int filter_idx) 4055 { 4056 return (struct efx_filter_spec *)(table->entry[filter_idx].spec & 4057 ~EFX_EF10_FILTER_FLAGS); 4058 } 4059 4060 static unsigned int 4061 efx_ef10_filter_entry_flags(const struct efx_ef10_filter_table *table, 4062 unsigned int filter_idx) 4063 { 4064 return table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAGS; 4065 } 4066 4067 static void 4068 efx_ef10_filter_set_entry(struct efx_ef10_filter_table *table, 4069 unsigned int filter_idx, 4070 const struct efx_filter_spec *spec, 4071 unsigned int flags) 4072 { 4073 table->entry[filter_idx].spec = (unsigned long)spec | flags; 4074 } 4075 4076 static void 4077 efx_ef10_filter_push_prep_set_match_fields(struct efx_nic *efx, 4078 const struct efx_filter_spec *spec, 4079 efx_dword_t *inbuf) 4080 { 4081 enum efx_encap_type encap_type = efx_filter_get_encap_type(spec); 4082 u32 match_fields = 0, uc_match, mc_match; 4083 4084 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 4085 efx_ef10_filter_is_exclusive(spec) ? 4086 MC_CMD_FILTER_OP_IN_OP_INSERT : 4087 MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE); 4088 4089 /* Convert match flags and values. Unlike almost 4090 * everything else in MCDI, these fields are in 4091 * network byte order. 4092 */ 4093 #define COPY_VALUE(value, mcdi_field) \ 4094 do { \ 4095 match_fields |= \ 4096 1 << MC_CMD_FILTER_OP_IN_MATCH_ ## \ 4097 mcdi_field ## _LBN; \ 4098 BUILD_BUG_ON( \ 4099 MC_CMD_FILTER_OP_IN_ ## mcdi_field ## _LEN < \ 4100 sizeof(value)); \ 4101 memcpy(MCDI_PTR(inbuf, FILTER_OP_IN_ ## mcdi_field), \ 4102 &value, sizeof(value)); \ 4103 } while (0) 4104 #define COPY_FIELD(gen_flag, gen_field, mcdi_field) \ 4105 if (spec->match_flags & EFX_FILTER_MATCH_ ## gen_flag) { \ 4106 COPY_VALUE(spec->gen_field, mcdi_field); \ 4107 } 4108 /* Handle encap filters first. They will always be mismatch 4109 * (unknown UC or MC) filters 4110 */ 4111 if (encap_type) { 4112 /* ether_type and outer_ip_proto need to be variables 4113 * because COPY_VALUE wants to memcpy them 4114 */ 4115 __be16 ether_type = 4116 htons(encap_type & EFX_ENCAP_FLAG_IPV6 ? 4117 ETH_P_IPV6 : ETH_P_IP); 4118 u8 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE; 4119 u8 outer_ip_proto; 4120 4121 switch (encap_type & EFX_ENCAP_TYPES_MASK) { 4122 case EFX_ENCAP_TYPE_VXLAN: 4123 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN; 4124 /* fallthrough */ 4125 case EFX_ENCAP_TYPE_GENEVE: 4126 COPY_VALUE(ether_type, ETHER_TYPE); 4127 outer_ip_proto = IPPROTO_UDP; 4128 COPY_VALUE(outer_ip_proto, IP_PROTO); 4129 /* We always need to set the type field, even 4130 * though we're not matching on the TNI. 4131 */ 4132 MCDI_POPULATE_DWORD_1(inbuf, 4133 FILTER_OP_EXT_IN_VNI_OR_VSID, 4134 FILTER_OP_EXT_IN_VNI_TYPE, 4135 vni_type); 4136 break; 4137 case EFX_ENCAP_TYPE_NVGRE: 4138 COPY_VALUE(ether_type, ETHER_TYPE); 4139 outer_ip_proto = IPPROTO_GRE; 4140 COPY_VALUE(outer_ip_proto, IP_PROTO); 4141 break; 4142 default: 4143 WARN_ON(1); 4144 } 4145 4146 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN; 4147 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN; 4148 } else { 4149 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN; 4150 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN; 4151 } 4152 4153 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) 4154 match_fields |= 4155 is_multicast_ether_addr(spec->loc_mac) ? 4156 1 << mc_match : 4157 1 << uc_match; 4158 COPY_FIELD(REM_HOST, rem_host, SRC_IP); 4159 COPY_FIELD(LOC_HOST, loc_host, DST_IP); 4160 COPY_FIELD(REM_MAC, rem_mac, SRC_MAC); 4161 COPY_FIELD(REM_PORT, rem_port, SRC_PORT); 4162 COPY_FIELD(LOC_MAC, loc_mac, DST_MAC); 4163 COPY_FIELD(LOC_PORT, loc_port, DST_PORT); 4164 COPY_FIELD(ETHER_TYPE, ether_type, ETHER_TYPE); 4165 COPY_FIELD(INNER_VID, inner_vid, INNER_VLAN); 4166 COPY_FIELD(OUTER_VID, outer_vid, OUTER_VLAN); 4167 COPY_FIELD(IP_PROTO, ip_proto, IP_PROTO); 4168 #undef COPY_FIELD 4169 #undef COPY_VALUE 4170 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_MATCH_FIELDS, 4171 match_fields); 4172 } 4173 4174 static void efx_ef10_filter_push_prep(struct efx_nic *efx, 4175 const struct efx_filter_spec *spec, 4176 efx_dword_t *inbuf, u64 handle, 4177 struct efx_rss_context *ctx, 4178 bool replacing) 4179 { 4180 struct efx_ef10_nic_data *nic_data = efx->nic_data; 4181 u32 flags = spec->flags; 4182 4183 memset(inbuf, 0, MC_CMD_FILTER_OP_EXT_IN_LEN); 4184 4185 /* If RSS filter, caller better have given us an RSS context */ 4186 if (flags & EFX_FILTER_FLAG_RX_RSS) { 4187 /* We don't have the ability to return an error, so we'll just 4188 * log a warning and disable RSS for the filter. 4189 */ 4190 if (WARN_ON_ONCE(!ctx)) 4191 flags &= ~EFX_FILTER_FLAG_RX_RSS; 4192 else if (WARN_ON_ONCE(ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID)) 4193 flags &= ~EFX_FILTER_FLAG_RX_RSS; 4194 } 4195 4196 if (replacing) { 4197 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 4198 MC_CMD_FILTER_OP_IN_OP_REPLACE); 4199 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, handle); 4200 } else { 4201 efx_ef10_filter_push_prep_set_match_fields(efx, spec, inbuf); 4202 } 4203 4204 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, nic_data->vport_id); 4205 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST, 4206 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ? 4207 MC_CMD_FILTER_OP_IN_RX_DEST_DROP : 4208 MC_CMD_FILTER_OP_IN_RX_DEST_HOST); 4209 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DOMAIN, 0); 4210 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DEST, 4211 MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT); 4212 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_QUEUE, 4213 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ? 4214 0 : spec->dmaq_id); 4215 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_MODE, 4216 (flags & EFX_FILTER_FLAG_RX_RSS) ? 4217 MC_CMD_FILTER_OP_IN_RX_MODE_RSS : 4218 MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE); 4219 if (flags & EFX_FILTER_FLAG_RX_RSS) 4220 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, ctx->context_id); 4221 } 4222 4223 static int efx_ef10_filter_push(struct efx_nic *efx, 4224 const struct efx_filter_spec *spec, u64 *handle, 4225 struct efx_rss_context *ctx, bool replacing) 4226 { 4227 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); 4228 MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_EXT_OUT_LEN); 4229 int rc; 4230 4231 efx_ef10_filter_push_prep(efx, spec, inbuf, *handle, ctx, replacing); 4232 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 4233 outbuf, sizeof(outbuf), NULL); 4234 if (rc == 0) 4235 *handle = MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE); 4236 if (rc == -ENOSPC) 4237 rc = -EBUSY; /* to match efx_farch_filter_insert() */ 4238 return rc; 4239 } 4240 4241 static u32 efx_ef10_filter_mcdi_flags_from_spec(const struct efx_filter_spec *spec) 4242 { 4243 enum efx_encap_type encap_type = efx_filter_get_encap_type(spec); 4244 unsigned int match_flags = spec->match_flags; 4245 unsigned int uc_match, mc_match; 4246 u32 mcdi_flags = 0; 4247 4248 #define MAP_FILTER_TO_MCDI_FLAG(gen_flag, mcdi_field, encap) { \ 4249 unsigned int old_match_flags = match_flags; \ 4250 match_flags &= ~EFX_FILTER_MATCH_ ## gen_flag; \ 4251 if (match_flags != old_match_flags) \ 4252 mcdi_flags |= \ 4253 (1 << ((encap) ? \ 4254 MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ ## \ 4255 mcdi_field ## _LBN : \ 4256 MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##\ 4257 mcdi_field ## _LBN)); \ 4258 } 4259 /* inner or outer based on encap type */ 4260 MAP_FILTER_TO_MCDI_FLAG(REM_HOST, SRC_IP, encap_type); 4261 MAP_FILTER_TO_MCDI_FLAG(LOC_HOST, DST_IP, encap_type); 4262 MAP_FILTER_TO_MCDI_FLAG(REM_MAC, SRC_MAC, encap_type); 4263 MAP_FILTER_TO_MCDI_FLAG(REM_PORT, SRC_PORT, encap_type); 4264 MAP_FILTER_TO_MCDI_FLAG(LOC_MAC, DST_MAC, encap_type); 4265 MAP_FILTER_TO_MCDI_FLAG(LOC_PORT, DST_PORT, encap_type); 4266 MAP_FILTER_TO_MCDI_FLAG(ETHER_TYPE, ETHER_TYPE, encap_type); 4267 MAP_FILTER_TO_MCDI_FLAG(IP_PROTO, IP_PROTO, encap_type); 4268 /* always outer */ 4269 MAP_FILTER_TO_MCDI_FLAG(INNER_VID, INNER_VLAN, false); 4270 MAP_FILTER_TO_MCDI_FLAG(OUTER_VID, OUTER_VLAN, false); 4271 #undef MAP_FILTER_TO_MCDI_FLAG 4272 4273 /* special handling for encap type, and mismatch */ 4274 if (encap_type) { 4275 match_flags &= ~EFX_FILTER_MATCH_ENCAP_TYPE; 4276 mcdi_flags |= 4277 (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN); 4278 mcdi_flags |= (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN); 4279 4280 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN; 4281 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN; 4282 } else { 4283 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN; 4284 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN; 4285 } 4286 4287 if (match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) { 4288 match_flags &= ~EFX_FILTER_MATCH_LOC_MAC_IG; 4289 mcdi_flags |= 4290 is_multicast_ether_addr(spec->loc_mac) ? 4291 1 << mc_match : 4292 1 << uc_match; 4293 } 4294 4295 /* Did we map them all? */ 4296 WARN_ON_ONCE(match_flags); 4297 4298 return mcdi_flags; 4299 } 4300 4301 static int efx_ef10_filter_pri(struct efx_ef10_filter_table *table, 4302 const struct efx_filter_spec *spec) 4303 { 4304 u32 mcdi_flags = efx_ef10_filter_mcdi_flags_from_spec(spec); 4305 unsigned int match_pri; 4306 4307 for (match_pri = 0; 4308 match_pri < table->rx_match_count; 4309 match_pri++) 4310 if (table->rx_match_mcdi_flags[match_pri] == mcdi_flags) 4311 return match_pri; 4312 4313 return -EPROTONOSUPPORT; 4314 } 4315 4316 static s32 efx_ef10_filter_insert(struct efx_nic *efx, 4317 struct efx_filter_spec *spec, 4318 bool replace_equal) 4319 { 4320 DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT); 4321 struct efx_ef10_nic_data *nic_data = efx->nic_data; 4322 struct efx_ef10_filter_table *table; 4323 struct efx_filter_spec *saved_spec; 4324 struct efx_rss_context *ctx = NULL; 4325 unsigned int match_pri, hash; 4326 unsigned int priv_flags; 4327 bool rss_locked = false; 4328 bool replacing = false; 4329 unsigned int depth, i; 4330 int ins_index = -1; 4331 DEFINE_WAIT(wait); 4332 bool is_mc_recip; 4333 s32 rc; 4334 4335 down_read(&efx->filter_sem); 4336 table = efx->filter_state; 4337 down_write(&table->lock); 4338 4339 /* For now, only support RX filters */ 4340 if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) != 4341 EFX_FILTER_FLAG_RX) { 4342 rc = -EINVAL; 4343 goto out_unlock; 4344 } 4345 4346 rc = efx_ef10_filter_pri(table, spec); 4347 if (rc < 0) 4348 goto out_unlock; 4349 match_pri = rc; 4350 4351 hash = efx_ef10_filter_hash(spec); 4352 is_mc_recip = efx_filter_is_mc_recipient(spec); 4353 if (is_mc_recip) 4354 bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT); 4355 4356 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) { 4357 mutex_lock(&efx->rss_lock); 4358 rss_locked = true; 4359 if (spec->rss_context) 4360 ctx = efx_find_rss_context_entry(efx, spec->rss_context); 4361 else 4362 ctx = &efx->rss_context; 4363 if (!ctx) { 4364 rc = -ENOENT; 4365 goto out_unlock; 4366 } 4367 if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) { 4368 rc = -EOPNOTSUPP; 4369 goto out_unlock; 4370 } 4371 } 4372 4373 /* Find any existing filters with the same match tuple or 4374 * else a free slot to insert at. 4375 */ 4376 for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) { 4377 i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1); 4378 saved_spec = efx_ef10_filter_entry_spec(table, i); 4379 4380 if (!saved_spec) { 4381 if (ins_index < 0) 4382 ins_index = i; 4383 } else if (efx_ef10_filter_equal(spec, saved_spec)) { 4384 if (spec->priority < saved_spec->priority && 4385 spec->priority != EFX_FILTER_PRI_AUTO) { 4386 rc = -EPERM; 4387 goto out_unlock; 4388 } 4389 if (!is_mc_recip) { 4390 /* This is the only one */ 4391 if (spec->priority == 4392 saved_spec->priority && 4393 !replace_equal) { 4394 rc = -EEXIST; 4395 goto out_unlock; 4396 } 4397 ins_index = i; 4398 break; 4399 } else if (spec->priority > 4400 saved_spec->priority || 4401 (spec->priority == 4402 saved_spec->priority && 4403 replace_equal)) { 4404 if (ins_index < 0) 4405 ins_index = i; 4406 else 4407 __set_bit(depth, mc_rem_map); 4408 } 4409 } 4410 } 4411 4412 /* Once we reach the maximum search depth, use the first suitable 4413 * slot, or return -EBUSY if there was none 4414 */ 4415 if (ins_index < 0) { 4416 rc = -EBUSY; 4417 goto out_unlock; 4418 } 4419 4420 /* Create a software table entry if necessary. */ 4421 saved_spec = efx_ef10_filter_entry_spec(table, ins_index); 4422 if (saved_spec) { 4423 if (spec->priority == EFX_FILTER_PRI_AUTO && 4424 saved_spec->priority >= EFX_FILTER_PRI_AUTO) { 4425 /* Just make sure it won't be removed */ 4426 if (saved_spec->priority > EFX_FILTER_PRI_AUTO) 4427 saved_spec->flags |= EFX_FILTER_FLAG_RX_OVER_AUTO; 4428 table->entry[ins_index].spec &= 4429 ~EFX_EF10_FILTER_FLAG_AUTO_OLD; 4430 rc = ins_index; 4431 goto out_unlock; 4432 } 4433 replacing = true; 4434 priv_flags = efx_ef10_filter_entry_flags(table, ins_index); 4435 } else { 4436 saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC); 4437 if (!saved_spec) { 4438 rc = -ENOMEM; 4439 goto out_unlock; 4440 } 4441 *saved_spec = *spec; 4442 priv_flags = 0; 4443 } 4444 efx_ef10_filter_set_entry(table, ins_index, saved_spec, priv_flags); 4445 4446 /* Actually insert the filter on the HW */ 4447 rc = efx_ef10_filter_push(efx, spec, &table->entry[ins_index].handle, 4448 ctx, replacing); 4449 4450 if (rc == -EINVAL && nic_data->must_realloc_vis) 4451 /* The MC rebooted under us, causing it to reject our filter 4452 * insertion as pointing to an invalid VI (spec->dmaq_id). 4453 */ 4454 rc = -EAGAIN; 4455 4456 /* Finalise the software table entry */ 4457 if (rc == 0) { 4458 if (replacing) { 4459 /* Update the fields that may differ */ 4460 if (saved_spec->priority == EFX_FILTER_PRI_AUTO) 4461 saved_spec->flags |= 4462 EFX_FILTER_FLAG_RX_OVER_AUTO; 4463 saved_spec->priority = spec->priority; 4464 saved_spec->flags &= EFX_FILTER_FLAG_RX_OVER_AUTO; 4465 saved_spec->flags |= spec->flags; 4466 saved_spec->rss_context = spec->rss_context; 4467 saved_spec->dmaq_id = spec->dmaq_id; 4468 } 4469 } else if (!replacing) { 4470 kfree(saved_spec); 4471 saved_spec = NULL; 4472 } else { 4473 /* We failed to replace, so the old filter is still present. 4474 * Roll back the software table to reflect this. In fact the 4475 * efx_ef10_filter_set_entry() call below will do the right 4476 * thing, so nothing extra is needed here. 4477 */ 4478 } 4479 efx_ef10_filter_set_entry(table, ins_index, saved_spec, priv_flags); 4480 4481 /* Remove and finalise entries for lower-priority multicast 4482 * recipients 4483 */ 4484 if (is_mc_recip) { 4485 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); 4486 unsigned int depth, i; 4487 4488 memset(inbuf, 0, sizeof(inbuf)); 4489 4490 for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) { 4491 if (!test_bit(depth, mc_rem_map)) 4492 continue; 4493 4494 i = (hash + depth) & (HUNT_FILTER_TBL_ROWS - 1); 4495 saved_spec = efx_ef10_filter_entry_spec(table, i); 4496 priv_flags = efx_ef10_filter_entry_flags(table, i); 4497 4498 if (rc == 0) { 4499 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 4500 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); 4501 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, 4502 table->entry[i].handle); 4503 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP, 4504 inbuf, sizeof(inbuf), 4505 NULL, 0, NULL); 4506 } 4507 4508 if (rc == 0) { 4509 kfree(saved_spec); 4510 saved_spec = NULL; 4511 priv_flags = 0; 4512 } 4513 efx_ef10_filter_set_entry(table, i, saved_spec, 4514 priv_flags); 4515 } 4516 } 4517 4518 /* If successful, return the inserted filter ID */ 4519 if (rc == 0) 4520 rc = efx_ef10_make_filter_id(match_pri, ins_index); 4521 4522 out_unlock: 4523 if (rss_locked) 4524 mutex_unlock(&efx->rss_lock); 4525 up_write(&table->lock); 4526 up_read(&efx->filter_sem); 4527 return rc; 4528 } 4529 4530 static void efx_ef10_filter_update_rx_scatter(struct efx_nic *efx) 4531 { 4532 /* no need to do anything here on EF10 */ 4533 } 4534 4535 /* Remove a filter. 4536 * If !by_index, remove by ID 4537 * If by_index, remove by index 4538 * Filter ID may come from userland and must be range-checked. 4539 * Caller must hold efx->filter_sem for read, and efx->filter_state->lock 4540 * for write. 4541 */ 4542 static int efx_ef10_filter_remove_internal(struct efx_nic *efx, 4543 unsigned int priority_mask, 4544 u32 filter_id, bool by_index) 4545 { 4546 unsigned int filter_idx = efx_ef10_filter_get_unsafe_id(filter_id); 4547 struct efx_ef10_filter_table *table = efx->filter_state; 4548 MCDI_DECLARE_BUF(inbuf, 4549 MC_CMD_FILTER_OP_IN_HANDLE_OFST + 4550 MC_CMD_FILTER_OP_IN_HANDLE_LEN); 4551 struct efx_filter_spec *spec; 4552 DEFINE_WAIT(wait); 4553 int rc; 4554 4555 spec = efx_ef10_filter_entry_spec(table, filter_idx); 4556 if (!spec || 4557 (!by_index && 4558 efx_ef10_filter_pri(table, spec) != 4559 efx_ef10_filter_get_unsafe_pri(filter_id))) 4560 return -ENOENT; 4561 4562 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO && 4563 priority_mask == (1U << EFX_FILTER_PRI_AUTO)) { 4564 /* Just remove flags */ 4565 spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO; 4566 table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD; 4567 return 0; 4568 } 4569 4570 if (!(priority_mask & (1U << spec->priority))) 4571 return -ENOENT; 4572 4573 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) { 4574 /* Reset to an automatic filter */ 4575 4576 struct efx_filter_spec new_spec = *spec; 4577 4578 new_spec.priority = EFX_FILTER_PRI_AUTO; 4579 new_spec.flags = (EFX_FILTER_FLAG_RX | 4580 (efx_rss_active(&efx->rss_context) ? 4581 EFX_FILTER_FLAG_RX_RSS : 0)); 4582 new_spec.dmaq_id = 0; 4583 new_spec.rss_context = 0; 4584 rc = efx_ef10_filter_push(efx, &new_spec, 4585 &table->entry[filter_idx].handle, 4586 &efx->rss_context, 4587 true); 4588 4589 if (rc == 0) 4590 *spec = new_spec; 4591 } else { 4592 /* Really remove the filter */ 4593 4594 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 4595 efx_ef10_filter_is_exclusive(spec) ? 4596 MC_CMD_FILTER_OP_IN_OP_REMOVE : 4597 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); 4598 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, 4599 table->entry[filter_idx].handle); 4600 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, 4601 inbuf, sizeof(inbuf), NULL, 0, NULL); 4602 4603 if ((rc == 0) || (rc == -ENOENT)) { 4604 /* Filter removed OK or didn't actually exist */ 4605 kfree(spec); 4606 efx_ef10_filter_set_entry(table, filter_idx, NULL, 0); 4607 } else { 4608 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, 4609 MC_CMD_FILTER_OP_EXT_IN_LEN, 4610 NULL, 0, rc); 4611 } 4612 } 4613 4614 return rc; 4615 } 4616 4617 static int efx_ef10_filter_remove_safe(struct efx_nic *efx, 4618 enum efx_filter_priority priority, 4619 u32 filter_id) 4620 { 4621 struct efx_ef10_filter_table *table; 4622 int rc; 4623 4624 down_read(&efx->filter_sem); 4625 table = efx->filter_state; 4626 down_write(&table->lock); 4627 rc = efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id, 4628 false); 4629 up_write(&table->lock); 4630 up_read(&efx->filter_sem); 4631 return rc; 4632 } 4633 4634 /* Caller must hold efx->filter_sem for read */ 4635 static void efx_ef10_filter_remove_unsafe(struct efx_nic *efx, 4636 enum efx_filter_priority priority, 4637 u32 filter_id) 4638 { 4639 struct efx_ef10_filter_table *table = efx->filter_state; 4640 4641 if (filter_id == EFX_EF10_FILTER_ID_INVALID) 4642 return; 4643 4644 down_write(&table->lock); 4645 efx_ef10_filter_remove_internal(efx, 1U << priority, filter_id, 4646 true); 4647 up_write(&table->lock); 4648 } 4649 4650 static int efx_ef10_filter_get_safe(struct efx_nic *efx, 4651 enum efx_filter_priority priority, 4652 u32 filter_id, struct efx_filter_spec *spec) 4653 { 4654 unsigned int filter_idx = efx_ef10_filter_get_unsafe_id(filter_id); 4655 const struct efx_filter_spec *saved_spec; 4656 struct efx_ef10_filter_table *table; 4657 int rc; 4658 4659 down_read(&efx->filter_sem); 4660 table = efx->filter_state; 4661 down_read(&table->lock); 4662 saved_spec = efx_ef10_filter_entry_spec(table, filter_idx); 4663 if (saved_spec && saved_spec->priority == priority && 4664 efx_ef10_filter_pri(table, saved_spec) == 4665 efx_ef10_filter_get_unsafe_pri(filter_id)) { 4666 *spec = *saved_spec; 4667 rc = 0; 4668 } else { 4669 rc = -ENOENT; 4670 } 4671 up_read(&table->lock); 4672 up_read(&efx->filter_sem); 4673 return rc; 4674 } 4675 4676 static int efx_ef10_filter_clear_rx(struct efx_nic *efx, 4677 enum efx_filter_priority priority) 4678 { 4679 struct efx_ef10_filter_table *table; 4680 unsigned int priority_mask; 4681 unsigned int i; 4682 int rc; 4683 4684 priority_mask = (((1U << (priority + 1)) - 1) & 4685 ~(1U << EFX_FILTER_PRI_AUTO)); 4686 4687 down_read(&efx->filter_sem); 4688 table = efx->filter_state; 4689 down_write(&table->lock); 4690 for (i = 0; i < HUNT_FILTER_TBL_ROWS; i++) { 4691 rc = efx_ef10_filter_remove_internal(efx, priority_mask, 4692 i, true); 4693 if (rc && rc != -ENOENT) 4694 break; 4695 rc = 0; 4696 } 4697 4698 up_write(&table->lock); 4699 up_read(&efx->filter_sem); 4700 return rc; 4701 } 4702 4703 static u32 efx_ef10_filter_count_rx_used(struct efx_nic *efx, 4704 enum efx_filter_priority priority) 4705 { 4706 struct efx_ef10_filter_table *table; 4707 unsigned int filter_idx; 4708 s32 count = 0; 4709 4710 down_read(&efx->filter_sem); 4711 table = efx->filter_state; 4712 down_read(&table->lock); 4713 for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) { 4714 if (table->entry[filter_idx].spec && 4715 efx_ef10_filter_entry_spec(table, filter_idx)->priority == 4716 priority) 4717 ++count; 4718 } 4719 up_read(&table->lock); 4720 up_read(&efx->filter_sem); 4721 return count; 4722 } 4723 4724 static u32 efx_ef10_filter_get_rx_id_limit(struct efx_nic *efx) 4725 { 4726 struct efx_ef10_filter_table *table = efx->filter_state; 4727 4728 return table->rx_match_count * HUNT_FILTER_TBL_ROWS * 2; 4729 } 4730 4731 static s32 efx_ef10_filter_get_rx_ids(struct efx_nic *efx, 4732 enum efx_filter_priority priority, 4733 u32 *buf, u32 size) 4734 { 4735 struct efx_ef10_filter_table *table; 4736 struct efx_filter_spec *spec; 4737 unsigned int filter_idx; 4738 s32 count = 0; 4739 4740 down_read(&efx->filter_sem); 4741 table = efx->filter_state; 4742 down_read(&table->lock); 4743 4744 for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) { 4745 spec = efx_ef10_filter_entry_spec(table, filter_idx); 4746 if (spec && spec->priority == priority) { 4747 if (count == size) { 4748 count = -EMSGSIZE; 4749 break; 4750 } 4751 buf[count++] = 4752 efx_ef10_make_filter_id( 4753 efx_ef10_filter_pri(table, spec), 4754 filter_idx); 4755 } 4756 } 4757 up_read(&table->lock); 4758 up_read(&efx->filter_sem); 4759 return count; 4760 } 4761 4762 #ifdef CONFIG_RFS_ACCEL 4763 4764 static bool efx_ef10_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, 4765 unsigned int filter_idx) 4766 { 4767 struct efx_ef10_filter_table *table; 4768 struct efx_filter_spec *spec; 4769 bool ret; 4770 4771 down_read(&efx->filter_sem); 4772 table = efx->filter_state; 4773 down_write(&table->lock); 4774 spec = efx_ef10_filter_entry_spec(table, filter_idx); 4775 4776 if (!spec || spec->priority != EFX_FILTER_PRI_HINT) { 4777 ret = true; 4778 goto out_unlock; 4779 } 4780 4781 if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, 4782 flow_id, filter_idx)) { 4783 ret = false; 4784 goto out_unlock; 4785 } 4786 4787 ret = efx_ef10_filter_remove_internal(efx, 1U << spec->priority, 4788 filter_idx, true) == 0; 4789 out_unlock: 4790 up_write(&table->lock); 4791 up_read(&efx->filter_sem); 4792 return ret; 4793 } 4794 4795 #endif /* CONFIG_RFS_ACCEL */ 4796 4797 static int efx_ef10_filter_match_flags_from_mcdi(bool encap, u32 mcdi_flags) 4798 { 4799 int match_flags = 0; 4800 4801 #define MAP_FLAG(gen_flag, mcdi_field) do { \ 4802 u32 old_mcdi_flags = mcdi_flags; \ 4803 mcdi_flags &= ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ ## \ 4804 mcdi_field ## _LBN); \ 4805 if (mcdi_flags != old_mcdi_flags) \ 4806 match_flags |= EFX_FILTER_MATCH_ ## gen_flag; \ 4807 } while (0) 4808 4809 if (encap) { 4810 /* encap filters must specify encap type */ 4811 match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE; 4812 /* and imply ethertype and ip proto */ 4813 mcdi_flags &= 4814 ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN); 4815 mcdi_flags &= 4816 ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN); 4817 /* VLAN tags refer to the outer packet */ 4818 MAP_FLAG(INNER_VID, INNER_VLAN); 4819 MAP_FLAG(OUTER_VID, OUTER_VLAN); 4820 /* everything else refers to the inner packet */ 4821 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_UCAST_DST); 4822 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_MCAST_DST); 4823 MAP_FLAG(REM_HOST, IFRM_SRC_IP); 4824 MAP_FLAG(LOC_HOST, IFRM_DST_IP); 4825 MAP_FLAG(REM_MAC, IFRM_SRC_MAC); 4826 MAP_FLAG(REM_PORT, IFRM_SRC_PORT); 4827 MAP_FLAG(LOC_MAC, IFRM_DST_MAC); 4828 MAP_FLAG(LOC_PORT, IFRM_DST_PORT); 4829 MAP_FLAG(ETHER_TYPE, IFRM_ETHER_TYPE); 4830 MAP_FLAG(IP_PROTO, IFRM_IP_PROTO); 4831 } else { 4832 MAP_FLAG(LOC_MAC_IG, UNKNOWN_UCAST_DST); 4833 MAP_FLAG(LOC_MAC_IG, UNKNOWN_MCAST_DST); 4834 MAP_FLAG(REM_HOST, SRC_IP); 4835 MAP_FLAG(LOC_HOST, DST_IP); 4836 MAP_FLAG(REM_MAC, SRC_MAC); 4837 MAP_FLAG(REM_PORT, SRC_PORT); 4838 MAP_FLAG(LOC_MAC, DST_MAC); 4839 MAP_FLAG(LOC_PORT, DST_PORT); 4840 MAP_FLAG(ETHER_TYPE, ETHER_TYPE); 4841 MAP_FLAG(INNER_VID, INNER_VLAN); 4842 MAP_FLAG(OUTER_VID, OUTER_VLAN); 4843 MAP_FLAG(IP_PROTO, IP_PROTO); 4844 } 4845 #undef MAP_FLAG 4846 4847 /* Did we map them all? */ 4848 if (mcdi_flags) 4849 return -EINVAL; 4850 4851 return match_flags; 4852 } 4853 4854 static void efx_ef10_filter_cleanup_vlans(struct efx_nic *efx) 4855 { 4856 struct efx_ef10_filter_table *table = efx->filter_state; 4857 struct efx_ef10_filter_vlan *vlan, *next_vlan; 4858 4859 /* See comment in efx_ef10_filter_table_remove() */ 4860 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 4861 return; 4862 4863 if (!table) 4864 return; 4865 4866 list_for_each_entry_safe(vlan, next_vlan, &table->vlan_list, list) 4867 efx_ef10_filter_del_vlan_internal(efx, vlan); 4868 } 4869 4870 static bool efx_ef10_filter_match_supported(struct efx_ef10_filter_table *table, 4871 bool encap, 4872 enum efx_filter_match_flags match_flags) 4873 { 4874 unsigned int match_pri; 4875 int mf; 4876 4877 for (match_pri = 0; 4878 match_pri < table->rx_match_count; 4879 match_pri++) { 4880 mf = efx_ef10_filter_match_flags_from_mcdi(encap, 4881 table->rx_match_mcdi_flags[match_pri]); 4882 if (mf == match_flags) 4883 return true; 4884 } 4885 4886 return false; 4887 } 4888 4889 static int 4890 efx_ef10_filter_table_probe_matches(struct efx_nic *efx, 4891 struct efx_ef10_filter_table *table, 4892 bool encap) 4893 { 4894 MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN); 4895 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX); 4896 unsigned int pd_match_pri, pd_match_count; 4897 size_t outlen; 4898 int rc; 4899 4900 /* Find out which RX filter types are supported, and their priorities */ 4901 MCDI_SET_DWORD(inbuf, GET_PARSER_DISP_INFO_IN_OP, 4902 encap ? 4903 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES : 4904 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES); 4905 rc = efx_mcdi_rpc(efx, MC_CMD_GET_PARSER_DISP_INFO, 4906 inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), 4907 &outlen); 4908 if (rc) 4909 return rc; 4910 4911 pd_match_count = MCDI_VAR_ARRAY_LEN( 4912 outlen, GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES); 4913 4914 for (pd_match_pri = 0; pd_match_pri < pd_match_count; pd_match_pri++) { 4915 u32 mcdi_flags = 4916 MCDI_ARRAY_DWORD( 4917 outbuf, 4918 GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES, 4919 pd_match_pri); 4920 rc = efx_ef10_filter_match_flags_from_mcdi(encap, mcdi_flags); 4921 if (rc < 0) { 4922 netif_dbg(efx, probe, efx->net_dev, 4923 "%s: fw flags %#x pri %u not supported in driver\n", 4924 __func__, mcdi_flags, pd_match_pri); 4925 } else { 4926 netif_dbg(efx, probe, efx->net_dev, 4927 "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n", 4928 __func__, mcdi_flags, pd_match_pri, 4929 rc, table->rx_match_count); 4930 table->rx_match_mcdi_flags[table->rx_match_count] = mcdi_flags; 4931 table->rx_match_count++; 4932 } 4933 } 4934 4935 return 0; 4936 } 4937 4938 static int efx_ef10_filter_table_probe(struct efx_nic *efx) 4939 { 4940 struct efx_ef10_nic_data *nic_data = efx->nic_data; 4941 struct net_device *net_dev = efx->net_dev; 4942 struct efx_ef10_filter_table *table; 4943 struct efx_ef10_vlan *vlan; 4944 int rc; 4945 4946 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 4947 return -EINVAL; 4948 4949 if (efx->filter_state) /* already probed */ 4950 return 0; 4951 4952 table = kzalloc(sizeof(*table), GFP_KERNEL); 4953 if (!table) 4954 return -ENOMEM; 4955 4956 table->rx_match_count = 0; 4957 rc = efx_ef10_filter_table_probe_matches(efx, table, false); 4958 if (rc) 4959 goto fail; 4960 if (nic_data->datapath_caps & 4961 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN)) 4962 rc = efx_ef10_filter_table_probe_matches(efx, table, true); 4963 if (rc) 4964 goto fail; 4965 if ((efx_supported_features(efx) & NETIF_F_HW_VLAN_CTAG_FILTER) && 4966 !(efx_ef10_filter_match_supported(table, false, 4967 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC)) && 4968 efx_ef10_filter_match_supported(table, false, 4969 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC_IG)))) { 4970 netif_info(efx, probe, net_dev, 4971 "VLAN filters are not supported in this firmware variant\n"); 4972 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4973 efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4974 net_dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4975 } 4976 4977 table->entry = vzalloc(HUNT_FILTER_TBL_ROWS * sizeof(*table->entry)); 4978 if (!table->entry) { 4979 rc = -ENOMEM; 4980 goto fail; 4981 } 4982 4983 table->mc_promisc_last = false; 4984 table->vlan_filter = 4985 !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER); 4986 INIT_LIST_HEAD(&table->vlan_list); 4987 init_rwsem(&table->lock); 4988 4989 efx->filter_state = table; 4990 4991 list_for_each_entry(vlan, &nic_data->vlan_list, list) { 4992 rc = efx_ef10_filter_add_vlan(efx, vlan->vid); 4993 if (rc) 4994 goto fail_add_vlan; 4995 } 4996 4997 return 0; 4998 4999 fail_add_vlan: 5000 efx_ef10_filter_cleanup_vlans(efx); 5001 efx->filter_state = NULL; 5002 fail: 5003 kfree(table); 5004 return rc; 5005 } 5006 5007 /* Caller must hold efx->filter_sem for read if race against 5008 * efx_ef10_filter_table_remove() is possible 5009 */ 5010 static void efx_ef10_filter_table_restore(struct efx_nic *efx) 5011 { 5012 struct efx_ef10_filter_table *table = efx->filter_state; 5013 struct efx_ef10_nic_data *nic_data = efx->nic_data; 5014 unsigned int invalid_filters = 0, failed = 0; 5015 struct efx_ef10_filter_vlan *vlan; 5016 struct efx_filter_spec *spec; 5017 struct efx_rss_context *ctx; 5018 unsigned int filter_idx; 5019 u32 mcdi_flags; 5020 int match_pri; 5021 int rc, i; 5022 5023 WARN_ON(!rwsem_is_locked(&efx->filter_sem)); 5024 5025 if (!nic_data->must_restore_filters) 5026 return; 5027 5028 if (!table) 5029 return; 5030 5031 down_write(&table->lock); 5032 mutex_lock(&efx->rss_lock); 5033 5034 for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) { 5035 spec = efx_ef10_filter_entry_spec(table, filter_idx); 5036 if (!spec) 5037 continue; 5038 5039 mcdi_flags = efx_ef10_filter_mcdi_flags_from_spec(spec); 5040 match_pri = 0; 5041 while (match_pri < table->rx_match_count && 5042 table->rx_match_mcdi_flags[match_pri] != mcdi_flags) 5043 ++match_pri; 5044 if (match_pri >= table->rx_match_count) { 5045 invalid_filters++; 5046 goto not_restored; 5047 } 5048 if (spec->rss_context) 5049 ctx = efx_find_rss_context_entry(efx, spec->rss_context); 5050 else 5051 ctx = &efx->rss_context; 5052 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) { 5053 if (!ctx) { 5054 netif_warn(efx, drv, efx->net_dev, 5055 "Warning: unable to restore a filter with nonexistent RSS context %u.\n", 5056 spec->rss_context); 5057 invalid_filters++; 5058 goto not_restored; 5059 } 5060 if (ctx->context_id == EFX_EF10_RSS_CONTEXT_INVALID) { 5061 netif_warn(efx, drv, efx->net_dev, 5062 "Warning: unable to restore a filter with RSS context %u as it was not created.\n", 5063 spec->rss_context); 5064 invalid_filters++; 5065 goto not_restored; 5066 } 5067 } 5068 5069 rc = efx_ef10_filter_push(efx, spec, 5070 &table->entry[filter_idx].handle, 5071 ctx, false); 5072 if (rc) 5073 failed++; 5074 5075 if (rc) { 5076 not_restored: 5077 list_for_each_entry(vlan, &table->vlan_list, list) 5078 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; ++i) 5079 if (vlan->default_filters[i] == filter_idx) 5080 vlan->default_filters[i] = 5081 EFX_EF10_FILTER_ID_INVALID; 5082 5083 kfree(spec); 5084 efx_ef10_filter_set_entry(table, filter_idx, NULL, 0); 5085 } 5086 } 5087 5088 mutex_unlock(&efx->rss_lock); 5089 up_write(&table->lock); 5090 5091 /* This can happen validly if the MC's capabilities have changed, so 5092 * is not an error. 5093 */ 5094 if (invalid_filters) 5095 netif_dbg(efx, drv, efx->net_dev, 5096 "Did not restore %u filters that are now unsupported.\n", 5097 invalid_filters); 5098 5099 if (failed) 5100 netif_err(efx, hw, efx->net_dev, 5101 "unable to restore %u filters\n", failed); 5102 else 5103 nic_data->must_restore_filters = false; 5104 } 5105 5106 static void efx_ef10_filter_table_remove(struct efx_nic *efx) 5107 { 5108 struct efx_ef10_filter_table *table = efx->filter_state; 5109 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); 5110 struct efx_filter_spec *spec; 5111 unsigned int filter_idx; 5112 int rc; 5113 5114 efx_ef10_filter_cleanup_vlans(efx); 5115 efx->filter_state = NULL; 5116 /* If we were called without locking, then it's not safe to free 5117 * the table as others might be using it. So we just WARN, leak 5118 * the memory, and potentially get an inconsistent filter table 5119 * state. 5120 * This should never actually happen. 5121 */ 5122 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 5123 return; 5124 5125 if (!table) 5126 return; 5127 5128 for (filter_idx = 0; filter_idx < HUNT_FILTER_TBL_ROWS; filter_idx++) { 5129 spec = efx_ef10_filter_entry_spec(table, filter_idx); 5130 if (!spec) 5131 continue; 5132 5133 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 5134 efx_ef10_filter_is_exclusive(spec) ? 5135 MC_CMD_FILTER_OP_IN_OP_REMOVE : 5136 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); 5137 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, 5138 table->entry[filter_idx].handle); 5139 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, 5140 sizeof(inbuf), NULL, 0, NULL); 5141 if (rc) 5142 netif_info(efx, drv, efx->net_dev, 5143 "%s: filter %04x remove failed\n", 5144 __func__, filter_idx); 5145 kfree(spec); 5146 } 5147 5148 vfree(table->entry); 5149 kfree(table); 5150 } 5151 5152 static void efx_ef10_filter_mark_one_old(struct efx_nic *efx, uint16_t *id) 5153 { 5154 struct efx_ef10_filter_table *table = efx->filter_state; 5155 unsigned int filter_idx; 5156 5157 efx_rwsem_assert_write_locked(&table->lock); 5158 5159 if (*id != EFX_EF10_FILTER_ID_INVALID) { 5160 filter_idx = efx_ef10_filter_get_unsafe_id(*id); 5161 if (!table->entry[filter_idx].spec) 5162 netif_dbg(efx, drv, efx->net_dev, 5163 "marked null spec old %04x:%04x\n", *id, 5164 filter_idx); 5165 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD; 5166 *id = EFX_EF10_FILTER_ID_INVALID; 5167 } 5168 } 5169 5170 /* Mark old per-VLAN filters that may need to be removed */ 5171 static void _efx_ef10_filter_vlan_mark_old(struct efx_nic *efx, 5172 struct efx_ef10_filter_vlan *vlan) 5173 { 5174 struct efx_ef10_filter_table *table = efx->filter_state; 5175 unsigned int i; 5176 5177 for (i = 0; i < table->dev_uc_count; i++) 5178 efx_ef10_filter_mark_one_old(efx, &vlan->uc[i]); 5179 for (i = 0; i < table->dev_mc_count; i++) 5180 efx_ef10_filter_mark_one_old(efx, &vlan->mc[i]); 5181 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) 5182 efx_ef10_filter_mark_one_old(efx, &vlan->default_filters[i]); 5183 } 5184 5185 /* Mark old filters that may need to be removed. 5186 * Caller must hold efx->filter_sem for read if race against 5187 * efx_ef10_filter_table_remove() is possible 5188 */ 5189 static void efx_ef10_filter_mark_old(struct efx_nic *efx) 5190 { 5191 struct efx_ef10_filter_table *table = efx->filter_state; 5192 struct efx_ef10_filter_vlan *vlan; 5193 5194 down_write(&table->lock); 5195 list_for_each_entry(vlan, &table->vlan_list, list) 5196 _efx_ef10_filter_vlan_mark_old(efx, vlan); 5197 up_write(&table->lock); 5198 } 5199 5200 static void efx_ef10_filter_uc_addr_list(struct efx_nic *efx) 5201 { 5202 struct efx_ef10_filter_table *table = efx->filter_state; 5203 struct net_device *net_dev = efx->net_dev; 5204 struct netdev_hw_addr *uc; 5205 unsigned int i; 5206 5207 table->uc_promisc = !!(net_dev->flags & IFF_PROMISC); 5208 ether_addr_copy(table->dev_uc_list[0].addr, net_dev->dev_addr); 5209 i = 1; 5210 netdev_for_each_uc_addr(uc, net_dev) { 5211 if (i >= EFX_EF10_FILTER_DEV_UC_MAX) { 5212 table->uc_promisc = true; 5213 break; 5214 } 5215 ether_addr_copy(table->dev_uc_list[i].addr, uc->addr); 5216 i++; 5217 } 5218 5219 table->dev_uc_count = i; 5220 } 5221 5222 static void efx_ef10_filter_mc_addr_list(struct efx_nic *efx) 5223 { 5224 struct efx_ef10_filter_table *table = efx->filter_state; 5225 struct net_device *net_dev = efx->net_dev; 5226 struct netdev_hw_addr *mc; 5227 unsigned int i; 5228 5229 table->mc_overflow = false; 5230 table->mc_promisc = !!(net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI)); 5231 5232 i = 0; 5233 netdev_for_each_mc_addr(mc, net_dev) { 5234 if (i >= EFX_EF10_FILTER_DEV_MC_MAX) { 5235 table->mc_promisc = true; 5236 table->mc_overflow = true; 5237 break; 5238 } 5239 ether_addr_copy(table->dev_mc_list[i].addr, mc->addr); 5240 i++; 5241 } 5242 5243 table->dev_mc_count = i; 5244 } 5245 5246 static int efx_ef10_filter_insert_addr_list(struct efx_nic *efx, 5247 struct efx_ef10_filter_vlan *vlan, 5248 bool multicast, bool rollback) 5249 { 5250 struct efx_ef10_filter_table *table = efx->filter_state; 5251 struct efx_ef10_dev_addr *addr_list; 5252 enum efx_filter_flags filter_flags; 5253 struct efx_filter_spec spec; 5254 u8 baddr[ETH_ALEN]; 5255 unsigned int i, j; 5256 int addr_count; 5257 u16 *ids; 5258 int rc; 5259 5260 if (multicast) { 5261 addr_list = table->dev_mc_list; 5262 addr_count = table->dev_mc_count; 5263 ids = vlan->mc; 5264 } else { 5265 addr_list = table->dev_uc_list; 5266 addr_count = table->dev_uc_count; 5267 ids = vlan->uc; 5268 } 5269 5270 filter_flags = efx_rss_enabled(efx) ? EFX_FILTER_FLAG_RX_RSS : 0; 5271 5272 /* Insert/renew filters */ 5273 for (i = 0; i < addr_count; i++) { 5274 EFX_WARN_ON_PARANOID(ids[i] != EFX_EF10_FILTER_ID_INVALID); 5275 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0); 5276 efx_filter_set_eth_local(&spec, vlan->vid, addr_list[i].addr); 5277 rc = efx_ef10_filter_insert(efx, &spec, true); 5278 if (rc < 0) { 5279 if (rollback) { 5280 netif_info(efx, drv, efx->net_dev, 5281 "efx_ef10_filter_insert failed rc=%d\n", 5282 rc); 5283 /* Fall back to promiscuous */ 5284 for (j = 0; j < i; j++) { 5285 efx_ef10_filter_remove_unsafe( 5286 efx, EFX_FILTER_PRI_AUTO, 5287 ids[j]); 5288 ids[j] = EFX_EF10_FILTER_ID_INVALID; 5289 } 5290 return rc; 5291 } else { 5292 /* keep invalid ID, and carry on */ 5293 } 5294 } else { 5295 ids[i] = efx_ef10_filter_get_unsafe_id(rc); 5296 } 5297 } 5298 5299 if (multicast && rollback) { 5300 /* Also need an Ethernet broadcast filter */ 5301 EFX_WARN_ON_PARANOID(vlan->default_filters[EFX_EF10_BCAST] != 5302 EFX_EF10_FILTER_ID_INVALID); 5303 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0); 5304 eth_broadcast_addr(baddr); 5305 efx_filter_set_eth_local(&spec, vlan->vid, baddr); 5306 rc = efx_ef10_filter_insert(efx, &spec, true); 5307 if (rc < 0) { 5308 netif_warn(efx, drv, efx->net_dev, 5309 "Broadcast filter insert failed rc=%d\n", rc); 5310 /* Fall back to promiscuous */ 5311 for (j = 0; j < i; j++) { 5312 efx_ef10_filter_remove_unsafe( 5313 efx, EFX_FILTER_PRI_AUTO, 5314 ids[j]); 5315 ids[j] = EFX_EF10_FILTER_ID_INVALID; 5316 } 5317 return rc; 5318 } else { 5319 vlan->default_filters[EFX_EF10_BCAST] = 5320 efx_ef10_filter_get_unsafe_id(rc); 5321 } 5322 } 5323 5324 return 0; 5325 } 5326 5327 static int efx_ef10_filter_insert_def(struct efx_nic *efx, 5328 struct efx_ef10_filter_vlan *vlan, 5329 enum efx_encap_type encap_type, 5330 bool multicast, bool rollback) 5331 { 5332 struct efx_ef10_nic_data *nic_data = efx->nic_data; 5333 enum efx_filter_flags filter_flags; 5334 struct efx_filter_spec spec; 5335 u8 baddr[ETH_ALEN]; 5336 int rc; 5337 u16 *id; 5338 5339 filter_flags = efx_rss_enabled(efx) ? EFX_FILTER_FLAG_RX_RSS : 0; 5340 5341 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0); 5342 5343 if (multicast) 5344 efx_filter_set_mc_def(&spec); 5345 else 5346 efx_filter_set_uc_def(&spec); 5347 5348 if (encap_type) { 5349 if (nic_data->datapath_caps & 5350 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN)) 5351 efx_filter_set_encap_type(&spec, encap_type); 5352 else 5353 /* don't insert encap filters on non-supporting 5354 * platforms. ID will be left as INVALID. 5355 */ 5356 return 0; 5357 } 5358 5359 if (vlan->vid != EFX_FILTER_VID_UNSPEC) 5360 efx_filter_set_eth_local(&spec, vlan->vid, NULL); 5361 5362 rc = efx_ef10_filter_insert(efx, &spec, true); 5363 if (rc < 0) { 5364 const char *um = multicast ? "Multicast" : "Unicast"; 5365 const char *encap_name = ""; 5366 const char *encap_ipv = ""; 5367 5368 if ((encap_type & EFX_ENCAP_TYPES_MASK) == 5369 EFX_ENCAP_TYPE_VXLAN) 5370 encap_name = "VXLAN "; 5371 else if ((encap_type & EFX_ENCAP_TYPES_MASK) == 5372 EFX_ENCAP_TYPE_NVGRE) 5373 encap_name = "NVGRE "; 5374 else if ((encap_type & EFX_ENCAP_TYPES_MASK) == 5375 EFX_ENCAP_TYPE_GENEVE) 5376 encap_name = "GENEVE "; 5377 if (encap_type & EFX_ENCAP_FLAG_IPV6) 5378 encap_ipv = "IPv6 "; 5379 else if (encap_type) 5380 encap_ipv = "IPv4 "; 5381 5382 /* unprivileged functions can't insert mismatch filters 5383 * for encapsulated or unicast traffic, so downgrade 5384 * those warnings to debug. 5385 */ 5386 netif_cond_dbg(efx, drv, efx->net_dev, 5387 rc == -EPERM && (encap_type || !multicast), warn, 5388 "%s%s%s mismatch filter insert failed rc=%d\n", 5389 encap_name, encap_ipv, um, rc); 5390 } else if (multicast) { 5391 /* mapping from encap types to default filter IDs (multicast) */ 5392 static enum efx_ef10_default_filters map[] = { 5393 [EFX_ENCAP_TYPE_NONE] = EFX_EF10_MCDEF, 5394 [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_MCDEF, 5395 [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_MCDEF, 5396 [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_MCDEF, 5397 [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] = 5398 EFX_EF10_VXLAN6_MCDEF, 5399 [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] = 5400 EFX_EF10_NVGRE6_MCDEF, 5401 [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] = 5402 EFX_EF10_GENEVE6_MCDEF, 5403 }; 5404 5405 /* quick bounds check (BCAST result impossible) */ 5406 BUILD_BUG_ON(EFX_EF10_BCAST != 0); 5407 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) { 5408 WARN_ON(1); 5409 return -EINVAL; 5410 } 5411 /* then follow map */ 5412 id = &vlan->default_filters[map[encap_type]]; 5413 5414 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID); 5415 *id = efx_ef10_filter_get_unsafe_id(rc); 5416 if (!nic_data->workaround_26807 && !encap_type) { 5417 /* Also need an Ethernet broadcast filter */ 5418 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, 5419 filter_flags, 0); 5420 eth_broadcast_addr(baddr); 5421 efx_filter_set_eth_local(&spec, vlan->vid, baddr); 5422 rc = efx_ef10_filter_insert(efx, &spec, true); 5423 if (rc < 0) { 5424 netif_warn(efx, drv, efx->net_dev, 5425 "Broadcast filter insert failed rc=%d\n", 5426 rc); 5427 if (rollback) { 5428 /* Roll back the mc_def filter */ 5429 efx_ef10_filter_remove_unsafe( 5430 efx, EFX_FILTER_PRI_AUTO, 5431 *id); 5432 *id = EFX_EF10_FILTER_ID_INVALID; 5433 return rc; 5434 } 5435 } else { 5436 EFX_WARN_ON_PARANOID( 5437 vlan->default_filters[EFX_EF10_BCAST] != 5438 EFX_EF10_FILTER_ID_INVALID); 5439 vlan->default_filters[EFX_EF10_BCAST] = 5440 efx_ef10_filter_get_unsafe_id(rc); 5441 } 5442 } 5443 rc = 0; 5444 } else { 5445 /* mapping from encap types to default filter IDs (unicast) */ 5446 static enum efx_ef10_default_filters map[] = { 5447 [EFX_ENCAP_TYPE_NONE] = EFX_EF10_UCDEF, 5448 [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_UCDEF, 5449 [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_UCDEF, 5450 [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_UCDEF, 5451 [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] = 5452 EFX_EF10_VXLAN6_UCDEF, 5453 [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] = 5454 EFX_EF10_NVGRE6_UCDEF, 5455 [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] = 5456 EFX_EF10_GENEVE6_UCDEF, 5457 }; 5458 5459 /* quick bounds check (BCAST result impossible) */ 5460 BUILD_BUG_ON(EFX_EF10_BCAST != 0); 5461 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) { 5462 WARN_ON(1); 5463 return -EINVAL; 5464 } 5465 /* then follow map */ 5466 id = &vlan->default_filters[map[encap_type]]; 5467 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID); 5468 *id = rc; 5469 rc = 0; 5470 } 5471 return rc; 5472 } 5473 5474 /* Remove filters that weren't renewed. */ 5475 static void efx_ef10_filter_remove_old(struct efx_nic *efx) 5476 { 5477 struct efx_ef10_filter_table *table = efx->filter_state; 5478 int remove_failed = 0; 5479 int remove_noent = 0; 5480 int rc; 5481 int i; 5482 5483 down_write(&table->lock); 5484 for (i = 0; i < HUNT_FILTER_TBL_ROWS; i++) { 5485 if (READ_ONCE(table->entry[i].spec) & 5486 EFX_EF10_FILTER_FLAG_AUTO_OLD) { 5487 rc = efx_ef10_filter_remove_internal(efx, 5488 1U << EFX_FILTER_PRI_AUTO, i, true); 5489 if (rc == -ENOENT) 5490 remove_noent++; 5491 else if (rc) 5492 remove_failed++; 5493 } 5494 } 5495 up_write(&table->lock); 5496 5497 if (remove_failed) 5498 netif_info(efx, drv, efx->net_dev, 5499 "%s: failed to remove %d filters\n", 5500 __func__, remove_failed); 5501 if (remove_noent) 5502 netif_info(efx, drv, efx->net_dev, 5503 "%s: failed to remove %d non-existent filters\n", 5504 __func__, remove_noent); 5505 } 5506 5507 static int efx_ef10_vport_set_mac_address(struct efx_nic *efx) 5508 { 5509 struct efx_ef10_nic_data *nic_data = efx->nic_data; 5510 u8 mac_old[ETH_ALEN]; 5511 int rc, rc2; 5512 5513 /* Only reconfigure a PF-created vport */ 5514 if (is_zero_ether_addr(nic_data->vport_mac)) 5515 return 0; 5516 5517 efx_device_detach_sync(efx); 5518 efx_net_stop(efx->net_dev); 5519 down_write(&efx->filter_sem); 5520 efx_ef10_filter_table_remove(efx); 5521 up_write(&efx->filter_sem); 5522 5523 rc = efx_ef10_vadaptor_free(efx, nic_data->vport_id); 5524 if (rc) 5525 goto restore_filters; 5526 5527 ether_addr_copy(mac_old, nic_data->vport_mac); 5528 rc = efx_ef10_vport_del_mac(efx, nic_data->vport_id, 5529 nic_data->vport_mac); 5530 if (rc) 5531 goto restore_vadaptor; 5532 5533 rc = efx_ef10_vport_add_mac(efx, nic_data->vport_id, 5534 efx->net_dev->dev_addr); 5535 if (!rc) { 5536 ether_addr_copy(nic_data->vport_mac, efx->net_dev->dev_addr); 5537 } else { 5538 rc2 = efx_ef10_vport_add_mac(efx, nic_data->vport_id, mac_old); 5539 if (rc2) { 5540 /* Failed to add original MAC, so clear vport_mac */ 5541 eth_zero_addr(nic_data->vport_mac); 5542 goto reset_nic; 5543 } 5544 } 5545 5546 restore_vadaptor: 5547 rc2 = efx_ef10_vadaptor_alloc(efx, nic_data->vport_id); 5548 if (rc2) 5549 goto reset_nic; 5550 restore_filters: 5551 down_write(&efx->filter_sem); 5552 rc2 = efx_ef10_filter_table_probe(efx); 5553 up_write(&efx->filter_sem); 5554 if (rc2) 5555 goto reset_nic; 5556 5557 rc2 = efx_net_open(efx->net_dev); 5558 if (rc2) 5559 goto reset_nic; 5560 5561 efx_device_attach_if_not_resetting(efx); 5562 5563 return rc; 5564 5565 reset_nic: 5566 netif_err(efx, drv, efx->net_dev, 5567 "Failed to restore when changing MAC address - scheduling reset\n"); 5568 efx_schedule_reset(efx, RESET_TYPE_DATAPATH); 5569 5570 return rc ? rc : rc2; 5571 } 5572 5573 /* Caller must hold efx->filter_sem for read if race against 5574 * efx_ef10_filter_table_remove() is possible 5575 */ 5576 static void efx_ef10_filter_vlan_sync_rx_mode(struct efx_nic *efx, 5577 struct efx_ef10_filter_vlan *vlan) 5578 { 5579 struct efx_ef10_filter_table *table = efx->filter_state; 5580 struct efx_ef10_nic_data *nic_data = efx->nic_data; 5581 5582 /* Do not install unspecified VID if VLAN filtering is enabled. 5583 * Do not install all specified VIDs if VLAN filtering is disabled. 5584 */ 5585 if ((vlan->vid == EFX_FILTER_VID_UNSPEC) == table->vlan_filter) 5586 return; 5587 5588 /* Insert/renew unicast filters */ 5589 if (table->uc_promisc) { 5590 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NONE, 5591 false, false); 5592 efx_ef10_filter_insert_addr_list(efx, vlan, false, false); 5593 } else { 5594 /* If any of the filters failed to insert, fall back to 5595 * promiscuous mode - add in the uc_def filter. But keep 5596 * our individual unicast filters. 5597 */ 5598 if (efx_ef10_filter_insert_addr_list(efx, vlan, false, false)) 5599 efx_ef10_filter_insert_def(efx, vlan, 5600 EFX_ENCAP_TYPE_NONE, 5601 false, false); 5602 } 5603 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN, 5604 false, false); 5605 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN | 5606 EFX_ENCAP_FLAG_IPV6, 5607 false, false); 5608 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE, 5609 false, false); 5610 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE | 5611 EFX_ENCAP_FLAG_IPV6, 5612 false, false); 5613 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE, 5614 false, false); 5615 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE | 5616 EFX_ENCAP_FLAG_IPV6, 5617 false, false); 5618 5619 /* Insert/renew multicast filters */ 5620 /* If changing promiscuous state with cascaded multicast filters, remove 5621 * old filters first, so that packets are dropped rather than duplicated 5622 */ 5623 if (nic_data->workaround_26807 && 5624 table->mc_promisc_last != table->mc_promisc) 5625 efx_ef10_filter_remove_old(efx); 5626 if (table->mc_promisc) { 5627 if (nic_data->workaround_26807) { 5628 /* If we failed to insert promiscuous filters, rollback 5629 * and fall back to individual multicast filters 5630 */ 5631 if (efx_ef10_filter_insert_def(efx, vlan, 5632 EFX_ENCAP_TYPE_NONE, 5633 true, true)) { 5634 /* Changing promisc state, so remove old filters */ 5635 efx_ef10_filter_remove_old(efx); 5636 efx_ef10_filter_insert_addr_list(efx, vlan, 5637 true, false); 5638 } 5639 } else { 5640 /* If we failed to insert promiscuous filters, don't 5641 * rollback. Regardless, also insert the mc_list, 5642 * unless it's incomplete due to overflow 5643 */ 5644 efx_ef10_filter_insert_def(efx, vlan, 5645 EFX_ENCAP_TYPE_NONE, 5646 true, false); 5647 if (!table->mc_overflow) 5648 efx_ef10_filter_insert_addr_list(efx, vlan, 5649 true, false); 5650 } 5651 } else { 5652 /* If any filters failed to insert, rollback and fall back to 5653 * promiscuous mode - mc_def filter and maybe broadcast. If 5654 * that fails, roll back again and insert as many of our 5655 * individual multicast filters as we can. 5656 */ 5657 if (efx_ef10_filter_insert_addr_list(efx, vlan, true, true)) { 5658 /* Changing promisc state, so remove old filters */ 5659 if (nic_data->workaround_26807) 5660 efx_ef10_filter_remove_old(efx); 5661 if (efx_ef10_filter_insert_def(efx, vlan, 5662 EFX_ENCAP_TYPE_NONE, 5663 true, true)) 5664 efx_ef10_filter_insert_addr_list(efx, vlan, 5665 true, false); 5666 } 5667 } 5668 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN, 5669 true, false); 5670 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN | 5671 EFX_ENCAP_FLAG_IPV6, 5672 true, false); 5673 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE, 5674 true, false); 5675 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE | 5676 EFX_ENCAP_FLAG_IPV6, 5677 true, false); 5678 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE, 5679 true, false); 5680 efx_ef10_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE | 5681 EFX_ENCAP_FLAG_IPV6, 5682 true, false); 5683 } 5684 5685 /* Caller must hold efx->filter_sem for read if race against 5686 * efx_ef10_filter_table_remove() is possible 5687 */ 5688 static void efx_ef10_filter_sync_rx_mode(struct efx_nic *efx) 5689 { 5690 struct efx_ef10_filter_table *table = efx->filter_state; 5691 struct net_device *net_dev = efx->net_dev; 5692 struct efx_ef10_filter_vlan *vlan; 5693 bool vlan_filter; 5694 5695 if (!efx_dev_registered(efx)) 5696 return; 5697 5698 if (!table) 5699 return; 5700 5701 efx_ef10_filter_mark_old(efx); 5702 5703 /* Copy/convert the address lists; add the primary station 5704 * address and broadcast address 5705 */ 5706 netif_addr_lock_bh(net_dev); 5707 efx_ef10_filter_uc_addr_list(efx); 5708 efx_ef10_filter_mc_addr_list(efx); 5709 netif_addr_unlock_bh(net_dev); 5710 5711 /* If VLAN filtering changes, all old filters are finally removed. 5712 * Do it in advance to avoid conflicts for unicast untagged and 5713 * VLAN 0 tagged filters. 5714 */ 5715 vlan_filter = !!(net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER); 5716 if (table->vlan_filter != vlan_filter) { 5717 table->vlan_filter = vlan_filter; 5718 efx_ef10_filter_remove_old(efx); 5719 } 5720 5721 list_for_each_entry(vlan, &table->vlan_list, list) 5722 efx_ef10_filter_vlan_sync_rx_mode(efx, vlan); 5723 5724 efx_ef10_filter_remove_old(efx); 5725 table->mc_promisc_last = table->mc_promisc; 5726 } 5727 5728 static struct efx_ef10_filter_vlan *efx_ef10_filter_find_vlan(struct efx_nic *efx, u16 vid) 5729 { 5730 struct efx_ef10_filter_table *table = efx->filter_state; 5731 struct efx_ef10_filter_vlan *vlan; 5732 5733 WARN_ON(!rwsem_is_locked(&efx->filter_sem)); 5734 5735 list_for_each_entry(vlan, &table->vlan_list, list) { 5736 if (vlan->vid == vid) 5737 return vlan; 5738 } 5739 5740 return NULL; 5741 } 5742 5743 static int efx_ef10_filter_add_vlan(struct efx_nic *efx, u16 vid) 5744 { 5745 struct efx_ef10_filter_table *table = efx->filter_state; 5746 struct efx_ef10_filter_vlan *vlan; 5747 unsigned int i; 5748 5749 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 5750 return -EINVAL; 5751 5752 vlan = efx_ef10_filter_find_vlan(efx, vid); 5753 if (WARN_ON(vlan)) { 5754 netif_err(efx, drv, efx->net_dev, 5755 "VLAN %u already added\n", vid); 5756 return -EALREADY; 5757 } 5758 5759 vlan = kzalloc(sizeof(*vlan), GFP_KERNEL); 5760 if (!vlan) 5761 return -ENOMEM; 5762 5763 vlan->vid = vid; 5764 5765 for (i = 0; i < ARRAY_SIZE(vlan->uc); i++) 5766 vlan->uc[i] = EFX_EF10_FILTER_ID_INVALID; 5767 for (i = 0; i < ARRAY_SIZE(vlan->mc); i++) 5768 vlan->mc[i] = EFX_EF10_FILTER_ID_INVALID; 5769 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) 5770 vlan->default_filters[i] = EFX_EF10_FILTER_ID_INVALID; 5771 5772 list_add_tail(&vlan->list, &table->vlan_list); 5773 5774 if (efx_dev_registered(efx)) 5775 efx_ef10_filter_vlan_sync_rx_mode(efx, vlan); 5776 5777 return 0; 5778 } 5779 5780 static void efx_ef10_filter_del_vlan_internal(struct efx_nic *efx, 5781 struct efx_ef10_filter_vlan *vlan) 5782 { 5783 unsigned int i; 5784 5785 /* See comment in efx_ef10_filter_table_remove() */ 5786 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 5787 return; 5788 5789 list_del(&vlan->list); 5790 5791 for (i = 0; i < ARRAY_SIZE(vlan->uc); i++) 5792 efx_ef10_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, 5793 vlan->uc[i]); 5794 for (i = 0; i < ARRAY_SIZE(vlan->mc); i++) 5795 efx_ef10_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, 5796 vlan->mc[i]); 5797 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) 5798 if (vlan->default_filters[i] != EFX_EF10_FILTER_ID_INVALID) 5799 efx_ef10_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, 5800 vlan->default_filters[i]); 5801 5802 kfree(vlan); 5803 } 5804 5805 static void efx_ef10_filter_del_vlan(struct efx_nic *efx, u16 vid) 5806 { 5807 struct efx_ef10_filter_vlan *vlan; 5808 5809 /* See comment in efx_ef10_filter_table_remove() */ 5810 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 5811 return; 5812 5813 vlan = efx_ef10_filter_find_vlan(efx, vid); 5814 if (!vlan) { 5815 netif_err(efx, drv, efx->net_dev, 5816 "VLAN %u not found in filter state\n", vid); 5817 return; 5818 } 5819 5820 efx_ef10_filter_del_vlan_internal(efx, vlan); 5821 } 5822 5823 static int efx_ef10_set_mac_address(struct efx_nic *efx) 5824 { 5825 MCDI_DECLARE_BUF(inbuf, MC_CMD_VADAPTOR_SET_MAC_IN_LEN); 5826 struct efx_ef10_nic_data *nic_data = efx->nic_data; 5827 bool was_enabled = efx->port_enabled; 5828 int rc; 5829 5830 efx_device_detach_sync(efx); 5831 efx_net_stop(efx->net_dev); 5832 5833 mutex_lock(&efx->mac_lock); 5834 down_write(&efx->filter_sem); 5835 efx_ef10_filter_table_remove(efx); 5836 5837 ether_addr_copy(MCDI_PTR(inbuf, VADAPTOR_SET_MAC_IN_MACADDR), 5838 efx->net_dev->dev_addr); 5839 MCDI_SET_DWORD(inbuf, VADAPTOR_SET_MAC_IN_UPSTREAM_PORT_ID, 5840 nic_data->vport_id); 5841 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_VADAPTOR_SET_MAC, inbuf, 5842 sizeof(inbuf), NULL, 0, NULL); 5843 5844 efx_ef10_filter_table_probe(efx); 5845 up_write(&efx->filter_sem); 5846 mutex_unlock(&efx->mac_lock); 5847 5848 if (was_enabled) 5849 efx_net_open(efx->net_dev); 5850 efx_device_attach_if_not_resetting(efx); 5851 5852 #ifdef CONFIG_SFC_SRIOV 5853 if (efx->pci_dev->is_virtfn && efx->pci_dev->physfn) { 5854 struct pci_dev *pci_dev_pf = efx->pci_dev->physfn; 5855 5856 if (rc == -EPERM) { 5857 struct efx_nic *efx_pf; 5858 5859 /* Switch to PF and change MAC address on vport */ 5860 efx_pf = pci_get_drvdata(pci_dev_pf); 5861 5862 rc = efx_ef10_sriov_set_vf_mac(efx_pf, 5863 nic_data->vf_index, 5864 efx->net_dev->dev_addr); 5865 } else if (!rc) { 5866 struct efx_nic *efx_pf = pci_get_drvdata(pci_dev_pf); 5867 struct efx_ef10_nic_data *nic_data = efx_pf->nic_data; 5868 unsigned int i; 5869 5870 /* MAC address successfully changed by VF (with MAC 5871 * spoofing) so update the parent PF if possible. 5872 */ 5873 for (i = 0; i < efx_pf->vf_count; ++i) { 5874 struct ef10_vf *vf = nic_data->vf + i; 5875 5876 if (vf->efx == efx) { 5877 ether_addr_copy(vf->mac, 5878 efx->net_dev->dev_addr); 5879 return 0; 5880 } 5881 } 5882 } 5883 } else 5884 #endif 5885 if (rc == -EPERM) { 5886 netif_err(efx, drv, efx->net_dev, 5887 "Cannot change MAC address; use sfboot to enable" 5888 " mac-spoofing on this interface\n"); 5889 } else if (rc == -ENOSYS && !efx_ef10_is_vf(efx)) { 5890 /* If the active MCFW does not support MC_CMD_VADAPTOR_SET_MAC 5891 * fall-back to the method of changing the MAC address on the 5892 * vport. This only applies to PFs because such versions of 5893 * MCFW do not support VFs. 5894 */ 5895 rc = efx_ef10_vport_set_mac_address(efx); 5896 } else if (rc) { 5897 efx_mcdi_display_error(efx, MC_CMD_VADAPTOR_SET_MAC, 5898 sizeof(inbuf), NULL, 0, rc); 5899 } 5900 5901 return rc; 5902 } 5903 5904 static int efx_ef10_mac_reconfigure(struct efx_nic *efx) 5905 { 5906 efx_ef10_filter_sync_rx_mode(efx); 5907 5908 return efx_mcdi_set_mac(efx); 5909 } 5910 5911 static int efx_ef10_mac_reconfigure_vf(struct efx_nic *efx) 5912 { 5913 efx_ef10_filter_sync_rx_mode(efx); 5914 5915 return 0; 5916 } 5917 5918 static int efx_ef10_start_bist(struct efx_nic *efx, u32 bist_type) 5919 { 5920 MCDI_DECLARE_BUF(inbuf, MC_CMD_START_BIST_IN_LEN); 5921 5922 MCDI_SET_DWORD(inbuf, START_BIST_IN_TYPE, bist_type); 5923 return efx_mcdi_rpc(efx, MC_CMD_START_BIST, inbuf, sizeof(inbuf), 5924 NULL, 0, NULL); 5925 } 5926 5927 /* MC BISTs follow a different poll mechanism to phy BISTs. 5928 * The BIST is done in the poll handler on the MC, and the MCDI command 5929 * will block until the BIST is done. 5930 */ 5931 static int efx_ef10_poll_bist(struct efx_nic *efx) 5932 { 5933 int rc; 5934 MCDI_DECLARE_BUF(outbuf, MC_CMD_POLL_BIST_OUT_LEN); 5935 size_t outlen; 5936 u32 result; 5937 5938 rc = efx_mcdi_rpc(efx, MC_CMD_POLL_BIST, NULL, 0, 5939 outbuf, sizeof(outbuf), &outlen); 5940 if (rc != 0) 5941 return rc; 5942 5943 if (outlen < MC_CMD_POLL_BIST_OUT_LEN) 5944 return -EIO; 5945 5946 result = MCDI_DWORD(outbuf, POLL_BIST_OUT_RESULT); 5947 switch (result) { 5948 case MC_CMD_POLL_BIST_PASSED: 5949 netif_dbg(efx, hw, efx->net_dev, "BIST passed.\n"); 5950 return 0; 5951 case MC_CMD_POLL_BIST_TIMEOUT: 5952 netif_err(efx, hw, efx->net_dev, "BIST timed out\n"); 5953 return -EIO; 5954 case MC_CMD_POLL_BIST_FAILED: 5955 netif_err(efx, hw, efx->net_dev, "BIST failed.\n"); 5956 return -EIO; 5957 default: 5958 netif_err(efx, hw, efx->net_dev, 5959 "BIST returned unknown result %u", result); 5960 return -EIO; 5961 } 5962 } 5963 5964 static int efx_ef10_run_bist(struct efx_nic *efx, u32 bist_type) 5965 { 5966 int rc; 5967 5968 netif_dbg(efx, drv, efx->net_dev, "starting BIST type %u\n", bist_type); 5969 5970 rc = efx_ef10_start_bist(efx, bist_type); 5971 if (rc != 0) 5972 return rc; 5973 5974 return efx_ef10_poll_bist(efx); 5975 } 5976 5977 static int 5978 efx_ef10_test_chip(struct efx_nic *efx, struct efx_self_tests *tests) 5979 { 5980 int rc, rc2; 5981 5982 efx_reset_down(efx, RESET_TYPE_WORLD); 5983 5984 rc = efx_mcdi_rpc(efx, MC_CMD_ENABLE_OFFLINE_BIST, 5985 NULL, 0, NULL, 0, NULL); 5986 if (rc != 0) 5987 goto out; 5988 5989 tests->memory = efx_ef10_run_bist(efx, MC_CMD_MC_MEM_BIST) ? -1 : 1; 5990 tests->registers = efx_ef10_run_bist(efx, MC_CMD_REG_BIST) ? -1 : 1; 5991 5992 rc = efx_mcdi_reset(efx, RESET_TYPE_WORLD); 5993 5994 out: 5995 if (rc == -EPERM) 5996 rc = 0; 5997 rc2 = efx_reset_up(efx, RESET_TYPE_WORLD, rc == 0); 5998 return rc ? rc : rc2; 5999 } 6000 6001 #ifdef CONFIG_SFC_MTD 6002 6003 struct efx_ef10_nvram_type_info { 6004 u16 type, type_mask; 6005 u8 port; 6006 const char *name; 6007 }; 6008 6009 static const struct efx_ef10_nvram_type_info efx_ef10_nvram_types[] = { 6010 { NVRAM_PARTITION_TYPE_MC_FIRMWARE, 0, 0, "sfc_mcfw" }, 6011 { NVRAM_PARTITION_TYPE_MC_FIRMWARE_BACKUP, 0, 0, "sfc_mcfw_backup" }, 6012 { NVRAM_PARTITION_TYPE_EXPANSION_ROM, 0, 0, "sfc_exp_rom" }, 6013 { NVRAM_PARTITION_TYPE_STATIC_CONFIG, 0, 0, "sfc_static_cfg" }, 6014 { NVRAM_PARTITION_TYPE_DYNAMIC_CONFIG, 0, 0, "sfc_dynamic_cfg" }, 6015 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT0, 0, 0, "sfc_exp_rom_cfg" }, 6016 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT1, 0, 1, "sfc_exp_rom_cfg" }, 6017 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT2, 0, 2, "sfc_exp_rom_cfg" }, 6018 { NVRAM_PARTITION_TYPE_EXPROM_CONFIG_PORT3, 0, 3, "sfc_exp_rom_cfg" }, 6019 { NVRAM_PARTITION_TYPE_LICENSE, 0, 0, "sfc_license" }, 6020 { NVRAM_PARTITION_TYPE_PHY_MIN, 0xff, 0, "sfc_phy_fw" }, 6021 }; 6022 6023 static int efx_ef10_mtd_probe_partition(struct efx_nic *efx, 6024 struct efx_mcdi_mtd_partition *part, 6025 unsigned int type) 6026 { 6027 MCDI_DECLARE_BUF(inbuf, MC_CMD_NVRAM_METADATA_IN_LEN); 6028 MCDI_DECLARE_BUF(outbuf, MC_CMD_NVRAM_METADATA_OUT_LENMAX); 6029 const struct efx_ef10_nvram_type_info *info; 6030 size_t size, erase_size, outlen; 6031 bool protected; 6032 int rc; 6033 6034 for (info = efx_ef10_nvram_types; ; info++) { 6035 if (info == 6036 efx_ef10_nvram_types + ARRAY_SIZE(efx_ef10_nvram_types)) 6037 return -ENODEV; 6038 if ((type & ~info->type_mask) == info->type) 6039 break; 6040 } 6041 if (info->port != efx_port_num(efx)) 6042 return -ENODEV; 6043 6044 rc = efx_mcdi_nvram_info(efx, type, &size, &erase_size, &protected); 6045 if (rc) 6046 return rc; 6047 if (protected) 6048 return -ENODEV; /* hide it */ 6049 6050 part->nvram_type = type; 6051 6052 MCDI_SET_DWORD(inbuf, NVRAM_METADATA_IN_TYPE, type); 6053 rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_METADATA, inbuf, sizeof(inbuf), 6054 outbuf, sizeof(outbuf), &outlen); 6055 if (rc) 6056 return rc; 6057 if (outlen < MC_CMD_NVRAM_METADATA_OUT_LENMIN) 6058 return -EIO; 6059 if (MCDI_DWORD(outbuf, NVRAM_METADATA_OUT_FLAGS) & 6060 (1 << MC_CMD_NVRAM_METADATA_OUT_SUBTYPE_VALID_LBN)) 6061 part->fw_subtype = MCDI_DWORD(outbuf, 6062 NVRAM_METADATA_OUT_SUBTYPE); 6063 6064 part->common.dev_type_name = "EF10 NVRAM manager"; 6065 part->common.type_name = info->name; 6066 6067 part->common.mtd.type = MTD_NORFLASH; 6068 part->common.mtd.flags = MTD_CAP_NORFLASH; 6069 part->common.mtd.size = size; 6070 part->common.mtd.erasesize = erase_size; 6071 6072 return 0; 6073 } 6074 6075 static int efx_ef10_mtd_probe(struct efx_nic *efx) 6076 { 6077 MCDI_DECLARE_BUF(outbuf, MC_CMD_NVRAM_PARTITIONS_OUT_LENMAX); 6078 struct efx_mcdi_mtd_partition *parts; 6079 size_t outlen, n_parts_total, i, n_parts; 6080 unsigned int type; 6081 int rc; 6082 6083 ASSERT_RTNL(); 6084 6085 BUILD_BUG_ON(MC_CMD_NVRAM_PARTITIONS_IN_LEN != 0); 6086 rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_PARTITIONS, NULL, 0, 6087 outbuf, sizeof(outbuf), &outlen); 6088 if (rc) 6089 return rc; 6090 if (outlen < MC_CMD_NVRAM_PARTITIONS_OUT_LENMIN) 6091 return -EIO; 6092 6093 n_parts_total = MCDI_DWORD(outbuf, NVRAM_PARTITIONS_OUT_NUM_PARTITIONS); 6094 if (n_parts_total > 6095 MCDI_VAR_ARRAY_LEN(outlen, NVRAM_PARTITIONS_OUT_TYPE_ID)) 6096 return -EIO; 6097 6098 parts = kcalloc(n_parts_total, sizeof(*parts), GFP_KERNEL); 6099 if (!parts) 6100 return -ENOMEM; 6101 6102 n_parts = 0; 6103 for (i = 0; i < n_parts_total; i++) { 6104 type = MCDI_ARRAY_DWORD(outbuf, NVRAM_PARTITIONS_OUT_TYPE_ID, 6105 i); 6106 rc = efx_ef10_mtd_probe_partition(efx, &parts[n_parts], type); 6107 if (rc == 0) 6108 n_parts++; 6109 else if (rc != -ENODEV) 6110 goto fail; 6111 } 6112 6113 rc = efx_mtd_add(efx, &parts[0].common, n_parts, sizeof(*parts)); 6114 fail: 6115 if (rc) 6116 kfree(parts); 6117 return rc; 6118 } 6119 6120 #endif /* CONFIG_SFC_MTD */ 6121 6122 static void efx_ef10_ptp_write_host_time(struct efx_nic *efx, u32 host_time) 6123 { 6124 _efx_writed(efx, cpu_to_le32(host_time), ER_DZ_MC_DB_LWRD); 6125 } 6126 6127 static void efx_ef10_ptp_write_host_time_vf(struct efx_nic *efx, 6128 u32 host_time) {} 6129 6130 static int efx_ef10_rx_enable_timestamping(struct efx_channel *channel, 6131 bool temp) 6132 { 6133 MCDI_DECLARE_BUF(inbuf, MC_CMD_PTP_IN_TIME_EVENT_SUBSCRIBE_LEN); 6134 int rc; 6135 6136 if (channel->sync_events_state == SYNC_EVENTS_REQUESTED || 6137 channel->sync_events_state == SYNC_EVENTS_VALID || 6138 (temp && channel->sync_events_state == SYNC_EVENTS_DISABLED)) 6139 return 0; 6140 channel->sync_events_state = SYNC_EVENTS_REQUESTED; 6141 6142 MCDI_SET_DWORD(inbuf, PTP_IN_OP, MC_CMD_PTP_OP_TIME_EVENT_SUBSCRIBE); 6143 MCDI_SET_DWORD(inbuf, PTP_IN_PERIPH_ID, 0); 6144 MCDI_SET_DWORD(inbuf, PTP_IN_TIME_EVENT_SUBSCRIBE_QUEUE, 6145 channel->channel); 6146 6147 rc = efx_mcdi_rpc(channel->efx, MC_CMD_PTP, 6148 inbuf, sizeof(inbuf), NULL, 0, NULL); 6149 6150 if (rc != 0) 6151 channel->sync_events_state = temp ? SYNC_EVENTS_QUIESCENT : 6152 SYNC_EVENTS_DISABLED; 6153 6154 return rc; 6155 } 6156 6157 static int efx_ef10_rx_disable_timestamping(struct efx_channel *channel, 6158 bool temp) 6159 { 6160 MCDI_DECLARE_BUF(inbuf, MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_LEN); 6161 int rc; 6162 6163 if (channel->sync_events_state == SYNC_EVENTS_DISABLED || 6164 (temp && channel->sync_events_state == SYNC_EVENTS_QUIESCENT)) 6165 return 0; 6166 if (channel->sync_events_state == SYNC_EVENTS_QUIESCENT) { 6167 channel->sync_events_state = SYNC_EVENTS_DISABLED; 6168 return 0; 6169 } 6170 channel->sync_events_state = temp ? SYNC_EVENTS_QUIESCENT : 6171 SYNC_EVENTS_DISABLED; 6172 6173 MCDI_SET_DWORD(inbuf, PTP_IN_OP, MC_CMD_PTP_OP_TIME_EVENT_UNSUBSCRIBE); 6174 MCDI_SET_DWORD(inbuf, PTP_IN_PERIPH_ID, 0); 6175 MCDI_SET_DWORD(inbuf, PTP_IN_TIME_EVENT_UNSUBSCRIBE_CONTROL, 6176 MC_CMD_PTP_IN_TIME_EVENT_UNSUBSCRIBE_SINGLE); 6177 MCDI_SET_DWORD(inbuf, PTP_IN_TIME_EVENT_UNSUBSCRIBE_QUEUE, 6178 channel->channel); 6179 6180 rc = efx_mcdi_rpc(channel->efx, MC_CMD_PTP, 6181 inbuf, sizeof(inbuf), NULL, 0, NULL); 6182 6183 return rc; 6184 } 6185 6186 static int efx_ef10_ptp_set_ts_sync_events(struct efx_nic *efx, bool en, 6187 bool temp) 6188 { 6189 int (*set)(struct efx_channel *channel, bool temp); 6190 struct efx_channel *channel; 6191 6192 set = en ? 6193 efx_ef10_rx_enable_timestamping : 6194 efx_ef10_rx_disable_timestamping; 6195 6196 channel = efx_ptp_channel(efx); 6197 if (channel) { 6198 int rc = set(channel, temp); 6199 if (en && rc != 0) { 6200 efx_ef10_ptp_set_ts_sync_events(efx, false, temp); 6201 return rc; 6202 } 6203 } 6204 6205 return 0; 6206 } 6207 6208 static int efx_ef10_ptp_set_ts_config_vf(struct efx_nic *efx, 6209 struct hwtstamp_config *init) 6210 { 6211 return -EOPNOTSUPP; 6212 } 6213 6214 static int efx_ef10_ptp_set_ts_config(struct efx_nic *efx, 6215 struct hwtstamp_config *init) 6216 { 6217 int rc; 6218 6219 switch (init->rx_filter) { 6220 case HWTSTAMP_FILTER_NONE: 6221 efx_ef10_ptp_set_ts_sync_events(efx, false, false); 6222 /* if TX timestamping is still requested then leave PTP on */ 6223 return efx_ptp_change_mode(efx, 6224 init->tx_type != HWTSTAMP_TX_OFF, 0); 6225 case HWTSTAMP_FILTER_ALL: 6226 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 6227 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 6228 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 6229 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 6230 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 6231 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 6232 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 6233 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 6234 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 6235 case HWTSTAMP_FILTER_PTP_V2_EVENT: 6236 case HWTSTAMP_FILTER_PTP_V2_SYNC: 6237 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 6238 case HWTSTAMP_FILTER_NTP_ALL: 6239 init->rx_filter = HWTSTAMP_FILTER_ALL; 6240 rc = efx_ptp_change_mode(efx, true, 0); 6241 if (!rc) 6242 rc = efx_ef10_ptp_set_ts_sync_events(efx, true, false); 6243 if (rc) 6244 efx_ptp_change_mode(efx, false, 0); 6245 return rc; 6246 default: 6247 return -ERANGE; 6248 } 6249 } 6250 6251 static int efx_ef10_get_phys_port_id(struct efx_nic *efx, 6252 struct netdev_phys_item_id *ppid) 6253 { 6254 struct efx_ef10_nic_data *nic_data = efx->nic_data; 6255 6256 if (!is_valid_ether_addr(nic_data->port_id)) 6257 return -EOPNOTSUPP; 6258 6259 ppid->id_len = ETH_ALEN; 6260 memcpy(ppid->id, nic_data->port_id, ppid->id_len); 6261 6262 return 0; 6263 } 6264 6265 static int efx_ef10_vlan_rx_add_vid(struct efx_nic *efx, __be16 proto, u16 vid) 6266 { 6267 if (proto != htons(ETH_P_8021Q)) 6268 return -EINVAL; 6269 6270 return efx_ef10_add_vlan(efx, vid); 6271 } 6272 6273 static int efx_ef10_vlan_rx_kill_vid(struct efx_nic *efx, __be16 proto, u16 vid) 6274 { 6275 if (proto != htons(ETH_P_8021Q)) 6276 return -EINVAL; 6277 6278 return efx_ef10_del_vlan(efx, vid); 6279 } 6280 6281 /* We rely on the MCDI wiping out our TX rings if it made any changes to the 6282 * ports table, ensuring that any TSO descriptors that were made on a now- 6283 * removed tunnel port will be blown away and won't break things when we try 6284 * to transmit them using the new ports table. 6285 */ 6286 static int efx_ef10_set_udp_tnl_ports(struct efx_nic *efx, bool unloading) 6287 { 6288 struct efx_ef10_nic_data *nic_data = efx->nic_data; 6289 MCDI_DECLARE_BUF(inbuf, MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LENMAX); 6290 MCDI_DECLARE_BUF(outbuf, MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_LEN); 6291 bool will_reset = false; 6292 size_t num_entries = 0; 6293 size_t inlen, outlen; 6294 size_t i; 6295 int rc; 6296 efx_dword_t flags_and_num_entries; 6297 6298 WARN_ON(!mutex_is_locked(&nic_data->udp_tunnels_lock)); 6299 6300 nic_data->udp_tunnels_dirty = false; 6301 6302 if (!(nic_data->datapath_caps & 6303 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN))) { 6304 efx_device_attach_if_not_resetting(efx); 6305 return 0; 6306 } 6307 6308 BUILD_BUG_ON(ARRAY_SIZE(nic_data->udp_tunnels) > 6309 MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES_MAXNUM); 6310 6311 for (i = 0; i < ARRAY_SIZE(nic_data->udp_tunnels); ++i) { 6312 if (nic_data->udp_tunnels[i].count && 6313 nic_data->udp_tunnels[i].port) { 6314 efx_dword_t entry; 6315 6316 EFX_POPULATE_DWORD_2(entry, 6317 TUNNEL_ENCAP_UDP_PORT_ENTRY_UDP_PORT, 6318 ntohs(nic_data->udp_tunnels[i].port), 6319 TUNNEL_ENCAP_UDP_PORT_ENTRY_PROTOCOL, 6320 nic_data->udp_tunnels[i].type); 6321 *_MCDI_ARRAY_DWORD(inbuf, 6322 SET_TUNNEL_ENCAP_UDP_PORTS_IN_ENTRIES, 6323 num_entries++) = entry; 6324 } 6325 } 6326 6327 BUILD_BUG_ON((MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_NUM_ENTRIES_OFST - 6328 MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_FLAGS_OFST) * 8 != 6329 EFX_WORD_1_LBN); 6330 BUILD_BUG_ON(MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_NUM_ENTRIES_LEN * 8 != 6331 EFX_WORD_1_WIDTH); 6332 EFX_POPULATE_DWORD_2(flags_and_num_entries, 6333 MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_UNLOADING, 6334 !!unloading, 6335 EFX_WORD_1, num_entries); 6336 *_MCDI_DWORD(inbuf, SET_TUNNEL_ENCAP_UDP_PORTS_IN_FLAGS) = 6337 flags_and_num_entries; 6338 6339 inlen = MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_IN_LEN(num_entries); 6340 6341 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS, 6342 inbuf, inlen, outbuf, sizeof(outbuf), &outlen); 6343 if (rc == -EIO) { 6344 /* Most likely the MC rebooted due to another function also 6345 * setting its tunnel port list. Mark the tunnel port list as 6346 * dirty, so it will be pushed upon coming up from the reboot. 6347 */ 6348 nic_data->udp_tunnels_dirty = true; 6349 return 0; 6350 } 6351 6352 if (rc) { 6353 /* expected not available on unprivileged functions */ 6354 if (rc != -EPERM) 6355 netif_warn(efx, drv, efx->net_dev, 6356 "Unable to set UDP tunnel ports; rc=%d.\n", rc); 6357 } else if (MCDI_DWORD(outbuf, SET_TUNNEL_ENCAP_UDP_PORTS_OUT_FLAGS) & 6358 (1 << MC_CMD_SET_TUNNEL_ENCAP_UDP_PORTS_OUT_RESETTING_LBN)) { 6359 netif_info(efx, drv, efx->net_dev, 6360 "Rebooting MC due to UDP tunnel port list change\n"); 6361 will_reset = true; 6362 if (unloading) 6363 /* Delay for the MC reset to complete. This will make 6364 * unloading other functions a bit smoother. This is a 6365 * race, but the other unload will work whichever way 6366 * it goes, this just avoids an unnecessary error 6367 * message. 6368 */ 6369 msleep(100); 6370 } 6371 if (!will_reset && !unloading) { 6372 /* The caller will have detached, relying on the MC reset to 6373 * trigger a re-attach. Since there won't be an MC reset, we 6374 * have to do the attach ourselves. 6375 */ 6376 efx_device_attach_if_not_resetting(efx); 6377 } 6378 6379 return rc; 6380 } 6381 6382 static int efx_ef10_udp_tnl_push_ports(struct efx_nic *efx) 6383 { 6384 struct efx_ef10_nic_data *nic_data = efx->nic_data; 6385 int rc = 0; 6386 6387 mutex_lock(&nic_data->udp_tunnels_lock); 6388 if (nic_data->udp_tunnels_dirty) { 6389 /* Make sure all TX are stopped while we modify the table, else 6390 * we might race against an efx_features_check(). 6391 */ 6392 efx_device_detach_sync(efx); 6393 rc = efx_ef10_set_udp_tnl_ports(efx, false); 6394 } 6395 mutex_unlock(&nic_data->udp_tunnels_lock); 6396 return rc; 6397 } 6398 6399 static struct efx_udp_tunnel *__efx_ef10_udp_tnl_lookup_port(struct efx_nic *efx, 6400 __be16 port) 6401 { 6402 struct efx_ef10_nic_data *nic_data = efx->nic_data; 6403 size_t i; 6404 6405 for (i = 0; i < ARRAY_SIZE(nic_data->udp_tunnels); ++i) { 6406 if (!nic_data->udp_tunnels[i].count) 6407 continue; 6408 if (nic_data->udp_tunnels[i].port == port) 6409 return &nic_data->udp_tunnels[i]; 6410 } 6411 return NULL; 6412 } 6413 6414 static int efx_ef10_udp_tnl_add_port(struct efx_nic *efx, 6415 struct efx_udp_tunnel tnl) 6416 { 6417 struct efx_ef10_nic_data *nic_data = efx->nic_data; 6418 struct efx_udp_tunnel *match; 6419 char typebuf[8]; 6420 size_t i; 6421 int rc; 6422 6423 if (!(nic_data->datapath_caps & 6424 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN))) 6425 return 0; 6426 6427 efx_get_udp_tunnel_type_name(tnl.type, typebuf, sizeof(typebuf)); 6428 netif_dbg(efx, drv, efx->net_dev, "Adding UDP tunnel (%s) port %d\n", 6429 typebuf, ntohs(tnl.port)); 6430 6431 mutex_lock(&nic_data->udp_tunnels_lock); 6432 /* Make sure all TX are stopped while we add to the table, else we 6433 * might race against an efx_features_check(). 6434 */ 6435 efx_device_detach_sync(efx); 6436 6437 match = __efx_ef10_udp_tnl_lookup_port(efx, tnl.port); 6438 if (match != NULL) { 6439 if (match->type == tnl.type) { 6440 netif_dbg(efx, drv, efx->net_dev, 6441 "Referencing existing tunnel entry\n"); 6442 match->count++; 6443 /* No need to cause an MCDI update */ 6444 rc = 0; 6445 goto unlock_out; 6446 } 6447 efx_get_udp_tunnel_type_name(match->type, 6448 typebuf, sizeof(typebuf)); 6449 netif_dbg(efx, drv, efx->net_dev, 6450 "UDP port %d is already in use by %s\n", 6451 ntohs(tnl.port), typebuf); 6452 rc = -EEXIST; 6453 goto unlock_out; 6454 } 6455 6456 for (i = 0; i < ARRAY_SIZE(nic_data->udp_tunnels); ++i) 6457 if (!nic_data->udp_tunnels[i].count) { 6458 nic_data->udp_tunnels[i] = tnl; 6459 nic_data->udp_tunnels[i].count = 1; 6460 rc = efx_ef10_set_udp_tnl_ports(efx, false); 6461 goto unlock_out; 6462 } 6463 6464 netif_dbg(efx, drv, efx->net_dev, 6465 "Unable to add UDP tunnel (%s) port %d; insufficient resources.\n", 6466 typebuf, ntohs(tnl.port)); 6467 6468 rc = -ENOMEM; 6469 6470 unlock_out: 6471 mutex_unlock(&nic_data->udp_tunnels_lock); 6472 return rc; 6473 } 6474 6475 /* Called under the TX lock with the TX queue running, hence no-one can be 6476 * in the middle of updating the UDP tunnels table. However, they could 6477 * have tried and failed the MCDI, in which case they'll have set the dirty 6478 * flag before dropping their locks. 6479 */ 6480 static bool efx_ef10_udp_tnl_has_port(struct efx_nic *efx, __be16 port) 6481 { 6482 struct efx_ef10_nic_data *nic_data = efx->nic_data; 6483 6484 if (!(nic_data->datapath_caps & 6485 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN))) 6486 return false; 6487 6488 if (nic_data->udp_tunnels_dirty) 6489 /* SW table may not match HW state, so just assume we can't 6490 * use any UDP tunnel offloads. 6491 */ 6492 return false; 6493 6494 return __efx_ef10_udp_tnl_lookup_port(efx, port) != NULL; 6495 } 6496 6497 static int efx_ef10_udp_tnl_del_port(struct efx_nic *efx, 6498 struct efx_udp_tunnel tnl) 6499 { 6500 struct efx_ef10_nic_data *nic_data = efx->nic_data; 6501 struct efx_udp_tunnel *match; 6502 char typebuf[8]; 6503 int rc; 6504 6505 if (!(nic_data->datapath_caps & 6506 (1 << MC_CMD_GET_CAPABILITIES_OUT_VXLAN_NVGRE_LBN))) 6507 return 0; 6508 6509 efx_get_udp_tunnel_type_name(tnl.type, typebuf, sizeof(typebuf)); 6510 netif_dbg(efx, drv, efx->net_dev, "Removing UDP tunnel (%s) port %d\n", 6511 typebuf, ntohs(tnl.port)); 6512 6513 mutex_lock(&nic_data->udp_tunnels_lock); 6514 /* Make sure all TX are stopped while we remove from the table, else we 6515 * might race against an efx_features_check(). 6516 */ 6517 efx_device_detach_sync(efx); 6518 6519 match = __efx_ef10_udp_tnl_lookup_port(efx, tnl.port); 6520 if (match != NULL) { 6521 if (match->type == tnl.type) { 6522 if (--match->count) { 6523 /* Port is still in use, so nothing to do */ 6524 netif_dbg(efx, drv, efx->net_dev, 6525 "UDP tunnel port %d remains active\n", 6526 ntohs(tnl.port)); 6527 rc = 0; 6528 goto out_unlock; 6529 } 6530 rc = efx_ef10_set_udp_tnl_ports(efx, false); 6531 goto out_unlock; 6532 } 6533 efx_get_udp_tunnel_type_name(match->type, 6534 typebuf, sizeof(typebuf)); 6535 netif_warn(efx, drv, efx->net_dev, 6536 "UDP port %d is actually in use by %s, not removing\n", 6537 ntohs(tnl.port), typebuf); 6538 } 6539 rc = -ENOENT; 6540 6541 out_unlock: 6542 mutex_unlock(&nic_data->udp_tunnels_lock); 6543 return rc; 6544 } 6545 6546 #define EF10_OFFLOAD_FEATURES \ 6547 (NETIF_F_IP_CSUM | \ 6548 NETIF_F_HW_VLAN_CTAG_FILTER | \ 6549 NETIF_F_IPV6_CSUM | \ 6550 NETIF_F_RXHASH | \ 6551 NETIF_F_NTUPLE) 6552 6553 const struct efx_nic_type efx_hunt_a0_vf_nic_type = { 6554 .is_vf = true, 6555 .mem_bar = efx_ef10_vf_mem_bar, 6556 .mem_map_size = efx_ef10_mem_map_size, 6557 .probe = efx_ef10_probe_vf, 6558 .remove = efx_ef10_remove, 6559 .dimension_resources = efx_ef10_dimension_resources, 6560 .init = efx_ef10_init_nic, 6561 .fini = efx_port_dummy_op_void, 6562 .map_reset_reason = efx_ef10_map_reset_reason, 6563 .map_reset_flags = efx_ef10_map_reset_flags, 6564 .reset = efx_ef10_reset, 6565 .probe_port = efx_mcdi_port_probe, 6566 .remove_port = efx_mcdi_port_remove, 6567 .fini_dmaq = efx_ef10_fini_dmaq, 6568 .prepare_flr = efx_ef10_prepare_flr, 6569 .finish_flr = efx_port_dummy_op_void, 6570 .describe_stats = efx_ef10_describe_stats, 6571 .update_stats = efx_ef10_update_stats_vf, 6572 .start_stats = efx_port_dummy_op_void, 6573 .pull_stats = efx_port_dummy_op_void, 6574 .stop_stats = efx_port_dummy_op_void, 6575 .set_id_led = efx_mcdi_set_id_led, 6576 .push_irq_moderation = efx_ef10_push_irq_moderation, 6577 .reconfigure_mac = efx_ef10_mac_reconfigure_vf, 6578 .check_mac_fault = efx_mcdi_mac_check_fault, 6579 .reconfigure_port = efx_mcdi_port_reconfigure, 6580 .get_wol = efx_ef10_get_wol_vf, 6581 .set_wol = efx_ef10_set_wol_vf, 6582 .resume_wol = efx_port_dummy_op_void, 6583 .mcdi_request = efx_ef10_mcdi_request, 6584 .mcdi_poll_response = efx_ef10_mcdi_poll_response, 6585 .mcdi_read_response = efx_ef10_mcdi_read_response, 6586 .mcdi_poll_reboot = efx_ef10_mcdi_poll_reboot, 6587 .mcdi_reboot_detected = efx_ef10_mcdi_reboot_detected, 6588 .irq_enable_master = efx_port_dummy_op_void, 6589 .irq_test_generate = efx_ef10_irq_test_generate, 6590 .irq_disable_non_ev = efx_port_dummy_op_void, 6591 .irq_handle_msi = efx_ef10_msi_interrupt, 6592 .irq_handle_legacy = efx_ef10_legacy_interrupt, 6593 .tx_probe = efx_ef10_tx_probe, 6594 .tx_init = efx_ef10_tx_init, 6595 .tx_remove = efx_ef10_tx_remove, 6596 .tx_write = efx_ef10_tx_write, 6597 .tx_limit_len = efx_ef10_tx_limit_len, 6598 .rx_push_rss_config = efx_ef10_vf_rx_push_rss_config, 6599 .rx_pull_rss_config = efx_ef10_rx_pull_rss_config, 6600 .rx_probe = efx_ef10_rx_probe, 6601 .rx_init = efx_ef10_rx_init, 6602 .rx_remove = efx_ef10_rx_remove, 6603 .rx_write = efx_ef10_rx_write, 6604 .rx_defer_refill = efx_ef10_rx_defer_refill, 6605 .ev_probe = efx_ef10_ev_probe, 6606 .ev_init = efx_ef10_ev_init, 6607 .ev_fini = efx_ef10_ev_fini, 6608 .ev_remove = efx_ef10_ev_remove, 6609 .ev_process = efx_ef10_ev_process, 6610 .ev_read_ack = efx_ef10_ev_read_ack, 6611 .ev_test_generate = efx_ef10_ev_test_generate, 6612 .filter_table_probe = efx_ef10_filter_table_probe, 6613 .filter_table_restore = efx_ef10_filter_table_restore, 6614 .filter_table_remove = efx_ef10_filter_table_remove, 6615 .filter_update_rx_scatter = efx_ef10_filter_update_rx_scatter, 6616 .filter_insert = efx_ef10_filter_insert, 6617 .filter_remove_safe = efx_ef10_filter_remove_safe, 6618 .filter_get_safe = efx_ef10_filter_get_safe, 6619 .filter_clear_rx = efx_ef10_filter_clear_rx, 6620 .filter_count_rx_used = efx_ef10_filter_count_rx_used, 6621 .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit, 6622 .filter_get_rx_ids = efx_ef10_filter_get_rx_ids, 6623 #ifdef CONFIG_RFS_ACCEL 6624 .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one, 6625 #endif 6626 #ifdef CONFIG_SFC_MTD 6627 .mtd_probe = efx_port_dummy_op_int, 6628 #endif 6629 .ptp_write_host_time = efx_ef10_ptp_write_host_time_vf, 6630 .ptp_set_ts_config = efx_ef10_ptp_set_ts_config_vf, 6631 .vlan_rx_add_vid = efx_ef10_vlan_rx_add_vid, 6632 .vlan_rx_kill_vid = efx_ef10_vlan_rx_kill_vid, 6633 #ifdef CONFIG_SFC_SRIOV 6634 .vswitching_probe = efx_ef10_vswitching_probe_vf, 6635 .vswitching_restore = efx_ef10_vswitching_restore_vf, 6636 .vswitching_remove = efx_ef10_vswitching_remove_vf, 6637 #endif 6638 .get_mac_address = efx_ef10_get_mac_address_vf, 6639 .set_mac_address = efx_ef10_set_mac_address, 6640 6641 .get_phys_port_id = efx_ef10_get_phys_port_id, 6642 .revision = EFX_REV_HUNT_A0, 6643 .max_dma_mask = DMA_BIT_MASK(ESF_DZ_TX_KER_BUF_ADDR_WIDTH), 6644 .rx_prefix_size = ES_DZ_RX_PREFIX_SIZE, 6645 .rx_hash_offset = ES_DZ_RX_PREFIX_HASH_OFST, 6646 .rx_ts_offset = ES_DZ_RX_PREFIX_TSTAMP_OFST, 6647 .can_rx_scatter = true, 6648 .always_rx_scatter = true, 6649 .min_interrupt_mode = EFX_INT_MODE_MSIX, 6650 .max_interrupt_mode = EFX_INT_MODE_MSIX, 6651 .timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH, 6652 .offload_features = EF10_OFFLOAD_FEATURES, 6653 .mcdi_max_ver = 2, 6654 .max_rx_ip_filters = HUNT_FILTER_TBL_ROWS, 6655 .hwtstamp_filters = 1 << HWTSTAMP_FILTER_NONE | 6656 1 << HWTSTAMP_FILTER_ALL, 6657 .rx_hash_key_size = 40, 6658 }; 6659 6660 const struct efx_nic_type efx_hunt_a0_nic_type = { 6661 .is_vf = false, 6662 .mem_bar = efx_ef10_pf_mem_bar, 6663 .mem_map_size = efx_ef10_mem_map_size, 6664 .probe = efx_ef10_probe_pf, 6665 .remove = efx_ef10_remove, 6666 .dimension_resources = efx_ef10_dimension_resources, 6667 .init = efx_ef10_init_nic, 6668 .fini = efx_port_dummy_op_void, 6669 .map_reset_reason = efx_ef10_map_reset_reason, 6670 .map_reset_flags = efx_ef10_map_reset_flags, 6671 .reset = efx_ef10_reset, 6672 .probe_port = efx_mcdi_port_probe, 6673 .remove_port = efx_mcdi_port_remove, 6674 .fini_dmaq = efx_ef10_fini_dmaq, 6675 .prepare_flr = efx_ef10_prepare_flr, 6676 .finish_flr = efx_port_dummy_op_void, 6677 .describe_stats = efx_ef10_describe_stats, 6678 .update_stats = efx_ef10_update_stats_pf, 6679 .start_stats = efx_mcdi_mac_start_stats, 6680 .pull_stats = efx_mcdi_mac_pull_stats, 6681 .stop_stats = efx_mcdi_mac_stop_stats, 6682 .set_id_led = efx_mcdi_set_id_led, 6683 .push_irq_moderation = efx_ef10_push_irq_moderation, 6684 .reconfigure_mac = efx_ef10_mac_reconfigure, 6685 .check_mac_fault = efx_mcdi_mac_check_fault, 6686 .reconfigure_port = efx_mcdi_port_reconfigure, 6687 .get_wol = efx_ef10_get_wol, 6688 .set_wol = efx_ef10_set_wol, 6689 .resume_wol = efx_port_dummy_op_void, 6690 .test_chip = efx_ef10_test_chip, 6691 .test_nvram = efx_mcdi_nvram_test_all, 6692 .mcdi_request = efx_ef10_mcdi_request, 6693 .mcdi_poll_response = efx_ef10_mcdi_poll_response, 6694 .mcdi_read_response = efx_ef10_mcdi_read_response, 6695 .mcdi_poll_reboot = efx_ef10_mcdi_poll_reboot, 6696 .mcdi_reboot_detected = efx_ef10_mcdi_reboot_detected, 6697 .irq_enable_master = efx_port_dummy_op_void, 6698 .irq_test_generate = efx_ef10_irq_test_generate, 6699 .irq_disable_non_ev = efx_port_dummy_op_void, 6700 .irq_handle_msi = efx_ef10_msi_interrupt, 6701 .irq_handle_legacy = efx_ef10_legacy_interrupt, 6702 .tx_probe = efx_ef10_tx_probe, 6703 .tx_init = efx_ef10_tx_init, 6704 .tx_remove = efx_ef10_tx_remove, 6705 .tx_write = efx_ef10_tx_write, 6706 .tx_limit_len = efx_ef10_tx_limit_len, 6707 .rx_push_rss_config = efx_ef10_pf_rx_push_rss_config, 6708 .rx_pull_rss_config = efx_ef10_rx_pull_rss_config, 6709 .rx_push_rss_context_config = efx_ef10_rx_push_rss_context_config, 6710 .rx_pull_rss_context_config = efx_ef10_rx_pull_rss_context_config, 6711 .rx_restore_rss_contexts = efx_ef10_rx_restore_rss_contexts, 6712 .rx_probe = efx_ef10_rx_probe, 6713 .rx_init = efx_ef10_rx_init, 6714 .rx_remove = efx_ef10_rx_remove, 6715 .rx_write = efx_ef10_rx_write, 6716 .rx_defer_refill = efx_ef10_rx_defer_refill, 6717 .ev_probe = efx_ef10_ev_probe, 6718 .ev_init = efx_ef10_ev_init, 6719 .ev_fini = efx_ef10_ev_fini, 6720 .ev_remove = efx_ef10_ev_remove, 6721 .ev_process = efx_ef10_ev_process, 6722 .ev_read_ack = efx_ef10_ev_read_ack, 6723 .ev_test_generate = efx_ef10_ev_test_generate, 6724 .filter_table_probe = efx_ef10_filter_table_probe, 6725 .filter_table_restore = efx_ef10_filter_table_restore, 6726 .filter_table_remove = efx_ef10_filter_table_remove, 6727 .filter_update_rx_scatter = efx_ef10_filter_update_rx_scatter, 6728 .filter_insert = efx_ef10_filter_insert, 6729 .filter_remove_safe = efx_ef10_filter_remove_safe, 6730 .filter_get_safe = efx_ef10_filter_get_safe, 6731 .filter_clear_rx = efx_ef10_filter_clear_rx, 6732 .filter_count_rx_used = efx_ef10_filter_count_rx_used, 6733 .filter_get_rx_id_limit = efx_ef10_filter_get_rx_id_limit, 6734 .filter_get_rx_ids = efx_ef10_filter_get_rx_ids, 6735 #ifdef CONFIG_RFS_ACCEL 6736 .filter_rfs_expire_one = efx_ef10_filter_rfs_expire_one, 6737 #endif 6738 #ifdef CONFIG_SFC_MTD 6739 .mtd_probe = efx_ef10_mtd_probe, 6740 .mtd_rename = efx_mcdi_mtd_rename, 6741 .mtd_read = efx_mcdi_mtd_read, 6742 .mtd_erase = efx_mcdi_mtd_erase, 6743 .mtd_write = efx_mcdi_mtd_write, 6744 .mtd_sync = efx_mcdi_mtd_sync, 6745 #endif 6746 .ptp_write_host_time = efx_ef10_ptp_write_host_time, 6747 .ptp_set_ts_sync_events = efx_ef10_ptp_set_ts_sync_events, 6748 .ptp_set_ts_config = efx_ef10_ptp_set_ts_config, 6749 .vlan_rx_add_vid = efx_ef10_vlan_rx_add_vid, 6750 .vlan_rx_kill_vid = efx_ef10_vlan_rx_kill_vid, 6751 .udp_tnl_push_ports = efx_ef10_udp_tnl_push_ports, 6752 .udp_tnl_add_port = efx_ef10_udp_tnl_add_port, 6753 .udp_tnl_has_port = efx_ef10_udp_tnl_has_port, 6754 .udp_tnl_del_port = efx_ef10_udp_tnl_del_port, 6755 #ifdef CONFIG_SFC_SRIOV 6756 .sriov_configure = efx_ef10_sriov_configure, 6757 .sriov_init = efx_ef10_sriov_init, 6758 .sriov_fini = efx_ef10_sriov_fini, 6759 .sriov_wanted = efx_ef10_sriov_wanted, 6760 .sriov_reset = efx_ef10_sriov_reset, 6761 .sriov_flr = efx_ef10_sriov_flr, 6762 .sriov_set_vf_mac = efx_ef10_sriov_set_vf_mac, 6763 .sriov_set_vf_vlan = efx_ef10_sriov_set_vf_vlan, 6764 .sriov_set_vf_spoofchk = efx_ef10_sriov_set_vf_spoofchk, 6765 .sriov_get_vf_config = efx_ef10_sriov_get_vf_config, 6766 .sriov_set_vf_link_state = efx_ef10_sriov_set_vf_link_state, 6767 .vswitching_probe = efx_ef10_vswitching_probe_pf, 6768 .vswitching_restore = efx_ef10_vswitching_restore_pf, 6769 .vswitching_remove = efx_ef10_vswitching_remove_pf, 6770 #endif 6771 .get_mac_address = efx_ef10_get_mac_address_pf, 6772 .set_mac_address = efx_ef10_set_mac_address, 6773 .tso_versions = efx_ef10_tso_versions, 6774 6775 .get_phys_port_id = efx_ef10_get_phys_port_id, 6776 .revision = EFX_REV_HUNT_A0, 6777 .max_dma_mask = DMA_BIT_MASK(ESF_DZ_TX_KER_BUF_ADDR_WIDTH), 6778 .rx_prefix_size = ES_DZ_RX_PREFIX_SIZE, 6779 .rx_hash_offset = ES_DZ_RX_PREFIX_HASH_OFST, 6780 .rx_ts_offset = ES_DZ_RX_PREFIX_TSTAMP_OFST, 6781 .can_rx_scatter = true, 6782 .always_rx_scatter = true, 6783 .option_descriptors = true, 6784 .min_interrupt_mode = EFX_INT_MODE_LEGACY, 6785 .max_interrupt_mode = EFX_INT_MODE_MSIX, 6786 .timer_period_max = 1 << ERF_DD_EVQ_IND_TIMER_VAL_WIDTH, 6787 .offload_features = EF10_OFFLOAD_FEATURES, 6788 .mcdi_max_ver = 2, 6789 .max_rx_ip_filters = HUNT_FILTER_TBL_ROWS, 6790 .hwtstamp_filters = 1 << HWTSTAMP_FILTER_NONE | 6791 1 << HWTSTAMP_FILTER_ALL, 6792 .rx_hash_key_size = 40, 6793 }; 6794