1 /* 2 * This file is part of the Chelsio T4 Ethernet driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/bitmap.h> 38 #include <linux/crc32.h> 39 #include <linux/ctype.h> 40 #include <linux/debugfs.h> 41 #include <linux/err.h> 42 #include <linux/etherdevice.h> 43 #include <linux/firmware.h> 44 #include <linux/if.h> 45 #include <linux/if_vlan.h> 46 #include <linux/init.h> 47 #include <linux/log2.h> 48 #include <linux/mdio.h> 49 #include <linux/module.h> 50 #include <linux/moduleparam.h> 51 #include <linux/mutex.h> 52 #include <linux/netdevice.h> 53 #include <linux/pci.h> 54 #include <linux/aer.h> 55 #include <linux/rtnetlink.h> 56 #include <linux/sched.h> 57 #include <linux/seq_file.h> 58 #include <linux/sockios.h> 59 #include <linux/vmalloc.h> 60 #include <linux/workqueue.h> 61 #include <net/neighbour.h> 62 #include <net/netevent.h> 63 #include <net/addrconf.h> 64 #include <net/bonding.h> 65 #include <net/addrconf.h> 66 #include <linux/uaccess.h> 67 #include <linux/crash_dump.h> 68 69 #include "cxgb4.h" 70 #include "cxgb4_filter.h" 71 #include "t4_regs.h" 72 #include "t4_values.h" 73 #include "t4_msg.h" 74 #include "t4fw_api.h" 75 #include "t4fw_version.h" 76 #include "cxgb4_dcb.h" 77 #include "cxgb4_debugfs.h" 78 #include "clip_tbl.h" 79 #include "l2t.h" 80 #include "sched.h" 81 #include "cxgb4_tc_u32.h" 82 #include "cxgb4_tc_flower.h" 83 #include "cxgb4_ptp.h" 84 #include "cxgb4_cudbg.h" 85 86 char cxgb4_driver_name[] = KBUILD_MODNAME; 87 88 #ifdef DRV_VERSION 89 #undef DRV_VERSION 90 #endif 91 #define DRV_VERSION "2.0.0-ko" 92 const char cxgb4_driver_version[] = DRV_VERSION; 93 #define DRV_DESC "Chelsio T4/T5/T6 Network Driver" 94 95 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ 96 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ 97 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) 98 99 /* Macros needed to support the PCI Device ID Table ... 100 */ 101 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ 102 static const struct pci_device_id cxgb4_pci_tbl[] = { 103 #define CH_PCI_DEVICE_ID_FUNCTION 0x4 104 105 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is 106 * called for both. 107 */ 108 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0 109 110 #define CH_PCI_ID_TABLE_ENTRY(devid) \ 111 {PCI_VDEVICE(CHELSIO, (devid)), 4} 112 113 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ 114 { 0, } \ 115 } 116 117 #include "t4_pci_id_tbl.h" 118 119 #define FW4_FNAME "cxgb4/t4fw.bin" 120 #define FW5_FNAME "cxgb4/t5fw.bin" 121 #define FW6_FNAME "cxgb4/t6fw.bin" 122 #define FW4_CFNAME "cxgb4/t4-config.txt" 123 #define FW5_CFNAME "cxgb4/t5-config.txt" 124 #define FW6_CFNAME "cxgb4/t6-config.txt" 125 #define PHY_AQ1202_FIRMWARE "cxgb4/aq1202_fw.cld" 126 #define PHY_BCM84834_FIRMWARE "cxgb4/bcm8483.bin" 127 #define PHY_AQ1202_DEVICEID 0x4409 128 #define PHY_BCM84834_DEVICEID 0x4486 129 130 MODULE_DESCRIPTION(DRV_DESC); 131 MODULE_AUTHOR("Chelsio Communications"); 132 MODULE_LICENSE("Dual BSD/GPL"); 133 MODULE_VERSION(DRV_VERSION); 134 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl); 135 MODULE_FIRMWARE(FW4_FNAME); 136 MODULE_FIRMWARE(FW5_FNAME); 137 MODULE_FIRMWARE(FW6_FNAME); 138 139 /* 140 * The driver uses the best interrupt scheme available on a platform in the 141 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which 142 * of these schemes the driver may consider as follows: 143 * 144 * msi = 2: choose from among all three options 145 * msi = 1: only consider MSI and INTx interrupts 146 * msi = 0: force INTx interrupts 147 */ 148 static int msi = 2; 149 150 module_param(msi, int, 0644); 151 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)"); 152 153 /* 154 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers 155 * offset by 2 bytes in order to have the IP headers line up on 4-byte 156 * boundaries. This is a requirement for many architectures which will throw 157 * a machine check fault if an attempt is made to access one of the 4-byte IP 158 * header fields on a non-4-byte boundary. And it's a major performance issue 159 * even on some architectures which allow it like some implementations of the 160 * x86 ISA. However, some architectures don't mind this and for some very 161 * edge-case performance sensitive applications (like forwarding large volumes 162 * of small packets), setting this DMA offset to 0 will decrease the number of 163 * PCI-E Bus transfers enough to measurably affect performance. 164 */ 165 static int rx_dma_offset = 2; 166 167 /* TX Queue select used to determine what algorithm to use for selecting TX 168 * queue. Select between the kernel provided function (select_queue=0) or user 169 * cxgb_select_queue function (select_queue=1) 170 * 171 * Default: select_queue=0 172 */ 173 static int select_queue; 174 module_param(select_queue, int, 0644); 175 MODULE_PARM_DESC(select_queue, 176 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method."); 177 178 static struct dentry *cxgb4_debugfs_root; 179 180 LIST_HEAD(adapter_list); 181 DEFINE_MUTEX(uld_mutex); 182 183 static void link_report(struct net_device *dev) 184 { 185 if (!netif_carrier_ok(dev)) 186 netdev_info(dev, "link down\n"); 187 else { 188 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" }; 189 190 const char *s; 191 const struct port_info *p = netdev_priv(dev); 192 193 switch (p->link_cfg.speed) { 194 case 100: 195 s = "100Mbps"; 196 break; 197 case 1000: 198 s = "1Gbps"; 199 break; 200 case 10000: 201 s = "10Gbps"; 202 break; 203 case 25000: 204 s = "25Gbps"; 205 break; 206 case 40000: 207 s = "40Gbps"; 208 break; 209 case 100000: 210 s = "100Gbps"; 211 break; 212 default: 213 pr_info("%s: unsupported speed: %d\n", 214 dev->name, p->link_cfg.speed); 215 return; 216 } 217 218 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, 219 fc[p->link_cfg.fc]); 220 } 221 } 222 223 #ifdef CONFIG_CHELSIO_T4_DCB 224 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */ 225 static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable) 226 { 227 struct port_info *pi = netdev_priv(dev); 228 struct adapter *adap = pi->adapter; 229 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset]; 230 int i; 231 232 /* We use a simple mapping of Port TX Queue Index to DCB 233 * Priority when we're enabling DCB. 234 */ 235 for (i = 0; i < pi->nqsets; i++, txq++) { 236 u32 name, value; 237 int err; 238 239 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 240 FW_PARAMS_PARAM_X_V( 241 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) | 242 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id)); 243 value = enable ? i : 0xffffffff; 244 245 /* Since we can be called while atomic (from "interrupt 246 * level") we need to issue the Set Parameters Commannd 247 * without sleeping (timeout < 0). 248 */ 249 err = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1, 250 &name, &value, 251 -FW_CMD_MAX_TIMEOUT); 252 253 if (err) 254 dev_err(adap->pdev_dev, 255 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n", 256 enable ? "set" : "unset", pi->port_id, i, -err); 257 else 258 txq->dcb_prio = value; 259 } 260 } 261 262 static int cxgb4_dcb_enabled(const struct net_device *dev) 263 { 264 struct port_info *pi = netdev_priv(dev); 265 266 if (!pi->dcb.enabled) 267 return 0; 268 269 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) || 270 (pi->dcb.state == CXGB4_DCB_STATE_HOST)); 271 } 272 #endif /* CONFIG_CHELSIO_T4_DCB */ 273 274 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat) 275 { 276 struct net_device *dev = adapter->port[port_id]; 277 278 /* Skip changes from disabled ports. */ 279 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) { 280 if (link_stat) 281 netif_carrier_on(dev); 282 else { 283 #ifdef CONFIG_CHELSIO_T4_DCB 284 if (cxgb4_dcb_enabled(dev)) { 285 cxgb4_dcb_reset(dev); 286 dcb_tx_queue_prio_enable(dev, false); 287 } 288 #endif /* CONFIG_CHELSIO_T4_DCB */ 289 netif_carrier_off(dev); 290 } 291 292 link_report(dev); 293 } 294 } 295 296 void t4_os_portmod_changed(const struct adapter *adap, int port_id) 297 { 298 static const char *mod_str[] = { 299 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM" 300 }; 301 302 const struct net_device *dev = adap->port[port_id]; 303 const struct port_info *pi = netdev_priv(dev); 304 305 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 306 netdev_info(dev, "port module unplugged\n"); 307 else if (pi->mod_type < ARRAY_SIZE(mod_str)) 308 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]); 309 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 310 netdev_info(dev, "%s: unsupported port module inserted\n", 311 dev->name); 312 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 313 netdev_info(dev, "%s: unknown port module inserted\n", 314 dev->name); 315 else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR) 316 netdev_info(dev, "%s: transceiver module error\n", dev->name); 317 else 318 netdev_info(dev, "%s: unknown module type %d inserted\n", 319 dev->name, pi->mod_type); 320 } 321 322 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */ 323 module_param(dbfifo_int_thresh, int, 0644); 324 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold"); 325 326 /* 327 * usecs to sleep while draining the dbfifo 328 */ 329 static int dbfifo_drain_delay = 1000; 330 module_param(dbfifo_drain_delay, int, 0644); 331 MODULE_PARM_DESC(dbfifo_drain_delay, 332 "usecs to sleep while draining the dbfifo"); 333 334 static inline int cxgb4_set_addr_hash(struct port_info *pi) 335 { 336 struct adapter *adap = pi->adapter; 337 u64 vec = 0; 338 bool ucast = false; 339 struct hash_mac_addr *entry; 340 341 /* Calculate the hash vector for the updated list and program it */ 342 list_for_each_entry(entry, &adap->mac_hlist, list) { 343 ucast |= is_unicast_ether_addr(entry->addr); 344 vec |= (1ULL << hash_mac_addr(entry->addr)); 345 } 346 return t4_set_addr_hash(adap, adap->mbox, pi->viid, ucast, 347 vec, false); 348 } 349 350 static int cxgb4_mac_sync(struct net_device *netdev, const u8 *mac_addr) 351 { 352 struct port_info *pi = netdev_priv(netdev); 353 struct adapter *adap = pi->adapter; 354 int ret; 355 u64 mhash = 0; 356 u64 uhash = 0; 357 bool free = false; 358 bool ucast = is_unicast_ether_addr(mac_addr); 359 const u8 *maclist[1] = {mac_addr}; 360 struct hash_mac_addr *new_entry; 361 362 ret = t4_alloc_mac_filt(adap, adap->mbox, pi->viid, free, 1, maclist, 363 NULL, ucast ? &uhash : &mhash, false); 364 if (ret < 0) 365 goto out; 366 /* if hash != 0, then add the addr to hash addr list 367 * so on the end we will calculate the hash for the 368 * list and program it 369 */ 370 if (uhash || mhash) { 371 new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC); 372 if (!new_entry) 373 return -ENOMEM; 374 ether_addr_copy(new_entry->addr, mac_addr); 375 list_add_tail(&new_entry->list, &adap->mac_hlist); 376 ret = cxgb4_set_addr_hash(pi); 377 } 378 out: 379 return ret < 0 ? ret : 0; 380 } 381 382 static int cxgb4_mac_unsync(struct net_device *netdev, const u8 *mac_addr) 383 { 384 struct port_info *pi = netdev_priv(netdev); 385 struct adapter *adap = pi->adapter; 386 int ret; 387 const u8 *maclist[1] = {mac_addr}; 388 struct hash_mac_addr *entry, *tmp; 389 390 /* If the MAC address to be removed is in the hash addr 391 * list, delete it from the list and update hash vector 392 */ 393 list_for_each_entry_safe(entry, tmp, &adap->mac_hlist, list) { 394 if (ether_addr_equal(entry->addr, mac_addr)) { 395 list_del(&entry->list); 396 kfree(entry); 397 return cxgb4_set_addr_hash(pi); 398 } 399 } 400 401 ret = t4_free_mac_filt(adap, adap->mbox, pi->viid, 1, maclist, false); 402 return ret < 0 ? -EINVAL : 0; 403 } 404 405 /* 406 * Set Rx properties of a port, such as promiscruity, address filters, and MTU. 407 * If @mtu is -1 it is left unchanged. 408 */ 409 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok) 410 { 411 struct port_info *pi = netdev_priv(dev); 412 struct adapter *adapter = pi->adapter; 413 414 __dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); 415 __dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); 416 417 return t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu, 418 (dev->flags & IFF_PROMISC) ? 1 : 0, 419 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1, 420 sleep_ok); 421 } 422 423 /** 424 * link_start - enable a port 425 * @dev: the port to enable 426 * 427 * Performs the MAC and PHY actions needed to enable a port. 428 */ 429 static int link_start(struct net_device *dev) 430 { 431 int ret; 432 struct port_info *pi = netdev_priv(dev); 433 unsigned int mb = pi->adapter->pf; 434 435 /* 436 * We do not set address filters and promiscuity here, the stack does 437 * that step explicitly. 438 */ 439 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1, 440 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true); 441 if (ret == 0) { 442 ret = t4_change_mac(pi->adapter, mb, pi->viid, 443 pi->xact_addr_filt, dev->dev_addr, true, 444 true); 445 if (ret >= 0) { 446 pi->xact_addr_filt = ret; 447 ret = 0; 448 } 449 } 450 if (ret == 0) 451 ret = t4_link_l1cfg(pi->adapter, mb, pi->tx_chan, 452 &pi->link_cfg); 453 if (ret == 0) { 454 local_bh_disable(); 455 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true, 456 true, CXGB4_DCB_ENABLED); 457 local_bh_enable(); 458 } 459 460 return ret; 461 } 462 463 #ifdef CONFIG_CHELSIO_T4_DCB 464 /* Handle a Data Center Bridging update message from the firmware. */ 465 static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd) 466 { 467 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid)); 468 struct net_device *dev = adap->port[adap->chan_map[port]]; 469 int old_dcb_enabled = cxgb4_dcb_enabled(dev); 470 int new_dcb_enabled; 471 472 cxgb4_dcb_handle_fw_update(adap, pcmd); 473 new_dcb_enabled = cxgb4_dcb_enabled(dev); 474 475 /* If the DCB has become enabled or disabled on the port then we're 476 * going to need to set up/tear down DCB Priority parameters for the 477 * TX Queues associated with the port. 478 */ 479 if (new_dcb_enabled != old_dcb_enabled) 480 dcb_tx_queue_prio_enable(dev, new_dcb_enabled); 481 } 482 #endif /* CONFIG_CHELSIO_T4_DCB */ 483 484 /* Response queue handler for the FW event queue. 485 */ 486 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp, 487 const struct pkt_gl *gl) 488 { 489 u8 opcode = ((const struct rss_header *)rsp)->opcode; 490 491 rsp++; /* skip RSS header */ 492 493 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. 494 */ 495 if (unlikely(opcode == CPL_FW4_MSG && 496 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) { 497 rsp++; 498 opcode = ((const struct rss_header *)rsp)->opcode; 499 rsp++; 500 if (opcode != CPL_SGE_EGR_UPDATE) { 501 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n" 502 , opcode); 503 goto out; 504 } 505 } 506 507 if (likely(opcode == CPL_SGE_EGR_UPDATE)) { 508 const struct cpl_sge_egr_update *p = (void *)rsp; 509 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid)); 510 struct sge_txq *txq; 511 512 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start]; 513 txq->restarts++; 514 if (txq->q_type == CXGB4_TXQ_ETH) { 515 struct sge_eth_txq *eq; 516 517 eq = container_of(txq, struct sge_eth_txq, q); 518 netif_tx_wake_queue(eq->txq); 519 } else { 520 struct sge_uld_txq *oq; 521 522 oq = container_of(txq, struct sge_uld_txq, q); 523 tasklet_schedule(&oq->qresume_tsk); 524 } 525 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) { 526 const struct cpl_fw6_msg *p = (void *)rsp; 527 528 #ifdef CONFIG_CHELSIO_T4_DCB 529 const struct fw_port_cmd *pcmd = (const void *)p->data; 530 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid)); 531 unsigned int action = 532 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16)); 533 534 if (cmd == FW_PORT_CMD && 535 (action == FW_PORT_ACTION_GET_PORT_INFO || 536 action == FW_PORT_ACTION_GET_PORT_INFO32)) { 537 int port = FW_PORT_CMD_PORTID_G( 538 be32_to_cpu(pcmd->op_to_portid)); 539 struct net_device *dev; 540 int dcbxdis, state_input; 541 542 dev = q->adap->port[q->adap->chan_map[port]]; 543 dcbxdis = (action == FW_PORT_ACTION_GET_PORT_INFO 544 ? !!(pcmd->u.info.dcbxdis_pkd & 545 FW_PORT_CMD_DCBXDIS_F) 546 : !!(pcmd->u.info32.lstatus32_to_cbllen32 & 547 FW_PORT_CMD_DCBXDIS32_F)); 548 state_input = (dcbxdis 549 ? CXGB4_DCB_INPUT_FW_DISABLED 550 : CXGB4_DCB_INPUT_FW_ENABLED); 551 552 cxgb4_dcb_state_fsm(dev, state_input); 553 } 554 555 if (cmd == FW_PORT_CMD && 556 action == FW_PORT_ACTION_L2_DCB_CFG) 557 dcb_rpl(q->adap, pcmd); 558 else 559 #endif 560 if (p->type == 0) 561 t4_handle_fw_rpl(q->adap, p->data); 562 } else if (opcode == CPL_L2T_WRITE_RPL) { 563 const struct cpl_l2t_write_rpl *p = (void *)rsp; 564 565 do_l2t_write_rpl(q->adap, p); 566 } else if (opcode == CPL_SET_TCB_RPL) { 567 const struct cpl_set_tcb_rpl *p = (void *)rsp; 568 569 filter_rpl(q->adap, p); 570 } else 571 dev_err(q->adap->pdev_dev, 572 "unexpected CPL %#x on FW event queue\n", opcode); 573 out: 574 return 0; 575 } 576 577 static void disable_msi(struct adapter *adapter) 578 { 579 if (adapter->flags & USING_MSIX) { 580 pci_disable_msix(adapter->pdev); 581 adapter->flags &= ~USING_MSIX; 582 } else if (adapter->flags & USING_MSI) { 583 pci_disable_msi(adapter->pdev); 584 adapter->flags &= ~USING_MSI; 585 } 586 } 587 588 /* 589 * Interrupt handler for non-data events used with MSI-X. 590 */ 591 static irqreturn_t t4_nondata_intr(int irq, void *cookie) 592 { 593 struct adapter *adap = cookie; 594 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A)); 595 596 if (v & PFSW_F) { 597 adap->swintr = 1; 598 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v); 599 } 600 if (adap->flags & MASTER_PF) 601 t4_slow_intr_handler(adap); 602 return IRQ_HANDLED; 603 } 604 605 /* 606 * Name the MSI-X interrupts. 607 */ 608 static void name_msix_vecs(struct adapter *adap) 609 { 610 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc); 611 612 /* non-data interrupts */ 613 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name); 614 615 /* FW events */ 616 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", 617 adap->port[0]->name); 618 619 /* Ethernet queues */ 620 for_each_port(adap, j) { 621 struct net_device *d = adap->port[j]; 622 const struct port_info *pi = netdev_priv(d); 623 624 for (i = 0; i < pi->nqsets; i++, msi_idx++) 625 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d", 626 d->name, i); 627 } 628 } 629 630 static int request_msix_queue_irqs(struct adapter *adap) 631 { 632 struct sge *s = &adap->sge; 633 int err, ethqidx; 634 int msi_index = 2; 635 636 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0, 637 adap->msix_info[1].desc, &s->fw_evtq); 638 if (err) 639 return err; 640 641 for_each_ethrxq(s, ethqidx) { 642 err = request_irq(adap->msix_info[msi_index].vec, 643 t4_sge_intr_msix, 0, 644 adap->msix_info[msi_index].desc, 645 &s->ethrxq[ethqidx].rspq); 646 if (err) 647 goto unwind; 648 msi_index++; 649 } 650 return 0; 651 652 unwind: 653 while (--ethqidx >= 0) 654 free_irq(adap->msix_info[--msi_index].vec, 655 &s->ethrxq[ethqidx].rspq); 656 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 657 return err; 658 } 659 660 static void free_msix_queue_irqs(struct adapter *adap) 661 { 662 int i, msi_index = 2; 663 struct sge *s = &adap->sge; 664 665 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 666 for_each_ethrxq(s, i) 667 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq); 668 } 669 670 /** 671 * cxgb4_write_rss - write the RSS table for a given port 672 * @pi: the port 673 * @queues: array of queue indices for RSS 674 * 675 * Sets up the portion of the HW RSS table for the port's VI to distribute 676 * packets to the Rx queues in @queues. 677 * Should never be called before setting up sge eth rx queues 678 */ 679 int cxgb4_write_rss(const struct port_info *pi, const u16 *queues) 680 { 681 u16 *rss; 682 int i, err; 683 struct adapter *adapter = pi->adapter; 684 const struct sge_eth_rxq *rxq; 685 686 rxq = &adapter->sge.ethrxq[pi->first_qset]; 687 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL); 688 if (!rss) 689 return -ENOMEM; 690 691 /* map the queue indices to queue ids */ 692 for (i = 0; i < pi->rss_size; i++, queues++) 693 rss[i] = rxq[*queues].rspq.abs_id; 694 695 err = t4_config_rss_range(adapter, adapter->pf, pi->viid, 0, 696 pi->rss_size, rss, pi->rss_size); 697 /* If Tunnel All Lookup isn't specified in the global RSS 698 * Configuration, then we need to specify a default Ingress 699 * Queue for any ingress packets which aren't hashed. We'll 700 * use our first ingress queue ... 701 */ 702 if (!err) 703 err = t4_config_vi_rss(adapter, adapter->mbox, pi->viid, 704 FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F | 705 FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F | 706 FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F | 707 FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F | 708 FW_RSS_VI_CONFIG_CMD_UDPEN_F, 709 rss[0]); 710 kfree(rss); 711 return err; 712 } 713 714 /** 715 * setup_rss - configure RSS 716 * @adap: the adapter 717 * 718 * Sets up RSS for each port. 719 */ 720 static int setup_rss(struct adapter *adap) 721 { 722 int i, j, err; 723 724 for_each_port(adap, i) { 725 const struct port_info *pi = adap2pinfo(adap, i); 726 727 /* Fill default values with equal distribution */ 728 for (j = 0; j < pi->rss_size; j++) 729 pi->rss[j] = j % pi->nqsets; 730 731 err = cxgb4_write_rss(pi, pi->rss); 732 if (err) 733 return err; 734 } 735 return 0; 736 } 737 738 /* 739 * Return the channel of the ingress queue with the given qid. 740 */ 741 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid) 742 { 743 qid -= p->ingr_start; 744 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan; 745 } 746 747 /* 748 * Wait until all NAPI handlers are descheduled. 749 */ 750 static void quiesce_rx(struct adapter *adap) 751 { 752 int i; 753 754 for (i = 0; i < adap->sge.ingr_sz; i++) { 755 struct sge_rspq *q = adap->sge.ingr_map[i]; 756 757 if (q && q->handler) 758 napi_disable(&q->napi); 759 } 760 } 761 762 /* Disable interrupt and napi handler */ 763 static void disable_interrupts(struct adapter *adap) 764 { 765 if (adap->flags & FULL_INIT_DONE) { 766 t4_intr_disable(adap); 767 if (adap->flags & USING_MSIX) { 768 free_msix_queue_irqs(adap); 769 free_irq(adap->msix_info[0].vec, adap); 770 } else { 771 free_irq(adap->pdev->irq, adap); 772 } 773 quiesce_rx(adap); 774 } 775 } 776 777 /* 778 * Enable NAPI scheduling and interrupt generation for all Rx queues. 779 */ 780 static void enable_rx(struct adapter *adap) 781 { 782 int i; 783 784 for (i = 0; i < adap->sge.ingr_sz; i++) { 785 struct sge_rspq *q = adap->sge.ingr_map[i]; 786 787 if (!q) 788 continue; 789 if (q->handler) 790 napi_enable(&q->napi); 791 792 /* 0-increment GTS to start the timer and enable interrupts */ 793 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A), 794 SEINTARM_V(q->intr_params) | 795 INGRESSQID_V(q->cntxt_id)); 796 } 797 } 798 799 800 static int setup_fw_sge_queues(struct adapter *adap) 801 { 802 struct sge *s = &adap->sge; 803 int err = 0; 804 805 bitmap_zero(s->starving_fl, s->egr_sz); 806 bitmap_zero(s->txq_maperr, s->egr_sz); 807 808 if (adap->flags & USING_MSIX) 809 adap->msi_idx = 1; /* vector 0 is for non-queue interrupts */ 810 else { 811 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0, 812 NULL, NULL, NULL, -1); 813 if (err) 814 return err; 815 adap->msi_idx = -((int)s->intrq.abs_id + 1); 816 } 817 818 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0], 819 adap->msi_idx, NULL, fwevtq_handler, NULL, -1); 820 if (err) 821 t4_free_sge_resources(adap); 822 return err; 823 } 824 825 /** 826 * setup_sge_queues - configure SGE Tx/Rx/response queues 827 * @adap: the adapter 828 * 829 * Determines how many sets of SGE queues to use and initializes them. 830 * We support multiple queue sets per port if we have MSI-X, otherwise 831 * just one queue set per port. 832 */ 833 static int setup_sge_queues(struct adapter *adap) 834 { 835 int err, i, j; 836 struct sge *s = &adap->sge; 837 struct sge_uld_rxq_info *rxq_info = NULL; 838 unsigned int cmplqid = 0; 839 840 if (is_uld(adap)) 841 rxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA]; 842 843 for_each_port(adap, i) { 844 struct net_device *dev = adap->port[i]; 845 struct port_info *pi = netdev_priv(dev); 846 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset]; 847 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset]; 848 849 for (j = 0; j < pi->nqsets; j++, q++) { 850 if (adap->msi_idx > 0) 851 adap->msi_idx++; 852 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, 853 adap->msi_idx, &q->fl, 854 t4_ethrx_handler, 855 NULL, 856 t4_get_tp_ch_map(adap, 857 pi->tx_chan)); 858 if (err) 859 goto freeout; 860 q->rspq.idx = j; 861 memset(&q->stats, 0, sizeof(q->stats)); 862 } 863 for (j = 0; j < pi->nqsets; j++, t++) { 864 err = t4_sge_alloc_eth_txq(adap, t, dev, 865 netdev_get_tx_queue(dev, j), 866 s->fw_evtq.cntxt_id); 867 if (err) 868 goto freeout; 869 } 870 } 871 872 for_each_port(adap, i) { 873 /* Note that cmplqid below is 0 if we don't 874 * have RDMA queues, and that's the right value. 875 */ 876 if (rxq_info) 877 cmplqid = rxq_info->uldrxq[i].rspq.cntxt_id; 878 879 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i], 880 s->fw_evtq.cntxt_id, cmplqid); 881 if (err) 882 goto freeout; 883 } 884 885 if (!is_t4(adap->params.chip)) { 886 err = t4_sge_alloc_eth_txq(adap, &s->ptptxq, adap->port[0], 887 netdev_get_tx_queue(adap->port[0], 0) 888 , s->fw_evtq.cntxt_id); 889 if (err) 890 goto freeout; 891 } 892 893 t4_write_reg(adap, is_t4(adap->params.chip) ? 894 MPS_TRC_RSS_CONTROL_A : 895 MPS_T5_TRC_RSS_CONTROL_A, 896 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) | 897 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id)); 898 return 0; 899 freeout: 900 t4_free_sge_resources(adap); 901 return err; 902 } 903 904 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb, 905 void *accel_priv, select_queue_fallback_t fallback) 906 { 907 int txq; 908 909 #ifdef CONFIG_CHELSIO_T4_DCB 910 /* If a Data Center Bridging has been successfully negotiated on this 911 * link then we'll use the skb's priority to map it to a TX Queue. 912 * The skb's priority is determined via the VLAN Tag Priority Code 913 * Point field. 914 */ 915 if (cxgb4_dcb_enabled(dev) && !is_kdump_kernel()) { 916 u16 vlan_tci; 917 int err; 918 919 err = vlan_get_tag(skb, &vlan_tci); 920 if (unlikely(err)) { 921 if (net_ratelimit()) 922 netdev_warn(dev, 923 "TX Packet without VLAN Tag on DCB Link\n"); 924 txq = 0; 925 } else { 926 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 927 #ifdef CONFIG_CHELSIO_T4_FCOE 928 if (skb->protocol == htons(ETH_P_FCOE)) 929 txq = skb->priority & 0x7; 930 #endif /* CONFIG_CHELSIO_T4_FCOE */ 931 } 932 return txq; 933 } 934 #endif /* CONFIG_CHELSIO_T4_DCB */ 935 936 if (select_queue) { 937 txq = (skb_rx_queue_recorded(skb) 938 ? skb_get_rx_queue(skb) 939 : smp_processor_id()); 940 941 while (unlikely(txq >= dev->real_num_tx_queues)) 942 txq -= dev->real_num_tx_queues; 943 944 return txq; 945 } 946 947 return fallback(dev, skb) % dev->real_num_tx_queues; 948 } 949 950 static int closest_timer(const struct sge *s, int time) 951 { 952 int i, delta, match = 0, min_delta = INT_MAX; 953 954 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { 955 delta = time - s->timer_val[i]; 956 if (delta < 0) 957 delta = -delta; 958 if (delta < min_delta) { 959 min_delta = delta; 960 match = i; 961 } 962 } 963 return match; 964 } 965 966 static int closest_thres(const struct sge *s, int thres) 967 { 968 int i, delta, match = 0, min_delta = INT_MAX; 969 970 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { 971 delta = thres - s->counter_val[i]; 972 if (delta < 0) 973 delta = -delta; 974 if (delta < min_delta) { 975 min_delta = delta; 976 match = i; 977 } 978 } 979 return match; 980 } 981 982 /** 983 * cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters 984 * @q: the Rx queue 985 * @us: the hold-off time in us, or 0 to disable timer 986 * @cnt: the hold-off packet count, or 0 to disable counter 987 * 988 * Sets an Rx queue's interrupt hold-off time and packet count. At least 989 * one of the two needs to be enabled for the queue to generate interrupts. 990 */ 991 int cxgb4_set_rspq_intr_params(struct sge_rspq *q, 992 unsigned int us, unsigned int cnt) 993 { 994 struct adapter *adap = q->adap; 995 996 if ((us | cnt) == 0) 997 cnt = 1; 998 999 if (cnt) { 1000 int err; 1001 u32 v, new_idx; 1002 1003 new_idx = closest_thres(&adap->sge, cnt); 1004 if (q->desc && q->pktcnt_idx != new_idx) { 1005 /* the queue has already been created, update it */ 1006 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 1007 FW_PARAMS_PARAM_X_V( 1008 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) | 1009 FW_PARAMS_PARAM_YZ_V(q->cntxt_id); 1010 err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, 1011 &v, &new_idx); 1012 if (err) 1013 return err; 1014 } 1015 q->pktcnt_idx = new_idx; 1016 } 1017 1018 us = us == 0 ? 6 : closest_timer(&adap->sge, us); 1019 q->intr_params = QINTR_TIMER_IDX_V(us) | QINTR_CNT_EN_V(cnt > 0); 1020 return 0; 1021 } 1022 1023 static int cxgb_set_features(struct net_device *dev, netdev_features_t features) 1024 { 1025 const struct port_info *pi = netdev_priv(dev); 1026 netdev_features_t changed = dev->features ^ features; 1027 int err; 1028 1029 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX)) 1030 return 0; 1031 1032 err = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, -1, 1033 -1, -1, -1, 1034 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true); 1035 if (unlikely(err)) 1036 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX; 1037 return err; 1038 } 1039 1040 static int setup_debugfs(struct adapter *adap) 1041 { 1042 if (IS_ERR_OR_NULL(adap->debugfs_root)) 1043 return -1; 1044 1045 #ifdef CONFIG_DEBUG_FS 1046 t4_setup_debugfs(adap); 1047 #endif 1048 return 0; 1049 } 1050 1051 /* 1052 * upper-layer driver support 1053 */ 1054 1055 /* 1056 * Allocate an active-open TID and set it to the supplied value. 1057 */ 1058 int cxgb4_alloc_atid(struct tid_info *t, void *data) 1059 { 1060 int atid = -1; 1061 1062 spin_lock_bh(&t->atid_lock); 1063 if (t->afree) { 1064 union aopen_entry *p = t->afree; 1065 1066 atid = (p - t->atid_tab) + t->atid_base; 1067 t->afree = p->next; 1068 p->data = data; 1069 t->atids_in_use++; 1070 } 1071 spin_unlock_bh(&t->atid_lock); 1072 return atid; 1073 } 1074 EXPORT_SYMBOL(cxgb4_alloc_atid); 1075 1076 /* 1077 * Release an active-open TID. 1078 */ 1079 void cxgb4_free_atid(struct tid_info *t, unsigned int atid) 1080 { 1081 union aopen_entry *p = &t->atid_tab[atid - t->atid_base]; 1082 1083 spin_lock_bh(&t->atid_lock); 1084 p->next = t->afree; 1085 t->afree = p; 1086 t->atids_in_use--; 1087 spin_unlock_bh(&t->atid_lock); 1088 } 1089 EXPORT_SYMBOL(cxgb4_free_atid); 1090 1091 /* 1092 * Allocate a server TID and set it to the supplied value. 1093 */ 1094 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data) 1095 { 1096 int stid; 1097 1098 spin_lock_bh(&t->stid_lock); 1099 if (family == PF_INET) { 1100 stid = find_first_zero_bit(t->stid_bmap, t->nstids); 1101 if (stid < t->nstids) 1102 __set_bit(stid, t->stid_bmap); 1103 else 1104 stid = -1; 1105 } else { 1106 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 1); 1107 if (stid < 0) 1108 stid = -1; 1109 } 1110 if (stid >= 0) { 1111 t->stid_tab[stid].data = data; 1112 stid += t->stid_base; 1113 /* IPv6 requires max of 520 bits or 16 cells in TCAM 1114 * This is equivalent to 4 TIDs. With CLIP enabled it 1115 * needs 2 TIDs. 1116 */ 1117 if (family == PF_INET6) { 1118 t->stids_in_use += 2; 1119 t->v6_stids_in_use += 2; 1120 } else { 1121 t->stids_in_use++; 1122 } 1123 } 1124 spin_unlock_bh(&t->stid_lock); 1125 return stid; 1126 } 1127 EXPORT_SYMBOL(cxgb4_alloc_stid); 1128 1129 /* Allocate a server filter TID and set it to the supplied value. 1130 */ 1131 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data) 1132 { 1133 int stid; 1134 1135 spin_lock_bh(&t->stid_lock); 1136 if (family == PF_INET) { 1137 stid = find_next_zero_bit(t->stid_bmap, 1138 t->nstids + t->nsftids, t->nstids); 1139 if (stid < (t->nstids + t->nsftids)) 1140 __set_bit(stid, t->stid_bmap); 1141 else 1142 stid = -1; 1143 } else { 1144 stid = -1; 1145 } 1146 if (stid >= 0) { 1147 t->stid_tab[stid].data = data; 1148 stid -= t->nstids; 1149 stid += t->sftid_base; 1150 t->sftids_in_use++; 1151 } 1152 spin_unlock_bh(&t->stid_lock); 1153 return stid; 1154 } 1155 EXPORT_SYMBOL(cxgb4_alloc_sftid); 1156 1157 /* Release a server TID. 1158 */ 1159 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family) 1160 { 1161 /* Is it a server filter TID? */ 1162 if (t->nsftids && (stid >= t->sftid_base)) { 1163 stid -= t->sftid_base; 1164 stid += t->nstids; 1165 } else { 1166 stid -= t->stid_base; 1167 } 1168 1169 spin_lock_bh(&t->stid_lock); 1170 if (family == PF_INET) 1171 __clear_bit(stid, t->stid_bmap); 1172 else 1173 bitmap_release_region(t->stid_bmap, stid, 1); 1174 t->stid_tab[stid].data = NULL; 1175 if (stid < t->nstids) { 1176 if (family == PF_INET6) { 1177 t->stids_in_use -= 2; 1178 t->v6_stids_in_use -= 2; 1179 } else { 1180 t->stids_in_use--; 1181 } 1182 } else { 1183 t->sftids_in_use--; 1184 } 1185 1186 spin_unlock_bh(&t->stid_lock); 1187 } 1188 EXPORT_SYMBOL(cxgb4_free_stid); 1189 1190 /* 1191 * Populate a TID_RELEASE WR. Caller must properly size the skb. 1192 */ 1193 static void mk_tid_release(struct sk_buff *skb, unsigned int chan, 1194 unsigned int tid) 1195 { 1196 struct cpl_tid_release *req; 1197 1198 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); 1199 req = __skb_put(skb, sizeof(*req)); 1200 INIT_TP_WR(req, tid); 1201 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 1202 } 1203 1204 /* 1205 * Queue a TID release request and if necessary schedule a work queue to 1206 * process it. 1207 */ 1208 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan, 1209 unsigned int tid) 1210 { 1211 void **p = &t->tid_tab[tid]; 1212 struct adapter *adap = container_of(t, struct adapter, tids); 1213 1214 spin_lock_bh(&adap->tid_release_lock); 1215 *p = adap->tid_release_head; 1216 /* Low 2 bits encode the Tx channel number */ 1217 adap->tid_release_head = (void **)((uintptr_t)p | chan); 1218 if (!adap->tid_release_task_busy) { 1219 adap->tid_release_task_busy = true; 1220 queue_work(adap->workq, &adap->tid_release_task); 1221 } 1222 spin_unlock_bh(&adap->tid_release_lock); 1223 } 1224 1225 /* 1226 * Process the list of pending TID release requests. 1227 */ 1228 static void process_tid_release_list(struct work_struct *work) 1229 { 1230 struct sk_buff *skb; 1231 struct adapter *adap; 1232 1233 adap = container_of(work, struct adapter, tid_release_task); 1234 1235 spin_lock_bh(&adap->tid_release_lock); 1236 while (adap->tid_release_head) { 1237 void **p = adap->tid_release_head; 1238 unsigned int chan = (uintptr_t)p & 3; 1239 p = (void *)p - chan; 1240 1241 adap->tid_release_head = *p; 1242 *p = NULL; 1243 spin_unlock_bh(&adap->tid_release_lock); 1244 1245 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release), 1246 GFP_KERNEL))) 1247 schedule_timeout_uninterruptible(1); 1248 1249 mk_tid_release(skb, chan, p - adap->tids.tid_tab); 1250 t4_ofld_send(adap, skb); 1251 spin_lock_bh(&adap->tid_release_lock); 1252 } 1253 adap->tid_release_task_busy = false; 1254 spin_unlock_bh(&adap->tid_release_lock); 1255 } 1256 1257 /* 1258 * Release a TID and inform HW. If we are unable to allocate the release 1259 * message we defer to a work queue. 1260 */ 1261 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid, 1262 unsigned short family) 1263 { 1264 struct sk_buff *skb; 1265 struct adapter *adap = container_of(t, struct adapter, tids); 1266 1267 WARN_ON(tid >= t->ntids); 1268 1269 if (t->tid_tab[tid]) { 1270 t->tid_tab[tid] = NULL; 1271 atomic_dec(&t->conns_in_use); 1272 if (t->hash_base && (tid >= t->hash_base)) { 1273 if (family == AF_INET6) 1274 atomic_sub(2, &t->hash_tids_in_use); 1275 else 1276 atomic_dec(&t->hash_tids_in_use); 1277 } else { 1278 if (family == AF_INET6) 1279 atomic_sub(2, &t->tids_in_use); 1280 else 1281 atomic_dec(&t->tids_in_use); 1282 } 1283 } 1284 1285 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); 1286 if (likely(skb)) { 1287 mk_tid_release(skb, chan, tid); 1288 t4_ofld_send(adap, skb); 1289 } else 1290 cxgb4_queue_tid_release(t, chan, tid); 1291 } 1292 EXPORT_SYMBOL(cxgb4_remove_tid); 1293 1294 /* 1295 * Allocate and initialize the TID tables. Returns 0 on success. 1296 */ 1297 static int tid_init(struct tid_info *t) 1298 { 1299 struct adapter *adap = container_of(t, struct adapter, tids); 1300 unsigned int max_ftids = t->nftids + t->nsftids; 1301 unsigned int natids = t->natids; 1302 unsigned int stid_bmap_size; 1303 unsigned int ftid_bmap_size; 1304 size_t size; 1305 1306 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids); 1307 ftid_bmap_size = BITS_TO_LONGS(t->nftids); 1308 size = t->ntids * sizeof(*t->tid_tab) + 1309 natids * sizeof(*t->atid_tab) + 1310 t->nstids * sizeof(*t->stid_tab) + 1311 t->nsftids * sizeof(*t->stid_tab) + 1312 stid_bmap_size * sizeof(long) + 1313 max_ftids * sizeof(*t->ftid_tab) + 1314 ftid_bmap_size * sizeof(long); 1315 1316 t->tid_tab = kvzalloc(size, GFP_KERNEL); 1317 if (!t->tid_tab) 1318 return -ENOMEM; 1319 1320 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; 1321 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids]; 1322 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids]; 1323 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size]; 1324 t->ftid_bmap = (unsigned long *)&t->ftid_tab[max_ftids]; 1325 spin_lock_init(&t->stid_lock); 1326 spin_lock_init(&t->atid_lock); 1327 spin_lock_init(&t->ftid_lock); 1328 1329 t->stids_in_use = 0; 1330 t->v6_stids_in_use = 0; 1331 t->sftids_in_use = 0; 1332 t->afree = NULL; 1333 t->atids_in_use = 0; 1334 atomic_set(&t->tids_in_use, 0); 1335 atomic_set(&t->conns_in_use, 0); 1336 atomic_set(&t->hash_tids_in_use, 0); 1337 1338 /* Setup the free list for atid_tab and clear the stid bitmap. */ 1339 if (natids) { 1340 while (--natids) 1341 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 1342 t->afree = t->atid_tab; 1343 } 1344 1345 if (is_offload(adap)) { 1346 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids); 1347 /* Reserve stid 0 for T4/T5 adapters */ 1348 if (!t->stid_base && 1349 CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 1350 __set_bit(0, t->stid_bmap); 1351 } 1352 1353 bitmap_zero(t->ftid_bmap, t->nftids); 1354 return 0; 1355 } 1356 1357 /** 1358 * cxgb4_create_server - create an IP server 1359 * @dev: the device 1360 * @stid: the server TID 1361 * @sip: local IP address to bind server to 1362 * @sport: the server's TCP port 1363 * @queue: queue to direct messages from this server to 1364 * 1365 * Create an IP server for the given port and address. 1366 * Returns <0 on error and one of the %NET_XMIT_* values on success. 1367 */ 1368 int cxgb4_create_server(const struct net_device *dev, unsigned int stid, 1369 __be32 sip, __be16 sport, __be16 vlan, 1370 unsigned int queue) 1371 { 1372 unsigned int chan; 1373 struct sk_buff *skb; 1374 struct adapter *adap; 1375 struct cpl_pass_open_req *req; 1376 int ret; 1377 1378 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1379 if (!skb) 1380 return -ENOMEM; 1381 1382 adap = netdev2adap(dev); 1383 req = __skb_put(skb, sizeof(*req)); 1384 INIT_TP_WR(req, 0); 1385 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid)); 1386 req->local_port = sport; 1387 req->peer_port = htons(0); 1388 req->local_ip = sip; 1389 req->peer_ip = htonl(0); 1390 chan = rxq_to_chan(&adap->sge, queue); 1391 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 1392 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 1393 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 1394 ret = t4_mgmt_tx(adap, skb); 1395 return net_xmit_eval(ret); 1396 } 1397 EXPORT_SYMBOL(cxgb4_create_server); 1398 1399 /* cxgb4_create_server6 - create an IPv6 server 1400 * @dev: the device 1401 * @stid: the server TID 1402 * @sip: local IPv6 address to bind server to 1403 * @sport: the server's TCP port 1404 * @queue: queue to direct messages from this server to 1405 * 1406 * Create an IPv6 server for the given port and address. 1407 * Returns <0 on error and one of the %NET_XMIT_* values on success. 1408 */ 1409 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid, 1410 const struct in6_addr *sip, __be16 sport, 1411 unsigned int queue) 1412 { 1413 unsigned int chan; 1414 struct sk_buff *skb; 1415 struct adapter *adap; 1416 struct cpl_pass_open_req6 *req; 1417 int ret; 1418 1419 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1420 if (!skb) 1421 return -ENOMEM; 1422 1423 adap = netdev2adap(dev); 1424 req = __skb_put(skb, sizeof(*req)); 1425 INIT_TP_WR(req, 0); 1426 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid)); 1427 req->local_port = sport; 1428 req->peer_port = htons(0); 1429 req->local_ip_hi = *(__be64 *)(sip->s6_addr); 1430 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8); 1431 req->peer_ip_hi = cpu_to_be64(0); 1432 req->peer_ip_lo = cpu_to_be64(0); 1433 chan = rxq_to_chan(&adap->sge, queue); 1434 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 1435 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 1436 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 1437 ret = t4_mgmt_tx(adap, skb); 1438 return net_xmit_eval(ret); 1439 } 1440 EXPORT_SYMBOL(cxgb4_create_server6); 1441 1442 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid, 1443 unsigned int queue, bool ipv6) 1444 { 1445 struct sk_buff *skb; 1446 struct adapter *adap; 1447 struct cpl_close_listsvr_req *req; 1448 int ret; 1449 1450 adap = netdev2adap(dev); 1451 1452 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1453 if (!skb) 1454 return -ENOMEM; 1455 1456 req = __skb_put(skb, sizeof(*req)); 1457 INIT_TP_WR(req, 0); 1458 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid)); 1459 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) : 1460 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue)); 1461 ret = t4_mgmt_tx(adap, skb); 1462 return net_xmit_eval(ret); 1463 } 1464 EXPORT_SYMBOL(cxgb4_remove_server); 1465 1466 /** 1467 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU 1468 * @mtus: the HW MTU table 1469 * @mtu: the target MTU 1470 * @idx: index of selected entry in the MTU table 1471 * 1472 * Returns the index and the value in the HW MTU table that is closest to 1473 * but does not exceed @mtu, unless @mtu is smaller than any value in the 1474 * table, in which case that smallest available value is selected. 1475 */ 1476 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu, 1477 unsigned int *idx) 1478 { 1479 unsigned int i = 0; 1480 1481 while (i < NMTUS - 1 && mtus[i + 1] <= mtu) 1482 ++i; 1483 if (idx) 1484 *idx = i; 1485 return mtus[i]; 1486 } 1487 EXPORT_SYMBOL(cxgb4_best_mtu); 1488 1489 /** 1490 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned 1491 * @mtus: the HW MTU table 1492 * @header_size: Header Size 1493 * @data_size_max: maximum Data Segment Size 1494 * @data_size_align: desired Data Segment Size Alignment (2^N) 1495 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL) 1496 * 1497 * Similar to cxgb4_best_mtu() but instead of searching the Hardware 1498 * MTU Table based solely on a Maximum MTU parameter, we break that 1499 * parameter up into a Header Size and Maximum Data Segment Size, and 1500 * provide a desired Data Segment Size Alignment. If we find an MTU in 1501 * the Hardware MTU Table which will result in a Data Segment Size with 1502 * the requested alignment _and_ that MTU isn't "too far" from the 1503 * closest MTU, then we'll return that rather than the closest MTU. 1504 */ 1505 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus, 1506 unsigned short header_size, 1507 unsigned short data_size_max, 1508 unsigned short data_size_align, 1509 unsigned int *mtu_idxp) 1510 { 1511 unsigned short max_mtu = header_size + data_size_max; 1512 unsigned short data_size_align_mask = data_size_align - 1; 1513 int mtu_idx, aligned_mtu_idx; 1514 1515 /* Scan the MTU Table till we find an MTU which is larger than our 1516 * Maximum MTU or we reach the end of the table. Along the way, 1517 * record the last MTU found, if any, which will result in a Data 1518 * Segment Length matching the requested alignment. 1519 */ 1520 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) { 1521 unsigned short data_size = mtus[mtu_idx] - header_size; 1522 1523 /* If this MTU minus the Header Size would result in a 1524 * Data Segment Size of the desired alignment, remember it. 1525 */ 1526 if ((data_size & data_size_align_mask) == 0) 1527 aligned_mtu_idx = mtu_idx; 1528 1529 /* If we're not at the end of the Hardware MTU Table and the 1530 * next element is larger than our Maximum MTU, drop out of 1531 * the loop. 1532 */ 1533 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu) 1534 break; 1535 } 1536 1537 /* If we fell out of the loop because we ran to the end of the table, 1538 * then we just have to use the last [largest] entry. 1539 */ 1540 if (mtu_idx == NMTUS) 1541 mtu_idx--; 1542 1543 /* If we found an MTU which resulted in the requested Data Segment 1544 * Length alignment and that's "not far" from the largest MTU which is 1545 * less than or equal to the maximum MTU, then use that. 1546 */ 1547 if (aligned_mtu_idx >= 0 && 1548 mtu_idx - aligned_mtu_idx <= 1) 1549 mtu_idx = aligned_mtu_idx; 1550 1551 /* If the caller has passed in an MTU Index pointer, pass the 1552 * MTU Index back. Return the MTU value. 1553 */ 1554 if (mtu_idxp) 1555 *mtu_idxp = mtu_idx; 1556 return mtus[mtu_idx]; 1557 } 1558 EXPORT_SYMBOL(cxgb4_best_aligned_mtu); 1559 1560 /** 1561 * cxgb4_tp_smt_idx - Get the Source Mac Table index for this VI 1562 * @chip: chip type 1563 * @viid: VI id of the given port 1564 * 1565 * Return the SMT index for this VI. 1566 */ 1567 unsigned int cxgb4_tp_smt_idx(enum chip_type chip, unsigned int viid) 1568 { 1569 /* In T4/T5, SMT contains 256 SMAC entries organized in 1570 * 128 rows of 2 entries each. 1571 * In T6, SMT contains 256 SMAC entries in 256 rows. 1572 * TODO: The below code needs to be updated when we add support 1573 * for 256 VFs. 1574 */ 1575 if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5) 1576 return ((viid & 0x7f) << 1); 1577 else 1578 return (viid & 0x7f); 1579 } 1580 EXPORT_SYMBOL(cxgb4_tp_smt_idx); 1581 1582 /** 1583 * cxgb4_port_chan - get the HW channel of a port 1584 * @dev: the net device for the port 1585 * 1586 * Return the HW Tx channel of the given port. 1587 */ 1588 unsigned int cxgb4_port_chan(const struct net_device *dev) 1589 { 1590 return netdev2pinfo(dev)->tx_chan; 1591 } 1592 EXPORT_SYMBOL(cxgb4_port_chan); 1593 1594 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo) 1595 { 1596 struct adapter *adap = netdev2adap(dev); 1597 u32 v1, v2, lp_count, hp_count; 1598 1599 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 1600 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 1601 if (is_t4(adap->params.chip)) { 1602 lp_count = LP_COUNT_G(v1); 1603 hp_count = HP_COUNT_G(v1); 1604 } else { 1605 lp_count = LP_COUNT_T5_G(v1); 1606 hp_count = HP_COUNT_T5_G(v2); 1607 } 1608 return lpfifo ? lp_count : hp_count; 1609 } 1610 EXPORT_SYMBOL(cxgb4_dbfifo_count); 1611 1612 /** 1613 * cxgb4_port_viid - get the VI id of a port 1614 * @dev: the net device for the port 1615 * 1616 * Return the VI id of the given port. 1617 */ 1618 unsigned int cxgb4_port_viid(const struct net_device *dev) 1619 { 1620 return netdev2pinfo(dev)->viid; 1621 } 1622 EXPORT_SYMBOL(cxgb4_port_viid); 1623 1624 /** 1625 * cxgb4_port_idx - get the index of a port 1626 * @dev: the net device for the port 1627 * 1628 * Return the index of the given port. 1629 */ 1630 unsigned int cxgb4_port_idx(const struct net_device *dev) 1631 { 1632 return netdev2pinfo(dev)->port_id; 1633 } 1634 EXPORT_SYMBOL(cxgb4_port_idx); 1635 1636 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4, 1637 struct tp_tcp_stats *v6) 1638 { 1639 struct adapter *adap = pci_get_drvdata(pdev); 1640 1641 spin_lock(&adap->stats_lock); 1642 t4_tp_get_tcp_stats(adap, v4, v6, false); 1643 spin_unlock(&adap->stats_lock); 1644 } 1645 EXPORT_SYMBOL(cxgb4_get_tcp_stats); 1646 1647 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask, 1648 const unsigned int *pgsz_order) 1649 { 1650 struct adapter *adap = netdev2adap(dev); 1651 1652 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask); 1653 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) | 1654 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) | 1655 HPZ3_V(pgsz_order[3])); 1656 } 1657 EXPORT_SYMBOL(cxgb4_iscsi_init); 1658 1659 int cxgb4_flush_eq_cache(struct net_device *dev) 1660 { 1661 struct adapter *adap = netdev2adap(dev); 1662 1663 return t4_sge_ctxt_flush(adap, adap->mbox); 1664 } 1665 EXPORT_SYMBOL(cxgb4_flush_eq_cache); 1666 1667 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx) 1668 { 1669 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8; 1670 __be64 indices; 1671 int ret; 1672 1673 spin_lock(&adap->win0_lock); 1674 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr, 1675 sizeof(indices), (__be32 *)&indices, 1676 T4_MEMORY_READ); 1677 spin_unlock(&adap->win0_lock); 1678 if (!ret) { 1679 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff; 1680 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff; 1681 } 1682 return ret; 1683 } 1684 1685 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, 1686 u16 size) 1687 { 1688 struct adapter *adap = netdev2adap(dev); 1689 u16 hw_pidx, hw_cidx; 1690 int ret; 1691 1692 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx); 1693 if (ret) 1694 goto out; 1695 1696 if (pidx != hw_pidx) { 1697 u16 delta; 1698 u32 val; 1699 1700 if (pidx >= hw_pidx) 1701 delta = pidx - hw_pidx; 1702 else 1703 delta = size - hw_pidx + pidx; 1704 1705 if (is_t4(adap->params.chip)) 1706 val = PIDX_V(delta); 1707 else 1708 val = PIDX_T5_V(delta); 1709 wmb(); 1710 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 1711 QID_V(qid) | val); 1712 } 1713 out: 1714 return ret; 1715 } 1716 EXPORT_SYMBOL(cxgb4_sync_txq_pidx); 1717 1718 int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte) 1719 { 1720 struct adapter *adap; 1721 u32 offset, memtype, memaddr; 1722 u32 edc0_size, edc1_size, mc0_size, mc1_size, size; 1723 u32 edc0_end, edc1_end, mc0_end, mc1_end; 1724 int ret; 1725 1726 adap = netdev2adap(dev); 1727 1728 offset = ((stag >> 8) * 32) + adap->vres.stag.start; 1729 1730 /* Figure out where the offset lands in the Memory Type/Address scheme. 1731 * This code assumes that the memory is laid out starting at offset 0 1732 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0 1733 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have 1734 * MC0, and some have both MC0 and MC1. 1735 */ 1736 size = t4_read_reg(adap, MA_EDRAM0_BAR_A); 1737 edc0_size = EDRAM0_SIZE_G(size) << 20; 1738 size = t4_read_reg(adap, MA_EDRAM1_BAR_A); 1739 edc1_size = EDRAM1_SIZE_G(size) << 20; 1740 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A); 1741 mc0_size = EXT_MEM0_SIZE_G(size) << 20; 1742 1743 edc0_end = edc0_size; 1744 edc1_end = edc0_end + edc1_size; 1745 mc0_end = edc1_end + mc0_size; 1746 1747 if (offset < edc0_end) { 1748 memtype = MEM_EDC0; 1749 memaddr = offset; 1750 } else if (offset < edc1_end) { 1751 memtype = MEM_EDC1; 1752 memaddr = offset - edc0_end; 1753 } else { 1754 if (offset < mc0_end) { 1755 memtype = MEM_MC0; 1756 memaddr = offset - edc1_end; 1757 } else if (is_t5(adap->params.chip)) { 1758 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); 1759 mc1_size = EXT_MEM1_SIZE_G(size) << 20; 1760 mc1_end = mc0_end + mc1_size; 1761 if (offset < mc1_end) { 1762 memtype = MEM_MC1; 1763 memaddr = offset - mc0_end; 1764 } else { 1765 /* offset beyond the end of any memory */ 1766 goto err; 1767 } 1768 } else { 1769 /* T4/T6 only has a single memory channel */ 1770 goto err; 1771 } 1772 } 1773 1774 spin_lock(&adap->win0_lock); 1775 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ); 1776 spin_unlock(&adap->win0_lock); 1777 return ret; 1778 1779 err: 1780 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n", 1781 stag, offset); 1782 return -EINVAL; 1783 } 1784 EXPORT_SYMBOL(cxgb4_read_tpte); 1785 1786 u64 cxgb4_read_sge_timestamp(struct net_device *dev) 1787 { 1788 u32 hi, lo; 1789 struct adapter *adap; 1790 1791 adap = netdev2adap(dev); 1792 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A); 1793 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A)); 1794 1795 return ((u64)hi << 32) | (u64)lo; 1796 } 1797 EXPORT_SYMBOL(cxgb4_read_sge_timestamp); 1798 1799 int cxgb4_bar2_sge_qregs(struct net_device *dev, 1800 unsigned int qid, 1801 enum cxgb4_bar2_qtype qtype, 1802 int user, 1803 u64 *pbar2_qoffset, 1804 unsigned int *pbar2_qid) 1805 { 1806 return t4_bar2_sge_qregs(netdev2adap(dev), 1807 qid, 1808 (qtype == CXGB4_BAR2_QTYPE_EGRESS 1809 ? T4_BAR2_QTYPE_EGRESS 1810 : T4_BAR2_QTYPE_INGRESS), 1811 user, 1812 pbar2_qoffset, 1813 pbar2_qid); 1814 } 1815 EXPORT_SYMBOL(cxgb4_bar2_sge_qregs); 1816 1817 static struct pci_driver cxgb4_driver; 1818 1819 static void check_neigh_update(struct neighbour *neigh) 1820 { 1821 const struct device *parent; 1822 const struct net_device *netdev = neigh->dev; 1823 1824 if (is_vlan_dev(netdev)) 1825 netdev = vlan_dev_real_dev(netdev); 1826 parent = netdev->dev.parent; 1827 if (parent && parent->driver == &cxgb4_driver.driver) 1828 t4_l2t_update(dev_get_drvdata(parent), neigh); 1829 } 1830 1831 static int netevent_cb(struct notifier_block *nb, unsigned long event, 1832 void *data) 1833 { 1834 switch (event) { 1835 case NETEVENT_NEIGH_UPDATE: 1836 check_neigh_update(data); 1837 break; 1838 case NETEVENT_REDIRECT: 1839 default: 1840 break; 1841 } 1842 return 0; 1843 } 1844 1845 static bool netevent_registered; 1846 static struct notifier_block cxgb4_netevent_nb = { 1847 .notifier_call = netevent_cb 1848 }; 1849 1850 static void drain_db_fifo(struct adapter *adap, int usecs) 1851 { 1852 u32 v1, v2, lp_count, hp_count; 1853 1854 do { 1855 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 1856 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 1857 if (is_t4(adap->params.chip)) { 1858 lp_count = LP_COUNT_G(v1); 1859 hp_count = HP_COUNT_G(v1); 1860 } else { 1861 lp_count = LP_COUNT_T5_G(v1); 1862 hp_count = HP_COUNT_T5_G(v2); 1863 } 1864 1865 if (lp_count == 0 && hp_count == 0) 1866 break; 1867 set_current_state(TASK_UNINTERRUPTIBLE); 1868 schedule_timeout(usecs_to_jiffies(usecs)); 1869 } while (1); 1870 } 1871 1872 static void disable_txq_db(struct sge_txq *q) 1873 { 1874 unsigned long flags; 1875 1876 spin_lock_irqsave(&q->db_lock, flags); 1877 q->db_disabled = 1; 1878 spin_unlock_irqrestore(&q->db_lock, flags); 1879 } 1880 1881 static void enable_txq_db(struct adapter *adap, struct sge_txq *q) 1882 { 1883 spin_lock_irq(&q->db_lock); 1884 if (q->db_pidx_inc) { 1885 /* Make sure that all writes to the TX descriptors 1886 * are committed before we tell HW about them. 1887 */ 1888 wmb(); 1889 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 1890 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc)); 1891 q->db_pidx_inc = 0; 1892 } 1893 q->db_disabled = 0; 1894 spin_unlock_irq(&q->db_lock); 1895 } 1896 1897 static void disable_dbs(struct adapter *adap) 1898 { 1899 int i; 1900 1901 for_each_ethrxq(&adap->sge, i) 1902 disable_txq_db(&adap->sge.ethtxq[i].q); 1903 if (is_offload(adap)) { 1904 struct sge_uld_txq_info *txq_info = 1905 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 1906 1907 if (txq_info) { 1908 for_each_ofldtxq(&adap->sge, i) { 1909 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 1910 1911 disable_txq_db(&txq->q); 1912 } 1913 } 1914 } 1915 for_each_port(adap, i) 1916 disable_txq_db(&adap->sge.ctrlq[i].q); 1917 } 1918 1919 static void enable_dbs(struct adapter *adap) 1920 { 1921 int i; 1922 1923 for_each_ethrxq(&adap->sge, i) 1924 enable_txq_db(adap, &adap->sge.ethtxq[i].q); 1925 if (is_offload(adap)) { 1926 struct sge_uld_txq_info *txq_info = 1927 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 1928 1929 if (txq_info) { 1930 for_each_ofldtxq(&adap->sge, i) { 1931 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 1932 1933 enable_txq_db(adap, &txq->q); 1934 } 1935 } 1936 } 1937 for_each_port(adap, i) 1938 enable_txq_db(adap, &adap->sge.ctrlq[i].q); 1939 } 1940 1941 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd) 1942 { 1943 enum cxgb4_uld type = CXGB4_ULD_RDMA; 1944 1945 if (adap->uld && adap->uld[type].handle) 1946 adap->uld[type].control(adap->uld[type].handle, cmd); 1947 } 1948 1949 static void process_db_full(struct work_struct *work) 1950 { 1951 struct adapter *adap; 1952 1953 adap = container_of(work, struct adapter, db_full_task); 1954 1955 drain_db_fifo(adap, dbfifo_drain_delay); 1956 enable_dbs(adap); 1957 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 1958 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 1959 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 1960 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 1961 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F); 1962 else 1963 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 1964 DBFIFO_LP_INT_F, DBFIFO_LP_INT_F); 1965 } 1966 1967 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q) 1968 { 1969 u16 hw_pidx, hw_cidx; 1970 int ret; 1971 1972 spin_lock_irq(&q->db_lock); 1973 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx); 1974 if (ret) 1975 goto out; 1976 if (q->db_pidx != hw_pidx) { 1977 u16 delta; 1978 u32 val; 1979 1980 if (q->db_pidx >= hw_pidx) 1981 delta = q->db_pidx - hw_pidx; 1982 else 1983 delta = q->size - hw_pidx + q->db_pidx; 1984 1985 if (is_t4(adap->params.chip)) 1986 val = PIDX_V(delta); 1987 else 1988 val = PIDX_T5_V(delta); 1989 wmb(); 1990 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 1991 QID_V(q->cntxt_id) | val); 1992 } 1993 out: 1994 q->db_disabled = 0; 1995 q->db_pidx_inc = 0; 1996 spin_unlock_irq(&q->db_lock); 1997 if (ret) 1998 CH_WARN(adap, "DB drop recovery failed.\n"); 1999 } 2000 2001 static void recover_all_queues(struct adapter *adap) 2002 { 2003 int i; 2004 2005 for_each_ethrxq(&adap->sge, i) 2006 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q); 2007 if (is_offload(adap)) { 2008 struct sge_uld_txq_info *txq_info = 2009 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 2010 if (txq_info) { 2011 for_each_ofldtxq(&adap->sge, i) { 2012 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 2013 2014 sync_txq_pidx(adap, &txq->q); 2015 } 2016 } 2017 } 2018 for_each_port(adap, i) 2019 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q); 2020 } 2021 2022 static void process_db_drop(struct work_struct *work) 2023 { 2024 struct adapter *adap; 2025 2026 adap = container_of(work, struct adapter, db_drop_task); 2027 2028 if (is_t4(adap->params.chip)) { 2029 drain_db_fifo(adap, dbfifo_drain_delay); 2030 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP); 2031 drain_db_fifo(adap, dbfifo_drain_delay); 2032 recover_all_queues(adap); 2033 drain_db_fifo(adap, dbfifo_drain_delay); 2034 enable_dbs(adap); 2035 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 2036 } else if (is_t5(adap->params.chip)) { 2037 u32 dropped_db = t4_read_reg(adap, 0x010ac); 2038 u16 qid = (dropped_db >> 15) & 0x1ffff; 2039 u16 pidx_inc = dropped_db & 0x1fff; 2040 u64 bar2_qoffset; 2041 unsigned int bar2_qid; 2042 int ret; 2043 2044 ret = t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS, 2045 0, &bar2_qoffset, &bar2_qid); 2046 if (ret) 2047 dev_err(adap->pdev_dev, "doorbell drop recovery: " 2048 "qid=%d, pidx_inc=%d\n", qid, pidx_inc); 2049 else 2050 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid), 2051 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL); 2052 2053 /* Re-enable BAR2 WC */ 2054 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15); 2055 } 2056 2057 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 2058 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0); 2059 } 2060 2061 void t4_db_full(struct adapter *adap) 2062 { 2063 if (is_t4(adap->params.chip)) { 2064 disable_dbs(adap); 2065 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 2066 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 2067 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0); 2068 queue_work(adap->workq, &adap->db_full_task); 2069 } 2070 } 2071 2072 void t4_db_dropped(struct adapter *adap) 2073 { 2074 if (is_t4(adap->params.chip)) { 2075 disable_dbs(adap); 2076 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 2077 } 2078 queue_work(adap->workq, &adap->db_drop_task); 2079 } 2080 2081 void t4_register_netevent_notifier(void) 2082 { 2083 if (!netevent_registered) { 2084 register_netevent_notifier(&cxgb4_netevent_nb); 2085 netevent_registered = true; 2086 } 2087 } 2088 2089 static void detach_ulds(struct adapter *adap) 2090 { 2091 unsigned int i; 2092 2093 mutex_lock(&uld_mutex); 2094 list_del(&adap->list_node); 2095 2096 for (i = 0; i < CXGB4_ULD_MAX; i++) 2097 if (adap->uld && adap->uld[i].handle) 2098 adap->uld[i].state_change(adap->uld[i].handle, 2099 CXGB4_STATE_DETACH); 2100 2101 if (netevent_registered && list_empty(&adapter_list)) { 2102 unregister_netevent_notifier(&cxgb4_netevent_nb); 2103 netevent_registered = false; 2104 } 2105 mutex_unlock(&uld_mutex); 2106 } 2107 2108 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state) 2109 { 2110 unsigned int i; 2111 2112 mutex_lock(&uld_mutex); 2113 for (i = 0; i < CXGB4_ULD_MAX; i++) 2114 if (adap->uld && adap->uld[i].handle) 2115 adap->uld[i].state_change(adap->uld[i].handle, 2116 new_state); 2117 mutex_unlock(&uld_mutex); 2118 } 2119 2120 #if IS_ENABLED(CONFIG_IPV6) 2121 static int cxgb4_inet6addr_handler(struct notifier_block *this, 2122 unsigned long event, void *data) 2123 { 2124 struct inet6_ifaddr *ifa = data; 2125 struct net_device *event_dev = ifa->idev->dev; 2126 const struct device *parent = NULL; 2127 #if IS_ENABLED(CONFIG_BONDING) 2128 struct adapter *adap; 2129 #endif 2130 if (is_vlan_dev(event_dev)) 2131 event_dev = vlan_dev_real_dev(event_dev); 2132 #if IS_ENABLED(CONFIG_BONDING) 2133 if (event_dev->flags & IFF_MASTER) { 2134 list_for_each_entry(adap, &adapter_list, list_node) { 2135 switch (event) { 2136 case NETDEV_UP: 2137 cxgb4_clip_get(adap->port[0], 2138 (const u32 *)ifa, 1); 2139 break; 2140 case NETDEV_DOWN: 2141 cxgb4_clip_release(adap->port[0], 2142 (const u32 *)ifa, 1); 2143 break; 2144 default: 2145 break; 2146 } 2147 } 2148 return NOTIFY_OK; 2149 } 2150 #endif 2151 2152 if (event_dev) 2153 parent = event_dev->dev.parent; 2154 2155 if (parent && parent->driver == &cxgb4_driver.driver) { 2156 switch (event) { 2157 case NETDEV_UP: 2158 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1); 2159 break; 2160 case NETDEV_DOWN: 2161 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1); 2162 break; 2163 default: 2164 break; 2165 } 2166 } 2167 return NOTIFY_OK; 2168 } 2169 2170 static bool inet6addr_registered; 2171 static struct notifier_block cxgb4_inet6addr_notifier = { 2172 .notifier_call = cxgb4_inet6addr_handler 2173 }; 2174 2175 static void update_clip(const struct adapter *adap) 2176 { 2177 int i; 2178 struct net_device *dev; 2179 int ret; 2180 2181 rcu_read_lock(); 2182 2183 for (i = 0; i < MAX_NPORTS; i++) { 2184 dev = adap->port[i]; 2185 ret = 0; 2186 2187 if (dev) 2188 ret = cxgb4_update_root_dev_clip(dev); 2189 2190 if (ret < 0) 2191 break; 2192 } 2193 rcu_read_unlock(); 2194 } 2195 #endif /* IS_ENABLED(CONFIG_IPV6) */ 2196 2197 /** 2198 * cxgb_up - enable the adapter 2199 * @adap: adapter being enabled 2200 * 2201 * Called when the first port is enabled, this function performs the 2202 * actions necessary to make an adapter operational, such as completing 2203 * the initialization of HW modules, and enabling interrupts. 2204 * 2205 * Must be called with the rtnl lock held. 2206 */ 2207 static int cxgb_up(struct adapter *adap) 2208 { 2209 int err; 2210 2211 mutex_lock(&uld_mutex); 2212 err = setup_sge_queues(adap); 2213 if (err) 2214 goto rel_lock; 2215 err = setup_rss(adap); 2216 if (err) 2217 goto freeq; 2218 2219 if (adap->flags & USING_MSIX) { 2220 name_msix_vecs(adap); 2221 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0, 2222 adap->msix_info[0].desc, adap); 2223 if (err) 2224 goto irq_err; 2225 err = request_msix_queue_irqs(adap); 2226 if (err) { 2227 free_irq(adap->msix_info[0].vec, adap); 2228 goto irq_err; 2229 } 2230 } else { 2231 err = request_irq(adap->pdev->irq, t4_intr_handler(adap), 2232 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED, 2233 adap->port[0]->name, adap); 2234 if (err) 2235 goto irq_err; 2236 } 2237 2238 enable_rx(adap); 2239 t4_sge_start(adap); 2240 t4_intr_enable(adap); 2241 adap->flags |= FULL_INIT_DONE; 2242 mutex_unlock(&uld_mutex); 2243 2244 notify_ulds(adap, CXGB4_STATE_UP); 2245 #if IS_ENABLED(CONFIG_IPV6) 2246 update_clip(adap); 2247 #endif 2248 /* Initialize hash mac addr list*/ 2249 INIT_LIST_HEAD(&adap->mac_hlist); 2250 return err; 2251 2252 irq_err: 2253 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err); 2254 freeq: 2255 t4_free_sge_resources(adap); 2256 rel_lock: 2257 mutex_unlock(&uld_mutex); 2258 return err; 2259 } 2260 2261 static void cxgb_down(struct adapter *adapter) 2262 { 2263 cancel_work_sync(&adapter->tid_release_task); 2264 cancel_work_sync(&adapter->db_full_task); 2265 cancel_work_sync(&adapter->db_drop_task); 2266 adapter->tid_release_task_busy = false; 2267 adapter->tid_release_head = NULL; 2268 2269 t4_sge_stop(adapter); 2270 t4_free_sge_resources(adapter); 2271 adapter->flags &= ~FULL_INIT_DONE; 2272 } 2273 2274 /* 2275 * net_device operations 2276 */ 2277 static int cxgb_open(struct net_device *dev) 2278 { 2279 int err; 2280 struct port_info *pi = netdev_priv(dev); 2281 struct adapter *adapter = pi->adapter; 2282 2283 netif_carrier_off(dev); 2284 2285 if (!(adapter->flags & FULL_INIT_DONE)) { 2286 err = cxgb_up(adapter); 2287 if (err < 0) 2288 return err; 2289 } 2290 2291 /* It's possible that the basic port information could have 2292 * changed since we first read it. 2293 */ 2294 err = t4_update_port_info(pi); 2295 if (err < 0) 2296 return err; 2297 2298 err = link_start(dev); 2299 if (!err) 2300 netif_tx_start_all_queues(dev); 2301 return err; 2302 } 2303 2304 static int cxgb_close(struct net_device *dev) 2305 { 2306 struct port_info *pi = netdev_priv(dev); 2307 struct adapter *adapter = pi->adapter; 2308 int ret; 2309 2310 netif_tx_stop_all_queues(dev); 2311 netif_carrier_off(dev); 2312 ret = t4_enable_vi(adapter, adapter->pf, pi->viid, false, false); 2313 #ifdef CONFIG_CHELSIO_T4_DCB 2314 cxgb4_dcb_reset(dev); 2315 dcb_tx_queue_prio_enable(dev, false); 2316 #endif 2317 return ret; 2318 } 2319 2320 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid, 2321 __be32 sip, __be16 sport, __be16 vlan, 2322 unsigned int queue, unsigned char port, unsigned char mask) 2323 { 2324 int ret; 2325 struct filter_entry *f; 2326 struct adapter *adap; 2327 int i; 2328 u8 *val; 2329 2330 adap = netdev2adap(dev); 2331 2332 /* Adjust stid to correct filter index */ 2333 stid -= adap->tids.sftid_base; 2334 stid += adap->tids.nftids; 2335 2336 /* Check to make sure the filter requested is writable ... 2337 */ 2338 f = &adap->tids.ftid_tab[stid]; 2339 ret = writable_filter(f); 2340 if (ret) 2341 return ret; 2342 2343 /* Clear out any old resources being used by the filter before 2344 * we start constructing the new filter. 2345 */ 2346 if (f->valid) 2347 clear_filter(adap, f); 2348 2349 /* Clear out filter specifications */ 2350 memset(&f->fs, 0, sizeof(struct ch_filter_specification)); 2351 f->fs.val.lport = cpu_to_be16(sport); 2352 f->fs.mask.lport = ~0; 2353 val = (u8 *)&sip; 2354 if ((val[0] | val[1] | val[2] | val[3]) != 0) { 2355 for (i = 0; i < 4; i++) { 2356 f->fs.val.lip[i] = val[i]; 2357 f->fs.mask.lip[i] = ~0; 2358 } 2359 if (adap->params.tp.vlan_pri_map & PORT_F) { 2360 f->fs.val.iport = port; 2361 f->fs.mask.iport = mask; 2362 } 2363 } 2364 2365 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) { 2366 f->fs.val.proto = IPPROTO_TCP; 2367 f->fs.mask.proto = ~0; 2368 } 2369 2370 f->fs.dirsteer = 1; 2371 f->fs.iq = queue; 2372 /* Mark filter as locked */ 2373 f->locked = 1; 2374 f->fs.rpttid = 1; 2375 2376 /* Save the actual tid. We need this to get the corresponding 2377 * filter entry structure in filter_rpl. 2378 */ 2379 f->tid = stid + adap->tids.ftid_base; 2380 ret = set_filter_wr(adap, stid); 2381 if (ret) { 2382 clear_filter(adap, f); 2383 return ret; 2384 } 2385 2386 return 0; 2387 } 2388 EXPORT_SYMBOL(cxgb4_create_server_filter); 2389 2390 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid, 2391 unsigned int queue, bool ipv6) 2392 { 2393 struct filter_entry *f; 2394 struct adapter *adap; 2395 2396 adap = netdev2adap(dev); 2397 2398 /* Adjust stid to correct filter index */ 2399 stid -= adap->tids.sftid_base; 2400 stid += adap->tids.nftids; 2401 2402 f = &adap->tids.ftid_tab[stid]; 2403 /* Unlock the filter */ 2404 f->locked = 0; 2405 2406 return delete_filter(adap, stid); 2407 } 2408 EXPORT_SYMBOL(cxgb4_remove_server_filter); 2409 2410 static void cxgb_get_stats(struct net_device *dev, 2411 struct rtnl_link_stats64 *ns) 2412 { 2413 struct port_stats stats; 2414 struct port_info *p = netdev_priv(dev); 2415 struct adapter *adapter = p->adapter; 2416 2417 /* Block retrieving statistics during EEH error 2418 * recovery. Otherwise, the recovery might fail 2419 * and the PCI device will be removed permanently 2420 */ 2421 spin_lock(&adapter->stats_lock); 2422 if (!netif_device_present(dev)) { 2423 spin_unlock(&adapter->stats_lock); 2424 return; 2425 } 2426 t4_get_port_stats_offset(adapter, p->tx_chan, &stats, 2427 &p->stats_base); 2428 spin_unlock(&adapter->stats_lock); 2429 2430 ns->tx_bytes = stats.tx_octets; 2431 ns->tx_packets = stats.tx_frames; 2432 ns->rx_bytes = stats.rx_octets; 2433 ns->rx_packets = stats.rx_frames; 2434 ns->multicast = stats.rx_mcast_frames; 2435 2436 /* detailed rx_errors */ 2437 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long + 2438 stats.rx_runt; 2439 ns->rx_over_errors = 0; 2440 ns->rx_crc_errors = stats.rx_fcs_err; 2441 ns->rx_frame_errors = stats.rx_symbol_err; 2442 ns->rx_dropped = stats.rx_ovflow0 + stats.rx_ovflow1 + 2443 stats.rx_ovflow2 + stats.rx_ovflow3 + 2444 stats.rx_trunc0 + stats.rx_trunc1 + 2445 stats.rx_trunc2 + stats.rx_trunc3; 2446 ns->rx_missed_errors = 0; 2447 2448 /* detailed tx_errors */ 2449 ns->tx_aborted_errors = 0; 2450 ns->tx_carrier_errors = 0; 2451 ns->tx_fifo_errors = 0; 2452 ns->tx_heartbeat_errors = 0; 2453 ns->tx_window_errors = 0; 2454 2455 ns->tx_errors = stats.tx_error_frames; 2456 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err + 2457 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors; 2458 } 2459 2460 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd) 2461 { 2462 unsigned int mbox; 2463 int ret = 0, prtad, devad; 2464 struct port_info *pi = netdev_priv(dev); 2465 struct adapter *adapter = pi->adapter; 2466 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data; 2467 2468 switch (cmd) { 2469 case SIOCGMIIPHY: 2470 if (pi->mdio_addr < 0) 2471 return -EOPNOTSUPP; 2472 data->phy_id = pi->mdio_addr; 2473 break; 2474 case SIOCGMIIREG: 2475 case SIOCSMIIREG: 2476 if (mdio_phy_id_is_c45(data->phy_id)) { 2477 prtad = mdio_phy_id_prtad(data->phy_id); 2478 devad = mdio_phy_id_devad(data->phy_id); 2479 } else if (data->phy_id < 32) { 2480 prtad = data->phy_id; 2481 devad = 0; 2482 data->reg_num &= 0x1f; 2483 } else 2484 return -EINVAL; 2485 2486 mbox = pi->adapter->pf; 2487 if (cmd == SIOCGMIIREG) 2488 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad, 2489 data->reg_num, &data->val_out); 2490 else 2491 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad, 2492 data->reg_num, data->val_in); 2493 break; 2494 case SIOCGHWTSTAMP: 2495 return copy_to_user(req->ifr_data, &pi->tstamp_config, 2496 sizeof(pi->tstamp_config)) ? 2497 -EFAULT : 0; 2498 case SIOCSHWTSTAMP: 2499 if (copy_from_user(&pi->tstamp_config, req->ifr_data, 2500 sizeof(pi->tstamp_config))) 2501 return -EFAULT; 2502 2503 if (!is_t4(adapter->params.chip)) { 2504 switch (pi->tstamp_config.tx_type) { 2505 case HWTSTAMP_TX_OFF: 2506 case HWTSTAMP_TX_ON: 2507 break; 2508 default: 2509 return -ERANGE; 2510 } 2511 2512 switch (pi->tstamp_config.rx_filter) { 2513 case HWTSTAMP_FILTER_NONE: 2514 pi->rxtstamp = false; 2515 break; 2516 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 2517 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 2518 cxgb4_ptprx_timestamping(pi, pi->port_id, 2519 PTP_TS_L4); 2520 break; 2521 case HWTSTAMP_FILTER_PTP_V2_EVENT: 2522 cxgb4_ptprx_timestamping(pi, pi->port_id, 2523 PTP_TS_L2_L4); 2524 break; 2525 case HWTSTAMP_FILTER_ALL: 2526 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 2527 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 2528 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 2529 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 2530 pi->rxtstamp = true; 2531 break; 2532 default: 2533 pi->tstamp_config.rx_filter = 2534 HWTSTAMP_FILTER_NONE; 2535 return -ERANGE; 2536 } 2537 2538 if ((pi->tstamp_config.tx_type == HWTSTAMP_TX_OFF) && 2539 (pi->tstamp_config.rx_filter == 2540 HWTSTAMP_FILTER_NONE)) { 2541 if (cxgb4_ptp_txtype(adapter, pi->port_id) >= 0) 2542 pi->ptp_enable = false; 2543 } 2544 2545 if (pi->tstamp_config.rx_filter != 2546 HWTSTAMP_FILTER_NONE) { 2547 if (cxgb4_ptp_redirect_rx_packet(adapter, 2548 pi) >= 0) 2549 pi->ptp_enable = true; 2550 } 2551 } else { 2552 /* For T4 Adapters */ 2553 switch (pi->tstamp_config.rx_filter) { 2554 case HWTSTAMP_FILTER_NONE: 2555 pi->rxtstamp = false; 2556 break; 2557 case HWTSTAMP_FILTER_ALL: 2558 pi->rxtstamp = true; 2559 break; 2560 default: 2561 pi->tstamp_config.rx_filter = 2562 HWTSTAMP_FILTER_NONE; 2563 return -ERANGE; 2564 } 2565 } 2566 return copy_to_user(req->ifr_data, &pi->tstamp_config, 2567 sizeof(pi->tstamp_config)) ? 2568 -EFAULT : 0; 2569 default: 2570 return -EOPNOTSUPP; 2571 } 2572 return ret; 2573 } 2574 2575 static void cxgb_set_rxmode(struct net_device *dev) 2576 { 2577 /* unfortunately we can't return errors to the stack */ 2578 set_rxmode(dev, -1, false); 2579 } 2580 2581 static int cxgb_change_mtu(struct net_device *dev, int new_mtu) 2582 { 2583 int ret; 2584 struct port_info *pi = netdev_priv(dev); 2585 2586 ret = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, new_mtu, -1, 2587 -1, -1, -1, true); 2588 if (!ret) 2589 dev->mtu = new_mtu; 2590 return ret; 2591 } 2592 2593 #ifdef CONFIG_PCI_IOV 2594 static int dummy_open(struct net_device *dev) 2595 { 2596 /* Turn carrier off since we don't have to transmit anything on this 2597 * interface. 2598 */ 2599 netif_carrier_off(dev); 2600 return 0; 2601 } 2602 2603 /* Fill MAC address that will be assigned by the FW */ 2604 static void fill_vf_station_mac_addr(struct adapter *adap) 2605 { 2606 unsigned int i; 2607 u8 hw_addr[ETH_ALEN], macaddr[ETH_ALEN]; 2608 int err; 2609 u8 *na; 2610 u16 a, b; 2611 2612 err = t4_get_raw_vpd_params(adap, &adap->params.vpd); 2613 if (!err) { 2614 na = adap->params.vpd.na; 2615 for (i = 0; i < ETH_ALEN; i++) 2616 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + 2617 hex2val(na[2 * i + 1])); 2618 a = (hw_addr[0] << 8) | hw_addr[1]; 2619 b = (hw_addr[1] << 8) | hw_addr[2]; 2620 a ^= b; 2621 a |= 0x0200; /* locally assigned Ethernet MAC address */ 2622 a &= ~0x0100; /* not a multicast Ethernet MAC address */ 2623 macaddr[0] = a >> 8; 2624 macaddr[1] = a & 0xff; 2625 2626 for (i = 2; i < 5; i++) 2627 macaddr[i] = hw_addr[i + 1]; 2628 2629 for (i = 0; i < adap->num_vfs; i++) { 2630 macaddr[5] = adap->pf * 16 + i; 2631 ether_addr_copy(adap->vfinfo[i].vf_mac_addr, macaddr); 2632 } 2633 } 2634 } 2635 2636 static int cxgb_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 2637 { 2638 struct port_info *pi = netdev_priv(dev); 2639 struct adapter *adap = pi->adapter; 2640 int ret; 2641 2642 /* verify MAC addr is valid */ 2643 if (!is_valid_ether_addr(mac)) { 2644 dev_err(pi->adapter->pdev_dev, 2645 "Invalid Ethernet address %pM for VF %d\n", 2646 mac, vf); 2647 return -EINVAL; 2648 } 2649 2650 dev_info(pi->adapter->pdev_dev, 2651 "Setting MAC %pM on VF %d\n", mac, vf); 2652 ret = t4_set_vf_mac_acl(adap, vf + 1, 1, mac); 2653 if (!ret) 2654 ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, mac); 2655 return ret; 2656 } 2657 2658 static int cxgb_get_vf_config(struct net_device *dev, 2659 int vf, struct ifla_vf_info *ivi) 2660 { 2661 struct port_info *pi = netdev_priv(dev); 2662 struct adapter *adap = pi->adapter; 2663 2664 if (vf >= adap->num_vfs) 2665 return -EINVAL; 2666 ivi->vf = vf; 2667 ivi->max_tx_rate = adap->vfinfo[vf].tx_rate; 2668 ivi->min_tx_rate = 0; 2669 ether_addr_copy(ivi->mac, adap->vfinfo[vf].vf_mac_addr); 2670 return 0; 2671 } 2672 2673 static int cxgb_get_phys_port_id(struct net_device *dev, 2674 struct netdev_phys_item_id *ppid) 2675 { 2676 struct port_info *pi = netdev_priv(dev); 2677 unsigned int phy_port_id; 2678 2679 phy_port_id = pi->adapter->adap_idx * 10 + pi->port_id; 2680 ppid->id_len = sizeof(phy_port_id); 2681 memcpy(ppid->id, &phy_port_id, ppid->id_len); 2682 return 0; 2683 } 2684 2685 static int cxgb_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 2686 int max_tx_rate) 2687 { 2688 struct port_info *pi = netdev_priv(dev); 2689 struct adapter *adap = pi->adapter; 2690 unsigned int link_ok, speed, mtu; 2691 u32 fw_pfvf, fw_class; 2692 int class_id = vf; 2693 int ret; 2694 u16 pktsize; 2695 2696 if (vf >= adap->num_vfs) 2697 return -EINVAL; 2698 2699 if (min_tx_rate) { 2700 dev_err(adap->pdev_dev, 2701 "Min tx rate (%d) (> 0) for VF %d is Invalid.\n", 2702 min_tx_rate, vf); 2703 return -EINVAL; 2704 } 2705 2706 ret = t4_get_link_params(pi, &link_ok, &speed, &mtu); 2707 if (ret != FW_SUCCESS) { 2708 dev_err(adap->pdev_dev, 2709 "Failed to get link information for VF %d\n", vf); 2710 return -EINVAL; 2711 } 2712 2713 if (!link_ok) { 2714 dev_err(adap->pdev_dev, "Link down for VF %d\n", vf); 2715 return -EINVAL; 2716 } 2717 2718 if (max_tx_rate > speed) { 2719 dev_err(adap->pdev_dev, 2720 "Max tx rate %d for VF %d can't be > link-speed %u", 2721 max_tx_rate, vf, speed); 2722 return -EINVAL; 2723 } 2724 2725 pktsize = mtu; 2726 /* subtract ethhdr size and 4 bytes crc since, f/w appends it */ 2727 pktsize = pktsize - sizeof(struct ethhdr) - 4; 2728 /* subtract ipv4 hdr size, tcp hdr size to get typical IPv4 MSS size */ 2729 pktsize = pktsize - sizeof(struct iphdr) - sizeof(struct tcphdr); 2730 /* configure Traffic Class for rate-limiting */ 2731 ret = t4_sched_params(adap, SCHED_CLASS_TYPE_PACKET, 2732 SCHED_CLASS_LEVEL_CL_RL, 2733 SCHED_CLASS_MODE_CLASS, 2734 SCHED_CLASS_RATEUNIT_BITS, 2735 SCHED_CLASS_RATEMODE_ABS, 2736 pi->tx_chan, class_id, 0, 2737 max_tx_rate * 1000, 0, pktsize); 2738 if (ret) { 2739 dev_err(adap->pdev_dev, "Err %d for Traffic Class config\n", 2740 ret); 2741 return -EINVAL; 2742 } 2743 dev_info(adap->pdev_dev, 2744 "Class %d with MSS %u configured with rate %u\n", 2745 class_id, pktsize, max_tx_rate); 2746 2747 /* bind VF to configured Traffic Class */ 2748 fw_pfvf = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | 2749 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH)); 2750 fw_class = class_id; 2751 ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1, &fw_pfvf, 2752 &fw_class); 2753 if (ret) { 2754 dev_err(adap->pdev_dev, 2755 "Err %d in binding VF %d to Traffic Class %d\n", 2756 ret, vf, class_id); 2757 return -EINVAL; 2758 } 2759 dev_info(adap->pdev_dev, "PF %d VF %d is bound to Class %d\n", 2760 adap->pf, vf, class_id); 2761 adap->vfinfo[vf].tx_rate = max_tx_rate; 2762 return 0; 2763 } 2764 2765 #endif 2766 2767 static int cxgb_set_mac_addr(struct net_device *dev, void *p) 2768 { 2769 int ret; 2770 struct sockaddr *addr = p; 2771 struct port_info *pi = netdev_priv(dev); 2772 2773 if (!is_valid_ether_addr(addr->sa_data)) 2774 return -EADDRNOTAVAIL; 2775 2776 ret = t4_change_mac(pi->adapter, pi->adapter->pf, pi->viid, 2777 pi->xact_addr_filt, addr->sa_data, true, true); 2778 if (ret < 0) 2779 return ret; 2780 2781 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 2782 pi->xact_addr_filt = ret; 2783 return 0; 2784 } 2785 2786 #ifdef CONFIG_NET_POLL_CONTROLLER 2787 static void cxgb_netpoll(struct net_device *dev) 2788 { 2789 struct port_info *pi = netdev_priv(dev); 2790 struct adapter *adap = pi->adapter; 2791 2792 if (adap->flags & USING_MSIX) { 2793 int i; 2794 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset]; 2795 2796 for (i = pi->nqsets; i; i--, rx++) 2797 t4_sge_intr_msix(0, &rx->rspq); 2798 } else 2799 t4_intr_handler(adap)(0, adap); 2800 } 2801 #endif 2802 2803 static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 2804 { 2805 struct port_info *pi = netdev_priv(dev); 2806 struct adapter *adap = pi->adapter; 2807 struct sched_class *e; 2808 struct ch_sched_params p; 2809 struct ch_sched_queue qe; 2810 u32 req_rate; 2811 int err = 0; 2812 2813 if (!can_sched(dev)) 2814 return -ENOTSUPP; 2815 2816 if (index < 0 || index > pi->nqsets - 1) 2817 return -EINVAL; 2818 2819 if (!(adap->flags & FULL_INIT_DONE)) { 2820 dev_err(adap->pdev_dev, 2821 "Failed to rate limit on queue %d. Link Down?\n", 2822 index); 2823 return -EINVAL; 2824 } 2825 2826 /* Convert from Mbps to Kbps */ 2827 req_rate = rate << 10; 2828 2829 /* Max rate is 10 Gbps */ 2830 if (req_rate >= SCHED_MAX_RATE_KBPS) { 2831 dev_err(adap->pdev_dev, 2832 "Invalid rate %u Mbps, Max rate is %u Gbps\n", 2833 rate, SCHED_MAX_RATE_KBPS); 2834 return -ERANGE; 2835 } 2836 2837 /* First unbind the queue from any existing class */ 2838 memset(&qe, 0, sizeof(qe)); 2839 qe.queue = index; 2840 qe.class = SCHED_CLS_NONE; 2841 2842 err = cxgb4_sched_class_unbind(dev, (void *)(&qe), SCHED_QUEUE); 2843 if (err) { 2844 dev_err(adap->pdev_dev, 2845 "Unbinding Queue %d on port %d fail. Err: %d\n", 2846 index, pi->port_id, err); 2847 return err; 2848 } 2849 2850 /* Queue already unbound */ 2851 if (!req_rate) 2852 return 0; 2853 2854 /* Fetch any available unused or matching scheduling class */ 2855 memset(&p, 0, sizeof(p)); 2856 p.type = SCHED_CLASS_TYPE_PACKET; 2857 p.u.params.level = SCHED_CLASS_LEVEL_CL_RL; 2858 p.u.params.mode = SCHED_CLASS_MODE_CLASS; 2859 p.u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS; 2860 p.u.params.ratemode = SCHED_CLASS_RATEMODE_ABS; 2861 p.u.params.channel = pi->tx_chan; 2862 p.u.params.class = SCHED_CLS_NONE; 2863 p.u.params.minrate = 0; 2864 p.u.params.maxrate = req_rate; 2865 p.u.params.weight = 0; 2866 p.u.params.pktsize = dev->mtu; 2867 2868 e = cxgb4_sched_class_alloc(dev, &p); 2869 if (!e) 2870 return -ENOMEM; 2871 2872 /* Bind the queue to a scheduling class */ 2873 memset(&qe, 0, sizeof(qe)); 2874 qe.queue = index; 2875 qe.class = e->idx; 2876 2877 err = cxgb4_sched_class_bind(dev, (void *)(&qe), SCHED_QUEUE); 2878 if (err) 2879 dev_err(adap->pdev_dev, 2880 "Queue rate limiting failed. Err: %d\n", err); 2881 return err; 2882 } 2883 2884 static int cxgb_setup_tc_flower(struct net_device *dev, 2885 struct tc_cls_flower_offload *cls_flower) 2886 { 2887 if (!is_classid_clsact_ingress(cls_flower->common.classid) || 2888 cls_flower->common.chain_index) 2889 return -EOPNOTSUPP; 2890 2891 switch (cls_flower->command) { 2892 case TC_CLSFLOWER_REPLACE: 2893 return cxgb4_tc_flower_replace(dev, cls_flower); 2894 case TC_CLSFLOWER_DESTROY: 2895 return cxgb4_tc_flower_destroy(dev, cls_flower); 2896 case TC_CLSFLOWER_STATS: 2897 return cxgb4_tc_flower_stats(dev, cls_flower); 2898 default: 2899 return -EOPNOTSUPP; 2900 } 2901 } 2902 2903 static int cxgb_setup_tc_cls_u32(struct net_device *dev, 2904 struct tc_cls_u32_offload *cls_u32) 2905 { 2906 if (!is_classid_clsact_ingress(cls_u32->common.classid) || 2907 cls_u32->common.chain_index) 2908 return -EOPNOTSUPP; 2909 2910 switch (cls_u32->command) { 2911 case TC_CLSU32_NEW_KNODE: 2912 case TC_CLSU32_REPLACE_KNODE: 2913 return cxgb4_config_knode(dev, cls_u32); 2914 case TC_CLSU32_DELETE_KNODE: 2915 return cxgb4_delete_knode(dev, cls_u32); 2916 default: 2917 return -EOPNOTSUPP; 2918 } 2919 } 2920 2921 static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type, 2922 void *type_data) 2923 { 2924 struct port_info *pi = netdev2pinfo(dev); 2925 struct adapter *adap = netdev2adap(dev); 2926 2927 if (!(adap->flags & FULL_INIT_DONE)) { 2928 dev_err(adap->pdev_dev, 2929 "Failed to setup tc on port %d. Link Down?\n", 2930 pi->port_id); 2931 return -EINVAL; 2932 } 2933 2934 switch (type) { 2935 case TC_SETUP_CLSU32: 2936 return cxgb_setup_tc_cls_u32(dev, type_data); 2937 case TC_SETUP_CLSFLOWER: 2938 return cxgb_setup_tc_flower(dev, type_data); 2939 default: 2940 return -EOPNOTSUPP; 2941 } 2942 } 2943 2944 static netdev_features_t cxgb_fix_features(struct net_device *dev, 2945 netdev_features_t features) 2946 { 2947 /* Disable GRO, if RX_CSUM is disabled */ 2948 if (!(features & NETIF_F_RXCSUM)) 2949 features &= ~NETIF_F_GRO; 2950 2951 return features; 2952 } 2953 2954 static const struct net_device_ops cxgb4_netdev_ops = { 2955 .ndo_open = cxgb_open, 2956 .ndo_stop = cxgb_close, 2957 .ndo_start_xmit = t4_eth_xmit, 2958 .ndo_select_queue = cxgb_select_queue, 2959 .ndo_get_stats64 = cxgb_get_stats, 2960 .ndo_set_rx_mode = cxgb_set_rxmode, 2961 .ndo_set_mac_address = cxgb_set_mac_addr, 2962 .ndo_set_features = cxgb_set_features, 2963 .ndo_validate_addr = eth_validate_addr, 2964 .ndo_do_ioctl = cxgb_ioctl, 2965 .ndo_change_mtu = cxgb_change_mtu, 2966 #ifdef CONFIG_NET_POLL_CONTROLLER 2967 .ndo_poll_controller = cxgb_netpoll, 2968 #endif 2969 #ifdef CONFIG_CHELSIO_T4_FCOE 2970 .ndo_fcoe_enable = cxgb_fcoe_enable, 2971 .ndo_fcoe_disable = cxgb_fcoe_disable, 2972 #endif /* CONFIG_CHELSIO_T4_FCOE */ 2973 .ndo_set_tx_maxrate = cxgb_set_tx_maxrate, 2974 .ndo_setup_tc = cxgb_setup_tc, 2975 .ndo_fix_features = cxgb_fix_features, 2976 }; 2977 2978 #ifdef CONFIG_PCI_IOV 2979 static const struct net_device_ops cxgb4_mgmt_netdev_ops = { 2980 .ndo_open = dummy_open, 2981 .ndo_set_vf_mac = cxgb_set_vf_mac, 2982 .ndo_get_vf_config = cxgb_get_vf_config, 2983 .ndo_set_vf_rate = cxgb_set_vf_rate, 2984 .ndo_get_phys_port_id = cxgb_get_phys_port_id, 2985 }; 2986 #endif 2987 2988 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 2989 { 2990 struct adapter *adapter = netdev2adap(dev); 2991 2992 strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); 2993 strlcpy(info->version, cxgb4_driver_version, 2994 sizeof(info->version)); 2995 strlcpy(info->bus_info, pci_name(adapter->pdev), 2996 sizeof(info->bus_info)); 2997 } 2998 2999 static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = { 3000 .get_drvinfo = get_drvinfo, 3001 }; 3002 3003 void t4_fatal_err(struct adapter *adap) 3004 { 3005 int port; 3006 3007 if (pci_channel_offline(adap->pdev)) 3008 return; 3009 3010 /* Disable the SGE since ULDs are going to free resources that 3011 * could be exposed to the adapter. RDMA MWs for example... 3012 */ 3013 t4_shutdown_adapter(adap); 3014 for_each_port(adap, port) { 3015 struct net_device *dev = adap->port[port]; 3016 3017 /* If we get here in very early initialization the network 3018 * devices may not have been set up yet. 3019 */ 3020 if (!dev) 3021 continue; 3022 3023 netif_tx_stop_all_queues(dev); 3024 netif_carrier_off(dev); 3025 } 3026 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n"); 3027 } 3028 3029 static void setup_memwin(struct adapter *adap) 3030 { 3031 u32 nic_win_base = t4_get_util_window(adap); 3032 3033 t4_setup_memwin(adap, nic_win_base, MEMWIN_NIC); 3034 } 3035 3036 static void setup_memwin_rdma(struct adapter *adap) 3037 { 3038 if (adap->vres.ocq.size) { 3039 u32 start; 3040 unsigned int sz_kb; 3041 3042 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2); 3043 start &= PCI_BASE_ADDRESS_MEM_MASK; 3044 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres); 3045 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10; 3046 t4_write_reg(adap, 3047 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3), 3048 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb))); 3049 t4_write_reg(adap, 3050 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3), 3051 adap->vres.ocq.start); 3052 t4_read_reg(adap, 3053 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3)); 3054 } 3055 } 3056 3057 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c) 3058 { 3059 u32 v; 3060 int ret; 3061 3062 /* get device capabilities */ 3063 memset(c, 0, sizeof(*c)); 3064 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3065 FW_CMD_REQUEST_F | FW_CMD_READ_F); 3066 c->cfvalid_to_len16 = htonl(FW_LEN16(*c)); 3067 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), c); 3068 if (ret < 0) 3069 return ret; 3070 3071 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3072 FW_CMD_REQUEST_F | FW_CMD_WRITE_F); 3073 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), NULL); 3074 if (ret < 0) 3075 return ret; 3076 3077 ret = t4_config_glbl_rss(adap, adap->pf, 3078 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL, 3079 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F | 3080 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F); 3081 if (ret < 0) 3082 return ret; 3083 3084 ret = t4_cfg_pfvf(adap, adap->mbox, adap->pf, 0, adap->sge.egr_sz, 64, 3085 MAX_INGQ, 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, 3086 FW_CMD_CAP_PF); 3087 if (ret < 0) 3088 return ret; 3089 3090 t4_sge_init(adap); 3091 3092 /* tweak some settings */ 3093 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849); 3094 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12)); 3095 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A); 3096 v = t4_read_reg(adap, TP_PIO_DATA_A); 3097 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F); 3098 3099 /* first 4 Tx modulation queues point to consecutive Tx channels */ 3100 adap->params.tp.tx_modq_map = 0xE4; 3101 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A, 3102 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map)); 3103 3104 /* associate each Tx modulation queue with consecutive Tx channels */ 3105 v = 0x84218421; 3106 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3107 &v, 1, TP_TX_SCHED_HDR_A); 3108 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3109 &v, 1, TP_TX_SCHED_FIFO_A); 3110 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3111 &v, 1, TP_TX_SCHED_PCMD_A); 3112 3113 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */ 3114 if (is_offload(adap)) { 3115 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A, 3116 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3117 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3118 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3119 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 3120 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A, 3121 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3122 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3123 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3124 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 3125 } 3126 3127 /* get basic stuff going */ 3128 return t4_early_init(adap, adap->pf); 3129 } 3130 3131 /* 3132 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower. 3133 */ 3134 #define MAX_ATIDS 8192U 3135 3136 /* 3137 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 3138 * 3139 * If the firmware we're dealing with has Configuration File support, then 3140 * we use that to perform all configuration 3141 */ 3142 3143 /* 3144 * Tweak configuration based on module parameters, etc. Most of these have 3145 * defaults assigned to them by Firmware Configuration Files (if we're using 3146 * them) but need to be explicitly set if we're using hard-coded 3147 * initialization. But even in the case of using Firmware Configuration 3148 * Files, we'd like to expose the ability to change these via module 3149 * parameters so these are essentially common tweaks/settings for 3150 * Configuration Files and hard-coded initialization ... 3151 */ 3152 static int adap_init0_tweaks(struct adapter *adapter) 3153 { 3154 /* 3155 * Fix up various Host-Dependent Parameters like Page Size, Cache 3156 * Line Size, etc. The firmware default is for a 4KB Page Size and 3157 * 64B Cache Line Size ... 3158 */ 3159 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES); 3160 3161 /* 3162 * Process module parameters which affect early initialization. 3163 */ 3164 if (rx_dma_offset != 2 && rx_dma_offset != 0) { 3165 dev_err(&adapter->pdev->dev, 3166 "Ignoring illegal rx_dma_offset=%d, using 2\n", 3167 rx_dma_offset); 3168 rx_dma_offset = 2; 3169 } 3170 t4_set_reg_field(adapter, SGE_CONTROL_A, 3171 PKTSHIFT_V(PKTSHIFT_M), 3172 PKTSHIFT_V(rx_dma_offset)); 3173 3174 /* 3175 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux 3176 * adds the pseudo header itself. 3177 */ 3178 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A, 3179 CSUM_HAS_PSEUDO_HDR_F, 0); 3180 3181 return 0; 3182 } 3183 3184 /* 10Gb/s-BT PHY Support. chip-external 10Gb/s-BT PHYs are complex chips 3185 * unto themselves and they contain their own firmware to perform their 3186 * tasks ... 3187 */ 3188 static int phy_aq1202_version(const u8 *phy_fw_data, 3189 size_t phy_fw_size) 3190 { 3191 int offset; 3192 3193 /* At offset 0x8 you're looking for the primary image's 3194 * starting offset which is 3 Bytes wide 3195 * 3196 * At offset 0xa of the primary image, you look for the offset 3197 * of the DRAM segment which is 3 Bytes wide. 3198 * 3199 * The FW version is at offset 0x27e of the DRAM and is 2 Bytes 3200 * wide 3201 */ 3202 #define be16(__p) (((__p)[0] << 8) | (__p)[1]) 3203 #define le16(__p) ((__p)[0] | ((__p)[1] << 8)) 3204 #define le24(__p) (le16(__p) | ((__p)[2] << 16)) 3205 3206 offset = le24(phy_fw_data + 0x8) << 12; 3207 offset = le24(phy_fw_data + offset + 0xa); 3208 return be16(phy_fw_data + offset + 0x27e); 3209 3210 #undef be16 3211 #undef le16 3212 #undef le24 3213 } 3214 3215 static struct info_10gbt_phy_fw { 3216 unsigned int phy_fw_id; /* PCI Device ID */ 3217 char *phy_fw_file; /* /lib/firmware/ PHY Firmware file */ 3218 int (*phy_fw_version)(const u8 *phy_fw_data, size_t phy_fw_size); 3219 int phy_flash; /* Has FLASH for PHY Firmware */ 3220 } phy_info_array[] = { 3221 { 3222 PHY_AQ1202_DEVICEID, 3223 PHY_AQ1202_FIRMWARE, 3224 phy_aq1202_version, 3225 1, 3226 }, 3227 { 3228 PHY_BCM84834_DEVICEID, 3229 PHY_BCM84834_FIRMWARE, 3230 NULL, 3231 0, 3232 }, 3233 { 0, NULL, NULL }, 3234 }; 3235 3236 static struct info_10gbt_phy_fw *find_phy_info(int devid) 3237 { 3238 int i; 3239 3240 for (i = 0; i < ARRAY_SIZE(phy_info_array); i++) { 3241 if (phy_info_array[i].phy_fw_id == devid) 3242 return &phy_info_array[i]; 3243 } 3244 return NULL; 3245 } 3246 3247 /* Handle updating of chip-external 10Gb/s-BT PHY firmware. This needs to 3248 * happen after the FW_RESET_CMD but before the FW_INITIALIZE_CMD. On error 3249 * we return a negative error number. If we transfer new firmware we return 1 3250 * (from t4_load_phy_fw()). If we don't do anything we return 0. 3251 */ 3252 static int adap_init0_phy(struct adapter *adap) 3253 { 3254 const struct firmware *phyf; 3255 int ret; 3256 struct info_10gbt_phy_fw *phy_info; 3257 3258 /* Use the device ID to determine which PHY file to flash. 3259 */ 3260 phy_info = find_phy_info(adap->pdev->device); 3261 if (!phy_info) { 3262 dev_warn(adap->pdev_dev, 3263 "No PHY Firmware file found for this PHY\n"); 3264 return -EOPNOTSUPP; 3265 } 3266 3267 /* If we have a T4 PHY firmware file under /lib/firmware/cxgb4/, then 3268 * use that. The adapter firmware provides us with a memory buffer 3269 * where we can load a PHY firmware file from the host if we want to 3270 * override the PHY firmware File in flash. 3271 */ 3272 ret = request_firmware_direct(&phyf, phy_info->phy_fw_file, 3273 adap->pdev_dev); 3274 if (ret < 0) { 3275 /* For adapters without FLASH attached to PHY for their 3276 * firmware, it's obviously a fatal error if we can't get the 3277 * firmware to the adapter. For adapters with PHY firmware 3278 * FLASH storage, it's worth a warning if we can't find the 3279 * PHY Firmware but we'll neuter the error ... 3280 */ 3281 dev_err(adap->pdev_dev, "unable to find PHY Firmware image " 3282 "/lib/firmware/%s, error %d\n", 3283 phy_info->phy_fw_file, -ret); 3284 if (phy_info->phy_flash) { 3285 int cur_phy_fw_ver = 0; 3286 3287 t4_phy_fw_ver(adap, &cur_phy_fw_ver); 3288 dev_warn(adap->pdev_dev, "continuing with, on-adapter " 3289 "FLASH copy, version %#x\n", cur_phy_fw_ver); 3290 ret = 0; 3291 } 3292 3293 return ret; 3294 } 3295 3296 /* Load PHY Firmware onto adapter. 3297 */ 3298 ret = t4_load_phy_fw(adap, MEMWIN_NIC, &adap->win0_lock, 3299 phy_info->phy_fw_version, 3300 (u8 *)phyf->data, phyf->size); 3301 if (ret < 0) 3302 dev_err(adap->pdev_dev, "PHY Firmware transfer error %d\n", 3303 -ret); 3304 else if (ret > 0) { 3305 int new_phy_fw_ver = 0; 3306 3307 if (phy_info->phy_fw_version) 3308 new_phy_fw_ver = phy_info->phy_fw_version(phyf->data, 3309 phyf->size); 3310 dev_info(adap->pdev_dev, "Successfully transferred PHY " 3311 "Firmware /lib/firmware/%s, version %#x\n", 3312 phy_info->phy_fw_file, new_phy_fw_ver); 3313 } 3314 3315 release_firmware(phyf); 3316 3317 return ret; 3318 } 3319 3320 /* 3321 * Attempt to initialize the adapter via a Firmware Configuration File. 3322 */ 3323 static int adap_init0_config(struct adapter *adapter, int reset) 3324 { 3325 struct fw_caps_config_cmd caps_cmd; 3326 const struct firmware *cf; 3327 unsigned long mtype = 0, maddr = 0; 3328 u32 finiver, finicsum, cfcsum; 3329 int ret; 3330 int config_issued = 0; 3331 char *fw_config_file, fw_config_file_path[256]; 3332 char *config_name = NULL; 3333 3334 /* 3335 * Reset device if necessary. 3336 */ 3337 if (reset) { 3338 ret = t4_fw_reset(adapter, adapter->mbox, 3339 PIORSTMODE_F | PIORST_F); 3340 if (ret < 0) 3341 goto bye; 3342 } 3343 3344 /* If this is a 10Gb/s-BT adapter make sure the chip-external 3345 * 10Gb/s-BT PHYs have up-to-date firmware. Note that this step needs 3346 * to be performed after any global adapter RESET above since some 3347 * PHYs only have local RAM copies of the PHY firmware. 3348 */ 3349 if (is_10gbt_device(adapter->pdev->device)) { 3350 ret = adap_init0_phy(adapter); 3351 if (ret < 0) 3352 goto bye; 3353 } 3354 /* 3355 * If we have a T4 configuration file under /lib/firmware/cxgb4/, 3356 * then use that. Otherwise, use the configuration file stored 3357 * in the adapter flash ... 3358 */ 3359 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) { 3360 case CHELSIO_T4: 3361 fw_config_file = FW4_CFNAME; 3362 break; 3363 case CHELSIO_T5: 3364 fw_config_file = FW5_CFNAME; 3365 break; 3366 case CHELSIO_T6: 3367 fw_config_file = FW6_CFNAME; 3368 break; 3369 default: 3370 dev_err(adapter->pdev_dev, "Device %d is not supported\n", 3371 adapter->pdev->device); 3372 ret = -EINVAL; 3373 goto bye; 3374 } 3375 3376 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev); 3377 if (ret < 0) { 3378 config_name = "On FLASH"; 3379 mtype = FW_MEMTYPE_CF_FLASH; 3380 maddr = t4_flash_cfg_addr(adapter); 3381 } else { 3382 u32 params[7], val[7]; 3383 3384 sprintf(fw_config_file_path, 3385 "/lib/firmware/%s", fw_config_file); 3386 config_name = fw_config_file_path; 3387 3388 if (cf->size >= FLASH_CFG_MAX_SIZE) 3389 ret = -ENOMEM; 3390 else { 3391 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3392 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 3393 ret = t4_query_params(adapter, adapter->mbox, 3394 adapter->pf, 0, 1, params, val); 3395 if (ret == 0) { 3396 /* 3397 * For t4_memory_rw() below addresses and 3398 * sizes have to be in terms of multiples of 4 3399 * bytes. So, if the Configuration File isn't 3400 * a multiple of 4 bytes in length we'll have 3401 * to write that out separately since we can't 3402 * guarantee that the bytes following the 3403 * residual byte in the buffer returned by 3404 * request_firmware() are zeroed out ... 3405 */ 3406 size_t resid = cf->size & 0x3; 3407 size_t size = cf->size & ~0x3; 3408 __be32 *data = (__be32 *)cf->data; 3409 3410 mtype = FW_PARAMS_PARAM_Y_G(val[0]); 3411 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16; 3412 3413 spin_lock(&adapter->win0_lock); 3414 ret = t4_memory_rw(adapter, 0, mtype, maddr, 3415 size, data, T4_MEMORY_WRITE); 3416 if (ret == 0 && resid != 0) { 3417 union { 3418 __be32 word; 3419 char buf[4]; 3420 } last; 3421 int i; 3422 3423 last.word = data[size >> 2]; 3424 for (i = resid; i < 4; i++) 3425 last.buf[i] = 0; 3426 ret = t4_memory_rw(adapter, 0, mtype, 3427 maddr + size, 3428 4, &last.word, 3429 T4_MEMORY_WRITE); 3430 } 3431 spin_unlock(&adapter->win0_lock); 3432 } 3433 } 3434 3435 release_firmware(cf); 3436 if (ret) 3437 goto bye; 3438 } 3439 3440 /* 3441 * Issue a Capability Configuration command to the firmware to get it 3442 * to parse the Configuration File. We don't use t4_fw_config_file() 3443 * because we want the ability to modify various features after we've 3444 * processed the configuration file ... 3445 */ 3446 memset(&caps_cmd, 0, sizeof(caps_cmd)); 3447 caps_cmd.op_to_write = 3448 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3449 FW_CMD_REQUEST_F | 3450 FW_CMD_READ_F); 3451 caps_cmd.cfvalid_to_len16 = 3452 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F | 3453 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) | 3454 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) | 3455 FW_LEN16(caps_cmd)); 3456 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 3457 &caps_cmd); 3458 3459 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware 3460 * Configuration File in FLASH), our last gasp effort is to use the 3461 * Firmware Configuration File which is embedded in the firmware. A 3462 * very few early versions of the firmware didn't have one embedded 3463 * but we can ignore those. 3464 */ 3465 if (ret == -ENOENT) { 3466 memset(&caps_cmd, 0, sizeof(caps_cmd)); 3467 caps_cmd.op_to_write = 3468 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3469 FW_CMD_REQUEST_F | 3470 FW_CMD_READ_F); 3471 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 3472 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, 3473 sizeof(caps_cmd), &caps_cmd); 3474 config_name = "Firmware Default"; 3475 } 3476 3477 config_issued = 1; 3478 if (ret < 0) 3479 goto bye; 3480 3481 finiver = ntohl(caps_cmd.finiver); 3482 finicsum = ntohl(caps_cmd.finicsum); 3483 cfcsum = ntohl(caps_cmd.cfcsum); 3484 if (finicsum != cfcsum) 3485 dev_warn(adapter->pdev_dev, "Configuration File checksum "\ 3486 "mismatch: [fini] csum=%#x, computed csum=%#x\n", 3487 finicsum, cfcsum); 3488 3489 /* 3490 * And now tell the firmware to use the configuration we just loaded. 3491 */ 3492 caps_cmd.op_to_write = 3493 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3494 FW_CMD_REQUEST_F | 3495 FW_CMD_WRITE_F); 3496 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 3497 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 3498 NULL); 3499 if (ret < 0) 3500 goto bye; 3501 3502 /* 3503 * Tweak configuration based on system architecture, module 3504 * parameters, etc. 3505 */ 3506 ret = adap_init0_tweaks(adapter); 3507 if (ret < 0) 3508 goto bye; 3509 3510 /* 3511 * And finally tell the firmware to initialize itself using the 3512 * parameters from the Configuration File. 3513 */ 3514 ret = t4_fw_initialize(adapter, adapter->mbox); 3515 if (ret < 0) 3516 goto bye; 3517 3518 /* Emit Firmware Configuration File information and return 3519 * successfully. 3520 */ 3521 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\ 3522 "Configuration File \"%s\", version %#x, computed checksum %#x\n", 3523 config_name, finiver, cfcsum); 3524 return 0; 3525 3526 /* 3527 * Something bad happened. Return the error ... (If the "error" 3528 * is that there's no Configuration File on the adapter we don't 3529 * want to issue a warning since this is fairly common.) 3530 */ 3531 bye: 3532 if (config_issued && ret != -ENOENT) 3533 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n", 3534 config_name, -ret); 3535 return ret; 3536 } 3537 3538 static struct fw_info fw_info_array[] = { 3539 { 3540 .chip = CHELSIO_T4, 3541 .fs_name = FW4_CFNAME, 3542 .fw_mod_name = FW4_FNAME, 3543 .fw_hdr = { 3544 .chip = FW_HDR_CHIP_T4, 3545 .fw_ver = __cpu_to_be32(FW_VERSION(T4)), 3546 .intfver_nic = FW_INTFVER(T4, NIC), 3547 .intfver_vnic = FW_INTFVER(T4, VNIC), 3548 .intfver_ri = FW_INTFVER(T4, RI), 3549 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 3550 .intfver_fcoe = FW_INTFVER(T4, FCOE), 3551 }, 3552 }, { 3553 .chip = CHELSIO_T5, 3554 .fs_name = FW5_CFNAME, 3555 .fw_mod_name = FW5_FNAME, 3556 .fw_hdr = { 3557 .chip = FW_HDR_CHIP_T5, 3558 .fw_ver = __cpu_to_be32(FW_VERSION(T5)), 3559 .intfver_nic = FW_INTFVER(T5, NIC), 3560 .intfver_vnic = FW_INTFVER(T5, VNIC), 3561 .intfver_ri = FW_INTFVER(T5, RI), 3562 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 3563 .intfver_fcoe = FW_INTFVER(T5, FCOE), 3564 }, 3565 }, { 3566 .chip = CHELSIO_T6, 3567 .fs_name = FW6_CFNAME, 3568 .fw_mod_name = FW6_FNAME, 3569 .fw_hdr = { 3570 .chip = FW_HDR_CHIP_T6, 3571 .fw_ver = __cpu_to_be32(FW_VERSION(T6)), 3572 .intfver_nic = FW_INTFVER(T6, NIC), 3573 .intfver_vnic = FW_INTFVER(T6, VNIC), 3574 .intfver_ofld = FW_INTFVER(T6, OFLD), 3575 .intfver_ri = FW_INTFVER(T6, RI), 3576 .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU), 3577 .intfver_iscsi = FW_INTFVER(T6, ISCSI), 3578 .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU), 3579 .intfver_fcoe = FW_INTFVER(T6, FCOE), 3580 }, 3581 } 3582 3583 }; 3584 3585 static struct fw_info *find_fw_info(int chip) 3586 { 3587 int i; 3588 3589 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) { 3590 if (fw_info_array[i].chip == chip) 3591 return &fw_info_array[i]; 3592 } 3593 return NULL; 3594 } 3595 3596 /* 3597 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 3598 */ 3599 static int adap_init0(struct adapter *adap) 3600 { 3601 int ret; 3602 u32 v, port_vec; 3603 enum dev_state state; 3604 u32 params[7], val[7]; 3605 struct fw_caps_config_cmd caps_cmd; 3606 int reset = 1; 3607 3608 /* Grab Firmware Device Log parameters as early as possible so we have 3609 * access to it for debugging, etc. 3610 */ 3611 ret = t4_init_devlog_params(adap); 3612 if (ret < 0) 3613 return ret; 3614 3615 /* Contact FW, advertising Master capability */ 3616 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, 3617 is_kdump_kernel() ? MASTER_MUST : MASTER_MAY, &state); 3618 if (ret < 0) { 3619 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n", 3620 ret); 3621 return ret; 3622 } 3623 if (ret == adap->mbox) 3624 adap->flags |= MASTER_PF; 3625 3626 /* 3627 * If we're the Master PF Driver and the device is uninitialized, 3628 * then let's consider upgrading the firmware ... (We always want 3629 * to check the firmware version number in order to A. get it for 3630 * later reporting and B. to warn if the currently loaded firmware 3631 * is excessively mismatched relative to the driver.) 3632 */ 3633 3634 t4_get_version_info(adap); 3635 ret = t4_check_fw_version(adap); 3636 /* If firmware is too old (not supported by driver) force an update. */ 3637 if (ret) 3638 state = DEV_STATE_UNINIT; 3639 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) { 3640 struct fw_info *fw_info; 3641 struct fw_hdr *card_fw; 3642 const struct firmware *fw; 3643 const u8 *fw_data = NULL; 3644 unsigned int fw_size = 0; 3645 3646 /* This is the firmware whose headers the driver was compiled 3647 * against 3648 */ 3649 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip)); 3650 if (fw_info == NULL) { 3651 dev_err(adap->pdev_dev, 3652 "unable to get firmware info for chip %d.\n", 3653 CHELSIO_CHIP_VERSION(adap->params.chip)); 3654 return -EINVAL; 3655 } 3656 3657 /* allocate memory to read the header of the firmware on the 3658 * card 3659 */ 3660 card_fw = kvzalloc(sizeof(*card_fw), GFP_KERNEL); 3661 3662 /* Get FW from from /lib/firmware/ */ 3663 ret = request_firmware(&fw, fw_info->fw_mod_name, 3664 adap->pdev_dev); 3665 if (ret < 0) { 3666 dev_err(adap->pdev_dev, 3667 "unable to load firmware image %s, error %d\n", 3668 fw_info->fw_mod_name, ret); 3669 } else { 3670 fw_data = fw->data; 3671 fw_size = fw->size; 3672 } 3673 3674 /* upgrade FW logic */ 3675 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw, 3676 state, &reset); 3677 3678 /* Cleaning up */ 3679 release_firmware(fw); 3680 kvfree(card_fw); 3681 3682 if (ret < 0) 3683 goto bye; 3684 } 3685 3686 /* 3687 * Grab VPD parameters. This should be done after we establish a 3688 * connection to the firmware since some of the VPD parameters 3689 * (notably the Core Clock frequency) are retrieved via requests to 3690 * the firmware. On the other hand, we need these fairly early on 3691 * so we do this right after getting ahold of the firmware. 3692 */ 3693 ret = t4_get_vpd_params(adap, &adap->params.vpd); 3694 if (ret < 0) 3695 goto bye; 3696 3697 /* 3698 * Find out what ports are available to us. Note that we need to do 3699 * this before calling adap_init0_no_config() since it needs nports 3700 * and portvec ... 3701 */ 3702 v = 3703 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3704 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC); 3705 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec); 3706 if (ret < 0) 3707 goto bye; 3708 3709 adap->params.nports = hweight32(port_vec); 3710 adap->params.portvec = port_vec; 3711 3712 /* If the firmware is initialized already, emit a simply note to that 3713 * effect. Otherwise, it's time to try initializing the adapter. 3714 */ 3715 if (state == DEV_STATE_INIT) { 3716 dev_info(adap->pdev_dev, "Coming up as %s: "\ 3717 "Adapter already initialized\n", 3718 adap->flags & MASTER_PF ? "MASTER" : "SLAVE"); 3719 } else { 3720 dev_info(adap->pdev_dev, "Coming up as MASTER: "\ 3721 "Initializing adapter\n"); 3722 3723 /* Find out whether we're dealing with a version of the 3724 * firmware which has configuration file support. 3725 */ 3726 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3727 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 3728 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, 3729 params, val); 3730 3731 /* If the firmware doesn't support Configuration Files, 3732 * return an error. 3733 */ 3734 if (ret < 0) { 3735 dev_err(adap->pdev_dev, "firmware doesn't support " 3736 "Firmware Configuration Files\n"); 3737 goto bye; 3738 } 3739 3740 /* The firmware provides us with a memory buffer where we can 3741 * load a Configuration File from the host if we want to 3742 * override the Configuration File in flash. 3743 */ 3744 ret = adap_init0_config(adap, reset); 3745 if (ret == -ENOENT) { 3746 dev_err(adap->pdev_dev, "no Configuration File " 3747 "present on adapter.\n"); 3748 goto bye; 3749 } 3750 if (ret < 0) { 3751 dev_err(adap->pdev_dev, "could not initialize " 3752 "adapter, error %d\n", -ret); 3753 goto bye; 3754 } 3755 } 3756 3757 /* Give the SGE code a chance to pull in anything that it needs ... 3758 * Note that this must be called after we retrieve our VPD parameters 3759 * in order to know how to convert core ticks to seconds, etc. 3760 */ 3761 ret = t4_sge_init(adap); 3762 if (ret < 0) 3763 goto bye; 3764 3765 if (is_bypass_device(adap->pdev->device)) 3766 adap->params.bypass = 1; 3767 3768 /* 3769 * Grab some of our basic fundamental operating parameters. 3770 */ 3771 #define FW_PARAM_DEV(param) \ 3772 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \ 3773 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param)) 3774 3775 #define FW_PARAM_PFVF(param) \ 3776 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \ 3777 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \ 3778 FW_PARAMS_PARAM_Y_V(0) | \ 3779 FW_PARAMS_PARAM_Z_V(0) 3780 3781 params[0] = FW_PARAM_PFVF(EQ_START); 3782 params[1] = FW_PARAM_PFVF(L2T_START); 3783 params[2] = FW_PARAM_PFVF(L2T_END); 3784 params[3] = FW_PARAM_PFVF(FILTER_START); 3785 params[4] = FW_PARAM_PFVF(FILTER_END); 3786 params[5] = FW_PARAM_PFVF(IQFLINT_START); 3787 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, val); 3788 if (ret < 0) 3789 goto bye; 3790 adap->sge.egr_start = val[0]; 3791 adap->l2t_start = val[1]; 3792 adap->l2t_end = val[2]; 3793 adap->tids.ftid_base = val[3]; 3794 adap->tids.nftids = val[4] - val[3] + 1; 3795 adap->sge.ingr_start = val[5]; 3796 3797 /* qids (ingress/egress) returned from firmware can be anywhere 3798 * in the range from EQ(IQFLINT)_START to EQ(IQFLINT)_END. 3799 * Hence driver needs to allocate memory for this range to 3800 * store the queue info. Get the highest IQFLINT/EQ index returned 3801 * in FW_EQ_*_CMD.alloc command. 3802 */ 3803 params[0] = FW_PARAM_PFVF(EQ_END); 3804 params[1] = FW_PARAM_PFVF(IQFLINT_END); 3805 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 3806 if (ret < 0) 3807 goto bye; 3808 adap->sge.egr_sz = val[0] - adap->sge.egr_start + 1; 3809 adap->sge.ingr_sz = val[1] - adap->sge.ingr_start + 1; 3810 3811 adap->sge.egr_map = kcalloc(adap->sge.egr_sz, 3812 sizeof(*adap->sge.egr_map), GFP_KERNEL); 3813 if (!adap->sge.egr_map) { 3814 ret = -ENOMEM; 3815 goto bye; 3816 } 3817 3818 adap->sge.ingr_map = kcalloc(adap->sge.ingr_sz, 3819 sizeof(*adap->sge.ingr_map), GFP_KERNEL); 3820 if (!adap->sge.ingr_map) { 3821 ret = -ENOMEM; 3822 goto bye; 3823 } 3824 3825 /* Allocate the memory for the vaious egress queue bitmaps 3826 * ie starving_fl, txq_maperr and blocked_fl. 3827 */ 3828 adap->sge.starving_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 3829 sizeof(long), GFP_KERNEL); 3830 if (!adap->sge.starving_fl) { 3831 ret = -ENOMEM; 3832 goto bye; 3833 } 3834 3835 adap->sge.txq_maperr = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 3836 sizeof(long), GFP_KERNEL); 3837 if (!adap->sge.txq_maperr) { 3838 ret = -ENOMEM; 3839 goto bye; 3840 } 3841 3842 #ifdef CONFIG_DEBUG_FS 3843 adap->sge.blocked_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 3844 sizeof(long), GFP_KERNEL); 3845 if (!adap->sge.blocked_fl) { 3846 ret = -ENOMEM; 3847 goto bye; 3848 } 3849 #endif 3850 3851 params[0] = FW_PARAM_PFVF(CLIP_START); 3852 params[1] = FW_PARAM_PFVF(CLIP_END); 3853 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 3854 if (ret < 0) 3855 goto bye; 3856 adap->clipt_start = val[0]; 3857 adap->clipt_end = val[1]; 3858 3859 /* We don't yet have a PARAMs calls to retrieve the number of Traffic 3860 * Classes supported by the hardware/firmware so we hard code it here 3861 * for now. 3862 */ 3863 adap->params.nsched_cls = is_t4(adap->params.chip) ? 15 : 16; 3864 3865 /* query params related to active filter region */ 3866 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START); 3867 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END); 3868 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 3869 /* If Active filter size is set we enable establishing 3870 * offload connection through firmware work request 3871 */ 3872 if ((val[0] != val[1]) && (ret >= 0)) { 3873 adap->flags |= FW_OFLD_CONN; 3874 adap->tids.aftid_base = val[0]; 3875 adap->tids.aftid_end = val[1]; 3876 } 3877 3878 /* If we're running on newer firmware, let it know that we're 3879 * prepared to deal with encapsulated CPL messages. Older 3880 * firmware won't understand this and we'll just get 3881 * unencapsulated messages ... 3882 */ 3883 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 3884 val[0] = 1; 3885 (void)t4_set_params(adap, adap->mbox, adap->pf, 0, 1, params, val); 3886 3887 /* 3888 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL 3889 * capability. Earlier versions of the firmware didn't have the 3890 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no 3891 * permission to use ULPTX MEMWRITE DSGL. 3892 */ 3893 if (is_t4(adap->params.chip)) { 3894 adap->params.ulptx_memwrite_dsgl = false; 3895 } else { 3896 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL); 3897 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 3898 1, params, val); 3899 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0); 3900 } 3901 3902 /* See if FW supports FW_RI_FR_NSMR_TPTE_WR work request */ 3903 params[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR); 3904 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 3905 1, params, val); 3906 adap->params.fr_nsmr_tpte_wr_support = (ret == 0 && val[0] != 0); 3907 3908 /* 3909 * Get device capabilities so we can determine what resources we need 3910 * to manage. 3911 */ 3912 memset(&caps_cmd, 0, sizeof(caps_cmd)); 3913 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3914 FW_CMD_REQUEST_F | FW_CMD_READ_F); 3915 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 3916 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd), 3917 &caps_cmd); 3918 if (ret < 0) 3919 goto bye; 3920 3921 if (caps_cmd.ofldcaps) { 3922 /* query offload-related parameters */ 3923 params[0] = FW_PARAM_DEV(NTID); 3924 params[1] = FW_PARAM_PFVF(SERVER_START); 3925 params[2] = FW_PARAM_PFVF(SERVER_END); 3926 params[3] = FW_PARAM_PFVF(TDDP_START); 3927 params[4] = FW_PARAM_PFVF(TDDP_END); 3928 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3929 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, 3930 params, val); 3931 if (ret < 0) 3932 goto bye; 3933 adap->tids.ntids = val[0]; 3934 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS); 3935 adap->tids.stid_base = val[1]; 3936 adap->tids.nstids = val[2] - val[1] + 1; 3937 /* 3938 * Setup server filter region. Divide the available filter 3939 * region into two parts. Regular filters get 1/3rd and server 3940 * filters get 2/3rd part. This is only enabled if workarond 3941 * path is enabled. 3942 * 1. For regular filters. 3943 * 2. Server filter: This are special filters which are used 3944 * to redirect SYN packets to offload queue. 3945 */ 3946 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) { 3947 adap->tids.sftid_base = adap->tids.ftid_base + 3948 DIV_ROUND_UP(adap->tids.nftids, 3); 3949 adap->tids.nsftids = adap->tids.nftids - 3950 DIV_ROUND_UP(adap->tids.nftids, 3); 3951 adap->tids.nftids = adap->tids.sftid_base - 3952 adap->tids.ftid_base; 3953 } 3954 adap->vres.ddp.start = val[3]; 3955 adap->vres.ddp.size = val[4] - val[3] + 1; 3956 adap->params.ofldq_wr_cred = val[5]; 3957 3958 adap->params.offload = 1; 3959 adap->num_ofld_uld += 1; 3960 } 3961 if (caps_cmd.rdmacaps) { 3962 params[0] = FW_PARAM_PFVF(STAG_START); 3963 params[1] = FW_PARAM_PFVF(STAG_END); 3964 params[2] = FW_PARAM_PFVF(RQ_START); 3965 params[3] = FW_PARAM_PFVF(RQ_END); 3966 params[4] = FW_PARAM_PFVF(PBL_START); 3967 params[5] = FW_PARAM_PFVF(PBL_END); 3968 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, 3969 params, val); 3970 if (ret < 0) 3971 goto bye; 3972 adap->vres.stag.start = val[0]; 3973 adap->vres.stag.size = val[1] - val[0] + 1; 3974 adap->vres.rq.start = val[2]; 3975 adap->vres.rq.size = val[3] - val[2] + 1; 3976 adap->vres.pbl.start = val[4]; 3977 adap->vres.pbl.size = val[5] - val[4] + 1; 3978 3979 params[0] = FW_PARAM_PFVF(SQRQ_START); 3980 params[1] = FW_PARAM_PFVF(SQRQ_END); 3981 params[2] = FW_PARAM_PFVF(CQ_START); 3982 params[3] = FW_PARAM_PFVF(CQ_END); 3983 params[4] = FW_PARAM_PFVF(OCQ_START); 3984 params[5] = FW_PARAM_PFVF(OCQ_END); 3985 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, 3986 val); 3987 if (ret < 0) 3988 goto bye; 3989 adap->vres.qp.start = val[0]; 3990 adap->vres.qp.size = val[1] - val[0] + 1; 3991 adap->vres.cq.start = val[2]; 3992 adap->vres.cq.size = val[3] - val[2] + 1; 3993 adap->vres.ocq.start = val[4]; 3994 adap->vres.ocq.size = val[5] - val[4] + 1; 3995 3996 params[0] = FW_PARAM_DEV(MAXORDIRD_QP); 3997 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER); 3998 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, 3999 val); 4000 if (ret < 0) { 4001 adap->params.max_ordird_qp = 8; 4002 adap->params.max_ird_adapter = 32 * adap->tids.ntids; 4003 ret = 0; 4004 } else { 4005 adap->params.max_ordird_qp = val[0]; 4006 adap->params.max_ird_adapter = val[1]; 4007 } 4008 dev_info(adap->pdev_dev, 4009 "max_ordird_qp %d max_ird_adapter %d\n", 4010 adap->params.max_ordird_qp, 4011 adap->params.max_ird_adapter); 4012 adap->num_ofld_uld += 2; 4013 } 4014 if (caps_cmd.iscsicaps) { 4015 params[0] = FW_PARAM_PFVF(ISCSI_START); 4016 params[1] = FW_PARAM_PFVF(ISCSI_END); 4017 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 4018 params, val); 4019 if (ret < 0) 4020 goto bye; 4021 adap->vres.iscsi.start = val[0]; 4022 adap->vres.iscsi.size = val[1] - val[0] + 1; 4023 /* LIO target and cxgb4i initiaitor */ 4024 adap->num_ofld_uld += 2; 4025 } 4026 if (caps_cmd.cryptocaps) { 4027 /* Should query params here...TODO */ 4028 params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE); 4029 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 4030 params, val); 4031 if (ret < 0) { 4032 if (ret != -EINVAL) 4033 goto bye; 4034 } else { 4035 adap->vres.ncrypto_fc = val[0]; 4036 } 4037 adap->params.crypto |= ULP_CRYPTO_LOOKASIDE; 4038 adap->num_uld += 1; 4039 } 4040 #undef FW_PARAM_PFVF 4041 #undef FW_PARAM_DEV 4042 4043 /* The MTU/MSS Table is initialized by now, so load their values. If 4044 * we're initializing the adapter, then we'll make any modifications 4045 * we want to the MTU/MSS Table and also initialize the congestion 4046 * parameters. 4047 */ 4048 t4_read_mtu_tbl(adap, adap->params.mtus, NULL); 4049 if (state != DEV_STATE_INIT) { 4050 int i; 4051 4052 /* The default MTU Table contains values 1492 and 1500. 4053 * However, for TCP, it's better to have two values which are 4054 * a multiple of 8 +/- 4 bytes apart near this popular MTU. 4055 * This allows us to have a TCP Data Payload which is a 4056 * multiple of 8 regardless of what combination of TCP Options 4057 * are in use (always a multiple of 4 bytes) which is 4058 * important for performance reasons. For instance, if no 4059 * options are in use, then we have a 20-byte IP header and a 4060 * 20-byte TCP header. In this case, a 1500-byte MSS would 4061 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes 4062 * which is not a multiple of 8. So using an MSS of 1488 in 4063 * this case results in a TCP Data Payload of 1448 bytes which 4064 * is a multiple of 8. On the other hand, if 12-byte TCP Time 4065 * Stamps have been negotiated, then an MTU of 1500 bytes 4066 * results in a TCP Data Payload of 1448 bytes which, as 4067 * above, is a multiple of 8 bytes ... 4068 */ 4069 for (i = 0; i < NMTUS; i++) 4070 if (adap->params.mtus[i] == 1492) { 4071 adap->params.mtus[i] = 1488; 4072 break; 4073 } 4074 4075 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 4076 adap->params.b_wnd); 4077 } 4078 t4_init_sge_params(adap); 4079 adap->flags |= FW_OK; 4080 t4_init_tp_params(adap, true); 4081 return 0; 4082 4083 /* 4084 * Something bad happened. If a command timed out or failed with EIO 4085 * FW does not operate within its spec or something catastrophic 4086 * happened to HW/FW, stop issuing commands. 4087 */ 4088 bye: 4089 kfree(adap->sge.egr_map); 4090 kfree(adap->sge.ingr_map); 4091 kfree(adap->sge.starving_fl); 4092 kfree(adap->sge.txq_maperr); 4093 #ifdef CONFIG_DEBUG_FS 4094 kfree(adap->sge.blocked_fl); 4095 #endif 4096 if (ret != -ETIMEDOUT && ret != -EIO) 4097 t4_fw_bye(adap, adap->mbox); 4098 return ret; 4099 } 4100 4101 /* EEH callbacks */ 4102 4103 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev, 4104 pci_channel_state_t state) 4105 { 4106 int i; 4107 struct adapter *adap = pci_get_drvdata(pdev); 4108 4109 if (!adap) 4110 goto out; 4111 4112 rtnl_lock(); 4113 adap->flags &= ~FW_OK; 4114 notify_ulds(adap, CXGB4_STATE_START_RECOVERY); 4115 spin_lock(&adap->stats_lock); 4116 for_each_port(adap, i) { 4117 struct net_device *dev = adap->port[i]; 4118 if (dev) { 4119 netif_device_detach(dev); 4120 netif_carrier_off(dev); 4121 } 4122 } 4123 spin_unlock(&adap->stats_lock); 4124 disable_interrupts(adap); 4125 if (adap->flags & FULL_INIT_DONE) 4126 cxgb_down(adap); 4127 rtnl_unlock(); 4128 if ((adap->flags & DEV_ENABLED)) { 4129 pci_disable_device(pdev); 4130 adap->flags &= ~DEV_ENABLED; 4131 } 4132 out: return state == pci_channel_io_perm_failure ? 4133 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 4134 } 4135 4136 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev) 4137 { 4138 int i, ret; 4139 struct fw_caps_config_cmd c; 4140 struct adapter *adap = pci_get_drvdata(pdev); 4141 4142 if (!adap) { 4143 pci_restore_state(pdev); 4144 pci_save_state(pdev); 4145 return PCI_ERS_RESULT_RECOVERED; 4146 } 4147 4148 if (!(adap->flags & DEV_ENABLED)) { 4149 if (pci_enable_device(pdev)) { 4150 dev_err(&pdev->dev, "Cannot reenable PCI " 4151 "device after reset\n"); 4152 return PCI_ERS_RESULT_DISCONNECT; 4153 } 4154 adap->flags |= DEV_ENABLED; 4155 } 4156 4157 pci_set_master(pdev); 4158 pci_restore_state(pdev); 4159 pci_save_state(pdev); 4160 pci_cleanup_aer_uncorrect_error_status(pdev); 4161 4162 if (t4_wait_dev_ready(adap->regs) < 0) 4163 return PCI_ERS_RESULT_DISCONNECT; 4164 if (t4_fw_hello(adap, adap->mbox, adap->pf, MASTER_MUST, NULL) < 0) 4165 return PCI_ERS_RESULT_DISCONNECT; 4166 adap->flags |= FW_OK; 4167 if (adap_init1(adap, &c)) 4168 return PCI_ERS_RESULT_DISCONNECT; 4169 4170 for_each_port(adap, i) { 4171 struct port_info *p = adap2pinfo(adap, i); 4172 4173 ret = t4_alloc_vi(adap, adap->mbox, p->tx_chan, adap->pf, 0, 1, 4174 NULL, NULL); 4175 if (ret < 0) 4176 return PCI_ERS_RESULT_DISCONNECT; 4177 p->viid = ret; 4178 p->xact_addr_filt = -1; 4179 } 4180 4181 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 4182 adap->params.b_wnd); 4183 setup_memwin(adap); 4184 if (cxgb_up(adap)) 4185 return PCI_ERS_RESULT_DISCONNECT; 4186 return PCI_ERS_RESULT_RECOVERED; 4187 } 4188 4189 static void eeh_resume(struct pci_dev *pdev) 4190 { 4191 int i; 4192 struct adapter *adap = pci_get_drvdata(pdev); 4193 4194 if (!adap) 4195 return; 4196 4197 rtnl_lock(); 4198 for_each_port(adap, i) { 4199 struct net_device *dev = adap->port[i]; 4200 if (dev) { 4201 if (netif_running(dev)) { 4202 link_start(dev); 4203 cxgb_set_rxmode(dev); 4204 } 4205 netif_device_attach(dev); 4206 } 4207 } 4208 rtnl_unlock(); 4209 } 4210 4211 static const struct pci_error_handlers cxgb4_eeh = { 4212 .error_detected = eeh_err_detected, 4213 .slot_reset = eeh_slot_reset, 4214 .resume = eeh_resume, 4215 }; 4216 4217 /* Return true if the Link Configuration supports "High Speeds" (those greater 4218 * than 1Gb/s). 4219 */ 4220 static inline bool is_x_10g_port(const struct link_config *lc) 4221 { 4222 unsigned int speeds, high_speeds; 4223 4224 speeds = FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_G(lc->pcaps)); 4225 high_speeds = speeds & 4226 ~(FW_PORT_CAP32_SPEED_100M | FW_PORT_CAP32_SPEED_1G); 4227 4228 return high_speeds != 0; 4229 } 4230 4231 /* 4232 * Perform default configuration of DMA queues depending on the number and type 4233 * of ports we found and the number of available CPUs. Most settings can be 4234 * modified by the admin prior to actual use. 4235 */ 4236 static void cfg_queues(struct adapter *adap) 4237 { 4238 struct sge *s = &adap->sge; 4239 int i = 0, n10g = 0, qidx = 0; 4240 #ifndef CONFIG_CHELSIO_T4_DCB 4241 int q10g = 0; 4242 #endif 4243 4244 /* Reduce memory usage in kdump environment, disable all offload. 4245 */ 4246 if (is_kdump_kernel() || (is_uld(adap) && t4_uld_mem_alloc(adap))) { 4247 adap->params.offload = 0; 4248 adap->params.crypto = 0; 4249 } 4250 4251 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg); 4252 #ifdef CONFIG_CHELSIO_T4_DCB 4253 /* For Data Center Bridging support we need to be able to support up 4254 * to 8 Traffic Priorities; each of which will be assigned to its 4255 * own TX Queue in order to prevent Head-Of-Line Blocking. 4256 */ 4257 if (adap->params.nports * 8 > MAX_ETH_QSETS) { 4258 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n", 4259 MAX_ETH_QSETS, adap->params.nports * 8); 4260 BUG_ON(1); 4261 } 4262 4263 for_each_port(adap, i) { 4264 struct port_info *pi = adap2pinfo(adap, i); 4265 4266 pi->first_qset = qidx; 4267 pi->nqsets = is_kdump_kernel() ? 1 : 8; 4268 qidx += pi->nqsets; 4269 } 4270 #else /* !CONFIG_CHELSIO_T4_DCB */ 4271 /* 4272 * We default to 1 queue per non-10G port and up to # of cores queues 4273 * per 10G port. 4274 */ 4275 if (n10g) 4276 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g; 4277 if (q10g > netif_get_num_default_rss_queues()) 4278 q10g = netif_get_num_default_rss_queues(); 4279 4280 if (is_kdump_kernel()) 4281 q10g = 1; 4282 4283 for_each_port(adap, i) { 4284 struct port_info *pi = adap2pinfo(adap, i); 4285 4286 pi->first_qset = qidx; 4287 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1; 4288 qidx += pi->nqsets; 4289 } 4290 #endif /* !CONFIG_CHELSIO_T4_DCB */ 4291 4292 s->ethqsets = qidx; 4293 s->max_ethqsets = qidx; /* MSI-X may lower it later */ 4294 4295 if (is_uld(adap)) { 4296 /* 4297 * For offload we use 1 queue/channel if all ports are up to 1G, 4298 * otherwise we divide all available queues amongst the channels 4299 * capped by the number of available cores. 4300 */ 4301 if (n10g) { 4302 i = min_t(int, MAX_OFLD_QSETS, num_online_cpus()); 4303 s->ofldqsets = roundup(i, adap->params.nports); 4304 } else { 4305 s->ofldqsets = adap->params.nports; 4306 } 4307 } 4308 4309 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { 4310 struct sge_eth_rxq *r = &s->ethrxq[i]; 4311 4312 init_rspq(adap, &r->rspq, 5, 10, 1024, 64); 4313 r->fl.size = 72; 4314 } 4315 4316 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++) 4317 s->ethtxq[i].q.size = 1024; 4318 4319 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) 4320 s->ctrlq[i].q.size = 512; 4321 4322 if (!is_t4(adap->params.chip)) 4323 s->ptptxq.q.size = 8; 4324 4325 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64); 4326 init_rspq(adap, &s->intrq, 0, 1, 512, 64); 4327 } 4328 4329 /* 4330 * Reduce the number of Ethernet queues across all ports to at most n. 4331 * n provides at least one queue per port. 4332 */ 4333 static void reduce_ethqs(struct adapter *adap, int n) 4334 { 4335 int i; 4336 struct port_info *pi; 4337 4338 while (n < adap->sge.ethqsets) 4339 for_each_port(adap, i) { 4340 pi = adap2pinfo(adap, i); 4341 if (pi->nqsets > 1) { 4342 pi->nqsets--; 4343 adap->sge.ethqsets--; 4344 if (adap->sge.ethqsets <= n) 4345 break; 4346 } 4347 } 4348 4349 n = 0; 4350 for_each_port(adap, i) { 4351 pi = adap2pinfo(adap, i); 4352 pi->first_qset = n; 4353 n += pi->nqsets; 4354 } 4355 } 4356 4357 static int get_msix_info(struct adapter *adap) 4358 { 4359 struct uld_msix_info *msix_info; 4360 unsigned int max_ingq = 0; 4361 4362 if (is_offload(adap)) 4363 max_ingq += MAX_OFLD_QSETS * adap->num_ofld_uld; 4364 if (is_pci_uld(adap)) 4365 max_ingq += MAX_OFLD_QSETS * adap->num_uld; 4366 4367 if (!max_ingq) 4368 goto out; 4369 4370 msix_info = kcalloc(max_ingq, sizeof(*msix_info), GFP_KERNEL); 4371 if (!msix_info) 4372 return -ENOMEM; 4373 4374 adap->msix_bmap_ulds.msix_bmap = kcalloc(BITS_TO_LONGS(max_ingq), 4375 sizeof(long), GFP_KERNEL); 4376 if (!adap->msix_bmap_ulds.msix_bmap) { 4377 kfree(msix_info); 4378 return -ENOMEM; 4379 } 4380 spin_lock_init(&adap->msix_bmap_ulds.lock); 4381 adap->msix_info_ulds = msix_info; 4382 out: 4383 return 0; 4384 } 4385 4386 static void free_msix_info(struct adapter *adap) 4387 { 4388 if (!(adap->num_uld && adap->num_ofld_uld)) 4389 return; 4390 4391 kfree(adap->msix_info_ulds); 4392 kfree(adap->msix_bmap_ulds.msix_bmap); 4393 } 4394 4395 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */ 4396 #define EXTRA_VECS 2 4397 4398 static int enable_msix(struct adapter *adap) 4399 { 4400 int ofld_need = 0, uld_need = 0; 4401 int i, j, want, need, allocated; 4402 struct sge *s = &adap->sge; 4403 unsigned int nchan = adap->params.nports; 4404 struct msix_entry *entries; 4405 int max_ingq = MAX_INGQ; 4406 4407 if (is_pci_uld(adap)) 4408 max_ingq += (MAX_OFLD_QSETS * adap->num_uld); 4409 if (is_offload(adap)) 4410 max_ingq += (MAX_OFLD_QSETS * adap->num_ofld_uld); 4411 entries = kmalloc(sizeof(*entries) * (max_ingq + 1), 4412 GFP_KERNEL); 4413 if (!entries) 4414 return -ENOMEM; 4415 4416 /* map for msix */ 4417 if (get_msix_info(adap)) { 4418 adap->params.offload = 0; 4419 adap->params.crypto = 0; 4420 } 4421 4422 for (i = 0; i < max_ingq + 1; ++i) 4423 entries[i].entry = i; 4424 4425 want = s->max_ethqsets + EXTRA_VECS; 4426 if (is_offload(adap)) { 4427 want += adap->num_ofld_uld * s->ofldqsets; 4428 ofld_need = adap->num_ofld_uld * nchan; 4429 } 4430 if (is_pci_uld(adap)) { 4431 want += adap->num_uld * s->ofldqsets; 4432 uld_need = adap->num_uld * nchan; 4433 } 4434 #ifdef CONFIG_CHELSIO_T4_DCB 4435 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for 4436 * each port. 4437 */ 4438 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need + uld_need; 4439 #else 4440 need = adap->params.nports + EXTRA_VECS + ofld_need + uld_need; 4441 #endif 4442 allocated = pci_enable_msix_range(adap->pdev, entries, need, want); 4443 if (allocated < 0) { 4444 dev_info(adap->pdev_dev, "not enough MSI-X vectors left," 4445 " not using MSI-X\n"); 4446 kfree(entries); 4447 return allocated; 4448 } 4449 4450 /* Distribute available vectors to the various queue groups. 4451 * Every group gets its minimum requirement and NIC gets top 4452 * priority for leftovers. 4453 */ 4454 i = allocated - EXTRA_VECS - ofld_need - uld_need; 4455 if (i < s->max_ethqsets) { 4456 s->max_ethqsets = i; 4457 if (i < s->ethqsets) 4458 reduce_ethqs(adap, i); 4459 } 4460 if (is_uld(adap)) { 4461 if (allocated < want) 4462 s->nqs_per_uld = nchan; 4463 else 4464 s->nqs_per_uld = s->ofldqsets; 4465 } 4466 4467 for (i = 0; i < (s->max_ethqsets + EXTRA_VECS); ++i) 4468 adap->msix_info[i].vec = entries[i].vector; 4469 if (is_uld(adap)) { 4470 for (j = 0 ; i < allocated; ++i, j++) { 4471 adap->msix_info_ulds[j].vec = entries[i].vector; 4472 adap->msix_info_ulds[j].idx = i; 4473 } 4474 adap->msix_bmap_ulds.mapsize = j; 4475 } 4476 dev_info(adap->pdev_dev, "%d MSI-X vectors allocated, " 4477 "nic %d per uld %d\n", 4478 allocated, s->max_ethqsets, s->nqs_per_uld); 4479 4480 kfree(entries); 4481 return 0; 4482 } 4483 4484 #undef EXTRA_VECS 4485 4486 static int init_rss(struct adapter *adap) 4487 { 4488 unsigned int i; 4489 int err; 4490 4491 err = t4_init_rss_mode(adap, adap->mbox); 4492 if (err) 4493 return err; 4494 4495 for_each_port(adap, i) { 4496 struct port_info *pi = adap2pinfo(adap, i); 4497 4498 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL); 4499 if (!pi->rss) 4500 return -ENOMEM; 4501 } 4502 return 0; 4503 } 4504 4505 static int cxgb4_get_pcie_dev_link_caps(struct adapter *adap, 4506 enum pci_bus_speed *speed, 4507 enum pcie_link_width *width) 4508 { 4509 u32 lnkcap1, lnkcap2; 4510 int err1, err2; 4511 4512 #define PCIE_MLW_CAP_SHIFT 4 /* start of MLW mask in link capabilities */ 4513 4514 *speed = PCI_SPEED_UNKNOWN; 4515 *width = PCIE_LNK_WIDTH_UNKNOWN; 4516 4517 err1 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP, 4518 &lnkcap1); 4519 err2 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP2, 4520 &lnkcap2); 4521 if (!err2 && lnkcap2) { /* PCIe r3.0-compliant */ 4522 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB) 4523 *speed = PCIE_SPEED_8_0GT; 4524 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB) 4525 *speed = PCIE_SPEED_5_0GT; 4526 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB) 4527 *speed = PCIE_SPEED_2_5GT; 4528 } 4529 if (!err1) { 4530 *width = (lnkcap1 & PCI_EXP_LNKCAP_MLW) >> PCIE_MLW_CAP_SHIFT; 4531 if (!lnkcap2) { /* pre-r3.0 */ 4532 if (lnkcap1 & PCI_EXP_LNKCAP_SLS_5_0GB) 4533 *speed = PCIE_SPEED_5_0GT; 4534 else if (lnkcap1 & PCI_EXP_LNKCAP_SLS_2_5GB) 4535 *speed = PCIE_SPEED_2_5GT; 4536 } 4537 } 4538 4539 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) 4540 return err1 ? err1 : err2 ? err2 : -EINVAL; 4541 return 0; 4542 } 4543 4544 static void cxgb4_check_pcie_caps(struct adapter *adap) 4545 { 4546 enum pcie_link_width width, width_cap; 4547 enum pci_bus_speed speed, speed_cap; 4548 4549 #define PCIE_SPEED_STR(speed) \ 4550 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : \ 4551 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : \ 4552 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : \ 4553 "Unknown") 4554 4555 if (cxgb4_get_pcie_dev_link_caps(adap, &speed_cap, &width_cap)) { 4556 dev_warn(adap->pdev_dev, 4557 "Unable to determine PCIe device BW capabilities\n"); 4558 return; 4559 } 4560 4561 if (pcie_get_minimum_link(adap->pdev, &speed, &width) || 4562 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) { 4563 dev_warn(adap->pdev_dev, 4564 "Unable to determine PCI Express bandwidth.\n"); 4565 return; 4566 } 4567 4568 dev_info(adap->pdev_dev, "PCIe link speed is %s, device supports %s\n", 4569 PCIE_SPEED_STR(speed), PCIE_SPEED_STR(speed_cap)); 4570 dev_info(adap->pdev_dev, "PCIe link width is x%d, device supports x%d\n", 4571 width, width_cap); 4572 if (speed < speed_cap || width < width_cap) 4573 dev_info(adap->pdev_dev, 4574 "A slot with more lanes and/or higher speed is " 4575 "suggested for optimal performance.\n"); 4576 } 4577 4578 /* Dump basic information about the adapter */ 4579 static void print_adapter_info(struct adapter *adapter) 4580 { 4581 /* Hardware/Firmware/etc. Version/Revision IDs */ 4582 t4_dump_version_info(adapter); 4583 4584 /* Software/Hardware configuration */ 4585 dev_info(adapter->pdev_dev, "Configuration: %sNIC %s, %s capable\n", 4586 is_offload(adapter) ? "R" : "", 4587 ((adapter->flags & USING_MSIX) ? "MSI-X" : 4588 (adapter->flags & USING_MSI) ? "MSI" : ""), 4589 is_offload(adapter) ? "Offload" : "non-Offload"); 4590 } 4591 4592 static void print_port_info(const struct net_device *dev) 4593 { 4594 char buf[80]; 4595 char *bufp = buf; 4596 const char *spd = ""; 4597 const struct port_info *pi = netdev_priv(dev); 4598 const struct adapter *adap = pi->adapter; 4599 4600 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB) 4601 spd = " 2.5 GT/s"; 4602 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB) 4603 spd = " 5 GT/s"; 4604 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB) 4605 spd = " 8 GT/s"; 4606 4607 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100M) 4608 bufp += sprintf(bufp, "100M/"); 4609 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_1G) 4610 bufp += sprintf(bufp, "1G/"); 4611 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G) 4612 bufp += sprintf(bufp, "10G/"); 4613 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G) 4614 bufp += sprintf(bufp, "25G/"); 4615 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G) 4616 bufp += sprintf(bufp, "40G/"); 4617 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G) 4618 bufp += sprintf(bufp, "50G/"); 4619 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G) 4620 bufp += sprintf(bufp, "100G/"); 4621 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_200G) 4622 bufp += sprintf(bufp, "200G/"); 4623 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_400G) 4624 bufp += sprintf(bufp, "400G/"); 4625 if (bufp != buf) 4626 --bufp; 4627 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type)); 4628 4629 netdev_info(dev, "%s: Chelsio %s (%s) %s\n", 4630 dev->name, adap->params.vpd.id, adap->name, buf); 4631 } 4632 4633 /* 4634 * Free the following resources: 4635 * - memory used for tables 4636 * - MSI/MSI-X 4637 * - net devices 4638 * - resources FW is holding for us 4639 */ 4640 static void free_some_resources(struct adapter *adapter) 4641 { 4642 unsigned int i; 4643 4644 kvfree(adapter->l2t); 4645 t4_cleanup_sched(adapter); 4646 kvfree(adapter->tids.tid_tab); 4647 cxgb4_cleanup_tc_flower(adapter); 4648 cxgb4_cleanup_tc_u32(adapter); 4649 kfree(adapter->sge.egr_map); 4650 kfree(adapter->sge.ingr_map); 4651 kfree(adapter->sge.starving_fl); 4652 kfree(adapter->sge.txq_maperr); 4653 #ifdef CONFIG_DEBUG_FS 4654 kfree(adapter->sge.blocked_fl); 4655 #endif 4656 disable_msi(adapter); 4657 4658 for_each_port(adapter, i) 4659 if (adapter->port[i]) { 4660 struct port_info *pi = adap2pinfo(adapter, i); 4661 4662 if (pi->viid != 0) 4663 t4_free_vi(adapter, adapter->mbox, adapter->pf, 4664 0, pi->viid); 4665 kfree(adap2pinfo(adapter, i)->rss); 4666 free_netdev(adapter->port[i]); 4667 } 4668 if (adapter->flags & FW_OK) 4669 t4_fw_bye(adapter, adapter->pf); 4670 } 4671 4672 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN) 4673 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \ 4674 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA) 4675 #define SEGMENT_SIZE 128 4676 4677 static int get_chip_type(struct pci_dev *pdev, u32 pl_rev) 4678 { 4679 u16 device_id; 4680 4681 /* Retrieve adapter's device ID */ 4682 pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id); 4683 4684 switch (device_id >> 12) { 4685 case CHELSIO_T4: 4686 return CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev); 4687 case CHELSIO_T5: 4688 return CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev); 4689 case CHELSIO_T6: 4690 return CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); 4691 default: 4692 dev_err(&pdev->dev, "Device %d is not supported\n", 4693 device_id); 4694 } 4695 return -EINVAL; 4696 } 4697 4698 #ifdef CONFIG_PCI_IOV 4699 static void dummy_setup(struct net_device *dev) 4700 { 4701 dev->type = ARPHRD_NONE; 4702 dev->mtu = 0; 4703 dev->hard_header_len = 0; 4704 dev->addr_len = 0; 4705 dev->tx_queue_len = 0; 4706 dev->flags |= IFF_NOARP; 4707 dev->priv_flags |= IFF_NO_QUEUE; 4708 4709 /* Initialize the device structure. */ 4710 dev->netdev_ops = &cxgb4_mgmt_netdev_ops; 4711 dev->ethtool_ops = &cxgb4_mgmt_ethtool_ops; 4712 dev->needs_free_netdev = true; 4713 } 4714 4715 static int config_mgmt_dev(struct pci_dev *pdev) 4716 { 4717 struct adapter *adap = pci_get_drvdata(pdev); 4718 struct net_device *netdev; 4719 struct port_info *pi; 4720 char name[IFNAMSIZ]; 4721 int err; 4722 4723 snprintf(name, IFNAMSIZ, "mgmtpf%d%d", adap->adap_idx, adap->pf); 4724 netdev = alloc_netdev(sizeof(struct port_info), name, NET_NAME_UNKNOWN, 4725 dummy_setup); 4726 if (!netdev) 4727 return -ENOMEM; 4728 4729 pi = netdev_priv(netdev); 4730 pi->adapter = adap; 4731 pi->tx_chan = adap->pf % adap->params.nports; 4732 SET_NETDEV_DEV(netdev, &pdev->dev); 4733 4734 adap->port[0] = netdev; 4735 pi->port_id = 0; 4736 4737 err = register_netdev(adap->port[0]); 4738 if (err) { 4739 pr_info("Unable to register VF mgmt netdev %s\n", name); 4740 free_netdev(adap->port[0]); 4741 adap->port[0] = NULL; 4742 return err; 4743 } 4744 return 0; 4745 } 4746 4747 static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs) 4748 { 4749 struct adapter *adap = pci_get_drvdata(pdev); 4750 int err = 0; 4751 int current_vfs = pci_num_vf(pdev); 4752 u32 pcie_fw; 4753 4754 pcie_fw = readl(adap->regs + PCIE_FW_A); 4755 /* Check if cxgb4 is the MASTER and fw is initialized */ 4756 if (!(pcie_fw & PCIE_FW_INIT_F) || 4757 !(pcie_fw & PCIE_FW_MASTER_VLD_F) || 4758 PCIE_FW_MASTER_G(pcie_fw) != 4) { 4759 dev_warn(&pdev->dev, 4760 "cxgb4 driver needs to be MASTER to support SRIOV\n"); 4761 return -EOPNOTSUPP; 4762 } 4763 4764 /* If any of the VF's is already assigned to Guest OS, then 4765 * SRIOV for the same cannot be modified 4766 */ 4767 if (current_vfs && pci_vfs_assigned(pdev)) { 4768 dev_err(&pdev->dev, 4769 "Cannot modify SR-IOV while VFs are assigned\n"); 4770 num_vfs = current_vfs; 4771 return num_vfs; 4772 } 4773 4774 /* Disable SRIOV when zero is passed. 4775 * One needs to disable SRIOV before modifying it, else 4776 * stack throws the below warning: 4777 * " 'n' VFs already enabled. Disable before enabling 'm' VFs." 4778 */ 4779 if (!num_vfs) { 4780 pci_disable_sriov(pdev); 4781 if (adap->port[0]) { 4782 unregister_netdev(adap->port[0]); 4783 adap->port[0] = NULL; 4784 } 4785 /* free VF resources */ 4786 kfree(adap->vfinfo); 4787 adap->vfinfo = NULL; 4788 adap->num_vfs = 0; 4789 return num_vfs; 4790 } 4791 4792 if (num_vfs != current_vfs) { 4793 err = pci_enable_sriov(pdev, num_vfs); 4794 if (err) 4795 return err; 4796 4797 adap->num_vfs = num_vfs; 4798 err = config_mgmt_dev(pdev); 4799 if (err) 4800 return err; 4801 } 4802 4803 adap->vfinfo = kcalloc(adap->num_vfs, 4804 sizeof(struct vf_info), GFP_KERNEL); 4805 if (adap->vfinfo) 4806 fill_vf_station_mac_addr(adap); 4807 return num_vfs; 4808 } 4809 #endif 4810 4811 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 4812 { 4813 int func, i, err, s_qpp, qpp, num_seg; 4814 struct port_info *pi; 4815 bool highdma = false; 4816 struct adapter *adapter = NULL; 4817 struct net_device *netdev; 4818 void __iomem *regs; 4819 u32 whoami, pl_rev; 4820 enum chip_type chip; 4821 static int adap_idx = 1; 4822 #ifdef CONFIG_PCI_IOV 4823 u32 v, port_vec; 4824 #endif 4825 4826 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION); 4827 4828 err = pci_request_regions(pdev, KBUILD_MODNAME); 4829 if (err) { 4830 /* Just info, some other driver may have claimed the device. */ 4831 dev_info(&pdev->dev, "cannot obtain PCI resources\n"); 4832 return err; 4833 } 4834 4835 err = pci_enable_device(pdev); 4836 if (err) { 4837 dev_err(&pdev->dev, "cannot enable PCI device\n"); 4838 goto out_release_regions; 4839 } 4840 4841 regs = pci_ioremap_bar(pdev, 0); 4842 if (!regs) { 4843 dev_err(&pdev->dev, "cannot map device registers\n"); 4844 err = -ENOMEM; 4845 goto out_disable_device; 4846 } 4847 4848 err = t4_wait_dev_ready(regs); 4849 if (err < 0) 4850 goto out_unmap_bar0; 4851 4852 /* We control everything through one PF */ 4853 whoami = readl(regs + PL_WHOAMI_A); 4854 pl_rev = REV_G(readl(regs + PL_REV_A)); 4855 chip = get_chip_type(pdev, pl_rev); 4856 func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ? 4857 SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami); 4858 if (func != ent->driver_data) { 4859 #ifndef CONFIG_PCI_IOV 4860 iounmap(regs); 4861 #endif 4862 pci_disable_device(pdev); 4863 pci_save_state(pdev); /* to restore SR-IOV later */ 4864 goto sriov; 4865 } 4866 4867 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 4868 highdma = true; 4869 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 4870 if (err) { 4871 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for " 4872 "coherent allocations\n"); 4873 goto out_unmap_bar0; 4874 } 4875 } else { 4876 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 4877 if (err) { 4878 dev_err(&pdev->dev, "no usable DMA configuration\n"); 4879 goto out_unmap_bar0; 4880 } 4881 } 4882 4883 pci_enable_pcie_error_reporting(pdev); 4884 pci_set_master(pdev); 4885 pci_save_state(pdev); 4886 4887 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 4888 if (!adapter) { 4889 err = -ENOMEM; 4890 goto out_unmap_bar0; 4891 } 4892 adap_idx++; 4893 4894 adapter->workq = create_singlethread_workqueue("cxgb4"); 4895 if (!adapter->workq) { 4896 err = -ENOMEM; 4897 goto out_free_adapter; 4898 } 4899 4900 adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) + 4901 (sizeof(struct mbox_cmd) * 4902 T4_OS_LOG_MBOX_CMDS), 4903 GFP_KERNEL); 4904 if (!adapter->mbox_log) { 4905 err = -ENOMEM; 4906 goto out_free_adapter; 4907 } 4908 adapter->mbox_log->size = T4_OS_LOG_MBOX_CMDS; 4909 4910 /* PCI device has been enabled */ 4911 adapter->flags |= DEV_ENABLED; 4912 4913 adapter->regs = regs; 4914 adapter->pdev = pdev; 4915 adapter->pdev_dev = &pdev->dev; 4916 adapter->name = pci_name(pdev); 4917 adapter->mbox = func; 4918 adapter->pf = func; 4919 adapter->msg_enable = DFLT_MSG_ENABLE; 4920 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map)); 4921 4922 /* If possible, we use PCIe Relaxed Ordering Attribute to deliver 4923 * Ingress Packet Data to Free List Buffers in order to allow for 4924 * chipset performance optimizations between the Root Complex and 4925 * Memory Controllers. (Messages to the associated Ingress Queue 4926 * notifying new Packet Placement in the Free Lists Buffers will be 4927 * send without the Relaxed Ordering Attribute thus guaranteeing that 4928 * all preceding PCIe Transaction Layer Packets will be processed 4929 * first.) But some Root Complexes have various issues with Upstream 4930 * Transaction Layer Packets with the Relaxed Ordering Attribute set. 4931 * The PCIe devices which under the Root Complexes will be cleared the 4932 * Relaxed Ordering bit in the configuration space, So we check our 4933 * PCIe configuration space to see if it's flagged with advice against 4934 * using Relaxed Ordering. 4935 */ 4936 if (!pcie_relaxed_ordering_enabled(pdev)) 4937 adapter->flags |= ROOT_NO_RELAXED_ORDERING; 4938 4939 spin_lock_init(&adapter->stats_lock); 4940 spin_lock_init(&adapter->tid_release_lock); 4941 spin_lock_init(&adapter->win0_lock); 4942 spin_lock_init(&adapter->mbox_lock); 4943 4944 INIT_LIST_HEAD(&adapter->mlist.list); 4945 4946 INIT_WORK(&adapter->tid_release_task, process_tid_release_list); 4947 INIT_WORK(&adapter->db_full_task, process_db_full); 4948 INIT_WORK(&adapter->db_drop_task, process_db_drop); 4949 4950 err = t4_prep_adapter(adapter); 4951 if (err) 4952 goto out_free_adapter; 4953 4954 4955 if (!is_t4(adapter->params.chip)) { 4956 s_qpp = (QUEUESPERPAGEPF0_S + 4957 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * 4958 adapter->pf); 4959 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter, 4960 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp); 4961 num_seg = PAGE_SIZE / SEGMENT_SIZE; 4962 4963 /* Each segment size is 128B. Write coalescing is enabled only 4964 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the 4965 * queue is less no of segments that can be accommodated in 4966 * a page size. 4967 */ 4968 if (qpp > num_seg) { 4969 dev_err(&pdev->dev, 4970 "Incorrect number of egress queues per page\n"); 4971 err = -EINVAL; 4972 goto out_free_adapter; 4973 } 4974 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2), 4975 pci_resource_len(pdev, 2)); 4976 if (!adapter->bar2) { 4977 dev_err(&pdev->dev, "cannot map device bar2 region\n"); 4978 err = -ENOMEM; 4979 goto out_free_adapter; 4980 } 4981 } 4982 4983 setup_memwin(adapter); 4984 err = adap_init0(adapter); 4985 #ifdef CONFIG_DEBUG_FS 4986 bitmap_zero(adapter->sge.blocked_fl, adapter->sge.egr_sz); 4987 #endif 4988 setup_memwin_rdma(adapter); 4989 if (err) 4990 goto out_unmap_bar; 4991 4992 /* configure SGE_STAT_CFG_A to read WC stats */ 4993 if (!is_t4(adapter->params.chip)) 4994 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7) | 4995 (is_t5(adapter->params.chip) ? STATMODE_V(0) : 4996 T6_STATMODE_V(0))); 4997 4998 for_each_port(adapter, i) { 4999 netdev = alloc_etherdev_mq(sizeof(struct port_info), 5000 MAX_ETH_QSETS); 5001 if (!netdev) { 5002 err = -ENOMEM; 5003 goto out_free_dev; 5004 } 5005 5006 SET_NETDEV_DEV(netdev, &pdev->dev); 5007 5008 adapter->port[i] = netdev; 5009 pi = netdev_priv(netdev); 5010 pi->adapter = adapter; 5011 pi->xact_addr_filt = -1; 5012 pi->port_id = i; 5013 netdev->irq = pdev->irq; 5014 5015 netdev->hw_features = NETIF_F_SG | TSO_FLAGS | 5016 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 5017 NETIF_F_RXCSUM | NETIF_F_RXHASH | 5018 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 5019 NETIF_F_HW_TC; 5020 if (highdma) 5021 netdev->hw_features |= NETIF_F_HIGHDMA; 5022 netdev->features |= netdev->hw_features; 5023 netdev->vlan_features = netdev->features & VLAN_FEAT; 5024 5025 netdev->priv_flags |= IFF_UNICAST_FLT; 5026 5027 /* MTU range: 81 - 9600 */ 5028 netdev->min_mtu = 81; /* accommodate SACK */ 5029 netdev->max_mtu = MAX_MTU; 5030 5031 netdev->netdev_ops = &cxgb4_netdev_ops; 5032 #ifdef CONFIG_CHELSIO_T4_DCB 5033 netdev->dcbnl_ops = &cxgb4_dcb_ops; 5034 cxgb4_dcb_state_init(netdev); 5035 #endif 5036 cxgb4_set_ethtool_ops(netdev); 5037 } 5038 5039 cxgb4_init_ethtool_dump(adapter); 5040 5041 pci_set_drvdata(pdev, adapter); 5042 5043 if (adapter->flags & FW_OK) { 5044 err = t4_port_init(adapter, func, func, 0); 5045 if (err) 5046 goto out_free_dev; 5047 } else if (adapter->params.nports == 1) { 5048 /* If we don't have a connection to the firmware -- possibly 5049 * because of an error -- grab the raw VPD parameters so we 5050 * can set the proper MAC Address on the debug network 5051 * interface that we've created. 5052 */ 5053 u8 hw_addr[ETH_ALEN]; 5054 u8 *na = adapter->params.vpd.na; 5055 5056 err = t4_get_raw_vpd_params(adapter, &adapter->params.vpd); 5057 if (!err) { 5058 for (i = 0; i < ETH_ALEN; i++) 5059 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + 5060 hex2val(na[2 * i + 1])); 5061 t4_set_hw_addr(adapter, 0, hw_addr); 5062 } 5063 } 5064 5065 /* Configure queues and allocate tables now, they can be needed as 5066 * soon as the first register_netdev completes. 5067 */ 5068 cfg_queues(adapter); 5069 5070 adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end); 5071 if (!adapter->l2t) { 5072 /* We tolerate a lack of L2T, giving up some functionality */ 5073 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n"); 5074 adapter->params.offload = 0; 5075 } 5076 5077 #if IS_ENABLED(CONFIG_IPV6) 5078 if ((CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) && 5079 (!(t4_read_reg(adapter, LE_DB_CONFIG_A) & ASLIPCOMPEN_F))) { 5080 /* CLIP functionality is not present in hardware, 5081 * hence disable all offload features 5082 */ 5083 dev_warn(&pdev->dev, 5084 "CLIP not enabled in hardware, continuing\n"); 5085 adapter->params.offload = 0; 5086 } else { 5087 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start, 5088 adapter->clipt_end); 5089 if (!adapter->clipt) { 5090 /* We tolerate a lack of clip_table, giving up 5091 * some functionality 5092 */ 5093 dev_warn(&pdev->dev, 5094 "could not allocate Clip table, continuing\n"); 5095 adapter->params.offload = 0; 5096 } 5097 } 5098 #endif 5099 5100 for_each_port(adapter, i) { 5101 pi = adap2pinfo(adapter, i); 5102 pi->sched_tbl = t4_init_sched(adapter->params.nsched_cls); 5103 if (!pi->sched_tbl) 5104 dev_warn(&pdev->dev, 5105 "could not activate scheduling on port %d\n", 5106 i); 5107 } 5108 5109 if (tid_init(&adapter->tids) < 0) { 5110 dev_warn(&pdev->dev, "could not allocate TID table, " 5111 "continuing\n"); 5112 adapter->params.offload = 0; 5113 } else { 5114 adapter->tc_u32 = cxgb4_init_tc_u32(adapter); 5115 if (!adapter->tc_u32) 5116 dev_warn(&pdev->dev, 5117 "could not offload tc u32, continuing\n"); 5118 5119 cxgb4_init_tc_flower(adapter); 5120 } 5121 5122 if (is_offload(adapter)) { 5123 if (t4_read_reg(adapter, LE_DB_CONFIG_A) & HASHEN_F) { 5124 u32 hash_base, hash_reg; 5125 5126 if (chip <= CHELSIO_T5) { 5127 hash_reg = LE_DB_TID_HASHBASE_A; 5128 hash_base = t4_read_reg(adapter, hash_reg); 5129 adapter->tids.hash_base = hash_base / 4; 5130 } else { 5131 hash_reg = T6_LE_DB_HASH_TID_BASE_A; 5132 hash_base = t4_read_reg(adapter, hash_reg); 5133 adapter->tids.hash_base = hash_base; 5134 } 5135 } 5136 } 5137 5138 /* See what interrupts we'll be using */ 5139 if (msi > 1 && enable_msix(adapter) == 0) 5140 adapter->flags |= USING_MSIX; 5141 else if (msi > 0 && pci_enable_msi(pdev) == 0) { 5142 adapter->flags |= USING_MSI; 5143 if (msi > 1) 5144 free_msix_info(adapter); 5145 } 5146 5147 /* check for PCI Express bandwidth capabiltites */ 5148 cxgb4_check_pcie_caps(adapter); 5149 5150 err = init_rss(adapter); 5151 if (err) 5152 goto out_free_dev; 5153 5154 /* 5155 * The card is now ready to go. If any errors occur during device 5156 * registration we do not fail the whole card but rather proceed only 5157 * with the ports we manage to register successfully. However we must 5158 * register at least one net device. 5159 */ 5160 for_each_port(adapter, i) { 5161 pi = adap2pinfo(adapter, i); 5162 adapter->port[i]->dev_port = pi->lport; 5163 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets); 5164 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets); 5165 5166 netif_carrier_off(adapter->port[i]); 5167 5168 err = register_netdev(adapter->port[i]); 5169 if (err) 5170 break; 5171 adapter->chan_map[pi->tx_chan] = i; 5172 print_port_info(adapter->port[i]); 5173 } 5174 if (i == 0) { 5175 dev_err(&pdev->dev, "could not register any net devices\n"); 5176 goto out_free_dev; 5177 } 5178 if (err) { 5179 dev_warn(&pdev->dev, "only %d net devices registered\n", i); 5180 err = 0; 5181 } 5182 5183 if (cxgb4_debugfs_root) { 5184 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev), 5185 cxgb4_debugfs_root); 5186 setup_debugfs(adapter); 5187 } 5188 5189 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */ 5190 pdev->needs_freset = 1; 5191 5192 if (is_uld(adapter)) { 5193 mutex_lock(&uld_mutex); 5194 list_add_tail(&adapter->list_node, &adapter_list); 5195 mutex_unlock(&uld_mutex); 5196 } 5197 5198 if (!is_t4(adapter->params.chip)) 5199 cxgb4_ptp_init(adapter); 5200 5201 print_adapter_info(adapter); 5202 setup_fw_sge_queues(adapter); 5203 return 0; 5204 5205 sriov: 5206 #ifdef CONFIG_PCI_IOV 5207 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 5208 if (!adapter) { 5209 err = -ENOMEM; 5210 goto free_pci_region; 5211 } 5212 5213 adapter->pdev = pdev; 5214 adapter->pdev_dev = &pdev->dev; 5215 adapter->name = pci_name(pdev); 5216 adapter->mbox = func; 5217 adapter->pf = func; 5218 adapter->regs = regs; 5219 adapter->adap_idx = adap_idx; 5220 adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) + 5221 (sizeof(struct mbox_cmd) * 5222 T4_OS_LOG_MBOX_CMDS), 5223 GFP_KERNEL); 5224 if (!adapter->mbox_log) { 5225 err = -ENOMEM; 5226 goto free_adapter; 5227 } 5228 spin_lock_init(&adapter->mbox_lock); 5229 INIT_LIST_HEAD(&adapter->mlist.list); 5230 5231 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 5232 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC); 5233 err = t4_query_params(adapter, adapter->mbox, adapter->pf, 0, 1, 5234 &v, &port_vec); 5235 if (err < 0) { 5236 dev_err(adapter->pdev_dev, "Could not fetch port params\n"); 5237 goto free_mbox_log; 5238 } 5239 5240 adapter->params.nports = hweight32(port_vec); 5241 pci_set_drvdata(pdev, adapter); 5242 return 0; 5243 5244 free_mbox_log: 5245 kfree(adapter->mbox_log); 5246 free_adapter: 5247 kfree(adapter); 5248 free_pci_region: 5249 iounmap(regs); 5250 pci_disable_sriov(pdev); 5251 pci_release_regions(pdev); 5252 return err; 5253 #else 5254 return 0; 5255 #endif 5256 5257 out_free_dev: 5258 free_some_resources(adapter); 5259 if (adapter->flags & USING_MSIX) 5260 free_msix_info(adapter); 5261 if (adapter->num_uld || adapter->num_ofld_uld) 5262 t4_uld_mem_free(adapter); 5263 out_unmap_bar: 5264 if (!is_t4(adapter->params.chip)) 5265 iounmap(adapter->bar2); 5266 out_free_adapter: 5267 if (adapter->workq) 5268 destroy_workqueue(adapter->workq); 5269 5270 kfree(adapter->mbox_log); 5271 kfree(adapter); 5272 out_unmap_bar0: 5273 iounmap(regs); 5274 out_disable_device: 5275 pci_disable_pcie_error_reporting(pdev); 5276 pci_disable_device(pdev); 5277 out_release_regions: 5278 pci_release_regions(pdev); 5279 return err; 5280 } 5281 5282 static void remove_one(struct pci_dev *pdev) 5283 { 5284 struct adapter *adapter = pci_get_drvdata(pdev); 5285 5286 if (!adapter) { 5287 pci_release_regions(pdev); 5288 return; 5289 } 5290 5291 adapter->flags |= SHUTTING_DOWN; 5292 5293 if (adapter->pf == 4) { 5294 int i; 5295 5296 /* Tear down per-adapter Work Queue first since it can contain 5297 * references to our adapter data structure. 5298 */ 5299 destroy_workqueue(adapter->workq); 5300 5301 if (is_uld(adapter)) { 5302 detach_ulds(adapter); 5303 t4_uld_clean_up(adapter); 5304 } 5305 5306 disable_interrupts(adapter); 5307 5308 for_each_port(adapter, i) 5309 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 5310 unregister_netdev(adapter->port[i]); 5311 5312 debugfs_remove_recursive(adapter->debugfs_root); 5313 5314 if (!is_t4(adapter->params.chip)) 5315 cxgb4_ptp_stop(adapter); 5316 5317 /* If we allocated filters, free up state associated with any 5318 * valid filters ... 5319 */ 5320 clear_all_filters(adapter); 5321 5322 if (adapter->flags & FULL_INIT_DONE) 5323 cxgb_down(adapter); 5324 5325 if (adapter->flags & USING_MSIX) 5326 free_msix_info(adapter); 5327 if (adapter->num_uld || adapter->num_ofld_uld) 5328 t4_uld_mem_free(adapter); 5329 free_some_resources(adapter); 5330 #if IS_ENABLED(CONFIG_IPV6) 5331 t4_cleanup_clip_tbl(adapter); 5332 #endif 5333 iounmap(adapter->regs); 5334 if (!is_t4(adapter->params.chip)) 5335 iounmap(adapter->bar2); 5336 pci_disable_pcie_error_reporting(pdev); 5337 if ((adapter->flags & DEV_ENABLED)) { 5338 pci_disable_device(pdev); 5339 adapter->flags &= ~DEV_ENABLED; 5340 } 5341 pci_release_regions(pdev); 5342 kfree(adapter->mbox_log); 5343 synchronize_rcu(); 5344 kfree(adapter); 5345 } 5346 #ifdef CONFIG_PCI_IOV 5347 else { 5348 if (adapter->port[0]) 5349 unregister_netdev(adapter->port[0]); 5350 iounmap(adapter->regs); 5351 kfree(adapter->vfinfo); 5352 kfree(adapter->mbox_log); 5353 kfree(adapter); 5354 pci_disable_sriov(pdev); 5355 pci_release_regions(pdev); 5356 } 5357 #endif 5358 } 5359 5360 /* "Shutdown" quiesces the device, stopping Ingress Packet and Interrupt 5361 * delivery. This is essentially a stripped down version of the PCI remove() 5362 * function where we do the minimal amount of work necessary to shutdown any 5363 * further activity. 5364 */ 5365 static void shutdown_one(struct pci_dev *pdev) 5366 { 5367 struct adapter *adapter = pci_get_drvdata(pdev); 5368 5369 /* As with remove_one() above (see extended comment), we only want do 5370 * do cleanup on PCI Devices which went all the way through init_one() 5371 * ... 5372 */ 5373 if (!adapter) { 5374 pci_release_regions(pdev); 5375 return; 5376 } 5377 5378 adapter->flags |= SHUTTING_DOWN; 5379 5380 if (adapter->pf == 4) { 5381 int i; 5382 5383 for_each_port(adapter, i) 5384 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 5385 cxgb_close(adapter->port[i]); 5386 5387 if (is_uld(adapter)) { 5388 detach_ulds(adapter); 5389 t4_uld_clean_up(adapter); 5390 } 5391 5392 disable_interrupts(adapter); 5393 disable_msi(adapter); 5394 5395 t4_sge_stop(adapter); 5396 if (adapter->flags & FW_OK) 5397 t4_fw_bye(adapter, adapter->mbox); 5398 } 5399 #ifdef CONFIG_PCI_IOV 5400 else { 5401 if (adapter->port[0]) 5402 unregister_netdev(adapter->port[0]); 5403 iounmap(adapter->regs); 5404 kfree(adapter->vfinfo); 5405 kfree(adapter->mbox_log); 5406 kfree(adapter); 5407 pci_disable_sriov(pdev); 5408 pci_release_regions(pdev); 5409 } 5410 #endif 5411 } 5412 5413 static struct pci_driver cxgb4_driver = { 5414 .name = KBUILD_MODNAME, 5415 .id_table = cxgb4_pci_tbl, 5416 .probe = init_one, 5417 .remove = remove_one, 5418 .shutdown = shutdown_one, 5419 #ifdef CONFIG_PCI_IOV 5420 .sriov_configure = cxgb4_iov_configure, 5421 #endif 5422 .err_handler = &cxgb4_eeh, 5423 }; 5424 5425 static int __init cxgb4_init_module(void) 5426 { 5427 int ret; 5428 5429 /* Debugfs support is optional, just warn if this fails */ 5430 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); 5431 if (!cxgb4_debugfs_root) 5432 pr_warn("could not create debugfs entry, continuing\n"); 5433 5434 ret = pci_register_driver(&cxgb4_driver); 5435 if (ret < 0) 5436 debugfs_remove(cxgb4_debugfs_root); 5437 5438 #if IS_ENABLED(CONFIG_IPV6) 5439 if (!inet6addr_registered) { 5440 register_inet6addr_notifier(&cxgb4_inet6addr_notifier); 5441 inet6addr_registered = true; 5442 } 5443 #endif 5444 5445 return ret; 5446 } 5447 5448 static void __exit cxgb4_cleanup_module(void) 5449 { 5450 #if IS_ENABLED(CONFIG_IPV6) 5451 if (inet6addr_registered) { 5452 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier); 5453 inet6addr_registered = false; 5454 } 5455 #endif 5456 pci_unregister_driver(&cxgb4_driver); 5457 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */ 5458 } 5459 5460 module_init(cxgb4_init_module); 5461 module_exit(cxgb4_cleanup_module); 5462