1 /* 2 * This file is part of the Chelsio T4 Ethernet driver for Linux. 3 * 4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/bitmap.h> 38 #include <linux/crc32.h> 39 #include <linux/ctype.h> 40 #include <linux/debugfs.h> 41 #include <linux/err.h> 42 #include <linux/etherdevice.h> 43 #include <linux/firmware.h> 44 #include <linux/if.h> 45 #include <linux/if_vlan.h> 46 #include <linux/init.h> 47 #include <linux/log2.h> 48 #include <linux/mdio.h> 49 #include <linux/module.h> 50 #include <linux/moduleparam.h> 51 #include <linux/mutex.h> 52 #include <linux/netdevice.h> 53 #include <linux/pci.h> 54 #include <linux/aer.h> 55 #include <linux/rtnetlink.h> 56 #include <linux/sched.h> 57 #include <linux/seq_file.h> 58 #include <linux/sockios.h> 59 #include <linux/vmalloc.h> 60 #include <linux/workqueue.h> 61 #include <net/neighbour.h> 62 #include <net/netevent.h> 63 #include <net/addrconf.h> 64 #include <net/bonding.h> 65 #include <net/addrconf.h> 66 #include <asm/uaccess.h> 67 68 #include "cxgb4.h" 69 #include "t4_regs.h" 70 #include "t4_values.h" 71 #include "t4_msg.h" 72 #include "t4fw_api.h" 73 #include "t4fw_version.h" 74 #include "cxgb4_dcb.h" 75 #include "cxgb4_debugfs.h" 76 #include "clip_tbl.h" 77 #include "l2t.h" 78 79 char cxgb4_driver_name[] = KBUILD_MODNAME; 80 81 #ifdef DRV_VERSION 82 #undef DRV_VERSION 83 #endif 84 #define DRV_VERSION "2.0.0-ko" 85 const char cxgb4_driver_version[] = DRV_VERSION; 86 #define DRV_DESC "Chelsio T4/T5/T6 Network Driver" 87 88 /* Host shadow copy of ingress filter entry. This is in host native format 89 * and doesn't match the ordering or bit order, etc. of the hardware of the 90 * firmware command. The use of bit-field structure elements is purely to 91 * remind ourselves of the field size limitations and save memory in the case 92 * where the filter table is large. 93 */ 94 struct filter_entry { 95 /* Administrative fields for filter. 96 */ 97 u32 valid:1; /* filter allocated and valid */ 98 u32 locked:1; /* filter is administratively locked */ 99 100 u32 pending:1; /* filter action is pending firmware reply */ 101 u32 smtidx:8; /* Source MAC Table index for smac */ 102 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 103 104 /* The filter itself. Most of this is a straight copy of information 105 * provided by the extended ioctl(). Some fields are translated to 106 * internal forms -- for instance the Ingress Queue ID passed in from 107 * the ioctl() is translated into the Absolute Ingress Queue ID. 108 */ 109 struct ch_filter_specification fs; 110 }; 111 112 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ 113 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ 114 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) 115 116 /* Macros needed to support the PCI Device ID Table ... 117 */ 118 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ 119 static const struct pci_device_id cxgb4_pci_tbl[] = { 120 #define CH_PCI_DEVICE_ID_FUNCTION 0x4 121 122 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is 123 * called for both. 124 */ 125 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0 126 127 #define CH_PCI_ID_TABLE_ENTRY(devid) \ 128 {PCI_VDEVICE(CHELSIO, (devid)), 4} 129 130 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ 131 { 0, } \ 132 } 133 134 #include "t4_pci_id_tbl.h" 135 136 #define FW4_FNAME "cxgb4/t4fw.bin" 137 #define FW5_FNAME "cxgb4/t5fw.bin" 138 #define FW6_FNAME "cxgb4/t6fw.bin" 139 #define FW4_CFNAME "cxgb4/t4-config.txt" 140 #define FW5_CFNAME "cxgb4/t5-config.txt" 141 #define FW6_CFNAME "cxgb4/t6-config.txt" 142 #define PHY_AQ1202_FIRMWARE "cxgb4/aq1202_fw.cld" 143 #define PHY_BCM84834_FIRMWARE "cxgb4/bcm8483.bin" 144 #define PHY_AQ1202_DEVICEID 0x4409 145 #define PHY_BCM84834_DEVICEID 0x4486 146 147 MODULE_DESCRIPTION(DRV_DESC); 148 MODULE_AUTHOR("Chelsio Communications"); 149 MODULE_LICENSE("Dual BSD/GPL"); 150 MODULE_VERSION(DRV_VERSION); 151 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl); 152 MODULE_FIRMWARE(FW4_FNAME); 153 MODULE_FIRMWARE(FW5_FNAME); 154 MODULE_FIRMWARE(FW6_FNAME); 155 156 /* 157 * Normally we're willing to become the firmware's Master PF but will be happy 158 * if another PF has already become the Master and initialized the adapter. 159 * Setting "force_init" will cause this driver to forcibly establish itself as 160 * the Master PF and initialize the adapter. 161 */ 162 static uint force_init; 163 164 module_param(force_init, uint, 0644); 165 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter," 166 "deprecated parameter"); 167 168 static int dflt_msg_enable = DFLT_MSG_ENABLE; 169 170 module_param(dflt_msg_enable, int, 0644); 171 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap, " 172 "deprecated parameter"); 173 174 /* 175 * The driver uses the best interrupt scheme available on a platform in the 176 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which 177 * of these schemes the driver may consider as follows: 178 * 179 * msi = 2: choose from among all three options 180 * msi = 1: only consider MSI and INTx interrupts 181 * msi = 0: force INTx interrupts 182 */ 183 static int msi = 2; 184 185 module_param(msi, int, 0644); 186 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)"); 187 188 /* 189 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers 190 * offset by 2 bytes in order to have the IP headers line up on 4-byte 191 * boundaries. This is a requirement for many architectures which will throw 192 * a machine check fault if an attempt is made to access one of the 4-byte IP 193 * header fields on a non-4-byte boundary. And it's a major performance issue 194 * even on some architectures which allow it like some implementations of the 195 * x86 ISA. However, some architectures don't mind this and for some very 196 * edge-case performance sensitive applications (like forwarding large volumes 197 * of small packets), setting this DMA offset to 0 will decrease the number of 198 * PCI-E Bus transfers enough to measurably affect performance. 199 */ 200 static int rx_dma_offset = 2; 201 202 #ifdef CONFIG_PCI_IOV 203 /* Configure the number of PCI-E Virtual Function which are to be instantiated 204 * on SR-IOV Capable Physical Functions. 205 */ 206 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV]; 207 208 module_param_array(num_vf, uint, NULL, 0644); 209 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3"); 210 #endif 211 212 /* TX Queue select used to determine what algorithm to use for selecting TX 213 * queue. Select between the kernel provided function (select_queue=0) or user 214 * cxgb_select_queue function (select_queue=1) 215 * 216 * Default: select_queue=0 217 */ 218 static int select_queue; 219 module_param(select_queue, int, 0644); 220 MODULE_PARM_DESC(select_queue, 221 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method."); 222 223 static struct dentry *cxgb4_debugfs_root; 224 225 static LIST_HEAD(adapter_list); 226 static DEFINE_MUTEX(uld_mutex); 227 /* Adapter list to be accessed from atomic context */ 228 static LIST_HEAD(adap_rcu_list); 229 static DEFINE_SPINLOCK(adap_rcu_lock); 230 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX]; 231 static const char *const uld_str[] = { "RDMA", "iSCSI", "iSCSIT" }; 232 233 static void link_report(struct net_device *dev) 234 { 235 if (!netif_carrier_ok(dev)) 236 netdev_info(dev, "link down\n"); 237 else { 238 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" }; 239 240 const char *s; 241 const struct port_info *p = netdev_priv(dev); 242 243 switch (p->link_cfg.speed) { 244 case 10000: 245 s = "10Gbps"; 246 break; 247 case 1000: 248 s = "1000Mbps"; 249 break; 250 case 100: 251 s = "100Mbps"; 252 break; 253 case 40000: 254 s = "40Gbps"; 255 break; 256 default: 257 pr_info("%s: unsupported speed: %d\n", 258 dev->name, p->link_cfg.speed); 259 return; 260 } 261 262 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, 263 fc[p->link_cfg.fc]); 264 } 265 } 266 267 #ifdef CONFIG_CHELSIO_T4_DCB 268 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */ 269 static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable) 270 { 271 struct port_info *pi = netdev_priv(dev); 272 struct adapter *adap = pi->adapter; 273 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset]; 274 int i; 275 276 /* We use a simple mapping of Port TX Queue Index to DCB 277 * Priority when we're enabling DCB. 278 */ 279 for (i = 0; i < pi->nqsets; i++, txq++) { 280 u32 name, value; 281 int err; 282 283 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 284 FW_PARAMS_PARAM_X_V( 285 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) | 286 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id)); 287 value = enable ? i : 0xffffffff; 288 289 /* Since we can be called while atomic (from "interrupt 290 * level") we need to issue the Set Parameters Commannd 291 * without sleeping (timeout < 0). 292 */ 293 err = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1, 294 &name, &value, 295 -FW_CMD_MAX_TIMEOUT); 296 297 if (err) 298 dev_err(adap->pdev_dev, 299 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n", 300 enable ? "set" : "unset", pi->port_id, i, -err); 301 else 302 txq->dcb_prio = value; 303 } 304 } 305 #endif /* CONFIG_CHELSIO_T4_DCB */ 306 307 int cxgb4_dcb_enabled(const struct net_device *dev) 308 { 309 #ifdef CONFIG_CHELSIO_T4_DCB 310 struct port_info *pi = netdev_priv(dev); 311 312 if (!pi->dcb.enabled) 313 return 0; 314 315 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) || 316 (pi->dcb.state == CXGB4_DCB_STATE_HOST)); 317 #else 318 return 0; 319 #endif 320 } 321 EXPORT_SYMBOL(cxgb4_dcb_enabled); 322 323 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat) 324 { 325 struct net_device *dev = adapter->port[port_id]; 326 327 /* Skip changes from disabled ports. */ 328 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) { 329 if (link_stat) 330 netif_carrier_on(dev); 331 else { 332 #ifdef CONFIG_CHELSIO_T4_DCB 333 if (cxgb4_dcb_enabled(dev)) { 334 cxgb4_dcb_state_init(dev); 335 dcb_tx_queue_prio_enable(dev, false); 336 } 337 #endif /* CONFIG_CHELSIO_T4_DCB */ 338 netif_carrier_off(dev); 339 } 340 341 link_report(dev); 342 } 343 } 344 345 void t4_os_portmod_changed(const struct adapter *adap, int port_id) 346 { 347 static const char *mod_str[] = { 348 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM" 349 }; 350 351 const struct net_device *dev = adap->port[port_id]; 352 const struct port_info *pi = netdev_priv(dev); 353 354 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 355 netdev_info(dev, "port module unplugged\n"); 356 else if (pi->mod_type < ARRAY_SIZE(mod_str)) 357 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]); 358 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 359 netdev_info(dev, "%s: unsupported port module inserted\n", 360 dev->name); 361 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 362 netdev_info(dev, "%s: unknown port module inserted\n", 363 dev->name); 364 else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR) 365 netdev_info(dev, "%s: transceiver module error\n", dev->name); 366 else 367 netdev_info(dev, "%s: unknown module type %d inserted\n", 368 dev->name, pi->mod_type); 369 } 370 371 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */ 372 module_param(dbfifo_int_thresh, int, 0644); 373 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold"); 374 375 /* 376 * usecs to sleep while draining the dbfifo 377 */ 378 static int dbfifo_drain_delay = 1000; 379 module_param(dbfifo_drain_delay, int, 0644); 380 MODULE_PARM_DESC(dbfifo_drain_delay, 381 "usecs to sleep while draining the dbfifo"); 382 383 static inline int cxgb4_set_addr_hash(struct port_info *pi) 384 { 385 struct adapter *adap = pi->adapter; 386 u64 vec = 0; 387 bool ucast = false; 388 struct hash_mac_addr *entry; 389 390 /* Calculate the hash vector for the updated list and program it */ 391 list_for_each_entry(entry, &adap->mac_hlist, list) { 392 ucast |= is_unicast_ether_addr(entry->addr); 393 vec |= (1ULL << hash_mac_addr(entry->addr)); 394 } 395 return t4_set_addr_hash(adap, adap->mbox, pi->viid, ucast, 396 vec, false); 397 } 398 399 static int cxgb4_mac_sync(struct net_device *netdev, const u8 *mac_addr) 400 { 401 struct port_info *pi = netdev_priv(netdev); 402 struct adapter *adap = pi->adapter; 403 int ret; 404 u64 mhash = 0; 405 u64 uhash = 0; 406 bool free = false; 407 bool ucast = is_unicast_ether_addr(mac_addr); 408 const u8 *maclist[1] = {mac_addr}; 409 struct hash_mac_addr *new_entry; 410 411 ret = t4_alloc_mac_filt(adap, adap->mbox, pi->viid, free, 1, maclist, 412 NULL, ucast ? &uhash : &mhash, false); 413 if (ret < 0) 414 goto out; 415 /* if hash != 0, then add the addr to hash addr list 416 * so on the end we will calculate the hash for the 417 * list and program it 418 */ 419 if (uhash || mhash) { 420 new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC); 421 if (!new_entry) 422 return -ENOMEM; 423 ether_addr_copy(new_entry->addr, mac_addr); 424 list_add_tail(&new_entry->list, &adap->mac_hlist); 425 ret = cxgb4_set_addr_hash(pi); 426 } 427 out: 428 return ret < 0 ? ret : 0; 429 } 430 431 static int cxgb4_mac_unsync(struct net_device *netdev, const u8 *mac_addr) 432 { 433 struct port_info *pi = netdev_priv(netdev); 434 struct adapter *adap = pi->adapter; 435 int ret; 436 const u8 *maclist[1] = {mac_addr}; 437 struct hash_mac_addr *entry, *tmp; 438 439 /* If the MAC address to be removed is in the hash addr 440 * list, delete it from the list and update hash vector 441 */ 442 list_for_each_entry_safe(entry, tmp, &adap->mac_hlist, list) { 443 if (ether_addr_equal(entry->addr, mac_addr)) { 444 list_del(&entry->list); 445 kfree(entry); 446 return cxgb4_set_addr_hash(pi); 447 } 448 } 449 450 ret = t4_free_mac_filt(adap, adap->mbox, pi->viid, 1, maclist, false); 451 return ret < 0 ? -EINVAL : 0; 452 } 453 454 /* 455 * Set Rx properties of a port, such as promiscruity, address filters, and MTU. 456 * If @mtu is -1 it is left unchanged. 457 */ 458 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok) 459 { 460 struct port_info *pi = netdev_priv(dev); 461 struct adapter *adapter = pi->adapter; 462 463 if (!(dev->flags & IFF_PROMISC)) { 464 __dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); 465 if (!(dev->flags & IFF_ALLMULTI)) 466 __dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); 467 } 468 469 return t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu, 470 (dev->flags & IFF_PROMISC) ? 1 : 0, 471 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1, 472 sleep_ok); 473 } 474 475 /** 476 * link_start - enable a port 477 * @dev: the port to enable 478 * 479 * Performs the MAC and PHY actions needed to enable a port. 480 */ 481 static int link_start(struct net_device *dev) 482 { 483 int ret; 484 struct port_info *pi = netdev_priv(dev); 485 unsigned int mb = pi->adapter->pf; 486 487 /* 488 * We do not set address filters and promiscuity here, the stack does 489 * that step explicitly. 490 */ 491 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1, 492 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true); 493 if (ret == 0) { 494 ret = t4_change_mac(pi->adapter, mb, pi->viid, 495 pi->xact_addr_filt, dev->dev_addr, true, 496 true); 497 if (ret >= 0) { 498 pi->xact_addr_filt = ret; 499 ret = 0; 500 } 501 } 502 if (ret == 0) 503 ret = t4_link_l1cfg(pi->adapter, mb, pi->tx_chan, 504 &pi->link_cfg); 505 if (ret == 0) { 506 local_bh_disable(); 507 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true, 508 true, CXGB4_DCB_ENABLED); 509 local_bh_enable(); 510 } 511 512 return ret; 513 } 514 515 #ifdef CONFIG_CHELSIO_T4_DCB 516 /* Handle a Data Center Bridging update message from the firmware. */ 517 static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd) 518 { 519 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid)); 520 struct net_device *dev = adap->port[adap->chan_map[port]]; 521 int old_dcb_enabled = cxgb4_dcb_enabled(dev); 522 int new_dcb_enabled; 523 524 cxgb4_dcb_handle_fw_update(adap, pcmd); 525 new_dcb_enabled = cxgb4_dcb_enabled(dev); 526 527 /* If the DCB has become enabled or disabled on the port then we're 528 * going to need to set up/tear down DCB Priority parameters for the 529 * TX Queues associated with the port. 530 */ 531 if (new_dcb_enabled != old_dcb_enabled) 532 dcb_tx_queue_prio_enable(dev, new_dcb_enabled); 533 } 534 #endif /* CONFIG_CHELSIO_T4_DCB */ 535 536 /* Clear a filter and release any of its resources that we own. This also 537 * clears the filter's "pending" status. 538 */ 539 static void clear_filter(struct adapter *adap, struct filter_entry *f) 540 { 541 /* If the new or old filter have loopback rewriteing rules then we'll 542 * need to free any existing Layer Two Table (L2T) entries of the old 543 * filter rule. The firmware will handle freeing up any Source MAC 544 * Table (SMT) entries used for rewriting Source MAC Addresses in 545 * loopback rules. 546 */ 547 if (f->l2t) 548 cxgb4_l2t_release(f->l2t); 549 550 /* The zeroing of the filter rule below clears the filter valid, 551 * pending, locked flags, l2t pointer, etc. so it's all we need for 552 * this operation. 553 */ 554 memset(f, 0, sizeof(*f)); 555 } 556 557 /* Handle a filter write/deletion reply. 558 */ 559 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl) 560 { 561 unsigned int idx = GET_TID(rpl); 562 unsigned int nidx = idx - adap->tids.ftid_base; 563 unsigned int ret; 564 struct filter_entry *f; 565 566 if (idx >= adap->tids.ftid_base && nidx < 567 (adap->tids.nftids + adap->tids.nsftids)) { 568 idx = nidx; 569 ret = TCB_COOKIE_G(rpl->cookie); 570 f = &adap->tids.ftid_tab[idx]; 571 572 if (ret == FW_FILTER_WR_FLT_DELETED) { 573 /* Clear the filter when we get confirmation from the 574 * hardware that the filter has been deleted. 575 */ 576 clear_filter(adap, f); 577 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) { 578 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n", 579 idx); 580 clear_filter(adap, f); 581 } else if (ret == FW_FILTER_WR_FLT_ADDED) { 582 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff; 583 f->pending = 0; /* asynchronous setup completed */ 584 f->valid = 1; 585 } else { 586 /* Something went wrong. Issue a warning about the 587 * problem and clear everything out. 588 */ 589 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n", 590 idx, ret); 591 clear_filter(adap, f); 592 } 593 } 594 } 595 596 /* Response queue handler for the FW event queue. 597 */ 598 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp, 599 const struct pkt_gl *gl) 600 { 601 u8 opcode = ((const struct rss_header *)rsp)->opcode; 602 603 rsp++; /* skip RSS header */ 604 605 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. 606 */ 607 if (unlikely(opcode == CPL_FW4_MSG && 608 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) { 609 rsp++; 610 opcode = ((const struct rss_header *)rsp)->opcode; 611 rsp++; 612 if (opcode != CPL_SGE_EGR_UPDATE) { 613 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n" 614 , opcode); 615 goto out; 616 } 617 } 618 619 if (likely(opcode == CPL_SGE_EGR_UPDATE)) { 620 const struct cpl_sge_egr_update *p = (void *)rsp; 621 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid)); 622 struct sge_txq *txq; 623 624 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start]; 625 txq->restarts++; 626 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) { 627 struct sge_eth_txq *eq; 628 629 eq = container_of(txq, struct sge_eth_txq, q); 630 netif_tx_wake_queue(eq->txq); 631 } else { 632 struct sge_ofld_txq *oq; 633 634 oq = container_of(txq, struct sge_ofld_txq, q); 635 tasklet_schedule(&oq->qresume_tsk); 636 } 637 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) { 638 const struct cpl_fw6_msg *p = (void *)rsp; 639 640 #ifdef CONFIG_CHELSIO_T4_DCB 641 const struct fw_port_cmd *pcmd = (const void *)p->data; 642 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid)); 643 unsigned int action = 644 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16)); 645 646 if (cmd == FW_PORT_CMD && 647 action == FW_PORT_ACTION_GET_PORT_INFO) { 648 int port = FW_PORT_CMD_PORTID_G( 649 be32_to_cpu(pcmd->op_to_portid)); 650 struct net_device *dev = 651 q->adap->port[q->adap->chan_map[port]]; 652 int state_input = ((pcmd->u.info.dcbxdis_pkd & 653 FW_PORT_CMD_DCBXDIS_F) 654 ? CXGB4_DCB_INPUT_FW_DISABLED 655 : CXGB4_DCB_INPUT_FW_ENABLED); 656 657 cxgb4_dcb_state_fsm(dev, state_input); 658 } 659 660 if (cmd == FW_PORT_CMD && 661 action == FW_PORT_ACTION_L2_DCB_CFG) 662 dcb_rpl(q->adap, pcmd); 663 else 664 #endif 665 if (p->type == 0) 666 t4_handle_fw_rpl(q->adap, p->data); 667 } else if (opcode == CPL_L2T_WRITE_RPL) { 668 const struct cpl_l2t_write_rpl *p = (void *)rsp; 669 670 do_l2t_write_rpl(q->adap, p); 671 } else if (opcode == CPL_SET_TCB_RPL) { 672 const struct cpl_set_tcb_rpl *p = (void *)rsp; 673 674 filter_rpl(q->adap, p); 675 } else 676 dev_err(q->adap->pdev_dev, 677 "unexpected CPL %#x on FW event queue\n", opcode); 678 out: 679 return 0; 680 } 681 682 /* Flush the aggregated lro sessions */ 683 static void uldrx_flush_handler(struct sge_rspq *q) 684 { 685 if (ulds[q->uld].lro_flush) 686 ulds[q->uld].lro_flush(&q->lro_mgr); 687 } 688 689 /** 690 * uldrx_handler - response queue handler for ULD queues 691 * @q: the response queue that received the packet 692 * @rsp: the response queue descriptor holding the offload message 693 * @gl: the gather list of packet fragments 694 * 695 * Deliver an ingress offload packet to a ULD. All processing is done by 696 * the ULD, we just maintain statistics. 697 */ 698 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp, 699 const struct pkt_gl *gl) 700 { 701 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq); 702 int ret; 703 704 /* FW can send CPLs encapsulated in a CPL_FW4_MSG. 705 */ 706 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG && 707 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL) 708 rsp += 2; 709 710 if (q->flush_handler) 711 ret = ulds[q->uld].lro_rx_handler(q->adap->uld_handle[q->uld], 712 rsp, gl, &q->lro_mgr, 713 &q->napi); 714 else 715 ret = ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], 716 rsp, gl); 717 718 if (ret) { 719 rxq->stats.nomem++; 720 return -1; 721 } 722 723 if (gl == NULL) 724 rxq->stats.imm++; 725 else if (gl == CXGB4_MSG_AN) 726 rxq->stats.an++; 727 else 728 rxq->stats.pkts++; 729 return 0; 730 } 731 732 static void disable_msi(struct adapter *adapter) 733 { 734 if (adapter->flags & USING_MSIX) { 735 pci_disable_msix(adapter->pdev); 736 adapter->flags &= ~USING_MSIX; 737 } else if (adapter->flags & USING_MSI) { 738 pci_disable_msi(adapter->pdev); 739 adapter->flags &= ~USING_MSI; 740 } 741 } 742 743 /* 744 * Interrupt handler for non-data events used with MSI-X. 745 */ 746 static irqreturn_t t4_nondata_intr(int irq, void *cookie) 747 { 748 struct adapter *adap = cookie; 749 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A)); 750 751 if (v & PFSW_F) { 752 adap->swintr = 1; 753 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v); 754 } 755 if (adap->flags & MASTER_PF) 756 t4_slow_intr_handler(adap); 757 return IRQ_HANDLED; 758 } 759 760 /* 761 * Name the MSI-X interrupts. 762 */ 763 static void name_msix_vecs(struct adapter *adap) 764 { 765 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc); 766 767 /* non-data interrupts */ 768 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name); 769 770 /* FW events */ 771 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", 772 adap->port[0]->name); 773 774 /* Ethernet queues */ 775 for_each_port(adap, j) { 776 struct net_device *d = adap->port[j]; 777 const struct port_info *pi = netdev_priv(d); 778 779 for (i = 0; i < pi->nqsets; i++, msi_idx++) 780 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d", 781 d->name, i); 782 } 783 784 /* offload queues */ 785 for_each_iscsirxq(&adap->sge, i) 786 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-iscsi%d", 787 adap->port[0]->name, i); 788 789 for_each_iscsitrxq(&adap->sge, i) 790 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-iSCSIT%d", 791 adap->port[0]->name, i); 792 793 for_each_rdmarxq(&adap->sge, i) 794 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d", 795 adap->port[0]->name, i); 796 797 for_each_rdmaciq(&adap->sge, i) 798 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d", 799 adap->port[0]->name, i); 800 } 801 802 static int request_msix_queue_irqs(struct adapter *adap) 803 { 804 struct sge *s = &adap->sge; 805 int err, ethqidx, iscsiqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0; 806 int iscsitqidx = 0; 807 int msi_index = 2; 808 809 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0, 810 adap->msix_info[1].desc, &s->fw_evtq); 811 if (err) 812 return err; 813 814 for_each_ethrxq(s, ethqidx) { 815 err = request_irq(adap->msix_info[msi_index].vec, 816 t4_sge_intr_msix, 0, 817 adap->msix_info[msi_index].desc, 818 &s->ethrxq[ethqidx].rspq); 819 if (err) 820 goto unwind; 821 msi_index++; 822 } 823 for_each_iscsirxq(s, iscsiqidx) { 824 err = request_irq(adap->msix_info[msi_index].vec, 825 t4_sge_intr_msix, 0, 826 adap->msix_info[msi_index].desc, 827 &s->iscsirxq[iscsiqidx].rspq); 828 if (err) 829 goto unwind; 830 msi_index++; 831 } 832 for_each_iscsitrxq(s, iscsitqidx) { 833 err = request_irq(adap->msix_info[msi_index].vec, 834 t4_sge_intr_msix, 0, 835 adap->msix_info[msi_index].desc, 836 &s->iscsitrxq[iscsitqidx].rspq); 837 if (err) 838 goto unwind; 839 msi_index++; 840 } 841 for_each_rdmarxq(s, rdmaqidx) { 842 err = request_irq(adap->msix_info[msi_index].vec, 843 t4_sge_intr_msix, 0, 844 adap->msix_info[msi_index].desc, 845 &s->rdmarxq[rdmaqidx].rspq); 846 if (err) 847 goto unwind; 848 msi_index++; 849 } 850 for_each_rdmaciq(s, rdmaciqqidx) { 851 err = request_irq(adap->msix_info[msi_index].vec, 852 t4_sge_intr_msix, 0, 853 adap->msix_info[msi_index].desc, 854 &s->rdmaciq[rdmaciqqidx].rspq); 855 if (err) 856 goto unwind; 857 msi_index++; 858 } 859 return 0; 860 861 unwind: 862 while (--rdmaciqqidx >= 0) 863 free_irq(adap->msix_info[--msi_index].vec, 864 &s->rdmaciq[rdmaciqqidx].rspq); 865 while (--rdmaqidx >= 0) 866 free_irq(adap->msix_info[--msi_index].vec, 867 &s->rdmarxq[rdmaqidx].rspq); 868 while (--iscsitqidx >= 0) 869 free_irq(adap->msix_info[--msi_index].vec, 870 &s->iscsitrxq[iscsitqidx].rspq); 871 while (--iscsiqidx >= 0) 872 free_irq(adap->msix_info[--msi_index].vec, 873 &s->iscsirxq[iscsiqidx].rspq); 874 while (--ethqidx >= 0) 875 free_irq(adap->msix_info[--msi_index].vec, 876 &s->ethrxq[ethqidx].rspq); 877 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 878 return err; 879 } 880 881 static void free_msix_queue_irqs(struct adapter *adap) 882 { 883 int i, msi_index = 2; 884 struct sge *s = &adap->sge; 885 886 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 887 for_each_ethrxq(s, i) 888 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq); 889 for_each_iscsirxq(s, i) 890 free_irq(adap->msix_info[msi_index++].vec, 891 &s->iscsirxq[i].rspq); 892 for_each_iscsitrxq(s, i) 893 free_irq(adap->msix_info[msi_index++].vec, 894 &s->iscsitrxq[i].rspq); 895 for_each_rdmarxq(s, i) 896 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq); 897 for_each_rdmaciq(s, i) 898 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq); 899 } 900 901 /** 902 * cxgb4_write_rss - write the RSS table for a given port 903 * @pi: the port 904 * @queues: array of queue indices for RSS 905 * 906 * Sets up the portion of the HW RSS table for the port's VI to distribute 907 * packets to the Rx queues in @queues. 908 * Should never be called before setting up sge eth rx queues 909 */ 910 int cxgb4_write_rss(const struct port_info *pi, const u16 *queues) 911 { 912 u16 *rss; 913 int i, err; 914 struct adapter *adapter = pi->adapter; 915 const struct sge_eth_rxq *rxq; 916 917 rxq = &adapter->sge.ethrxq[pi->first_qset]; 918 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL); 919 if (!rss) 920 return -ENOMEM; 921 922 /* map the queue indices to queue ids */ 923 for (i = 0; i < pi->rss_size; i++, queues++) 924 rss[i] = rxq[*queues].rspq.abs_id; 925 926 err = t4_config_rss_range(adapter, adapter->pf, pi->viid, 0, 927 pi->rss_size, rss, pi->rss_size); 928 /* If Tunnel All Lookup isn't specified in the global RSS 929 * Configuration, then we need to specify a default Ingress 930 * Queue for any ingress packets which aren't hashed. We'll 931 * use our first ingress queue ... 932 */ 933 if (!err) 934 err = t4_config_vi_rss(adapter, adapter->mbox, pi->viid, 935 FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F | 936 FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F | 937 FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F | 938 FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F | 939 FW_RSS_VI_CONFIG_CMD_UDPEN_F, 940 rss[0]); 941 kfree(rss); 942 return err; 943 } 944 945 /** 946 * setup_rss - configure RSS 947 * @adap: the adapter 948 * 949 * Sets up RSS for each port. 950 */ 951 static int setup_rss(struct adapter *adap) 952 { 953 int i, j, err; 954 955 for_each_port(adap, i) { 956 const struct port_info *pi = adap2pinfo(adap, i); 957 958 /* Fill default values with equal distribution */ 959 for (j = 0; j < pi->rss_size; j++) 960 pi->rss[j] = j % pi->nqsets; 961 962 err = cxgb4_write_rss(pi, pi->rss); 963 if (err) 964 return err; 965 } 966 return 0; 967 } 968 969 /* 970 * Return the channel of the ingress queue with the given qid. 971 */ 972 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid) 973 { 974 qid -= p->ingr_start; 975 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan; 976 } 977 978 /* 979 * Wait until all NAPI handlers are descheduled. 980 */ 981 static void quiesce_rx(struct adapter *adap) 982 { 983 int i; 984 985 for (i = 0; i < adap->sge.ingr_sz; i++) { 986 struct sge_rspq *q = adap->sge.ingr_map[i]; 987 988 if (q && q->handler) { 989 napi_disable(&q->napi); 990 local_bh_disable(); 991 while (!cxgb_poll_lock_napi(q)) 992 mdelay(1); 993 local_bh_enable(); 994 } 995 996 } 997 } 998 999 /* Disable interrupt and napi handler */ 1000 static void disable_interrupts(struct adapter *adap) 1001 { 1002 if (adap->flags & FULL_INIT_DONE) { 1003 t4_intr_disable(adap); 1004 if (adap->flags & USING_MSIX) { 1005 free_msix_queue_irqs(adap); 1006 free_irq(adap->msix_info[0].vec, adap); 1007 } else { 1008 free_irq(adap->pdev->irq, adap); 1009 } 1010 quiesce_rx(adap); 1011 } 1012 } 1013 1014 /* 1015 * Enable NAPI scheduling and interrupt generation for all Rx queues. 1016 */ 1017 static void enable_rx(struct adapter *adap) 1018 { 1019 int i; 1020 1021 for (i = 0; i < adap->sge.ingr_sz; i++) { 1022 struct sge_rspq *q = adap->sge.ingr_map[i]; 1023 1024 if (!q) 1025 continue; 1026 if (q->handler) { 1027 cxgb_busy_poll_init_lock(q); 1028 napi_enable(&q->napi); 1029 } 1030 /* 0-increment GTS to start the timer and enable interrupts */ 1031 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A), 1032 SEINTARM_V(q->intr_params) | 1033 INGRESSQID_V(q->cntxt_id)); 1034 } 1035 } 1036 1037 static int alloc_ofld_rxqs(struct adapter *adap, struct sge_ofld_rxq *q, 1038 unsigned int nq, unsigned int per_chan, int msi_idx, 1039 u16 *ids, bool lro) 1040 { 1041 int i, err; 1042 1043 for (i = 0; i < nq; i++, q++) { 1044 if (msi_idx > 0) 1045 msi_idx++; 1046 err = t4_sge_alloc_rxq(adap, &q->rspq, false, 1047 adap->port[i / per_chan], 1048 msi_idx, q->fl.size ? &q->fl : NULL, 1049 uldrx_handler, 1050 lro ? uldrx_flush_handler : NULL, 1051 0); 1052 if (err) 1053 return err; 1054 memset(&q->stats, 0, sizeof(q->stats)); 1055 if (ids) 1056 ids[i] = q->rspq.abs_id; 1057 } 1058 return 0; 1059 } 1060 1061 /** 1062 * setup_sge_queues - configure SGE Tx/Rx/response queues 1063 * @adap: the adapter 1064 * 1065 * Determines how many sets of SGE queues to use and initializes them. 1066 * We support multiple queue sets per port if we have MSI-X, otherwise 1067 * just one queue set per port. 1068 */ 1069 static int setup_sge_queues(struct adapter *adap) 1070 { 1071 int err, msi_idx, i, j; 1072 struct sge *s = &adap->sge; 1073 1074 bitmap_zero(s->starving_fl, s->egr_sz); 1075 bitmap_zero(s->txq_maperr, s->egr_sz); 1076 1077 if (adap->flags & USING_MSIX) 1078 msi_idx = 1; /* vector 0 is for non-queue interrupts */ 1079 else { 1080 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0, 1081 NULL, NULL, NULL, -1); 1082 if (err) 1083 return err; 1084 msi_idx = -((int)s->intrq.abs_id + 1); 1085 } 1086 1087 /* NOTE: If you add/delete any Ingress/Egress Queue allocations in here, 1088 * don't forget to update the following which need to be 1089 * synchronized to and changes here. 1090 * 1091 * 1. The calculations of MAX_INGQ in cxgb4.h. 1092 * 1093 * 2. Update enable_msix/name_msix_vecs/request_msix_queue_irqs 1094 * to accommodate any new/deleted Ingress Queues 1095 * which need MSI-X Vectors. 1096 * 1097 * 3. Update sge_qinfo_show() to include information on the 1098 * new/deleted queues. 1099 */ 1100 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0], 1101 msi_idx, NULL, fwevtq_handler, NULL, -1); 1102 if (err) { 1103 freeout: t4_free_sge_resources(adap); 1104 return err; 1105 } 1106 1107 for_each_port(adap, i) { 1108 struct net_device *dev = adap->port[i]; 1109 struct port_info *pi = netdev_priv(dev); 1110 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset]; 1111 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset]; 1112 1113 for (j = 0; j < pi->nqsets; j++, q++) { 1114 if (msi_idx > 0) 1115 msi_idx++; 1116 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, 1117 msi_idx, &q->fl, 1118 t4_ethrx_handler, 1119 NULL, 1120 t4_get_mps_bg_map(adap, 1121 pi->tx_chan)); 1122 if (err) 1123 goto freeout; 1124 q->rspq.idx = j; 1125 memset(&q->stats, 0, sizeof(q->stats)); 1126 } 1127 for (j = 0; j < pi->nqsets; j++, t++) { 1128 err = t4_sge_alloc_eth_txq(adap, t, dev, 1129 netdev_get_tx_queue(dev, j), 1130 s->fw_evtq.cntxt_id); 1131 if (err) 1132 goto freeout; 1133 } 1134 } 1135 1136 j = s->iscsiqsets / adap->params.nports; /* iscsi queues per channel */ 1137 for_each_iscsirxq(s, i) { 1138 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], 1139 adap->port[i / j], 1140 s->fw_evtq.cntxt_id); 1141 if (err) 1142 goto freeout; 1143 } 1144 1145 #define ALLOC_OFLD_RXQS(firstq, nq, per_chan, ids, lro) do { \ 1146 err = alloc_ofld_rxqs(adap, firstq, nq, per_chan, msi_idx, ids, lro); \ 1147 if (err) \ 1148 goto freeout; \ 1149 if (msi_idx > 0) \ 1150 msi_idx += nq; \ 1151 } while (0) 1152 1153 ALLOC_OFLD_RXQS(s->iscsirxq, s->iscsiqsets, j, s->iscsi_rxq, false); 1154 ALLOC_OFLD_RXQS(s->iscsitrxq, s->niscsitq, j, s->iscsit_rxq, true); 1155 ALLOC_OFLD_RXQS(s->rdmarxq, s->rdmaqs, 1, s->rdma_rxq, false); 1156 j = s->rdmaciqs / adap->params.nports; /* rdmaq queues per channel */ 1157 ALLOC_OFLD_RXQS(s->rdmaciq, s->rdmaciqs, j, s->rdma_ciq, false); 1158 1159 #undef ALLOC_OFLD_RXQS 1160 1161 for_each_port(adap, i) { 1162 /* 1163 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't 1164 * have RDMA queues, and that's the right value. 1165 */ 1166 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i], 1167 s->fw_evtq.cntxt_id, 1168 s->rdmarxq[i].rspq.cntxt_id); 1169 if (err) 1170 goto freeout; 1171 } 1172 1173 t4_write_reg(adap, is_t4(adap->params.chip) ? 1174 MPS_TRC_RSS_CONTROL_A : 1175 MPS_T5_TRC_RSS_CONTROL_A, 1176 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) | 1177 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id)); 1178 return 0; 1179 } 1180 1181 /* 1182 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc. 1183 * The allocated memory is cleared. 1184 */ 1185 void *t4_alloc_mem(size_t size) 1186 { 1187 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 1188 1189 if (!p) 1190 p = vzalloc(size); 1191 return p; 1192 } 1193 1194 /* 1195 * Free memory allocated through alloc_mem(). 1196 */ 1197 void t4_free_mem(void *addr) 1198 { 1199 kvfree(addr); 1200 } 1201 1202 /* Send a Work Request to write the filter at a specified index. We construct 1203 * a Firmware Filter Work Request to have the work done and put the indicated 1204 * filter into "pending" mode which will prevent any further actions against 1205 * it till we get a reply from the firmware on the completion status of the 1206 * request. 1207 */ 1208 static int set_filter_wr(struct adapter *adapter, int fidx) 1209 { 1210 struct filter_entry *f = &adapter->tids.ftid_tab[fidx]; 1211 struct sk_buff *skb; 1212 struct fw_filter_wr *fwr; 1213 unsigned int ftid; 1214 1215 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL); 1216 if (!skb) 1217 return -ENOMEM; 1218 1219 /* If the new filter requires loopback Destination MAC and/or VLAN 1220 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for 1221 * the filter. 1222 */ 1223 if (f->fs.newdmac || f->fs.newvlan) { 1224 /* allocate L2T entry for new filter */ 1225 f->l2t = t4_l2t_alloc_switching(adapter, f->fs.vlan, 1226 f->fs.eport, f->fs.dmac); 1227 if (f->l2t == NULL) { 1228 kfree_skb(skb); 1229 return -ENOMEM; 1230 } 1231 } 1232 1233 ftid = adapter->tids.ftid_base + fidx; 1234 1235 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr)); 1236 memset(fwr, 0, sizeof(*fwr)); 1237 1238 /* It would be nice to put most of the following in t4_hw.c but most 1239 * of the work is translating the cxgbtool ch_filter_specification 1240 * into the Work Request and the definition of that structure is 1241 * currently in cxgbtool.h which isn't appropriate to pull into the 1242 * common code. We may eventually try to come up with a more neutral 1243 * filter specification structure but for now it's easiest to simply 1244 * put this fairly direct code in line ... 1245 */ 1246 fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR)); 1247 fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16)); 1248 fwr->tid_to_iq = 1249 htonl(FW_FILTER_WR_TID_V(ftid) | 1250 FW_FILTER_WR_RQTYPE_V(f->fs.type) | 1251 FW_FILTER_WR_NOREPLY_V(0) | 1252 FW_FILTER_WR_IQ_V(f->fs.iq)); 1253 fwr->del_filter_to_l2tix = 1254 htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) | 1255 FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) | 1256 FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) | 1257 FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) | 1258 FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) | 1259 FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) | 1260 FW_FILTER_WR_DMAC_V(f->fs.newdmac) | 1261 FW_FILTER_WR_SMAC_V(f->fs.newsmac) | 1262 FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT || 1263 f->fs.newvlan == VLAN_REWRITE) | 1264 FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE || 1265 f->fs.newvlan == VLAN_REWRITE) | 1266 FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) | 1267 FW_FILTER_WR_TXCHAN_V(f->fs.eport) | 1268 FW_FILTER_WR_PRIO_V(f->fs.prio) | 1269 FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0)); 1270 fwr->ethtype = htons(f->fs.val.ethtype); 1271 fwr->ethtypem = htons(f->fs.mask.ethtype); 1272 fwr->frag_to_ovlan_vldm = 1273 (FW_FILTER_WR_FRAG_V(f->fs.val.frag) | 1274 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) | 1275 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) | 1276 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) | 1277 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) | 1278 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld)); 1279 fwr->smac_sel = 0; 1280 fwr->rx_chan_rx_rpl_iq = 1281 htons(FW_FILTER_WR_RX_CHAN_V(0) | 1282 FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id)); 1283 fwr->maci_to_matchtypem = 1284 htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) | 1285 FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) | 1286 FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) | 1287 FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) | 1288 FW_FILTER_WR_PORT_V(f->fs.val.iport) | 1289 FW_FILTER_WR_PORTM_V(f->fs.mask.iport) | 1290 FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) | 1291 FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype)); 1292 fwr->ptcl = f->fs.val.proto; 1293 fwr->ptclm = f->fs.mask.proto; 1294 fwr->ttyp = f->fs.val.tos; 1295 fwr->ttypm = f->fs.mask.tos; 1296 fwr->ivlan = htons(f->fs.val.ivlan); 1297 fwr->ivlanm = htons(f->fs.mask.ivlan); 1298 fwr->ovlan = htons(f->fs.val.ovlan); 1299 fwr->ovlanm = htons(f->fs.mask.ovlan); 1300 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip)); 1301 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm)); 1302 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip)); 1303 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm)); 1304 fwr->lp = htons(f->fs.val.lport); 1305 fwr->lpm = htons(f->fs.mask.lport); 1306 fwr->fp = htons(f->fs.val.fport); 1307 fwr->fpm = htons(f->fs.mask.fport); 1308 if (f->fs.newsmac) 1309 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma)); 1310 1311 /* Mark the filter as "pending" and ship off the Filter Work Request. 1312 * When we get the Work Request Reply we'll clear the pending status. 1313 */ 1314 f->pending = 1; 1315 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3); 1316 t4_ofld_send(adapter, skb); 1317 return 0; 1318 } 1319 1320 /* Delete the filter at a specified index. 1321 */ 1322 static int del_filter_wr(struct adapter *adapter, int fidx) 1323 { 1324 struct filter_entry *f = &adapter->tids.ftid_tab[fidx]; 1325 struct sk_buff *skb; 1326 struct fw_filter_wr *fwr; 1327 unsigned int len, ftid; 1328 1329 len = sizeof(*fwr); 1330 ftid = adapter->tids.ftid_base + fidx; 1331 1332 skb = alloc_skb(len, GFP_KERNEL); 1333 if (!skb) 1334 return -ENOMEM; 1335 1336 fwr = (struct fw_filter_wr *)__skb_put(skb, len); 1337 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id); 1338 1339 /* Mark the filter as "pending" and ship off the Filter Work Request. 1340 * When we get the Work Request Reply we'll clear the pending status. 1341 */ 1342 f->pending = 1; 1343 t4_mgmt_tx(adapter, skb); 1344 return 0; 1345 } 1346 1347 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb, 1348 void *accel_priv, select_queue_fallback_t fallback) 1349 { 1350 int txq; 1351 1352 #ifdef CONFIG_CHELSIO_T4_DCB 1353 /* If a Data Center Bridging has been successfully negotiated on this 1354 * link then we'll use the skb's priority to map it to a TX Queue. 1355 * The skb's priority is determined via the VLAN Tag Priority Code 1356 * Point field. 1357 */ 1358 if (cxgb4_dcb_enabled(dev)) { 1359 u16 vlan_tci; 1360 int err; 1361 1362 err = vlan_get_tag(skb, &vlan_tci); 1363 if (unlikely(err)) { 1364 if (net_ratelimit()) 1365 netdev_warn(dev, 1366 "TX Packet without VLAN Tag on DCB Link\n"); 1367 txq = 0; 1368 } else { 1369 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 1370 #ifdef CONFIG_CHELSIO_T4_FCOE 1371 if (skb->protocol == htons(ETH_P_FCOE)) 1372 txq = skb->priority & 0x7; 1373 #endif /* CONFIG_CHELSIO_T4_FCOE */ 1374 } 1375 return txq; 1376 } 1377 #endif /* CONFIG_CHELSIO_T4_DCB */ 1378 1379 if (select_queue) { 1380 txq = (skb_rx_queue_recorded(skb) 1381 ? skb_get_rx_queue(skb) 1382 : smp_processor_id()); 1383 1384 while (unlikely(txq >= dev->real_num_tx_queues)) 1385 txq -= dev->real_num_tx_queues; 1386 1387 return txq; 1388 } 1389 1390 return fallback(dev, skb) % dev->real_num_tx_queues; 1391 } 1392 1393 static int closest_timer(const struct sge *s, int time) 1394 { 1395 int i, delta, match = 0, min_delta = INT_MAX; 1396 1397 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { 1398 delta = time - s->timer_val[i]; 1399 if (delta < 0) 1400 delta = -delta; 1401 if (delta < min_delta) { 1402 min_delta = delta; 1403 match = i; 1404 } 1405 } 1406 return match; 1407 } 1408 1409 static int closest_thres(const struct sge *s, int thres) 1410 { 1411 int i, delta, match = 0, min_delta = INT_MAX; 1412 1413 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { 1414 delta = thres - s->counter_val[i]; 1415 if (delta < 0) 1416 delta = -delta; 1417 if (delta < min_delta) { 1418 min_delta = delta; 1419 match = i; 1420 } 1421 } 1422 return match; 1423 } 1424 1425 /** 1426 * cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters 1427 * @q: the Rx queue 1428 * @us: the hold-off time in us, or 0 to disable timer 1429 * @cnt: the hold-off packet count, or 0 to disable counter 1430 * 1431 * Sets an Rx queue's interrupt hold-off time and packet count. At least 1432 * one of the two needs to be enabled for the queue to generate interrupts. 1433 */ 1434 int cxgb4_set_rspq_intr_params(struct sge_rspq *q, 1435 unsigned int us, unsigned int cnt) 1436 { 1437 struct adapter *adap = q->adap; 1438 1439 if ((us | cnt) == 0) 1440 cnt = 1; 1441 1442 if (cnt) { 1443 int err; 1444 u32 v, new_idx; 1445 1446 new_idx = closest_thres(&adap->sge, cnt); 1447 if (q->desc && q->pktcnt_idx != new_idx) { 1448 /* the queue has already been created, update it */ 1449 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 1450 FW_PARAMS_PARAM_X_V( 1451 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) | 1452 FW_PARAMS_PARAM_YZ_V(q->cntxt_id); 1453 err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, 1454 &v, &new_idx); 1455 if (err) 1456 return err; 1457 } 1458 q->pktcnt_idx = new_idx; 1459 } 1460 1461 us = us == 0 ? 6 : closest_timer(&adap->sge, us); 1462 q->intr_params = QINTR_TIMER_IDX_V(us) | QINTR_CNT_EN_V(cnt > 0); 1463 return 0; 1464 } 1465 1466 static int cxgb_set_features(struct net_device *dev, netdev_features_t features) 1467 { 1468 const struct port_info *pi = netdev_priv(dev); 1469 netdev_features_t changed = dev->features ^ features; 1470 int err; 1471 1472 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX)) 1473 return 0; 1474 1475 err = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, -1, 1476 -1, -1, -1, 1477 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true); 1478 if (unlikely(err)) 1479 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX; 1480 return err; 1481 } 1482 1483 static int setup_debugfs(struct adapter *adap) 1484 { 1485 if (IS_ERR_OR_NULL(adap->debugfs_root)) 1486 return -1; 1487 1488 #ifdef CONFIG_DEBUG_FS 1489 t4_setup_debugfs(adap); 1490 #endif 1491 return 0; 1492 } 1493 1494 /* 1495 * upper-layer driver support 1496 */ 1497 1498 /* 1499 * Allocate an active-open TID and set it to the supplied value. 1500 */ 1501 int cxgb4_alloc_atid(struct tid_info *t, void *data) 1502 { 1503 int atid = -1; 1504 1505 spin_lock_bh(&t->atid_lock); 1506 if (t->afree) { 1507 union aopen_entry *p = t->afree; 1508 1509 atid = (p - t->atid_tab) + t->atid_base; 1510 t->afree = p->next; 1511 p->data = data; 1512 t->atids_in_use++; 1513 } 1514 spin_unlock_bh(&t->atid_lock); 1515 return atid; 1516 } 1517 EXPORT_SYMBOL(cxgb4_alloc_atid); 1518 1519 /* 1520 * Release an active-open TID. 1521 */ 1522 void cxgb4_free_atid(struct tid_info *t, unsigned int atid) 1523 { 1524 union aopen_entry *p = &t->atid_tab[atid - t->atid_base]; 1525 1526 spin_lock_bh(&t->atid_lock); 1527 p->next = t->afree; 1528 t->afree = p; 1529 t->atids_in_use--; 1530 spin_unlock_bh(&t->atid_lock); 1531 } 1532 EXPORT_SYMBOL(cxgb4_free_atid); 1533 1534 /* 1535 * Allocate a server TID and set it to the supplied value. 1536 */ 1537 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data) 1538 { 1539 int stid; 1540 1541 spin_lock_bh(&t->stid_lock); 1542 if (family == PF_INET) { 1543 stid = find_first_zero_bit(t->stid_bmap, t->nstids); 1544 if (stid < t->nstids) 1545 __set_bit(stid, t->stid_bmap); 1546 else 1547 stid = -1; 1548 } else { 1549 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 1); 1550 if (stid < 0) 1551 stid = -1; 1552 } 1553 if (stid >= 0) { 1554 t->stid_tab[stid].data = data; 1555 stid += t->stid_base; 1556 /* IPv6 requires max of 520 bits or 16 cells in TCAM 1557 * This is equivalent to 4 TIDs. With CLIP enabled it 1558 * needs 2 TIDs. 1559 */ 1560 if (family == PF_INET) 1561 t->stids_in_use++; 1562 else 1563 t->stids_in_use += 2; 1564 } 1565 spin_unlock_bh(&t->stid_lock); 1566 return stid; 1567 } 1568 EXPORT_SYMBOL(cxgb4_alloc_stid); 1569 1570 /* Allocate a server filter TID and set it to the supplied value. 1571 */ 1572 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data) 1573 { 1574 int stid; 1575 1576 spin_lock_bh(&t->stid_lock); 1577 if (family == PF_INET) { 1578 stid = find_next_zero_bit(t->stid_bmap, 1579 t->nstids + t->nsftids, t->nstids); 1580 if (stid < (t->nstids + t->nsftids)) 1581 __set_bit(stid, t->stid_bmap); 1582 else 1583 stid = -1; 1584 } else { 1585 stid = -1; 1586 } 1587 if (stid >= 0) { 1588 t->stid_tab[stid].data = data; 1589 stid -= t->nstids; 1590 stid += t->sftid_base; 1591 t->sftids_in_use++; 1592 } 1593 spin_unlock_bh(&t->stid_lock); 1594 return stid; 1595 } 1596 EXPORT_SYMBOL(cxgb4_alloc_sftid); 1597 1598 /* Release a server TID. 1599 */ 1600 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family) 1601 { 1602 /* Is it a server filter TID? */ 1603 if (t->nsftids && (stid >= t->sftid_base)) { 1604 stid -= t->sftid_base; 1605 stid += t->nstids; 1606 } else { 1607 stid -= t->stid_base; 1608 } 1609 1610 spin_lock_bh(&t->stid_lock); 1611 if (family == PF_INET) 1612 __clear_bit(stid, t->stid_bmap); 1613 else 1614 bitmap_release_region(t->stid_bmap, stid, 1); 1615 t->stid_tab[stid].data = NULL; 1616 if (stid < t->nstids) { 1617 if (family == PF_INET) 1618 t->stids_in_use--; 1619 else 1620 t->stids_in_use -= 2; 1621 } else { 1622 t->sftids_in_use--; 1623 } 1624 spin_unlock_bh(&t->stid_lock); 1625 } 1626 EXPORT_SYMBOL(cxgb4_free_stid); 1627 1628 /* 1629 * Populate a TID_RELEASE WR. Caller must properly size the skb. 1630 */ 1631 static void mk_tid_release(struct sk_buff *skb, unsigned int chan, 1632 unsigned int tid) 1633 { 1634 struct cpl_tid_release *req; 1635 1636 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); 1637 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req)); 1638 INIT_TP_WR(req, tid); 1639 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 1640 } 1641 1642 /* 1643 * Queue a TID release request and if necessary schedule a work queue to 1644 * process it. 1645 */ 1646 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan, 1647 unsigned int tid) 1648 { 1649 void **p = &t->tid_tab[tid]; 1650 struct adapter *adap = container_of(t, struct adapter, tids); 1651 1652 spin_lock_bh(&adap->tid_release_lock); 1653 *p = adap->tid_release_head; 1654 /* Low 2 bits encode the Tx channel number */ 1655 adap->tid_release_head = (void **)((uintptr_t)p | chan); 1656 if (!adap->tid_release_task_busy) { 1657 adap->tid_release_task_busy = true; 1658 queue_work(adap->workq, &adap->tid_release_task); 1659 } 1660 spin_unlock_bh(&adap->tid_release_lock); 1661 } 1662 1663 /* 1664 * Process the list of pending TID release requests. 1665 */ 1666 static void process_tid_release_list(struct work_struct *work) 1667 { 1668 struct sk_buff *skb; 1669 struct adapter *adap; 1670 1671 adap = container_of(work, struct adapter, tid_release_task); 1672 1673 spin_lock_bh(&adap->tid_release_lock); 1674 while (adap->tid_release_head) { 1675 void **p = adap->tid_release_head; 1676 unsigned int chan = (uintptr_t)p & 3; 1677 p = (void *)p - chan; 1678 1679 adap->tid_release_head = *p; 1680 *p = NULL; 1681 spin_unlock_bh(&adap->tid_release_lock); 1682 1683 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release), 1684 GFP_KERNEL))) 1685 schedule_timeout_uninterruptible(1); 1686 1687 mk_tid_release(skb, chan, p - adap->tids.tid_tab); 1688 t4_ofld_send(adap, skb); 1689 spin_lock_bh(&adap->tid_release_lock); 1690 } 1691 adap->tid_release_task_busy = false; 1692 spin_unlock_bh(&adap->tid_release_lock); 1693 } 1694 1695 /* 1696 * Release a TID and inform HW. If we are unable to allocate the release 1697 * message we defer to a work queue. 1698 */ 1699 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid) 1700 { 1701 struct sk_buff *skb; 1702 struct adapter *adap = container_of(t, struct adapter, tids); 1703 1704 WARN_ON(tid >= t->ntids); 1705 1706 if (t->tid_tab[tid]) { 1707 t->tid_tab[tid] = NULL; 1708 if (t->hash_base && (tid >= t->hash_base)) 1709 atomic_dec(&t->hash_tids_in_use); 1710 else 1711 atomic_dec(&t->tids_in_use); 1712 } 1713 1714 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); 1715 if (likely(skb)) { 1716 mk_tid_release(skb, chan, tid); 1717 t4_ofld_send(adap, skb); 1718 } else 1719 cxgb4_queue_tid_release(t, chan, tid); 1720 } 1721 EXPORT_SYMBOL(cxgb4_remove_tid); 1722 1723 /* 1724 * Allocate and initialize the TID tables. Returns 0 on success. 1725 */ 1726 static int tid_init(struct tid_info *t) 1727 { 1728 size_t size; 1729 unsigned int stid_bmap_size; 1730 unsigned int natids = t->natids; 1731 struct adapter *adap = container_of(t, struct adapter, tids); 1732 1733 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids); 1734 size = t->ntids * sizeof(*t->tid_tab) + 1735 natids * sizeof(*t->atid_tab) + 1736 t->nstids * sizeof(*t->stid_tab) + 1737 t->nsftids * sizeof(*t->stid_tab) + 1738 stid_bmap_size * sizeof(long) + 1739 t->nftids * sizeof(*t->ftid_tab) + 1740 t->nsftids * sizeof(*t->ftid_tab); 1741 1742 t->tid_tab = t4_alloc_mem(size); 1743 if (!t->tid_tab) 1744 return -ENOMEM; 1745 1746 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; 1747 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids]; 1748 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids]; 1749 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size]; 1750 spin_lock_init(&t->stid_lock); 1751 spin_lock_init(&t->atid_lock); 1752 1753 t->stids_in_use = 0; 1754 t->sftids_in_use = 0; 1755 t->afree = NULL; 1756 t->atids_in_use = 0; 1757 atomic_set(&t->tids_in_use, 0); 1758 atomic_set(&t->hash_tids_in_use, 0); 1759 1760 /* Setup the free list for atid_tab and clear the stid bitmap. */ 1761 if (natids) { 1762 while (--natids) 1763 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 1764 t->afree = t->atid_tab; 1765 } 1766 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids); 1767 /* Reserve stid 0 for T4/T5 adapters */ 1768 if (!t->stid_base && 1769 (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)) 1770 __set_bit(0, t->stid_bmap); 1771 1772 return 0; 1773 } 1774 1775 /** 1776 * cxgb4_create_server - create an IP server 1777 * @dev: the device 1778 * @stid: the server TID 1779 * @sip: local IP address to bind server to 1780 * @sport: the server's TCP port 1781 * @queue: queue to direct messages from this server to 1782 * 1783 * Create an IP server for the given port and address. 1784 * Returns <0 on error and one of the %NET_XMIT_* values on success. 1785 */ 1786 int cxgb4_create_server(const struct net_device *dev, unsigned int stid, 1787 __be32 sip, __be16 sport, __be16 vlan, 1788 unsigned int queue) 1789 { 1790 unsigned int chan; 1791 struct sk_buff *skb; 1792 struct adapter *adap; 1793 struct cpl_pass_open_req *req; 1794 int ret; 1795 1796 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1797 if (!skb) 1798 return -ENOMEM; 1799 1800 adap = netdev2adap(dev); 1801 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req)); 1802 INIT_TP_WR(req, 0); 1803 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid)); 1804 req->local_port = sport; 1805 req->peer_port = htons(0); 1806 req->local_ip = sip; 1807 req->peer_ip = htonl(0); 1808 chan = rxq_to_chan(&adap->sge, queue); 1809 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 1810 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 1811 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 1812 ret = t4_mgmt_tx(adap, skb); 1813 return net_xmit_eval(ret); 1814 } 1815 EXPORT_SYMBOL(cxgb4_create_server); 1816 1817 /* cxgb4_create_server6 - create an IPv6 server 1818 * @dev: the device 1819 * @stid: the server TID 1820 * @sip: local IPv6 address to bind server to 1821 * @sport: the server's TCP port 1822 * @queue: queue to direct messages from this server to 1823 * 1824 * Create an IPv6 server for the given port and address. 1825 * Returns <0 on error and one of the %NET_XMIT_* values on success. 1826 */ 1827 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid, 1828 const struct in6_addr *sip, __be16 sport, 1829 unsigned int queue) 1830 { 1831 unsigned int chan; 1832 struct sk_buff *skb; 1833 struct adapter *adap; 1834 struct cpl_pass_open_req6 *req; 1835 int ret; 1836 1837 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1838 if (!skb) 1839 return -ENOMEM; 1840 1841 adap = netdev2adap(dev); 1842 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req)); 1843 INIT_TP_WR(req, 0); 1844 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid)); 1845 req->local_port = sport; 1846 req->peer_port = htons(0); 1847 req->local_ip_hi = *(__be64 *)(sip->s6_addr); 1848 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8); 1849 req->peer_ip_hi = cpu_to_be64(0); 1850 req->peer_ip_lo = cpu_to_be64(0); 1851 chan = rxq_to_chan(&adap->sge, queue); 1852 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 1853 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 1854 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 1855 ret = t4_mgmt_tx(adap, skb); 1856 return net_xmit_eval(ret); 1857 } 1858 EXPORT_SYMBOL(cxgb4_create_server6); 1859 1860 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid, 1861 unsigned int queue, bool ipv6) 1862 { 1863 struct sk_buff *skb; 1864 struct adapter *adap; 1865 struct cpl_close_listsvr_req *req; 1866 int ret; 1867 1868 adap = netdev2adap(dev); 1869 1870 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1871 if (!skb) 1872 return -ENOMEM; 1873 1874 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req)); 1875 INIT_TP_WR(req, 0); 1876 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid)); 1877 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) : 1878 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue)); 1879 ret = t4_mgmt_tx(adap, skb); 1880 return net_xmit_eval(ret); 1881 } 1882 EXPORT_SYMBOL(cxgb4_remove_server); 1883 1884 /** 1885 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU 1886 * @mtus: the HW MTU table 1887 * @mtu: the target MTU 1888 * @idx: index of selected entry in the MTU table 1889 * 1890 * Returns the index and the value in the HW MTU table that is closest to 1891 * but does not exceed @mtu, unless @mtu is smaller than any value in the 1892 * table, in which case that smallest available value is selected. 1893 */ 1894 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu, 1895 unsigned int *idx) 1896 { 1897 unsigned int i = 0; 1898 1899 while (i < NMTUS - 1 && mtus[i + 1] <= mtu) 1900 ++i; 1901 if (idx) 1902 *idx = i; 1903 return mtus[i]; 1904 } 1905 EXPORT_SYMBOL(cxgb4_best_mtu); 1906 1907 /** 1908 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned 1909 * @mtus: the HW MTU table 1910 * @header_size: Header Size 1911 * @data_size_max: maximum Data Segment Size 1912 * @data_size_align: desired Data Segment Size Alignment (2^N) 1913 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL) 1914 * 1915 * Similar to cxgb4_best_mtu() but instead of searching the Hardware 1916 * MTU Table based solely on a Maximum MTU parameter, we break that 1917 * parameter up into a Header Size and Maximum Data Segment Size, and 1918 * provide a desired Data Segment Size Alignment. If we find an MTU in 1919 * the Hardware MTU Table which will result in a Data Segment Size with 1920 * the requested alignment _and_ that MTU isn't "too far" from the 1921 * closest MTU, then we'll return that rather than the closest MTU. 1922 */ 1923 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus, 1924 unsigned short header_size, 1925 unsigned short data_size_max, 1926 unsigned short data_size_align, 1927 unsigned int *mtu_idxp) 1928 { 1929 unsigned short max_mtu = header_size + data_size_max; 1930 unsigned short data_size_align_mask = data_size_align - 1; 1931 int mtu_idx, aligned_mtu_idx; 1932 1933 /* Scan the MTU Table till we find an MTU which is larger than our 1934 * Maximum MTU or we reach the end of the table. Along the way, 1935 * record the last MTU found, if any, which will result in a Data 1936 * Segment Length matching the requested alignment. 1937 */ 1938 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) { 1939 unsigned short data_size = mtus[mtu_idx] - header_size; 1940 1941 /* If this MTU minus the Header Size would result in a 1942 * Data Segment Size of the desired alignment, remember it. 1943 */ 1944 if ((data_size & data_size_align_mask) == 0) 1945 aligned_mtu_idx = mtu_idx; 1946 1947 /* If we're not at the end of the Hardware MTU Table and the 1948 * next element is larger than our Maximum MTU, drop out of 1949 * the loop. 1950 */ 1951 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu) 1952 break; 1953 } 1954 1955 /* If we fell out of the loop because we ran to the end of the table, 1956 * then we just have to use the last [largest] entry. 1957 */ 1958 if (mtu_idx == NMTUS) 1959 mtu_idx--; 1960 1961 /* If we found an MTU which resulted in the requested Data Segment 1962 * Length alignment and that's "not far" from the largest MTU which is 1963 * less than or equal to the maximum MTU, then use that. 1964 */ 1965 if (aligned_mtu_idx >= 0 && 1966 mtu_idx - aligned_mtu_idx <= 1) 1967 mtu_idx = aligned_mtu_idx; 1968 1969 /* If the caller has passed in an MTU Index pointer, pass the 1970 * MTU Index back. Return the MTU value. 1971 */ 1972 if (mtu_idxp) 1973 *mtu_idxp = mtu_idx; 1974 return mtus[mtu_idx]; 1975 } 1976 EXPORT_SYMBOL(cxgb4_best_aligned_mtu); 1977 1978 /** 1979 * cxgb4_tp_smt_idx - Get the Source Mac Table index for this VI 1980 * @chip: chip type 1981 * @viid: VI id of the given port 1982 * 1983 * Return the SMT index for this VI. 1984 */ 1985 unsigned int cxgb4_tp_smt_idx(enum chip_type chip, unsigned int viid) 1986 { 1987 /* In T4/T5, SMT contains 256 SMAC entries organized in 1988 * 128 rows of 2 entries each. 1989 * In T6, SMT contains 256 SMAC entries in 256 rows. 1990 * TODO: The below code needs to be updated when we add support 1991 * for 256 VFs. 1992 */ 1993 if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5) 1994 return ((viid & 0x7f) << 1); 1995 else 1996 return (viid & 0x7f); 1997 } 1998 EXPORT_SYMBOL(cxgb4_tp_smt_idx); 1999 2000 /** 2001 * cxgb4_port_chan - get the HW channel of a port 2002 * @dev: the net device for the port 2003 * 2004 * Return the HW Tx channel of the given port. 2005 */ 2006 unsigned int cxgb4_port_chan(const struct net_device *dev) 2007 { 2008 return netdev2pinfo(dev)->tx_chan; 2009 } 2010 EXPORT_SYMBOL(cxgb4_port_chan); 2011 2012 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo) 2013 { 2014 struct adapter *adap = netdev2adap(dev); 2015 u32 v1, v2, lp_count, hp_count; 2016 2017 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 2018 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 2019 if (is_t4(adap->params.chip)) { 2020 lp_count = LP_COUNT_G(v1); 2021 hp_count = HP_COUNT_G(v1); 2022 } else { 2023 lp_count = LP_COUNT_T5_G(v1); 2024 hp_count = HP_COUNT_T5_G(v2); 2025 } 2026 return lpfifo ? lp_count : hp_count; 2027 } 2028 EXPORT_SYMBOL(cxgb4_dbfifo_count); 2029 2030 /** 2031 * cxgb4_port_viid - get the VI id of a port 2032 * @dev: the net device for the port 2033 * 2034 * Return the VI id of the given port. 2035 */ 2036 unsigned int cxgb4_port_viid(const struct net_device *dev) 2037 { 2038 return netdev2pinfo(dev)->viid; 2039 } 2040 EXPORT_SYMBOL(cxgb4_port_viid); 2041 2042 /** 2043 * cxgb4_port_idx - get the index of a port 2044 * @dev: the net device for the port 2045 * 2046 * Return the index of the given port. 2047 */ 2048 unsigned int cxgb4_port_idx(const struct net_device *dev) 2049 { 2050 return netdev2pinfo(dev)->port_id; 2051 } 2052 EXPORT_SYMBOL(cxgb4_port_idx); 2053 2054 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4, 2055 struct tp_tcp_stats *v6) 2056 { 2057 struct adapter *adap = pci_get_drvdata(pdev); 2058 2059 spin_lock(&adap->stats_lock); 2060 t4_tp_get_tcp_stats(adap, v4, v6); 2061 spin_unlock(&adap->stats_lock); 2062 } 2063 EXPORT_SYMBOL(cxgb4_get_tcp_stats); 2064 2065 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask, 2066 const unsigned int *pgsz_order) 2067 { 2068 struct adapter *adap = netdev2adap(dev); 2069 2070 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask); 2071 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) | 2072 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) | 2073 HPZ3_V(pgsz_order[3])); 2074 } 2075 EXPORT_SYMBOL(cxgb4_iscsi_init); 2076 2077 int cxgb4_flush_eq_cache(struct net_device *dev) 2078 { 2079 struct adapter *adap = netdev2adap(dev); 2080 2081 return t4_sge_ctxt_flush(adap, adap->mbox); 2082 } 2083 EXPORT_SYMBOL(cxgb4_flush_eq_cache); 2084 2085 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx) 2086 { 2087 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8; 2088 __be64 indices; 2089 int ret; 2090 2091 spin_lock(&adap->win0_lock); 2092 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr, 2093 sizeof(indices), (__be32 *)&indices, 2094 T4_MEMORY_READ); 2095 spin_unlock(&adap->win0_lock); 2096 if (!ret) { 2097 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff; 2098 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff; 2099 } 2100 return ret; 2101 } 2102 2103 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, 2104 u16 size) 2105 { 2106 struct adapter *adap = netdev2adap(dev); 2107 u16 hw_pidx, hw_cidx; 2108 int ret; 2109 2110 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx); 2111 if (ret) 2112 goto out; 2113 2114 if (pidx != hw_pidx) { 2115 u16 delta; 2116 u32 val; 2117 2118 if (pidx >= hw_pidx) 2119 delta = pidx - hw_pidx; 2120 else 2121 delta = size - hw_pidx + pidx; 2122 2123 if (is_t4(adap->params.chip)) 2124 val = PIDX_V(delta); 2125 else 2126 val = PIDX_T5_V(delta); 2127 wmb(); 2128 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 2129 QID_V(qid) | val); 2130 } 2131 out: 2132 return ret; 2133 } 2134 EXPORT_SYMBOL(cxgb4_sync_txq_pidx); 2135 2136 int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte) 2137 { 2138 struct adapter *adap; 2139 u32 offset, memtype, memaddr; 2140 u32 edc0_size, edc1_size, mc0_size, mc1_size, size; 2141 u32 edc0_end, edc1_end, mc0_end, mc1_end; 2142 int ret; 2143 2144 adap = netdev2adap(dev); 2145 2146 offset = ((stag >> 8) * 32) + adap->vres.stag.start; 2147 2148 /* Figure out where the offset lands in the Memory Type/Address scheme. 2149 * This code assumes that the memory is laid out starting at offset 0 2150 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0 2151 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have 2152 * MC0, and some have both MC0 and MC1. 2153 */ 2154 size = t4_read_reg(adap, MA_EDRAM0_BAR_A); 2155 edc0_size = EDRAM0_SIZE_G(size) << 20; 2156 size = t4_read_reg(adap, MA_EDRAM1_BAR_A); 2157 edc1_size = EDRAM1_SIZE_G(size) << 20; 2158 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A); 2159 mc0_size = EXT_MEM0_SIZE_G(size) << 20; 2160 2161 edc0_end = edc0_size; 2162 edc1_end = edc0_end + edc1_size; 2163 mc0_end = edc1_end + mc0_size; 2164 2165 if (offset < edc0_end) { 2166 memtype = MEM_EDC0; 2167 memaddr = offset; 2168 } else if (offset < edc1_end) { 2169 memtype = MEM_EDC1; 2170 memaddr = offset - edc0_end; 2171 } else { 2172 if (offset < mc0_end) { 2173 memtype = MEM_MC0; 2174 memaddr = offset - edc1_end; 2175 } else if (is_t5(adap->params.chip)) { 2176 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); 2177 mc1_size = EXT_MEM1_SIZE_G(size) << 20; 2178 mc1_end = mc0_end + mc1_size; 2179 if (offset < mc1_end) { 2180 memtype = MEM_MC1; 2181 memaddr = offset - mc0_end; 2182 } else { 2183 /* offset beyond the end of any memory */ 2184 goto err; 2185 } 2186 } else { 2187 /* T4/T6 only has a single memory channel */ 2188 goto err; 2189 } 2190 } 2191 2192 spin_lock(&adap->win0_lock); 2193 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ); 2194 spin_unlock(&adap->win0_lock); 2195 return ret; 2196 2197 err: 2198 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n", 2199 stag, offset); 2200 return -EINVAL; 2201 } 2202 EXPORT_SYMBOL(cxgb4_read_tpte); 2203 2204 u64 cxgb4_read_sge_timestamp(struct net_device *dev) 2205 { 2206 u32 hi, lo; 2207 struct adapter *adap; 2208 2209 adap = netdev2adap(dev); 2210 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A); 2211 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A)); 2212 2213 return ((u64)hi << 32) | (u64)lo; 2214 } 2215 EXPORT_SYMBOL(cxgb4_read_sge_timestamp); 2216 2217 int cxgb4_bar2_sge_qregs(struct net_device *dev, 2218 unsigned int qid, 2219 enum cxgb4_bar2_qtype qtype, 2220 int user, 2221 u64 *pbar2_qoffset, 2222 unsigned int *pbar2_qid) 2223 { 2224 return t4_bar2_sge_qregs(netdev2adap(dev), 2225 qid, 2226 (qtype == CXGB4_BAR2_QTYPE_EGRESS 2227 ? T4_BAR2_QTYPE_EGRESS 2228 : T4_BAR2_QTYPE_INGRESS), 2229 user, 2230 pbar2_qoffset, 2231 pbar2_qid); 2232 } 2233 EXPORT_SYMBOL(cxgb4_bar2_sge_qregs); 2234 2235 static struct pci_driver cxgb4_driver; 2236 2237 static void check_neigh_update(struct neighbour *neigh) 2238 { 2239 const struct device *parent; 2240 const struct net_device *netdev = neigh->dev; 2241 2242 if (netdev->priv_flags & IFF_802_1Q_VLAN) 2243 netdev = vlan_dev_real_dev(netdev); 2244 parent = netdev->dev.parent; 2245 if (parent && parent->driver == &cxgb4_driver.driver) 2246 t4_l2t_update(dev_get_drvdata(parent), neigh); 2247 } 2248 2249 static int netevent_cb(struct notifier_block *nb, unsigned long event, 2250 void *data) 2251 { 2252 switch (event) { 2253 case NETEVENT_NEIGH_UPDATE: 2254 check_neigh_update(data); 2255 break; 2256 case NETEVENT_REDIRECT: 2257 default: 2258 break; 2259 } 2260 return 0; 2261 } 2262 2263 static bool netevent_registered; 2264 static struct notifier_block cxgb4_netevent_nb = { 2265 .notifier_call = netevent_cb 2266 }; 2267 2268 static void drain_db_fifo(struct adapter *adap, int usecs) 2269 { 2270 u32 v1, v2, lp_count, hp_count; 2271 2272 do { 2273 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 2274 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 2275 if (is_t4(adap->params.chip)) { 2276 lp_count = LP_COUNT_G(v1); 2277 hp_count = HP_COUNT_G(v1); 2278 } else { 2279 lp_count = LP_COUNT_T5_G(v1); 2280 hp_count = HP_COUNT_T5_G(v2); 2281 } 2282 2283 if (lp_count == 0 && hp_count == 0) 2284 break; 2285 set_current_state(TASK_UNINTERRUPTIBLE); 2286 schedule_timeout(usecs_to_jiffies(usecs)); 2287 } while (1); 2288 } 2289 2290 static void disable_txq_db(struct sge_txq *q) 2291 { 2292 unsigned long flags; 2293 2294 spin_lock_irqsave(&q->db_lock, flags); 2295 q->db_disabled = 1; 2296 spin_unlock_irqrestore(&q->db_lock, flags); 2297 } 2298 2299 static void enable_txq_db(struct adapter *adap, struct sge_txq *q) 2300 { 2301 spin_lock_irq(&q->db_lock); 2302 if (q->db_pidx_inc) { 2303 /* Make sure that all writes to the TX descriptors 2304 * are committed before we tell HW about them. 2305 */ 2306 wmb(); 2307 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 2308 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc)); 2309 q->db_pidx_inc = 0; 2310 } 2311 q->db_disabled = 0; 2312 spin_unlock_irq(&q->db_lock); 2313 } 2314 2315 static void disable_dbs(struct adapter *adap) 2316 { 2317 int i; 2318 2319 for_each_ethrxq(&adap->sge, i) 2320 disable_txq_db(&adap->sge.ethtxq[i].q); 2321 for_each_iscsirxq(&adap->sge, i) 2322 disable_txq_db(&adap->sge.ofldtxq[i].q); 2323 for_each_port(adap, i) 2324 disable_txq_db(&adap->sge.ctrlq[i].q); 2325 } 2326 2327 static void enable_dbs(struct adapter *adap) 2328 { 2329 int i; 2330 2331 for_each_ethrxq(&adap->sge, i) 2332 enable_txq_db(adap, &adap->sge.ethtxq[i].q); 2333 for_each_iscsirxq(&adap->sge, i) 2334 enable_txq_db(adap, &adap->sge.ofldtxq[i].q); 2335 for_each_port(adap, i) 2336 enable_txq_db(adap, &adap->sge.ctrlq[i].q); 2337 } 2338 2339 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd) 2340 { 2341 if (adap->uld_handle[CXGB4_ULD_RDMA]) 2342 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA], 2343 cmd); 2344 } 2345 2346 static void process_db_full(struct work_struct *work) 2347 { 2348 struct adapter *adap; 2349 2350 adap = container_of(work, struct adapter, db_full_task); 2351 2352 drain_db_fifo(adap, dbfifo_drain_delay); 2353 enable_dbs(adap); 2354 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 2355 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 2356 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 2357 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 2358 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F); 2359 else 2360 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 2361 DBFIFO_LP_INT_F, DBFIFO_LP_INT_F); 2362 } 2363 2364 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q) 2365 { 2366 u16 hw_pidx, hw_cidx; 2367 int ret; 2368 2369 spin_lock_irq(&q->db_lock); 2370 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx); 2371 if (ret) 2372 goto out; 2373 if (q->db_pidx != hw_pidx) { 2374 u16 delta; 2375 u32 val; 2376 2377 if (q->db_pidx >= hw_pidx) 2378 delta = q->db_pidx - hw_pidx; 2379 else 2380 delta = q->size - hw_pidx + q->db_pidx; 2381 2382 if (is_t4(adap->params.chip)) 2383 val = PIDX_V(delta); 2384 else 2385 val = PIDX_T5_V(delta); 2386 wmb(); 2387 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 2388 QID_V(q->cntxt_id) | val); 2389 } 2390 out: 2391 q->db_disabled = 0; 2392 q->db_pidx_inc = 0; 2393 spin_unlock_irq(&q->db_lock); 2394 if (ret) 2395 CH_WARN(adap, "DB drop recovery failed.\n"); 2396 } 2397 static void recover_all_queues(struct adapter *adap) 2398 { 2399 int i; 2400 2401 for_each_ethrxq(&adap->sge, i) 2402 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q); 2403 for_each_iscsirxq(&adap->sge, i) 2404 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q); 2405 for_each_port(adap, i) 2406 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q); 2407 } 2408 2409 static void process_db_drop(struct work_struct *work) 2410 { 2411 struct adapter *adap; 2412 2413 adap = container_of(work, struct adapter, db_drop_task); 2414 2415 if (is_t4(adap->params.chip)) { 2416 drain_db_fifo(adap, dbfifo_drain_delay); 2417 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP); 2418 drain_db_fifo(adap, dbfifo_drain_delay); 2419 recover_all_queues(adap); 2420 drain_db_fifo(adap, dbfifo_drain_delay); 2421 enable_dbs(adap); 2422 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 2423 } else if (is_t5(adap->params.chip)) { 2424 u32 dropped_db = t4_read_reg(adap, 0x010ac); 2425 u16 qid = (dropped_db >> 15) & 0x1ffff; 2426 u16 pidx_inc = dropped_db & 0x1fff; 2427 u64 bar2_qoffset; 2428 unsigned int bar2_qid; 2429 int ret; 2430 2431 ret = t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS, 2432 0, &bar2_qoffset, &bar2_qid); 2433 if (ret) 2434 dev_err(adap->pdev_dev, "doorbell drop recovery: " 2435 "qid=%d, pidx_inc=%d\n", qid, pidx_inc); 2436 else 2437 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid), 2438 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL); 2439 2440 /* Re-enable BAR2 WC */ 2441 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15); 2442 } 2443 2444 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 2445 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0); 2446 } 2447 2448 void t4_db_full(struct adapter *adap) 2449 { 2450 if (is_t4(adap->params.chip)) { 2451 disable_dbs(adap); 2452 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 2453 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 2454 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0); 2455 queue_work(adap->workq, &adap->db_full_task); 2456 } 2457 } 2458 2459 void t4_db_dropped(struct adapter *adap) 2460 { 2461 if (is_t4(adap->params.chip)) { 2462 disable_dbs(adap); 2463 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 2464 } 2465 queue_work(adap->workq, &adap->db_drop_task); 2466 } 2467 2468 static void uld_attach(struct adapter *adap, unsigned int uld) 2469 { 2470 void *handle; 2471 struct cxgb4_lld_info lli; 2472 unsigned short i; 2473 2474 lli.pdev = adap->pdev; 2475 lli.pf = adap->pf; 2476 lli.l2t = adap->l2t; 2477 lli.tids = &adap->tids; 2478 lli.ports = adap->port; 2479 lli.vr = &adap->vres; 2480 lli.mtus = adap->params.mtus; 2481 if (uld == CXGB4_ULD_RDMA) { 2482 lli.rxq_ids = adap->sge.rdma_rxq; 2483 lli.ciq_ids = adap->sge.rdma_ciq; 2484 lli.nrxq = adap->sge.rdmaqs; 2485 lli.nciq = adap->sge.rdmaciqs; 2486 } else if (uld == CXGB4_ULD_ISCSI) { 2487 lli.rxq_ids = adap->sge.iscsi_rxq; 2488 lli.nrxq = adap->sge.iscsiqsets; 2489 } else if (uld == CXGB4_ULD_ISCSIT) { 2490 lli.rxq_ids = adap->sge.iscsit_rxq; 2491 lli.nrxq = adap->sge.niscsitq; 2492 } 2493 lli.ntxq = adap->sge.iscsiqsets; 2494 lli.nchan = adap->params.nports; 2495 lli.nports = adap->params.nports; 2496 lli.wr_cred = adap->params.ofldq_wr_cred; 2497 lli.adapter_type = adap->params.chip; 2498 lli.iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A)); 2499 lli.iscsi_tagmask = t4_read_reg(adap, ULP_RX_ISCSI_TAGMASK_A); 2500 lli.iscsi_pgsz_order = t4_read_reg(adap, ULP_RX_ISCSI_PSZ_A); 2501 lli.iscsi_llimit = t4_read_reg(adap, ULP_RX_ISCSI_LLIMIT_A); 2502 lli.iscsi_ppm = &adap->iscsi_ppm; 2503 lli.cclk_ps = 1000000000 / adap->params.vpd.cclk; 2504 lli.udb_density = 1 << adap->params.sge.eq_qpp; 2505 lli.ucq_density = 1 << adap->params.sge.iq_qpp; 2506 lli.filt_mode = adap->params.tp.vlan_pri_map; 2507 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */ 2508 for (i = 0; i < NCHAN; i++) 2509 lli.tx_modq[i] = i; 2510 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A); 2511 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A); 2512 lli.fw_vers = adap->params.fw_vers; 2513 lli.dbfifo_int_thresh = dbfifo_int_thresh; 2514 lli.sge_ingpadboundary = adap->sge.fl_align; 2515 lli.sge_egrstatuspagesize = adap->sge.stat_len; 2516 lli.sge_pktshift = adap->sge.pktshift; 2517 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN; 2518 lli.max_ordird_qp = adap->params.max_ordird_qp; 2519 lli.max_ird_adapter = adap->params.max_ird_adapter; 2520 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl; 2521 lli.nodeid = dev_to_node(adap->pdev_dev); 2522 2523 handle = ulds[uld].add(&lli); 2524 if (IS_ERR(handle)) { 2525 dev_warn(adap->pdev_dev, 2526 "could not attach to the %s driver, error %ld\n", 2527 uld_str[uld], PTR_ERR(handle)); 2528 return; 2529 } 2530 2531 adap->uld_handle[uld] = handle; 2532 2533 if (!netevent_registered) { 2534 register_netevent_notifier(&cxgb4_netevent_nb); 2535 netevent_registered = true; 2536 } 2537 2538 if (adap->flags & FULL_INIT_DONE) 2539 ulds[uld].state_change(handle, CXGB4_STATE_UP); 2540 } 2541 2542 static void attach_ulds(struct adapter *adap) 2543 { 2544 unsigned int i; 2545 2546 spin_lock(&adap_rcu_lock); 2547 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list); 2548 spin_unlock(&adap_rcu_lock); 2549 2550 mutex_lock(&uld_mutex); 2551 list_add_tail(&adap->list_node, &adapter_list); 2552 for (i = 0; i < CXGB4_ULD_MAX; i++) 2553 if (ulds[i].add) 2554 uld_attach(adap, i); 2555 mutex_unlock(&uld_mutex); 2556 } 2557 2558 static void detach_ulds(struct adapter *adap) 2559 { 2560 unsigned int i; 2561 2562 mutex_lock(&uld_mutex); 2563 list_del(&adap->list_node); 2564 for (i = 0; i < CXGB4_ULD_MAX; i++) 2565 if (adap->uld_handle[i]) { 2566 ulds[i].state_change(adap->uld_handle[i], 2567 CXGB4_STATE_DETACH); 2568 adap->uld_handle[i] = NULL; 2569 } 2570 if (netevent_registered && list_empty(&adapter_list)) { 2571 unregister_netevent_notifier(&cxgb4_netevent_nb); 2572 netevent_registered = false; 2573 } 2574 mutex_unlock(&uld_mutex); 2575 2576 spin_lock(&adap_rcu_lock); 2577 list_del_rcu(&adap->rcu_node); 2578 spin_unlock(&adap_rcu_lock); 2579 } 2580 2581 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state) 2582 { 2583 unsigned int i; 2584 2585 mutex_lock(&uld_mutex); 2586 for (i = 0; i < CXGB4_ULD_MAX; i++) 2587 if (adap->uld_handle[i]) 2588 ulds[i].state_change(adap->uld_handle[i], new_state); 2589 mutex_unlock(&uld_mutex); 2590 } 2591 2592 /** 2593 * cxgb4_register_uld - register an upper-layer driver 2594 * @type: the ULD type 2595 * @p: the ULD methods 2596 * 2597 * Registers an upper-layer driver with this driver and notifies the ULD 2598 * about any presently available devices that support its type. Returns 2599 * %-EBUSY if a ULD of the same type is already registered. 2600 */ 2601 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p) 2602 { 2603 int ret = 0; 2604 struct adapter *adap; 2605 2606 if (type >= CXGB4_ULD_MAX) 2607 return -EINVAL; 2608 mutex_lock(&uld_mutex); 2609 if (ulds[type].add) { 2610 ret = -EBUSY; 2611 goto out; 2612 } 2613 ulds[type] = *p; 2614 list_for_each_entry(adap, &adapter_list, list_node) 2615 uld_attach(adap, type); 2616 out: mutex_unlock(&uld_mutex); 2617 return ret; 2618 } 2619 EXPORT_SYMBOL(cxgb4_register_uld); 2620 2621 /** 2622 * cxgb4_unregister_uld - unregister an upper-layer driver 2623 * @type: the ULD type 2624 * 2625 * Unregisters an existing upper-layer driver. 2626 */ 2627 int cxgb4_unregister_uld(enum cxgb4_uld type) 2628 { 2629 struct adapter *adap; 2630 2631 if (type >= CXGB4_ULD_MAX) 2632 return -EINVAL; 2633 mutex_lock(&uld_mutex); 2634 list_for_each_entry(adap, &adapter_list, list_node) 2635 adap->uld_handle[type] = NULL; 2636 ulds[type].add = NULL; 2637 mutex_unlock(&uld_mutex); 2638 return 0; 2639 } 2640 EXPORT_SYMBOL(cxgb4_unregister_uld); 2641 2642 #if IS_ENABLED(CONFIG_IPV6) 2643 static int cxgb4_inet6addr_handler(struct notifier_block *this, 2644 unsigned long event, void *data) 2645 { 2646 struct inet6_ifaddr *ifa = data; 2647 struct net_device *event_dev = ifa->idev->dev; 2648 const struct device *parent = NULL; 2649 #if IS_ENABLED(CONFIG_BONDING) 2650 struct adapter *adap; 2651 #endif 2652 if (event_dev->priv_flags & IFF_802_1Q_VLAN) 2653 event_dev = vlan_dev_real_dev(event_dev); 2654 #if IS_ENABLED(CONFIG_BONDING) 2655 if (event_dev->flags & IFF_MASTER) { 2656 list_for_each_entry(adap, &adapter_list, list_node) { 2657 switch (event) { 2658 case NETDEV_UP: 2659 cxgb4_clip_get(adap->port[0], 2660 (const u32 *)ifa, 1); 2661 break; 2662 case NETDEV_DOWN: 2663 cxgb4_clip_release(adap->port[0], 2664 (const u32 *)ifa, 1); 2665 break; 2666 default: 2667 break; 2668 } 2669 } 2670 return NOTIFY_OK; 2671 } 2672 #endif 2673 2674 if (event_dev) 2675 parent = event_dev->dev.parent; 2676 2677 if (parent && parent->driver == &cxgb4_driver.driver) { 2678 switch (event) { 2679 case NETDEV_UP: 2680 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1); 2681 break; 2682 case NETDEV_DOWN: 2683 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1); 2684 break; 2685 default: 2686 break; 2687 } 2688 } 2689 return NOTIFY_OK; 2690 } 2691 2692 static bool inet6addr_registered; 2693 static struct notifier_block cxgb4_inet6addr_notifier = { 2694 .notifier_call = cxgb4_inet6addr_handler 2695 }; 2696 2697 static void update_clip(const struct adapter *adap) 2698 { 2699 int i; 2700 struct net_device *dev; 2701 int ret; 2702 2703 rcu_read_lock(); 2704 2705 for (i = 0; i < MAX_NPORTS; i++) { 2706 dev = adap->port[i]; 2707 ret = 0; 2708 2709 if (dev) 2710 ret = cxgb4_update_root_dev_clip(dev); 2711 2712 if (ret < 0) 2713 break; 2714 } 2715 rcu_read_unlock(); 2716 } 2717 #endif /* IS_ENABLED(CONFIG_IPV6) */ 2718 2719 /** 2720 * cxgb_up - enable the adapter 2721 * @adap: adapter being enabled 2722 * 2723 * Called when the first port is enabled, this function performs the 2724 * actions necessary to make an adapter operational, such as completing 2725 * the initialization of HW modules, and enabling interrupts. 2726 * 2727 * Must be called with the rtnl lock held. 2728 */ 2729 static int cxgb_up(struct adapter *adap) 2730 { 2731 int err; 2732 2733 err = setup_sge_queues(adap); 2734 if (err) 2735 goto out; 2736 err = setup_rss(adap); 2737 if (err) 2738 goto freeq; 2739 2740 if (adap->flags & USING_MSIX) { 2741 name_msix_vecs(adap); 2742 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0, 2743 adap->msix_info[0].desc, adap); 2744 if (err) 2745 goto irq_err; 2746 2747 err = request_msix_queue_irqs(adap); 2748 if (err) { 2749 free_irq(adap->msix_info[0].vec, adap); 2750 goto irq_err; 2751 } 2752 } else { 2753 err = request_irq(adap->pdev->irq, t4_intr_handler(adap), 2754 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED, 2755 adap->port[0]->name, adap); 2756 if (err) 2757 goto irq_err; 2758 } 2759 enable_rx(adap); 2760 t4_sge_start(adap); 2761 t4_intr_enable(adap); 2762 adap->flags |= FULL_INIT_DONE; 2763 notify_ulds(adap, CXGB4_STATE_UP); 2764 #if IS_ENABLED(CONFIG_IPV6) 2765 update_clip(adap); 2766 #endif 2767 /* Initialize hash mac addr list*/ 2768 INIT_LIST_HEAD(&adap->mac_hlist); 2769 out: 2770 return err; 2771 irq_err: 2772 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err); 2773 freeq: 2774 t4_free_sge_resources(adap); 2775 goto out; 2776 } 2777 2778 static void cxgb_down(struct adapter *adapter) 2779 { 2780 cancel_work_sync(&adapter->tid_release_task); 2781 cancel_work_sync(&adapter->db_full_task); 2782 cancel_work_sync(&adapter->db_drop_task); 2783 adapter->tid_release_task_busy = false; 2784 adapter->tid_release_head = NULL; 2785 2786 t4_sge_stop(adapter); 2787 t4_free_sge_resources(adapter); 2788 adapter->flags &= ~FULL_INIT_DONE; 2789 } 2790 2791 /* 2792 * net_device operations 2793 */ 2794 static int cxgb_open(struct net_device *dev) 2795 { 2796 int err; 2797 struct port_info *pi = netdev_priv(dev); 2798 struct adapter *adapter = pi->adapter; 2799 2800 netif_carrier_off(dev); 2801 2802 if (!(adapter->flags & FULL_INIT_DONE)) { 2803 err = cxgb_up(adapter); 2804 if (err < 0) 2805 return err; 2806 } 2807 2808 err = link_start(dev); 2809 if (!err) 2810 netif_tx_start_all_queues(dev); 2811 return err; 2812 } 2813 2814 static int cxgb_close(struct net_device *dev) 2815 { 2816 struct port_info *pi = netdev_priv(dev); 2817 struct adapter *adapter = pi->adapter; 2818 2819 netif_tx_stop_all_queues(dev); 2820 netif_carrier_off(dev); 2821 return t4_enable_vi(adapter, adapter->pf, pi->viid, false, false); 2822 } 2823 2824 /* Return an error number if the indicated filter isn't writable ... 2825 */ 2826 static int writable_filter(struct filter_entry *f) 2827 { 2828 if (f->locked) 2829 return -EPERM; 2830 if (f->pending) 2831 return -EBUSY; 2832 2833 return 0; 2834 } 2835 2836 /* Delete the filter at the specified index (if valid). The checks for all 2837 * the common problems with doing this like the filter being locked, currently 2838 * pending in another operation, etc. 2839 */ 2840 static int delete_filter(struct adapter *adapter, unsigned int fidx) 2841 { 2842 struct filter_entry *f; 2843 int ret; 2844 2845 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids) 2846 return -EINVAL; 2847 2848 f = &adapter->tids.ftid_tab[fidx]; 2849 ret = writable_filter(f); 2850 if (ret) 2851 return ret; 2852 if (f->valid) 2853 return del_filter_wr(adapter, fidx); 2854 2855 return 0; 2856 } 2857 2858 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid, 2859 __be32 sip, __be16 sport, __be16 vlan, 2860 unsigned int queue, unsigned char port, unsigned char mask) 2861 { 2862 int ret; 2863 struct filter_entry *f; 2864 struct adapter *adap; 2865 int i; 2866 u8 *val; 2867 2868 adap = netdev2adap(dev); 2869 2870 /* Adjust stid to correct filter index */ 2871 stid -= adap->tids.sftid_base; 2872 stid += adap->tids.nftids; 2873 2874 /* Check to make sure the filter requested is writable ... 2875 */ 2876 f = &adap->tids.ftid_tab[stid]; 2877 ret = writable_filter(f); 2878 if (ret) 2879 return ret; 2880 2881 /* Clear out any old resources being used by the filter before 2882 * we start constructing the new filter. 2883 */ 2884 if (f->valid) 2885 clear_filter(adap, f); 2886 2887 /* Clear out filter specifications */ 2888 memset(&f->fs, 0, sizeof(struct ch_filter_specification)); 2889 f->fs.val.lport = cpu_to_be16(sport); 2890 f->fs.mask.lport = ~0; 2891 val = (u8 *)&sip; 2892 if ((val[0] | val[1] | val[2] | val[3]) != 0) { 2893 for (i = 0; i < 4; i++) { 2894 f->fs.val.lip[i] = val[i]; 2895 f->fs.mask.lip[i] = ~0; 2896 } 2897 if (adap->params.tp.vlan_pri_map & PORT_F) { 2898 f->fs.val.iport = port; 2899 f->fs.mask.iport = mask; 2900 } 2901 } 2902 2903 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) { 2904 f->fs.val.proto = IPPROTO_TCP; 2905 f->fs.mask.proto = ~0; 2906 } 2907 2908 f->fs.dirsteer = 1; 2909 f->fs.iq = queue; 2910 /* Mark filter as locked */ 2911 f->locked = 1; 2912 f->fs.rpttid = 1; 2913 2914 ret = set_filter_wr(adap, stid); 2915 if (ret) { 2916 clear_filter(adap, f); 2917 return ret; 2918 } 2919 2920 return 0; 2921 } 2922 EXPORT_SYMBOL(cxgb4_create_server_filter); 2923 2924 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid, 2925 unsigned int queue, bool ipv6) 2926 { 2927 int ret; 2928 struct filter_entry *f; 2929 struct adapter *adap; 2930 2931 adap = netdev2adap(dev); 2932 2933 /* Adjust stid to correct filter index */ 2934 stid -= adap->tids.sftid_base; 2935 stid += adap->tids.nftids; 2936 2937 f = &adap->tids.ftid_tab[stid]; 2938 /* Unlock the filter */ 2939 f->locked = 0; 2940 2941 ret = delete_filter(adap, stid); 2942 if (ret) 2943 return ret; 2944 2945 return 0; 2946 } 2947 EXPORT_SYMBOL(cxgb4_remove_server_filter); 2948 2949 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev, 2950 struct rtnl_link_stats64 *ns) 2951 { 2952 struct port_stats stats; 2953 struct port_info *p = netdev_priv(dev); 2954 struct adapter *adapter = p->adapter; 2955 2956 /* Block retrieving statistics during EEH error 2957 * recovery. Otherwise, the recovery might fail 2958 * and the PCI device will be removed permanently 2959 */ 2960 spin_lock(&adapter->stats_lock); 2961 if (!netif_device_present(dev)) { 2962 spin_unlock(&adapter->stats_lock); 2963 return ns; 2964 } 2965 t4_get_port_stats_offset(adapter, p->tx_chan, &stats, 2966 &p->stats_base); 2967 spin_unlock(&adapter->stats_lock); 2968 2969 ns->tx_bytes = stats.tx_octets; 2970 ns->tx_packets = stats.tx_frames; 2971 ns->rx_bytes = stats.rx_octets; 2972 ns->rx_packets = stats.rx_frames; 2973 ns->multicast = stats.rx_mcast_frames; 2974 2975 /* detailed rx_errors */ 2976 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long + 2977 stats.rx_runt; 2978 ns->rx_over_errors = 0; 2979 ns->rx_crc_errors = stats.rx_fcs_err; 2980 ns->rx_frame_errors = stats.rx_symbol_err; 2981 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 + 2982 stats.rx_ovflow2 + stats.rx_ovflow3 + 2983 stats.rx_trunc0 + stats.rx_trunc1 + 2984 stats.rx_trunc2 + stats.rx_trunc3; 2985 ns->rx_missed_errors = 0; 2986 2987 /* detailed tx_errors */ 2988 ns->tx_aborted_errors = 0; 2989 ns->tx_carrier_errors = 0; 2990 ns->tx_fifo_errors = 0; 2991 ns->tx_heartbeat_errors = 0; 2992 ns->tx_window_errors = 0; 2993 2994 ns->tx_errors = stats.tx_error_frames; 2995 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err + 2996 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors; 2997 return ns; 2998 } 2999 3000 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd) 3001 { 3002 unsigned int mbox; 3003 int ret = 0, prtad, devad; 3004 struct port_info *pi = netdev_priv(dev); 3005 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data; 3006 3007 switch (cmd) { 3008 case SIOCGMIIPHY: 3009 if (pi->mdio_addr < 0) 3010 return -EOPNOTSUPP; 3011 data->phy_id = pi->mdio_addr; 3012 break; 3013 case SIOCGMIIREG: 3014 case SIOCSMIIREG: 3015 if (mdio_phy_id_is_c45(data->phy_id)) { 3016 prtad = mdio_phy_id_prtad(data->phy_id); 3017 devad = mdio_phy_id_devad(data->phy_id); 3018 } else if (data->phy_id < 32) { 3019 prtad = data->phy_id; 3020 devad = 0; 3021 data->reg_num &= 0x1f; 3022 } else 3023 return -EINVAL; 3024 3025 mbox = pi->adapter->pf; 3026 if (cmd == SIOCGMIIREG) 3027 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad, 3028 data->reg_num, &data->val_out); 3029 else 3030 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad, 3031 data->reg_num, data->val_in); 3032 break; 3033 case SIOCGHWTSTAMP: 3034 return copy_to_user(req->ifr_data, &pi->tstamp_config, 3035 sizeof(pi->tstamp_config)) ? 3036 -EFAULT : 0; 3037 case SIOCSHWTSTAMP: 3038 if (copy_from_user(&pi->tstamp_config, req->ifr_data, 3039 sizeof(pi->tstamp_config))) 3040 return -EFAULT; 3041 3042 switch (pi->tstamp_config.rx_filter) { 3043 case HWTSTAMP_FILTER_NONE: 3044 pi->rxtstamp = false; 3045 break; 3046 case HWTSTAMP_FILTER_ALL: 3047 pi->rxtstamp = true; 3048 break; 3049 default: 3050 pi->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; 3051 return -ERANGE; 3052 } 3053 3054 return copy_to_user(req->ifr_data, &pi->tstamp_config, 3055 sizeof(pi->tstamp_config)) ? 3056 -EFAULT : 0; 3057 default: 3058 return -EOPNOTSUPP; 3059 } 3060 return ret; 3061 } 3062 3063 static void cxgb_set_rxmode(struct net_device *dev) 3064 { 3065 /* unfortunately we can't return errors to the stack */ 3066 set_rxmode(dev, -1, false); 3067 } 3068 3069 static int cxgb_change_mtu(struct net_device *dev, int new_mtu) 3070 { 3071 int ret; 3072 struct port_info *pi = netdev_priv(dev); 3073 3074 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */ 3075 return -EINVAL; 3076 ret = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, new_mtu, -1, 3077 -1, -1, -1, true); 3078 if (!ret) 3079 dev->mtu = new_mtu; 3080 return ret; 3081 } 3082 3083 static int cxgb_set_mac_addr(struct net_device *dev, void *p) 3084 { 3085 int ret; 3086 struct sockaddr *addr = p; 3087 struct port_info *pi = netdev_priv(dev); 3088 3089 if (!is_valid_ether_addr(addr->sa_data)) 3090 return -EADDRNOTAVAIL; 3091 3092 ret = t4_change_mac(pi->adapter, pi->adapter->pf, pi->viid, 3093 pi->xact_addr_filt, addr->sa_data, true, true); 3094 if (ret < 0) 3095 return ret; 3096 3097 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 3098 pi->xact_addr_filt = ret; 3099 return 0; 3100 } 3101 3102 #ifdef CONFIG_NET_POLL_CONTROLLER 3103 static void cxgb_netpoll(struct net_device *dev) 3104 { 3105 struct port_info *pi = netdev_priv(dev); 3106 struct adapter *adap = pi->adapter; 3107 3108 if (adap->flags & USING_MSIX) { 3109 int i; 3110 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset]; 3111 3112 for (i = pi->nqsets; i; i--, rx++) 3113 t4_sge_intr_msix(0, &rx->rspq); 3114 } else 3115 t4_intr_handler(adap)(0, adap); 3116 } 3117 #endif 3118 3119 static const struct net_device_ops cxgb4_netdev_ops = { 3120 .ndo_open = cxgb_open, 3121 .ndo_stop = cxgb_close, 3122 .ndo_start_xmit = t4_eth_xmit, 3123 .ndo_select_queue = cxgb_select_queue, 3124 .ndo_get_stats64 = cxgb_get_stats, 3125 .ndo_set_rx_mode = cxgb_set_rxmode, 3126 .ndo_set_mac_address = cxgb_set_mac_addr, 3127 .ndo_set_features = cxgb_set_features, 3128 .ndo_validate_addr = eth_validate_addr, 3129 .ndo_do_ioctl = cxgb_ioctl, 3130 .ndo_change_mtu = cxgb_change_mtu, 3131 #ifdef CONFIG_NET_POLL_CONTROLLER 3132 .ndo_poll_controller = cxgb_netpoll, 3133 #endif 3134 #ifdef CONFIG_CHELSIO_T4_FCOE 3135 .ndo_fcoe_enable = cxgb_fcoe_enable, 3136 .ndo_fcoe_disable = cxgb_fcoe_disable, 3137 #endif /* CONFIG_CHELSIO_T4_FCOE */ 3138 #ifdef CONFIG_NET_RX_BUSY_POLL 3139 .ndo_busy_poll = cxgb_busy_poll, 3140 #endif 3141 3142 }; 3143 3144 void t4_fatal_err(struct adapter *adap) 3145 { 3146 t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0); 3147 t4_intr_disable(adap); 3148 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n"); 3149 } 3150 3151 static void setup_memwin(struct adapter *adap) 3152 { 3153 u32 nic_win_base = t4_get_util_window(adap); 3154 3155 t4_setup_memwin(adap, nic_win_base, MEMWIN_NIC); 3156 } 3157 3158 static void setup_memwin_rdma(struct adapter *adap) 3159 { 3160 if (adap->vres.ocq.size) { 3161 u32 start; 3162 unsigned int sz_kb; 3163 3164 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2); 3165 start &= PCI_BASE_ADDRESS_MEM_MASK; 3166 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres); 3167 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10; 3168 t4_write_reg(adap, 3169 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3), 3170 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb))); 3171 t4_write_reg(adap, 3172 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3), 3173 adap->vres.ocq.start); 3174 t4_read_reg(adap, 3175 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3)); 3176 } 3177 } 3178 3179 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c) 3180 { 3181 u32 v; 3182 int ret; 3183 3184 /* get device capabilities */ 3185 memset(c, 0, sizeof(*c)); 3186 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3187 FW_CMD_REQUEST_F | FW_CMD_READ_F); 3188 c->cfvalid_to_len16 = htonl(FW_LEN16(*c)); 3189 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), c); 3190 if (ret < 0) 3191 return ret; 3192 3193 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3194 FW_CMD_REQUEST_F | FW_CMD_WRITE_F); 3195 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), NULL); 3196 if (ret < 0) 3197 return ret; 3198 3199 ret = t4_config_glbl_rss(adap, adap->pf, 3200 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL, 3201 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F | 3202 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F); 3203 if (ret < 0) 3204 return ret; 3205 3206 ret = t4_cfg_pfvf(adap, adap->mbox, adap->pf, 0, adap->sge.egr_sz, 64, 3207 MAX_INGQ, 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, 3208 FW_CMD_CAP_PF); 3209 if (ret < 0) 3210 return ret; 3211 3212 t4_sge_init(adap); 3213 3214 /* tweak some settings */ 3215 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849); 3216 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12)); 3217 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A); 3218 v = t4_read_reg(adap, TP_PIO_DATA_A); 3219 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F); 3220 3221 /* first 4 Tx modulation queues point to consecutive Tx channels */ 3222 adap->params.tp.tx_modq_map = 0xE4; 3223 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A, 3224 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map)); 3225 3226 /* associate each Tx modulation queue with consecutive Tx channels */ 3227 v = 0x84218421; 3228 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3229 &v, 1, TP_TX_SCHED_HDR_A); 3230 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3231 &v, 1, TP_TX_SCHED_FIFO_A); 3232 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3233 &v, 1, TP_TX_SCHED_PCMD_A); 3234 3235 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */ 3236 if (is_offload(adap)) { 3237 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A, 3238 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3239 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3240 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3241 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 3242 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A, 3243 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3244 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3245 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3246 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 3247 } 3248 3249 /* get basic stuff going */ 3250 return t4_early_init(adap, adap->pf); 3251 } 3252 3253 /* 3254 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower. 3255 */ 3256 #define MAX_ATIDS 8192U 3257 3258 /* 3259 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 3260 * 3261 * If the firmware we're dealing with has Configuration File support, then 3262 * we use that to perform all configuration 3263 */ 3264 3265 /* 3266 * Tweak configuration based on module parameters, etc. Most of these have 3267 * defaults assigned to them by Firmware Configuration Files (if we're using 3268 * them) but need to be explicitly set if we're using hard-coded 3269 * initialization. But even in the case of using Firmware Configuration 3270 * Files, we'd like to expose the ability to change these via module 3271 * parameters so these are essentially common tweaks/settings for 3272 * Configuration Files and hard-coded initialization ... 3273 */ 3274 static int adap_init0_tweaks(struct adapter *adapter) 3275 { 3276 /* 3277 * Fix up various Host-Dependent Parameters like Page Size, Cache 3278 * Line Size, etc. The firmware default is for a 4KB Page Size and 3279 * 64B Cache Line Size ... 3280 */ 3281 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES); 3282 3283 /* 3284 * Process module parameters which affect early initialization. 3285 */ 3286 if (rx_dma_offset != 2 && rx_dma_offset != 0) { 3287 dev_err(&adapter->pdev->dev, 3288 "Ignoring illegal rx_dma_offset=%d, using 2\n", 3289 rx_dma_offset); 3290 rx_dma_offset = 2; 3291 } 3292 t4_set_reg_field(adapter, SGE_CONTROL_A, 3293 PKTSHIFT_V(PKTSHIFT_M), 3294 PKTSHIFT_V(rx_dma_offset)); 3295 3296 /* 3297 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux 3298 * adds the pseudo header itself. 3299 */ 3300 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A, 3301 CSUM_HAS_PSEUDO_HDR_F, 0); 3302 3303 return 0; 3304 } 3305 3306 /* 10Gb/s-BT PHY Support. chip-external 10Gb/s-BT PHYs are complex chips 3307 * unto themselves and they contain their own firmware to perform their 3308 * tasks ... 3309 */ 3310 static int phy_aq1202_version(const u8 *phy_fw_data, 3311 size_t phy_fw_size) 3312 { 3313 int offset; 3314 3315 /* At offset 0x8 you're looking for the primary image's 3316 * starting offset which is 3 Bytes wide 3317 * 3318 * At offset 0xa of the primary image, you look for the offset 3319 * of the DRAM segment which is 3 Bytes wide. 3320 * 3321 * The FW version is at offset 0x27e of the DRAM and is 2 Bytes 3322 * wide 3323 */ 3324 #define be16(__p) (((__p)[0] << 8) | (__p)[1]) 3325 #define le16(__p) ((__p)[0] | ((__p)[1] << 8)) 3326 #define le24(__p) (le16(__p) | ((__p)[2] << 16)) 3327 3328 offset = le24(phy_fw_data + 0x8) << 12; 3329 offset = le24(phy_fw_data + offset + 0xa); 3330 return be16(phy_fw_data + offset + 0x27e); 3331 3332 #undef be16 3333 #undef le16 3334 #undef le24 3335 } 3336 3337 static struct info_10gbt_phy_fw { 3338 unsigned int phy_fw_id; /* PCI Device ID */ 3339 char *phy_fw_file; /* /lib/firmware/ PHY Firmware file */ 3340 int (*phy_fw_version)(const u8 *phy_fw_data, size_t phy_fw_size); 3341 int phy_flash; /* Has FLASH for PHY Firmware */ 3342 } phy_info_array[] = { 3343 { 3344 PHY_AQ1202_DEVICEID, 3345 PHY_AQ1202_FIRMWARE, 3346 phy_aq1202_version, 3347 1, 3348 }, 3349 { 3350 PHY_BCM84834_DEVICEID, 3351 PHY_BCM84834_FIRMWARE, 3352 NULL, 3353 0, 3354 }, 3355 { 0, NULL, NULL }, 3356 }; 3357 3358 static struct info_10gbt_phy_fw *find_phy_info(int devid) 3359 { 3360 int i; 3361 3362 for (i = 0; i < ARRAY_SIZE(phy_info_array); i++) { 3363 if (phy_info_array[i].phy_fw_id == devid) 3364 return &phy_info_array[i]; 3365 } 3366 return NULL; 3367 } 3368 3369 /* Handle updating of chip-external 10Gb/s-BT PHY firmware. This needs to 3370 * happen after the FW_RESET_CMD but before the FW_INITIALIZE_CMD. On error 3371 * we return a negative error number. If we transfer new firmware we return 1 3372 * (from t4_load_phy_fw()). If we don't do anything we return 0. 3373 */ 3374 static int adap_init0_phy(struct adapter *adap) 3375 { 3376 const struct firmware *phyf; 3377 int ret; 3378 struct info_10gbt_phy_fw *phy_info; 3379 3380 /* Use the device ID to determine which PHY file to flash. 3381 */ 3382 phy_info = find_phy_info(adap->pdev->device); 3383 if (!phy_info) { 3384 dev_warn(adap->pdev_dev, 3385 "No PHY Firmware file found for this PHY\n"); 3386 return -EOPNOTSUPP; 3387 } 3388 3389 /* If we have a T4 PHY firmware file under /lib/firmware/cxgb4/, then 3390 * use that. The adapter firmware provides us with a memory buffer 3391 * where we can load a PHY firmware file from the host if we want to 3392 * override the PHY firmware File in flash. 3393 */ 3394 ret = request_firmware_direct(&phyf, phy_info->phy_fw_file, 3395 adap->pdev_dev); 3396 if (ret < 0) { 3397 /* For adapters without FLASH attached to PHY for their 3398 * firmware, it's obviously a fatal error if we can't get the 3399 * firmware to the adapter. For adapters with PHY firmware 3400 * FLASH storage, it's worth a warning if we can't find the 3401 * PHY Firmware but we'll neuter the error ... 3402 */ 3403 dev_err(adap->pdev_dev, "unable to find PHY Firmware image " 3404 "/lib/firmware/%s, error %d\n", 3405 phy_info->phy_fw_file, -ret); 3406 if (phy_info->phy_flash) { 3407 int cur_phy_fw_ver = 0; 3408 3409 t4_phy_fw_ver(adap, &cur_phy_fw_ver); 3410 dev_warn(adap->pdev_dev, "continuing with, on-adapter " 3411 "FLASH copy, version %#x\n", cur_phy_fw_ver); 3412 ret = 0; 3413 } 3414 3415 return ret; 3416 } 3417 3418 /* Load PHY Firmware onto adapter. 3419 */ 3420 ret = t4_load_phy_fw(adap, MEMWIN_NIC, &adap->win0_lock, 3421 phy_info->phy_fw_version, 3422 (u8 *)phyf->data, phyf->size); 3423 if (ret < 0) 3424 dev_err(adap->pdev_dev, "PHY Firmware transfer error %d\n", 3425 -ret); 3426 else if (ret > 0) { 3427 int new_phy_fw_ver = 0; 3428 3429 if (phy_info->phy_fw_version) 3430 new_phy_fw_ver = phy_info->phy_fw_version(phyf->data, 3431 phyf->size); 3432 dev_info(adap->pdev_dev, "Successfully transferred PHY " 3433 "Firmware /lib/firmware/%s, version %#x\n", 3434 phy_info->phy_fw_file, new_phy_fw_ver); 3435 } 3436 3437 release_firmware(phyf); 3438 3439 return ret; 3440 } 3441 3442 /* 3443 * Attempt to initialize the adapter via a Firmware Configuration File. 3444 */ 3445 static int adap_init0_config(struct adapter *adapter, int reset) 3446 { 3447 struct fw_caps_config_cmd caps_cmd; 3448 const struct firmware *cf; 3449 unsigned long mtype = 0, maddr = 0; 3450 u32 finiver, finicsum, cfcsum; 3451 int ret; 3452 int config_issued = 0; 3453 char *fw_config_file, fw_config_file_path[256]; 3454 char *config_name = NULL; 3455 3456 /* 3457 * Reset device if necessary. 3458 */ 3459 if (reset) { 3460 ret = t4_fw_reset(adapter, adapter->mbox, 3461 PIORSTMODE_F | PIORST_F); 3462 if (ret < 0) 3463 goto bye; 3464 } 3465 3466 /* If this is a 10Gb/s-BT adapter make sure the chip-external 3467 * 10Gb/s-BT PHYs have up-to-date firmware. Note that this step needs 3468 * to be performed after any global adapter RESET above since some 3469 * PHYs only have local RAM copies of the PHY firmware. 3470 */ 3471 if (is_10gbt_device(adapter->pdev->device)) { 3472 ret = adap_init0_phy(adapter); 3473 if (ret < 0) 3474 goto bye; 3475 } 3476 /* 3477 * If we have a T4 configuration file under /lib/firmware/cxgb4/, 3478 * then use that. Otherwise, use the configuration file stored 3479 * in the adapter flash ... 3480 */ 3481 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) { 3482 case CHELSIO_T4: 3483 fw_config_file = FW4_CFNAME; 3484 break; 3485 case CHELSIO_T5: 3486 fw_config_file = FW5_CFNAME; 3487 break; 3488 case CHELSIO_T6: 3489 fw_config_file = FW6_CFNAME; 3490 break; 3491 default: 3492 dev_err(adapter->pdev_dev, "Device %d is not supported\n", 3493 adapter->pdev->device); 3494 ret = -EINVAL; 3495 goto bye; 3496 } 3497 3498 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev); 3499 if (ret < 0) { 3500 config_name = "On FLASH"; 3501 mtype = FW_MEMTYPE_CF_FLASH; 3502 maddr = t4_flash_cfg_addr(adapter); 3503 } else { 3504 u32 params[7], val[7]; 3505 3506 sprintf(fw_config_file_path, 3507 "/lib/firmware/%s", fw_config_file); 3508 config_name = fw_config_file_path; 3509 3510 if (cf->size >= FLASH_CFG_MAX_SIZE) 3511 ret = -ENOMEM; 3512 else { 3513 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3514 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 3515 ret = t4_query_params(adapter, adapter->mbox, 3516 adapter->pf, 0, 1, params, val); 3517 if (ret == 0) { 3518 /* 3519 * For t4_memory_rw() below addresses and 3520 * sizes have to be in terms of multiples of 4 3521 * bytes. So, if the Configuration File isn't 3522 * a multiple of 4 bytes in length we'll have 3523 * to write that out separately since we can't 3524 * guarantee that the bytes following the 3525 * residual byte in the buffer returned by 3526 * request_firmware() are zeroed out ... 3527 */ 3528 size_t resid = cf->size & 0x3; 3529 size_t size = cf->size & ~0x3; 3530 __be32 *data = (__be32 *)cf->data; 3531 3532 mtype = FW_PARAMS_PARAM_Y_G(val[0]); 3533 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16; 3534 3535 spin_lock(&adapter->win0_lock); 3536 ret = t4_memory_rw(adapter, 0, mtype, maddr, 3537 size, data, T4_MEMORY_WRITE); 3538 if (ret == 0 && resid != 0) { 3539 union { 3540 __be32 word; 3541 char buf[4]; 3542 } last; 3543 int i; 3544 3545 last.word = data[size >> 2]; 3546 for (i = resid; i < 4; i++) 3547 last.buf[i] = 0; 3548 ret = t4_memory_rw(adapter, 0, mtype, 3549 maddr + size, 3550 4, &last.word, 3551 T4_MEMORY_WRITE); 3552 } 3553 spin_unlock(&adapter->win0_lock); 3554 } 3555 } 3556 3557 release_firmware(cf); 3558 if (ret) 3559 goto bye; 3560 } 3561 3562 /* 3563 * Issue a Capability Configuration command to the firmware to get it 3564 * to parse the Configuration File. We don't use t4_fw_config_file() 3565 * because we want the ability to modify various features after we've 3566 * processed the configuration file ... 3567 */ 3568 memset(&caps_cmd, 0, sizeof(caps_cmd)); 3569 caps_cmd.op_to_write = 3570 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3571 FW_CMD_REQUEST_F | 3572 FW_CMD_READ_F); 3573 caps_cmd.cfvalid_to_len16 = 3574 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F | 3575 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) | 3576 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) | 3577 FW_LEN16(caps_cmd)); 3578 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 3579 &caps_cmd); 3580 3581 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware 3582 * Configuration File in FLASH), our last gasp effort is to use the 3583 * Firmware Configuration File which is embedded in the firmware. A 3584 * very few early versions of the firmware didn't have one embedded 3585 * but we can ignore those. 3586 */ 3587 if (ret == -ENOENT) { 3588 memset(&caps_cmd, 0, sizeof(caps_cmd)); 3589 caps_cmd.op_to_write = 3590 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3591 FW_CMD_REQUEST_F | 3592 FW_CMD_READ_F); 3593 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 3594 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, 3595 sizeof(caps_cmd), &caps_cmd); 3596 config_name = "Firmware Default"; 3597 } 3598 3599 config_issued = 1; 3600 if (ret < 0) 3601 goto bye; 3602 3603 finiver = ntohl(caps_cmd.finiver); 3604 finicsum = ntohl(caps_cmd.finicsum); 3605 cfcsum = ntohl(caps_cmd.cfcsum); 3606 if (finicsum != cfcsum) 3607 dev_warn(adapter->pdev_dev, "Configuration File checksum "\ 3608 "mismatch: [fini] csum=%#x, computed csum=%#x\n", 3609 finicsum, cfcsum); 3610 3611 /* 3612 * And now tell the firmware to use the configuration we just loaded. 3613 */ 3614 caps_cmd.op_to_write = 3615 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3616 FW_CMD_REQUEST_F | 3617 FW_CMD_WRITE_F); 3618 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 3619 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 3620 NULL); 3621 if (ret < 0) 3622 goto bye; 3623 3624 /* 3625 * Tweak configuration based on system architecture, module 3626 * parameters, etc. 3627 */ 3628 ret = adap_init0_tweaks(adapter); 3629 if (ret < 0) 3630 goto bye; 3631 3632 /* 3633 * And finally tell the firmware to initialize itself using the 3634 * parameters from the Configuration File. 3635 */ 3636 ret = t4_fw_initialize(adapter, adapter->mbox); 3637 if (ret < 0) 3638 goto bye; 3639 3640 /* Emit Firmware Configuration File information and return 3641 * successfully. 3642 */ 3643 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\ 3644 "Configuration File \"%s\", version %#x, computed checksum %#x\n", 3645 config_name, finiver, cfcsum); 3646 return 0; 3647 3648 /* 3649 * Something bad happened. Return the error ... (If the "error" 3650 * is that there's no Configuration File on the adapter we don't 3651 * want to issue a warning since this is fairly common.) 3652 */ 3653 bye: 3654 if (config_issued && ret != -ENOENT) 3655 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n", 3656 config_name, -ret); 3657 return ret; 3658 } 3659 3660 static struct fw_info fw_info_array[] = { 3661 { 3662 .chip = CHELSIO_T4, 3663 .fs_name = FW4_CFNAME, 3664 .fw_mod_name = FW4_FNAME, 3665 .fw_hdr = { 3666 .chip = FW_HDR_CHIP_T4, 3667 .fw_ver = __cpu_to_be32(FW_VERSION(T4)), 3668 .intfver_nic = FW_INTFVER(T4, NIC), 3669 .intfver_vnic = FW_INTFVER(T4, VNIC), 3670 .intfver_ri = FW_INTFVER(T4, RI), 3671 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 3672 .intfver_fcoe = FW_INTFVER(T4, FCOE), 3673 }, 3674 }, { 3675 .chip = CHELSIO_T5, 3676 .fs_name = FW5_CFNAME, 3677 .fw_mod_name = FW5_FNAME, 3678 .fw_hdr = { 3679 .chip = FW_HDR_CHIP_T5, 3680 .fw_ver = __cpu_to_be32(FW_VERSION(T5)), 3681 .intfver_nic = FW_INTFVER(T5, NIC), 3682 .intfver_vnic = FW_INTFVER(T5, VNIC), 3683 .intfver_ri = FW_INTFVER(T5, RI), 3684 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 3685 .intfver_fcoe = FW_INTFVER(T5, FCOE), 3686 }, 3687 }, { 3688 .chip = CHELSIO_T6, 3689 .fs_name = FW6_CFNAME, 3690 .fw_mod_name = FW6_FNAME, 3691 .fw_hdr = { 3692 .chip = FW_HDR_CHIP_T6, 3693 .fw_ver = __cpu_to_be32(FW_VERSION(T6)), 3694 .intfver_nic = FW_INTFVER(T6, NIC), 3695 .intfver_vnic = FW_INTFVER(T6, VNIC), 3696 .intfver_ofld = FW_INTFVER(T6, OFLD), 3697 .intfver_ri = FW_INTFVER(T6, RI), 3698 .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU), 3699 .intfver_iscsi = FW_INTFVER(T6, ISCSI), 3700 .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU), 3701 .intfver_fcoe = FW_INTFVER(T6, FCOE), 3702 }, 3703 } 3704 3705 }; 3706 3707 static struct fw_info *find_fw_info(int chip) 3708 { 3709 int i; 3710 3711 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) { 3712 if (fw_info_array[i].chip == chip) 3713 return &fw_info_array[i]; 3714 } 3715 return NULL; 3716 } 3717 3718 /* 3719 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 3720 */ 3721 static int adap_init0(struct adapter *adap) 3722 { 3723 int ret; 3724 u32 v, port_vec; 3725 enum dev_state state; 3726 u32 params[7], val[7]; 3727 struct fw_caps_config_cmd caps_cmd; 3728 int reset = 1; 3729 3730 /* Grab Firmware Device Log parameters as early as possible so we have 3731 * access to it for debugging, etc. 3732 */ 3733 ret = t4_init_devlog_params(adap); 3734 if (ret < 0) 3735 return ret; 3736 3737 /* Contact FW, advertising Master capability */ 3738 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state); 3739 if (ret < 0) { 3740 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n", 3741 ret); 3742 return ret; 3743 } 3744 if (ret == adap->mbox) 3745 adap->flags |= MASTER_PF; 3746 3747 /* 3748 * If we're the Master PF Driver and the device is uninitialized, 3749 * then let's consider upgrading the firmware ... (We always want 3750 * to check the firmware version number in order to A. get it for 3751 * later reporting and B. to warn if the currently loaded firmware 3752 * is excessively mismatched relative to the driver.) 3753 */ 3754 t4_get_fw_version(adap, &adap->params.fw_vers); 3755 t4_get_bs_version(adap, &adap->params.bs_vers); 3756 t4_get_tp_version(adap, &adap->params.tp_vers); 3757 t4_get_exprom_version(adap, &adap->params.er_vers); 3758 3759 ret = t4_check_fw_version(adap); 3760 /* If firmware is too old (not supported by driver) force an update. */ 3761 if (ret) 3762 state = DEV_STATE_UNINIT; 3763 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) { 3764 struct fw_info *fw_info; 3765 struct fw_hdr *card_fw; 3766 const struct firmware *fw; 3767 const u8 *fw_data = NULL; 3768 unsigned int fw_size = 0; 3769 3770 /* This is the firmware whose headers the driver was compiled 3771 * against 3772 */ 3773 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip)); 3774 if (fw_info == NULL) { 3775 dev_err(adap->pdev_dev, 3776 "unable to get firmware info for chip %d.\n", 3777 CHELSIO_CHIP_VERSION(adap->params.chip)); 3778 return -EINVAL; 3779 } 3780 3781 /* allocate memory to read the header of the firmware on the 3782 * card 3783 */ 3784 card_fw = t4_alloc_mem(sizeof(*card_fw)); 3785 3786 /* Get FW from from /lib/firmware/ */ 3787 ret = request_firmware(&fw, fw_info->fw_mod_name, 3788 adap->pdev_dev); 3789 if (ret < 0) { 3790 dev_err(adap->pdev_dev, 3791 "unable to load firmware image %s, error %d\n", 3792 fw_info->fw_mod_name, ret); 3793 } else { 3794 fw_data = fw->data; 3795 fw_size = fw->size; 3796 } 3797 3798 /* upgrade FW logic */ 3799 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw, 3800 state, &reset); 3801 3802 /* Cleaning up */ 3803 release_firmware(fw); 3804 t4_free_mem(card_fw); 3805 3806 if (ret < 0) 3807 goto bye; 3808 } 3809 3810 /* 3811 * Grab VPD parameters. This should be done after we establish a 3812 * connection to the firmware since some of the VPD parameters 3813 * (notably the Core Clock frequency) are retrieved via requests to 3814 * the firmware. On the other hand, we need these fairly early on 3815 * so we do this right after getting ahold of the firmware. 3816 */ 3817 ret = t4_get_vpd_params(adap, &adap->params.vpd); 3818 if (ret < 0) 3819 goto bye; 3820 3821 /* 3822 * Find out what ports are available to us. Note that we need to do 3823 * this before calling adap_init0_no_config() since it needs nports 3824 * and portvec ... 3825 */ 3826 v = 3827 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3828 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC); 3829 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec); 3830 if (ret < 0) 3831 goto bye; 3832 3833 adap->params.nports = hweight32(port_vec); 3834 adap->params.portvec = port_vec; 3835 3836 /* If the firmware is initialized already, emit a simply note to that 3837 * effect. Otherwise, it's time to try initializing the adapter. 3838 */ 3839 if (state == DEV_STATE_INIT) { 3840 dev_info(adap->pdev_dev, "Coming up as %s: "\ 3841 "Adapter already initialized\n", 3842 adap->flags & MASTER_PF ? "MASTER" : "SLAVE"); 3843 } else { 3844 dev_info(adap->pdev_dev, "Coming up as MASTER: "\ 3845 "Initializing adapter\n"); 3846 3847 /* Find out whether we're dealing with a version of the 3848 * firmware which has configuration file support. 3849 */ 3850 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3851 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 3852 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, 3853 params, val); 3854 3855 /* If the firmware doesn't support Configuration Files, 3856 * return an error. 3857 */ 3858 if (ret < 0) { 3859 dev_err(adap->pdev_dev, "firmware doesn't support " 3860 "Firmware Configuration Files\n"); 3861 goto bye; 3862 } 3863 3864 /* The firmware provides us with a memory buffer where we can 3865 * load a Configuration File from the host if we want to 3866 * override the Configuration File in flash. 3867 */ 3868 ret = adap_init0_config(adap, reset); 3869 if (ret == -ENOENT) { 3870 dev_err(adap->pdev_dev, "no Configuration File " 3871 "present on adapter.\n"); 3872 goto bye; 3873 } 3874 if (ret < 0) { 3875 dev_err(adap->pdev_dev, "could not initialize " 3876 "adapter, error %d\n", -ret); 3877 goto bye; 3878 } 3879 } 3880 3881 /* Give the SGE code a chance to pull in anything that it needs ... 3882 * Note that this must be called after we retrieve our VPD parameters 3883 * in order to know how to convert core ticks to seconds, etc. 3884 */ 3885 ret = t4_sge_init(adap); 3886 if (ret < 0) 3887 goto bye; 3888 3889 if (is_bypass_device(adap->pdev->device)) 3890 adap->params.bypass = 1; 3891 3892 /* 3893 * Grab some of our basic fundamental operating parameters. 3894 */ 3895 #define FW_PARAM_DEV(param) \ 3896 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \ 3897 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param)) 3898 3899 #define FW_PARAM_PFVF(param) \ 3900 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \ 3901 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \ 3902 FW_PARAMS_PARAM_Y_V(0) | \ 3903 FW_PARAMS_PARAM_Z_V(0) 3904 3905 params[0] = FW_PARAM_PFVF(EQ_START); 3906 params[1] = FW_PARAM_PFVF(L2T_START); 3907 params[2] = FW_PARAM_PFVF(L2T_END); 3908 params[3] = FW_PARAM_PFVF(FILTER_START); 3909 params[4] = FW_PARAM_PFVF(FILTER_END); 3910 params[5] = FW_PARAM_PFVF(IQFLINT_START); 3911 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, val); 3912 if (ret < 0) 3913 goto bye; 3914 adap->sge.egr_start = val[0]; 3915 adap->l2t_start = val[1]; 3916 adap->l2t_end = val[2]; 3917 adap->tids.ftid_base = val[3]; 3918 adap->tids.nftids = val[4] - val[3] + 1; 3919 adap->sge.ingr_start = val[5]; 3920 3921 /* qids (ingress/egress) returned from firmware can be anywhere 3922 * in the range from EQ(IQFLINT)_START to EQ(IQFLINT)_END. 3923 * Hence driver needs to allocate memory for this range to 3924 * store the queue info. Get the highest IQFLINT/EQ index returned 3925 * in FW_EQ_*_CMD.alloc command. 3926 */ 3927 params[0] = FW_PARAM_PFVF(EQ_END); 3928 params[1] = FW_PARAM_PFVF(IQFLINT_END); 3929 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 3930 if (ret < 0) 3931 goto bye; 3932 adap->sge.egr_sz = val[0] - adap->sge.egr_start + 1; 3933 adap->sge.ingr_sz = val[1] - adap->sge.ingr_start + 1; 3934 3935 adap->sge.egr_map = kcalloc(adap->sge.egr_sz, 3936 sizeof(*adap->sge.egr_map), GFP_KERNEL); 3937 if (!adap->sge.egr_map) { 3938 ret = -ENOMEM; 3939 goto bye; 3940 } 3941 3942 adap->sge.ingr_map = kcalloc(adap->sge.ingr_sz, 3943 sizeof(*adap->sge.ingr_map), GFP_KERNEL); 3944 if (!adap->sge.ingr_map) { 3945 ret = -ENOMEM; 3946 goto bye; 3947 } 3948 3949 /* Allocate the memory for the vaious egress queue bitmaps 3950 * ie starving_fl, txq_maperr and blocked_fl. 3951 */ 3952 adap->sge.starving_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 3953 sizeof(long), GFP_KERNEL); 3954 if (!adap->sge.starving_fl) { 3955 ret = -ENOMEM; 3956 goto bye; 3957 } 3958 3959 adap->sge.txq_maperr = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 3960 sizeof(long), GFP_KERNEL); 3961 if (!adap->sge.txq_maperr) { 3962 ret = -ENOMEM; 3963 goto bye; 3964 } 3965 3966 #ifdef CONFIG_DEBUG_FS 3967 adap->sge.blocked_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 3968 sizeof(long), GFP_KERNEL); 3969 if (!adap->sge.blocked_fl) { 3970 ret = -ENOMEM; 3971 goto bye; 3972 } 3973 #endif 3974 3975 params[0] = FW_PARAM_PFVF(CLIP_START); 3976 params[1] = FW_PARAM_PFVF(CLIP_END); 3977 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 3978 if (ret < 0) 3979 goto bye; 3980 adap->clipt_start = val[0]; 3981 adap->clipt_end = val[1]; 3982 3983 /* query params related to active filter region */ 3984 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START); 3985 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END); 3986 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 3987 /* If Active filter size is set we enable establishing 3988 * offload connection through firmware work request 3989 */ 3990 if ((val[0] != val[1]) && (ret >= 0)) { 3991 adap->flags |= FW_OFLD_CONN; 3992 adap->tids.aftid_base = val[0]; 3993 adap->tids.aftid_end = val[1]; 3994 } 3995 3996 /* If we're running on newer firmware, let it know that we're 3997 * prepared to deal with encapsulated CPL messages. Older 3998 * firmware won't understand this and we'll just get 3999 * unencapsulated messages ... 4000 */ 4001 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 4002 val[0] = 1; 4003 (void)t4_set_params(adap, adap->mbox, adap->pf, 0, 1, params, val); 4004 4005 /* 4006 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL 4007 * capability. Earlier versions of the firmware didn't have the 4008 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no 4009 * permission to use ULPTX MEMWRITE DSGL. 4010 */ 4011 if (is_t4(adap->params.chip)) { 4012 adap->params.ulptx_memwrite_dsgl = false; 4013 } else { 4014 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL); 4015 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 4016 1, params, val); 4017 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0); 4018 } 4019 4020 /* 4021 * Get device capabilities so we can determine what resources we need 4022 * to manage. 4023 */ 4024 memset(&caps_cmd, 0, sizeof(caps_cmd)); 4025 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4026 FW_CMD_REQUEST_F | FW_CMD_READ_F); 4027 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 4028 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd), 4029 &caps_cmd); 4030 if (ret < 0) 4031 goto bye; 4032 4033 if (caps_cmd.ofldcaps) { 4034 /* query offload-related parameters */ 4035 params[0] = FW_PARAM_DEV(NTID); 4036 params[1] = FW_PARAM_PFVF(SERVER_START); 4037 params[2] = FW_PARAM_PFVF(SERVER_END); 4038 params[3] = FW_PARAM_PFVF(TDDP_START); 4039 params[4] = FW_PARAM_PFVF(TDDP_END); 4040 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 4041 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, 4042 params, val); 4043 if (ret < 0) 4044 goto bye; 4045 adap->tids.ntids = val[0]; 4046 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS); 4047 adap->tids.stid_base = val[1]; 4048 adap->tids.nstids = val[2] - val[1] + 1; 4049 /* 4050 * Setup server filter region. Divide the available filter 4051 * region into two parts. Regular filters get 1/3rd and server 4052 * filters get 2/3rd part. This is only enabled if workarond 4053 * path is enabled. 4054 * 1. For regular filters. 4055 * 2. Server filter: This are special filters which are used 4056 * to redirect SYN packets to offload queue. 4057 */ 4058 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) { 4059 adap->tids.sftid_base = adap->tids.ftid_base + 4060 DIV_ROUND_UP(adap->tids.nftids, 3); 4061 adap->tids.nsftids = adap->tids.nftids - 4062 DIV_ROUND_UP(adap->tids.nftids, 3); 4063 adap->tids.nftids = adap->tids.sftid_base - 4064 adap->tids.ftid_base; 4065 } 4066 adap->vres.ddp.start = val[3]; 4067 adap->vres.ddp.size = val[4] - val[3] + 1; 4068 adap->params.ofldq_wr_cred = val[5]; 4069 4070 adap->params.offload = 1; 4071 } 4072 if (caps_cmd.rdmacaps) { 4073 params[0] = FW_PARAM_PFVF(STAG_START); 4074 params[1] = FW_PARAM_PFVF(STAG_END); 4075 params[2] = FW_PARAM_PFVF(RQ_START); 4076 params[3] = FW_PARAM_PFVF(RQ_END); 4077 params[4] = FW_PARAM_PFVF(PBL_START); 4078 params[5] = FW_PARAM_PFVF(PBL_END); 4079 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, 4080 params, val); 4081 if (ret < 0) 4082 goto bye; 4083 adap->vres.stag.start = val[0]; 4084 adap->vres.stag.size = val[1] - val[0] + 1; 4085 adap->vres.rq.start = val[2]; 4086 adap->vres.rq.size = val[3] - val[2] + 1; 4087 adap->vres.pbl.start = val[4]; 4088 adap->vres.pbl.size = val[5] - val[4] + 1; 4089 4090 params[0] = FW_PARAM_PFVF(SQRQ_START); 4091 params[1] = FW_PARAM_PFVF(SQRQ_END); 4092 params[2] = FW_PARAM_PFVF(CQ_START); 4093 params[3] = FW_PARAM_PFVF(CQ_END); 4094 params[4] = FW_PARAM_PFVF(OCQ_START); 4095 params[5] = FW_PARAM_PFVF(OCQ_END); 4096 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, 4097 val); 4098 if (ret < 0) 4099 goto bye; 4100 adap->vres.qp.start = val[0]; 4101 adap->vres.qp.size = val[1] - val[0] + 1; 4102 adap->vres.cq.start = val[2]; 4103 adap->vres.cq.size = val[3] - val[2] + 1; 4104 adap->vres.ocq.start = val[4]; 4105 adap->vres.ocq.size = val[5] - val[4] + 1; 4106 4107 params[0] = FW_PARAM_DEV(MAXORDIRD_QP); 4108 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER); 4109 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, 4110 val); 4111 if (ret < 0) { 4112 adap->params.max_ordird_qp = 8; 4113 adap->params.max_ird_adapter = 32 * adap->tids.ntids; 4114 ret = 0; 4115 } else { 4116 adap->params.max_ordird_qp = val[0]; 4117 adap->params.max_ird_adapter = val[1]; 4118 } 4119 dev_info(adap->pdev_dev, 4120 "max_ordird_qp %d max_ird_adapter %d\n", 4121 adap->params.max_ordird_qp, 4122 adap->params.max_ird_adapter); 4123 } 4124 if (caps_cmd.iscsicaps) { 4125 params[0] = FW_PARAM_PFVF(ISCSI_START); 4126 params[1] = FW_PARAM_PFVF(ISCSI_END); 4127 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 4128 params, val); 4129 if (ret < 0) 4130 goto bye; 4131 adap->vres.iscsi.start = val[0]; 4132 adap->vres.iscsi.size = val[1] - val[0] + 1; 4133 } 4134 #undef FW_PARAM_PFVF 4135 #undef FW_PARAM_DEV 4136 4137 /* The MTU/MSS Table is initialized by now, so load their values. If 4138 * we're initializing the adapter, then we'll make any modifications 4139 * we want to the MTU/MSS Table and also initialize the congestion 4140 * parameters. 4141 */ 4142 t4_read_mtu_tbl(adap, adap->params.mtus, NULL); 4143 if (state != DEV_STATE_INIT) { 4144 int i; 4145 4146 /* The default MTU Table contains values 1492 and 1500. 4147 * However, for TCP, it's better to have two values which are 4148 * a multiple of 8 +/- 4 bytes apart near this popular MTU. 4149 * This allows us to have a TCP Data Payload which is a 4150 * multiple of 8 regardless of what combination of TCP Options 4151 * are in use (always a multiple of 4 bytes) which is 4152 * important for performance reasons. For instance, if no 4153 * options are in use, then we have a 20-byte IP header and a 4154 * 20-byte TCP header. In this case, a 1500-byte MSS would 4155 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes 4156 * which is not a multiple of 8. So using an MSS of 1488 in 4157 * this case results in a TCP Data Payload of 1448 bytes which 4158 * is a multiple of 8. On the other hand, if 12-byte TCP Time 4159 * Stamps have been negotiated, then an MTU of 1500 bytes 4160 * results in a TCP Data Payload of 1448 bytes which, as 4161 * above, is a multiple of 8 bytes ... 4162 */ 4163 for (i = 0; i < NMTUS; i++) 4164 if (adap->params.mtus[i] == 1492) { 4165 adap->params.mtus[i] = 1488; 4166 break; 4167 } 4168 4169 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 4170 adap->params.b_wnd); 4171 } 4172 t4_init_sge_params(adap); 4173 adap->flags |= FW_OK; 4174 t4_init_tp_params(adap); 4175 return 0; 4176 4177 /* 4178 * Something bad happened. If a command timed out or failed with EIO 4179 * FW does not operate within its spec or something catastrophic 4180 * happened to HW/FW, stop issuing commands. 4181 */ 4182 bye: 4183 kfree(adap->sge.egr_map); 4184 kfree(adap->sge.ingr_map); 4185 kfree(adap->sge.starving_fl); 4186 kfree(adap->sge.txq_maperr); 4187 #ifdef CONFIG_DEBUG_FS 4188 kfree(adap->sge.blocked_fl); 4189 #endif 4190 if (ret != -ETIMEDOUT && ret != -EIO) 4191 t4_fw_bye(adap, adap->mbox); 4192 return ret; 4193 } 4194 4195 /* EEH callbacks */ 4196 4197 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev, 4198 pci_channel_state_t state) 4199 { 4200 int i; 4201 struct adapter *adap = pci_get_drvdata(pdev); 4202 4203 if (!adap) 4204 goto out; 4205 4206 rtnl_lock(); 4207 adap->flags &= ~FW_OK; 4208 notify_ulds(adap, CXGB4_STATE_START_RECOVERY); 4209 spin_lock(&adap->stats_lock); 4210 for_each_port(adap, i) { 4211 struct net_device *dev = adap->port[i]; 4212 4213 netif_device_detach(dev); 4214 netif_carrier_off(dev); 4215 } 4216 spin_unlock(&adap->stats_lock); 4217 disable_interrupts(adap); 4218 if (adap->flags & FULL_INIT_DONE) 4219 cxgb_down(adap); 4220 rtnl_unlock(); 4221 if ((adap->flags & DEV_ENABLED)) { 4222 pci_disable_device(pdev); 4223 adap->flags &= ~DEV_ENABLED; 4224 } 4225 out: return state == pci_channel_io_perm_failure ? 4226 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 4227 } 4228 4229 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev) 4230 { 4231 int i, ret; 4232 struct fw_caps_config_cmd c; 4233 struct adapter *adap = pci_get_drvdata(pdev); 4234 4235 if (!adap) { 4236 pci_restore_state(pdev); 4237 pci_save_state(pdev); 4238 return PCI_ERS_RESULT_RECOVERED; 4239 } 4240 4241 if (!(adap->flags & DEV_ENABLED)) { 4242 if (pci_enable_device(pdev)) { 4243 dev_err(&pdev->dev, "Cannot reenable PCI " 4244 "device after reset\n"); 4245 return PCI_ERS_RESULT_DISCONNECT; 4246 } 4247 adap->flags |= DEV_ENABLED; 4248 } 4249 4250 pci_set_master(pdev); 4251 pci_restore_state(pdev); 4252 pci_save_state(pdev); 4253 pci_cleanup_aer_uncorrect_error_status(pdev); 4254 4255 if (t4_wait_dev_ready(adap->regs) < 0) 4256 return PCI_ERS_RESULT_DISCONNECT; 4257 if (t4_fw_hello(adap, adap->mbox, adap->pf, MASTER_MUST, NULL) < 0) 4258 return PCI_ERS_RESULT_DISCONNECT; 4259 adap->flags |= FW_OK; 4260 if (adap_init1(adap, &c)) 4261 return PCI_ERS_RESULT_DISCONNECT; 4262 4263 for_each_port(adap, i) { 4264 struct port_info *p = adap2pinfo(adap, i); 4265 4266 ret = t4_alloc_vi(adap, adap->mbox, p->tx_chan, adap->pf, 0, 1, 4267 NULL, NULL); 4268 if (ret < 0) 4269 return PCI_ERS_RESULT_DISCONNECT; 4270 p->viid = ret; 4271 p->xact_addr_filt = -1; 4272 } 4273 4274 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 4275 adap->params.b_wnd); 4276 setup_memwin(adap); 4277 if (cxgb_up(adap)) 4278 return PCI_ERS_RESULT_DISCONNECT; 4279 return PCI_ERS_RESULT_RECOVERED; 4280 } 4281 4282 static void eeh_resume(struct pci_dev *pdev) 4283 { 4284 int i; 4285 struct adapter *adap = pci_get_drvdata(pdev); 4286 4287 if (!adap) 4288 return; 4289 4290 rtnl_lock(); 4291 for_each_port(adap, i) { 4292 struct net_device *dev = adap->port[i]; 4293 4294 if (netif_running(dev)) { 4295 link_start(dev); 4296 cxgb_set_rxmode(dev); 4297 } 4298 netif_device_attach(dev); 4299 } 4300 rtnl_unlock(); 4301 } 4302 4303 static const struct pci_error_handlers cxgb4_eeh = { 4304 .error_detected = eeh_err_detected, 4305 .slot_reset = eeh_slot_reset, 4306 .resume = eeh_resume, 4307 }; 4308 4309 static inline bool is_x_10g_port(const struct link_config *lc) 4310 { 4311 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 || 4312 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0; 4313 } 4314 4315 static inline void init_rspq(struct adapter *adap, struct sge_rspq *q, 4316 unsigned int us, unsigned int cnt, 4317 unsigned int size, unsigned int iqe_size) 4318 { 4319 q->adap = adap; 4320 cxgb4_set_rspq_intr_params(q, us, cnt); 4321 q->iqe_len = iqe_size; 4322 q->size = size; 4323 } 4324 4325 /* 4326 * Perform default configuration of DMA queues depending on the number and type 4327 * of ports we found and the number of available CPUs. Most settings can be 4328 * modified by the admin prior to actual use. 4329 */ 4330 static void cfg_queues(struct adapter *adap) 4331 { 4332 struct sge *s = &adap->sge; 4333 int i, n10g = 0, qidx = 0; 4334 #ifndef CONFIG_CHELSIO_T4_DCB 4335 int q10g = 0; 4336 #endif 4337 int ciq_size; 4338 4339 for_each_port(adap, i) 4340 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg); 4341 #ifdef CONFIG_CHELSIO_T4_DCB 4342 /* For Data Center Bridging support we need to be able to support up 4343 * to 8 Traffic Priorities; each of which will be assigned to its 4344 * own TX Queue in order to prevent Head-Of-Line Blocking. 4345 */ 4346 if (adap->params.nports * 8 > MAX_ETH_QSETS) { 4347 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n", 4348 MAX_ETH_QSETS, adap->params.nports * 8); 4349 BUG_ON(1); 4350 } 4351 4352 for_each_port(adap, i) { 4353 struct port_info *pi = adap2pinfo(adap, i); 4354 4355 pi->first_qset = qidx; 4356 pi->nqsets = 8; 4357 qidx += pi->nqsets; 4358 } 4359 #else /* !CONFIG_CHELSIO_T4_DCB */ 4360 /* 4361 * We default to 1 queue per non-10G port and up to # of cores queues 4362 * per 10G port. 4363 */ 4364 if (n10g) 4365 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g; 4366 if (q10g > netif_get_num_default_rss_queues()) 4367 q10g = netif_get_num_default_rss_queues(); 4368 4369 for_each_port(adap, i) { 4370 struct port_info *pi = adap2pinfo(adap, i); 4371 4372 pi->first_qset = qidx; 4373 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1; 4374 qidx += pi->nqsets; 4375 } 4376 #endif /* !CONFIG_CHELSIO_T4_DCB */ 4377 4378 s->ethqsets = qidx; 4379 s->max_ethqsets = qidx; /* MSI-X may lower it later */ 4380 4381 if (is_offload(adap)) { 4382 /* 4383 * For offload we use 1 queue/channel if all ports are up to 1G, 4384 * otherwise we divide all available queues amongst the channels 4385 * capped by the number of available cores. 4386 */ 4387 if (n10g) { 4388 i = min_t(int, ARRAY_SIZE(s->iscsirxq), 4389 num_online_cpus()); 4390 s->iscsiqsets = roundup(i, adap->params.nports); 4391 } else 4392 s->iscsiqsets = adap->params.nports; 4393 /* For RDMA one Rx queue per channel suffices */ 4394 s->rdmaqs = adap->params.nports; 4395 /* Try and allow at least 1 CIQ per cpu rounding down 4396 * to the number of ports, with a minimum of 1 per port. 4397 * A 2 port card in a 6 cpu system: 6 CIQs, 3 / port. 4398 * A 4 port card in a 6 cpu system: 4 CIQs, 1 / port. 4399 * A 4 port card in a 2 cpu system: 4 CIQs, 1 / port. 4400 */ 4401 s->rdmaciqs = min_t(int, MAX_RDMA_CIQS, num_online_cpus()); 4402 s->rdmaciqs = (s->rdmaciqs / adap->params.nports) * 4403 adap->params.nports; 4404 s->rdmaciqs = max_t(int, s->rdmaciqs, adap->params.nports); 4405 4406 if (!is_t4(adap->params.chip)) 4407 s->niscsitq = s->iscsiqsets; 4408 } 4409 4410 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { 4411 struct sge_eth_rxq *r = &s->ethrxq[i]; 4412 4413 init_rspq(adap, &r->rspq, 5, 10, 1024, 64); 4414 r->fl.size = 72; 4415 } 4416 4417 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++) 4418 s->ethtxq[i].q.size = 1024; 4419 4420 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) 4421 s->ctrlq[i].q.size = 512; 4422 4423 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++) 4424 s->ofldtxq[i].q.size = 1024; 4425 4426 for (i = 0; i < ARRAY_SIZE(s->iscsirxq); i++) { 4427 struct sge_ofld_rxq *r = &s->iscsirxq[i]; 4428 4429 init_rspq(adap, &r->rspq, 5, 1, 1024, 64); 4430 r->rspq.uld = CXGB4_ULD_ISCSI; 4431 r->fl.size = 72; 4432 } 4433 4434 if (!is_t4(adap->params.chip)) { 4435 for (i = 0; i < ARRAY_SIZE(s->iscsitrxq); i++) { 4436 struct sge_ofld_rxq *r = &s->iscsitrxq[i]; 4437 4438 init_rspq(adap, &r->rspq, 5, 1, 1024, 64); 4439 r->rspq.uld = CXGB4_ULD_ISCSIT; 4440 r->fl.size = 72; 4441 } 4442 } 4443 4444 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) { 4445 struct sge_ofld_rxq *r = &s->rdmarxq[i]; 4446 4447 init_rspq(adap, &r->rspq, 5, 1, 511, 64); 4448 r->rspq.uld = CXGB4_ULD_RDMA; 4449 r->fl.size = 72; 4450 } 4451 4452 ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids; 4453 if (ciq_size > SGE_MAX_IQ_SIZE) { 4454 CH_WARN(adap, "CIQ size too small for available IQs\n"); 4455 ciq_size = SGE_MAX_IQ_SIZE; 4456 } 4457 4458 for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) { 4459 struct sge_ofld_rxq *r = &s->rdmaciq[i]; 4460 4461 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64); 4462 r->rspq.uld = CXGB4_ULD_RDMA; 4463 } 4464 4465 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64); 4466 init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64); 4467 } 4468 4469 /* 4470 * Reduce the number of Ethernet queues across all ports to at most n. 4471 * n provides at least one queue per port. 4472 */ 4473 static void reduce_ethqs(struct adapter *adap, int n) 4474 { 4475 int i; 4476 struct port_info *pi; 4477 4478 while (n < adap->sge.ethqsets) 4479 for_each_port(adap, i) { 4480 pi = adap2pinfo(adap, i); 4481 if (pi->nqsets > 1) { 4482 pi->nqsets--; 4483 adap->sge.ethqsets--; 4484 if (adap->sge.ethqsets <= n) 4485 break; 4486 } 4487 } 4488 4489 n = 0; 4490 for_each_port(adap, i) { 4491 pi = adap2pinfo(adap, i); 4492 pi->first_qset = n; 4493 n += pi->nqsets; 4494 } 4495 } 4496 4497 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */ 4498 #define EXTRA_VECS 2 4499 4500 static int enable_msix(struct adapter *adap) 4501 { 4502 int ofld_need = 0; 4503 int i, want, need, allocated; 4504 struct sge *s = &adap->sge; 4505 unsigned int nchan = adap->params.nports; 4506 struct msix_entry *entries; 4507 4508 entries = kmalloc(sizeof(*entries) * (MAX_INGQ + 1), 4509 GFP_KERNEL); 4510 if (!entries) 4511 return -ENOMEM; 4512 4513 for (i = 0; i < MAX_INGQ + 1; ++i) 4514 entries[i].entry = i; 4515 4516 want = s->max_ethqsets + EXTRA_VECS; 4517 if (is_offload(adap)) { 4518 want += s->rdmaqs + s->rdmaciqs + s->iscsiqsets + 4519 s->niscsitq; 4520 /* need nchan for each possible ULD */ 4521 if (is_t4(adap->params.chip)) 4522 ofld_need = 3 * nchan; 4523 else 4524 ofld_need = 4 * nchan; 4525 } 4526 #ifdef CONFIG_CHELSIO_T4_DCB 4527 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for 4528 * each port. 4529 */ 4530 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need; 4531 #else 4532 need = adap->params.nports + EXTRA_VECS + ofld_need; 4533 #endif 4534 allocated = pci_enable_msix_range(adap->pdev, entries, need, want); 4535 if (allocated < 0) { 4536 dev_info(adap->pdev_dev, "not enough MSI-X vectors left," 4537 " not using MSI-X\n"); 4538 kfree(entries); 4539 return allocated; 4540 } 4541 4542 /* Distribute available vectors to the various queue groups. 4543 * Every group gets its minimum requirement and NIC gets top 4544 * priority for leftovers. 4545 */ 4546 i = allocated - EXTRA_VECS - ofld_need; 4547 if (i < s->max_ethqsets) { 4548 s->max_ethqsets = i; 4549 if (i < s->ethqsets) 4550 reduce_ethqs(adap, i); 4551 } 4552 if (is_offload(adap)) { 4553 if (allocated < want) { 4554 s->rdmaqs = nchan; 4555 s->rdmaciqs = nchan; 4556 4557 if (!is_t4(adap->params.chip)) 4558 s->niscsitq = nchan; 4559 } 4560 4561 /* leftovers go to OFLD */ 4562 i = allocated - EXTRA_VECS - s->max_ethqsets - 4563 s->rdmaqs - s->rdmaciqs - s->niscsitq; 4564 s->iscsiqsets = (i / nchan) * nchan; /* round down */ 4565 4566 } 4567 for (i = 0; i < allocated; ++i) 4568 adap->msix_info[i].vec = entries[i].vector; 4569 dev_info(adap->pdev_dev, "%d MSI-X vectors allocated, " 4570 "nic %d iscsi %d rdma cpl %d rdma ciq %d\n", 4571 allocated, s->max_ethqsets, s->iscsiqsets, s->rdmaqs, 4572 s->rdmaciqs); 4573 4574 kfree(entries); 4575 return 0; 4576 } 4577 4578 #undef EXTRA_VECS 4579 4580 static int init_rss(struct adapter *adap) 4581 { 4582 unsigned int i; 4583 int err; 4584 4585 err = t4_init_rss_mode(adap, adap->mbox); 4586 if (err) 4587 return err; 4588 4589 for_each_port(adap, i) { 4590 struct port_info *pi = adap2pinfo(adap, i); 4591 4592 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL); 4593 if (!pi->rss) 4594 return -ENOMEM; 4595 } 4596 return 0; 4597 } 4598 4599 static int cxgb4_get_pcie_dev_link_caps(struct adapter *adap, 4600 enum pci_bus_speed *speed, 4601 enum pcie_link_width *width) 4602 { 4603 u32 lnkcap1, lnkcap2; 4604 int err1, err2; 4605 4606 #define PCIE_MLW_CAP_SHIFT 4 /* start of MLW mask in link capabilities */ 4607 4608 *speed = PCI_SPEED_UNKNOWN; 4609 *width = PCIE_LNK_WIDTH_UNKNOWN; 4610 4611 err1 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP, 4612 &lnkcap1); 4613 err2 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP2, 4614 &lnkcap2); 4615 if (!err2 && lnkcap2) { /* PCIe r3.0-compliant */ 4616 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB) 4617 *speed = PCIE_SPEED_8_0GT; 4618 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB) 4619 *speed = PCIE_SPEED_5_0GT; 4620 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB) 4621 *speed = PCIE_SPEED_2_5GT; 4622 } 4623 if (!err1) { 4624 *width = (lnkcap1 & PCI_EXP_LNKCAP_MLW) >> PCIE_MLW_CAP_SHIFT; 4625 if (!lnkcap2) { /* pre-r3.0 */ 4626 if (lnkcap1 & PCI_EXP_LNKCAP_SLS_5_0GB) 4627 *speed = PCIE_SPEED_5_0GT; 4628 else if (lnkcap1 & PCI_EXP_LNKCAP_SLS_2_5GB) 4629 *speed = PCIE_SPEED_2_5GT; 4630 } 4631 } 4632 4633 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) 4634 return err1 ? err1 : err2 ? err2 : -EINVAL; 4635 return 0; 4636 } 4637 4638 static void cxgb4_check_pcie_caps(struct adapter *adap) 4639 { 4640 enum pcie_link_width width, width_cap; 4641 enum pci_bus_speed speed, speed_cap; 4642 4643 #define PCIE_SPEED_STR(speed) \ 4644 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : \ 4645 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : \ 4646 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : \ 4647 "Unknown") 4648 4649 if (cxgb4_get_pcie_dev_link_caps(adap, &speed_cap, &width_cap)) { 4650 dev_warn(adap->pdev_dev, 4651 "Unable to determine PCIe device BW capabilities\n"); 4652 return; 4653 } 4654 4655 if (pcie_get_minimum_link(adap->pdev, &speed, &width) || 4656 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) { 4657 dev_warn(adap->pdev_dev, 4658 "Unable to determine PCI Express bandwidth.\n"); 4659 return; 4660 } 4661 4662 dev_info(adap->pdev_dev, "PCIe link speed is %s, device supports %s\n", 4663 PCIE_SPEED_STR(speed), PCIE_SPEED_STR(speed_cap)); 4664 dev_info(adap->pdev_dev, "PCIe link width is x%d, device supports x%d\n", 4665 width, width_cap); 4666 if (speed < speed_cap || width < width_cap) 4667 dev_info(adap->pdev_dev, 4668 "A slot with more lanes and/or higher speed is " 4669 "suggested for optimal performance.\n"); 4670 } 4671 4672 /* Dump basic information about the adapter */ 4673 static void print_adapter_info(struct adapter *adapter) 4674 { 4675 /* Device information */ 4676 dev_info(adapter->pdev_dev, "Chelsio %s rev %d\n", 4677 adapter->params.vpd.id, 4678 CHELSIO_CHIP_RELEASE(adapter->params.chip)); 4679 dev_info(adapter->pdev_dev, "S/N: %s, P/N: %s\n", 4680 adapter->params.vpd.sn, adapter->params.vpd.pn); 4681 4682 /* Firmware Version */ 4683 if (!adapter->params.fw_vers) 4684 dev_warn(adapter->pdev_dev, "No firmware loaded\n"); 4685 else 4686 dev_info(adapter->pdev_dev, "Firmware version: %u.%u.%u.%u\n", 4687 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers), 4688 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers), 4689 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers), 4690 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers)); 4691 4692 /* Bootstrap Firmware Version. (Some adapters don't have Bootstrap 4693 * Firmware, so dev_info() is more appropriate here.) 4694 */ 4695 if (!adapter->params.bs_vers) 4696 dev_info(adapter->pdev_dev, "No bootstrap loaded\n"); 4697 else 4698 dev_info(adapter->pdev_dev, "Bootstrap version: %u.%u.%u.%u\n", 4699 FW_HDR_FW_VER_MAJOR_G(adapter->params.bs_vers), 4700 FW_HDR_FW_VER_MINOR_G(adapter->params.bs_vers), 4701 FW_HDR_FW_VER_MICRO_G(adapter->params.bs_vers), 4702 FW_HDR_FW_VER_BUILD_G(adapter->params.bs_vers)); 4703 4704 /* TP Microcode Version */ 4705 if (!adapter->params.tp_vers) 4706 dev_warn(adapter->pdev_dev, "No TP Microcode loaded\n"); 4707 else 4708 dev_info(adapter->pdev_dev, 4709 "TP Microcode version: %u.%u.%u.%u\n", 4710 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers), 4711 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers), 4712 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers), 4713 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers)); 4714 4715 /* Expansion ROM version */ 4716 if (!adapter->params.er_vers) 4717 dev_info(adapter->pdev_dev, "No Expansion ROM loaded\n"); 4718 else 4719 dev_info(adapter->pdev_dev, 4720 "Expansion ROM version: %u.%u.%u.%u\n", 4721 FW_HDR_FW_VER_MAJOR_G(adapter->params.er_vers), 4722 FW_HDR_FW_VER_MINOR_G(adapter->params.er_vers), 4723 FW_HDR_FW_VER_MICRO_G(adapter->params.er_vers), 4724 FW_HDR_FW_VER_BUILD_G(adapter->params.er_vers)); 4725 4726 /* Software/Hardware configuration */ 4727 dev_info(adapter->pdev_dev, "Configuration: %sNIC %s, %s capable\n", 4728 is_offload(adapter) ? "R" : "", 4729 ((adapter->flags & USING_MSIX) ? "MSI-X" : 4730 (adapter->flags & USING_MSI) ? "MSI" : ""), 4731 is_offload(adapter) ? "Offload" : "non-Offload"); 4732 } 4733 4734 static void print_port_info(const struct net_device *dev) 4735 { 4736 char buf[80]; 4737 char *bufp = buf; 4738 const char *spd = ""; 4739 const struct port_info *pi = netdev_priv(dev); 4740 const struct adapter *adap = pi->adapter; 4741 4742 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB) 4743 spd = " 2.5 GT/s"; 4744 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB) 4745 spd = " 5 GT/s"; 4746 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB) 4747 spd = " 8 GT/s"; 4748 4749 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M) 4750 bufp += sprintf(bufp, "100/"); 4751 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G) 4752 bufp += sprintf(bufp, "1000/"); 4753 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) 4754 bufp += sprintf(bufp, "10G/"); 4755 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) 4756 bufp += sprintf(bufp, "40G/"); 4757 if (bufp != buf) 4758 --bufp; 4759 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type)); 4760 4761 netdev_info(dev, "%s: Chelsio %s (%s) %s\n", 4762 dev->name, adap->params.vpd.id, adap->name, buf); 4763 } 4764 4765 static void enable_pcie_relaxed_ordering(struct pci_dev *dev) 4766 { 4767 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); 4768 } 4769 4770 /* 4771 * Free the following resources: 4772 * - memory used for tables 4773 * - MSI/MSI-X 4774 * - net devices 4775 * - resources FW is holding for us 4776 */ 4777 static void free_some_resources(struct adapter *adapter) 4778 { 4779 unsigned int i; 4780 4781 t4_free_mem(adapter->l2t); 4782 t4_free_mem(adapter->tids.tid_tab); 4783 kfree(adapter->sge.egr_map); 4784 kfree(adapter->sge.ingr_map); 4785 kfree(adapter->sge.starving_fl); 4786 kfree(adapter->sge.txq_maperr); 4787 #ifdef CONFIG_DEBUG_FS 4788 kfree(adapter->sge.blocked_fl); 4789 #endif 4790 disable_msi(adapter); 4791 4792 for_each_port(adapter, i) 4793 if (adapter->port[i]) { 4794 struct port_info *pi = adap2pinfo(adapter, i); 4795 4796 if (pi->viid != 0) 4797 t4_free_vi(adapter, adapter->mbox, adapter->pf, 4798 0, pi->viid); 4799 kfree(adap2pinfo(adapter, i)->rss); 4800 free_netdev(adapter->port[i]); 4801 } 4802 if (adapter->flags & FW_OK) 4803 t4_fw_bye(adapter, adapter->pf); 4804 } 4805 4806 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN) 4807 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \ 4808 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA) 4809 #define SEGMENT_SIZE 128 4810 4811 static int get_chip_type(struct pci_dev *pdev, u32 pl_rev) 4812 { 4813 u16 device_id; 4814 4815 /* Retrieve adapter's device ID */ 4816 pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id); 4817 4818 switch (device_id >> 12) { 4819 case CHELSIO_T4: 4820 return CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev); 4821 case CHELSIO_T5: 4822 return CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev); 4823 case CHELSIO_T6: 4824 return CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); 4825 default: 4826 dev_err(&pdev->dev, "Device %d is not supported\n", 4827 device_id); 4828 } 4829 return -EINVAL; 4830 } 4831 4832 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 4833 { 4834 int func, i, err, s_qpp, qpp, num_seg; 4835 struct port_info *pi; 4836 bool highdma = false; 4837 struct adapter *adapter = NULL; 4838 void __iomem *regs; 4839 u32 whoami, pl_rev; 4840 enum chip_type chip; 4841 4842 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION); 4843 4844 err = pci_request_regions(pdev, KBUILD_MODNAME); 4845 if (err) { 4846 /* Just info, some other driver may have claimed the device. */ 4847 dev_info(&pdev->dev, "cannot obtain PCI resources\n"); 4848 return err; 4849 } 4850 4851 err = pci_enable_device(pdev); 4852 if (err) { 4853 dev_err(&pdev->dev, "cannot enable PCI device\n"); 4854 goto out_release_regions; 4855 } 4856 4857 regs = pci_ioremap_bar(pdev, 0); 4858 if (!regs) { 4859 dev_err(&pdev->dev, "cannot map device registers\n"); 4860 err = -ENOMEM; 4861 goto out_disable_device; 4862 } 4863 4864 err = t4_wait_dev_ready(regs); 4865 if (err < 0) 4866 goto out_unmap_bar0; 4867 4868 /* We control everything through one PF */ 4869 whoami = readl(regs + PL_WHOAMI_A); 4870 pl_rev = REV_G(readl(regs + PL_REV_A)); 4871 chip = get_chip_type(pdev, pl_rev); 4872 func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ? 4873 SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami); 4874 if (func != ent->driver_data) { 4875 iounmap(regs); 4876 pci_disable_device(pdev); 4877 pci_save_state(pdev); /* to restore SR-IOV later */ 4878 goto sriov; 4879 } 4880 4881 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 4882 highdma = true; 4883 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 4884 if (err) { 4885 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for " 4886 "coherent allocations\n"); 4887 goto out_unmap_bar0; 4888 } 4889 } else { 4890 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 4891 if (err) { 4892 dev_err(&pdev->dev, "no usable DMA configuration\n"); 4893 goto out_unmap_bar0; 4894 } 4895 } 4896 4897 pci_enable_pcie_error_reporting(pdev); 4898 enable_pcie_relaxed_ordering(pdev); 4899 pci_set_master(pdev); 4900 pci_save_state(pdev); 4901 4902 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 4903 if (!adapter) { 4904 err = -ENOMEM; 4905 goto out_unmap_bar0; 4906 } 4907 4908 adapter->workq = create_singlethread_workqueue("cxgb4"); 4909 if (!adapter->workq) { 4910 err = -ENOMEM; 4911 goto out_free_adapter; 4912 } 4913 4914 adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) + 4915 (sizeof(struct mbox_cmd) * 4916 T4_OS_LOG_MBOX_CMDS), 4917 GFP_KERNEL); 4918 if (!adapter->mbox_log) { 4919 err = -ENOMEM; 4920 goto out_free_adapter; 4921 } 4922 adapter->mbox_log->size = T4_OS_LOG_MBOX_CMDS; 4923 4924 /* PCI device has been enabled */ 4925 adapter->flags |= DEV_ENABLED; 4926 4927 adapter->regs = regs; 4928 adapter->pdev = pdev; 4929 adapter->pdev_dev = &pdev->dev; 4930 adapter->name = pci_name(pdev); 4931 adapter->mbox = func; 4932 adapter->pf = func; 4933 adapter->msg_enable = dflt_msg_enable; 4934 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map)); 4935 4936 spin_lock_init(&adapter->stats_lock); 4937 spin_lock_init(&adapter->tid_release_lock); 4938 spin_lock_init(&adapter->win0_lock); 4939 4940 INIT_WORK(&adapter->tid_release_task, process_tid_release_list); 4941 INIT_WORK(&adapter->db_full_task, process_db_full); 4942 INIT_WORK(&adapter->db_drop_task, process_db_drop); 4943 4944 err = t4_prep_adapter(adapter); 4945 if (err) 4946 goto out_free_adapter; 4947 4948 4949 if (!is_t4(adapter->params.chip)) { 4950 s_qpp = (QUEUESPERPAGEPF0_S + 4951 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * 4952 adapter->pf); 4953 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter, 4954 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp); 4955 num_seg = PAGE_SIZE / SEGMENT_SIZE; 4956 4957 /* Each segment size is 128B. Write coalescing is enabled only 4958 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the 4959 * queue is less no of segments that can be accommodated in 4960 * a page size. 4961 */ 4962 if (qpp > num_seg) { 4963 dev_err(&pdev->dev, 4964 "Incorrect number of egress queues per page\n"); 4965 err = -EINVAL; 4966 goto out_free_adapter; 4967 } 4968 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2), 4969 pci_resource_len(pdev, 2)); 4970 if (!adapter->bar2) { 4971 dev_err(&pdev->dev, "cannot map device bar2 region\n"); 4972 err = -ENOMEM; 4973 goto out_free_adapter; 4974 } 4975 } 4976 4977 setup_memwin(adapter); 4978 err = adap_init0(adapter); 4979 #ifdef CONFIG_DEBUG_FS 4980 bitmap_zero(adapter->sge.blocked_fl, adapter->sge.egr_sz); 4981 #endif 4982 setup_memwin_rdma(adapter); 4983 if (err) 4984 goto out_unmap_bar; 4985 4986 /* configure SGE_STAT_CFG_A to read WC stats */ 4987 if (!is_t4(adapter->params.chip)) 4988 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7) | 4989 (is_t5(adapter->params.chip) ? STATMODE_V(0) : 4990 T6_STATMODE_V(0))); 4991 4992 for_each_port(adapter, i) { 4993 struct net_device *netdev; 4994 4995 netdev = alloc_etherdev_mq(sizeof(struct port_info), 4996 MAX_ETH_QSETS); 4997 if (!netdev) { 4998 err = -ENOMEM; 4999 goto out_free_dev; 5000 } 5001 5002 SET_NETDEV_DEV(netdev, &pdev->dev); 5003 5004 adapter->port[i] = netdev; 5005 pi = netdev_priv(netdev); 5006 pi->adapter = adapter; 5007 pi->xact_addr_filt = -1; 5008 pi->port_id = i; 5009 netdev->irq = pdev->irq; 5010 5011 netdev->hw_features = NETIF_F_SG | TSO_FLAGS | 5012 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 5013 NETIF_F_RXCSUM | NETIF_F_RXHASH | 5014 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; 5015 if (highdma) 5016 netdev->hw_features |= NETIF_F_HIGHDMA; 5017 netdev->features |= netdev->hw_features; 5018 netdev->vlan_features = netdev->features & VLAN_FEAT; 5019 5020 netdev->priv_flags |= IFF_UNICAST_FLT; 5021 5022 netdev->netdev_ops = &cxgb4_netdev_ops; 5023 #ifdef CONFIG_CHELSIO_T4_DCB 5024 netdev->dcbnl_ops = &cxgb4_dcb_ops; 5025 cxgb4_dcb_state_init(netdev); 5026 #endif 5027 cxgb4_set_ethtool_ops(netdev); 5028 } 5029 5030 pci_set_drvdata(pdev, adapter); 5031 5032 if (adapter->flags & FW_OK) { 5033 err = t4_port_init(adapter, func, func, 0); 5034 if (err) 5035 goto out_free_dev; 5036 } else if (adapter->params.nports == 1) { 5037 /* If we don't have a connection to the firmware -- possibly 5038 * because of an error -- grab the raw VPD parameters so we 5039 * can set the proper MAC Address on the debug network 5040 * interface that we've created. 5041 */ 5042 u8 hw_addr[ETH_ALEN]; 5043 u8 *na = adapter->params.vpd.na; 5044 5045 err = t4_get_raw_vpd_params(adapter, &adapter->params.vpd); 5046 if (!err) { 5047 for (i = 0; i < ETH_ALEN; i++) 5048 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + 5049 hex2val(na[2 * i + 1])); 5050 t4_set_hw_addr(adapter, 0, hw_addr); 5051 } 5052 } 5053 5054 /* Configure queues and allocate tables now, they can be needed as 5055 * soon as the first register_netdev completes. 5056 */ 5057 cfg_queues(adapter); 5058 5059 adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end); 5060 if (!adapter->l2t) { 5061 /* We tolerate a lack of L2T, giving up some functionality */ 5062 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n"); 5063 adapter->params.offload = 0; 5064 } 5065 5066 #if IS_ENABLED(CONFIG_IPV6) 5067 if ((CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) && 5068 (!(t4_read_reg(adapter, LE_DB_CONFIG_A) & ASLIPCOMPEN_F))) { 5069 /* CLIP functionality is not present in hardware, 5070 * hence disable all offload features 5071 */ 5072 dev_warn(&pdev->dev, 5073 "CLIP not enabled in hardware, continuing\n"); 5074 adapter->params.offload = 0; 5075 } else { 5076 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start, 5077 adapter->clipt_end); 5078 if (!adapter->clipt) { 5079 /* We tolerate a lack of clip_table, giving up 5080 * some functionality 5081 */ 5082 dev_warn(&pdev->dev, 5083 "could not allocate Clip table, continuing\n"); 5084 adapter->params.offload = 0; 5085 } 5086 } 5087 #endif 5088 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) { 5089 dev_warn(&pdev->dev, "could not allocate TID table, " 5090 "continuing\n"); 5091 adapter->params.offload = 0; 5092 } 5093 5094 if (is_offload(adapter)) { 5095 if (t4_read_reg(adapter, LE_DB_CONFIG_A) & HASHEN_F) { 5096 u32 hash_base, hash_reg; 5097 5098 if (chip <= CHELSIO_T5) { 5099 hash_reg = LE_DB_TID_HASHBASE_A; 5100 hash_base = t4_read_reg(adapter, hash_reg); 5101 adapter->tids.hash_base = hash_base / 4; 5102 } else { 5103 hash_reg = T6_LE_DB_HASH_TID_BASE_A; 5104 hash_base = t4_read_reg(adapter, hash_reg); 5105 adapter->tids.hash_base = hash_base; 5106 } 5107 } 5108 } 5109 5110 /* See what interrupts we'll be using */ 5111 if (msi > 1 && enable_msix(adapter) == 0) 5112 adapter->flags |= USING_MSIX; 5113 else if (msi > 0 && pci_enable_msi(pdev) == 0) 5114 adapter->flags |= USING_MSI; 5115 5116 /* check for PCI Express bandwidth capabiltites */ 5117 cxgb4_check_pcie_caps(adapter); 5118 5119 err = init_rss(adapter); 5120 if (err) 5121 goto out_free_dev; 5122 5123 /* 5124 * The card is now ready to go. If any errors occur during device 5125 * registration we do not fail the whole card but rather proceed only 5126 * with the ports we manage to register successfully. However we must 5127 * register at least one net device. 5128 */ 5129 for_each_port(adapter, i) { 5130 pi = adap2pinfo(adapter, i); 5131 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets); 5132 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets); 5133 5134 err = register_netdev(adapter->port[i]); 5135 if (err) 5136 break; 5137 adapter->chan_map[pi->tx_chan] = i; 5138 print_port_info(adapter->port[i]); 5139 } 5140 if (i == 0) { 5141 dev_err(&pdev->dev, "could not register any net devices\n"); 5142 goto out_free_dev; 5143 } 5144 if (err) { 5145 dev_warn(&pdev->dev, "only %d net devices registered\n", i); 5146 err = 0; 5147 } 5148 5149 if (cxgb4_debugfs_root) { 5150 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev), 5151 cxgb4_debugfs_root); 5152 setup_debugfs(adapter); 5153 } 5154 5155 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */ 5156 pdev->needs_freset = 1; 5157 5158 if (is_offload(adapter)) 5159 attach_ulds(adapter); 5160 5161 print_adapter_info(adapter); 5162 5163 sriov: 5164 #ifdef CONFIG_PCI_IOV 5165 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0) 5166 if (pci_enable_sriov(pdev, num_vf[func]) == 0) 5167 dev_info(&pdev->dev, 5168 "instantiated %u virtual functions\n", 5169 num_vf[func]); 5170 #endif 5171 return 0; 5172 5173 out_free_dev: 5174 free_some_resources(adapter); 5175 out_unmap_bar: 5176 if (!is_t4(adapter->params.chip)) 5177 iounmap(adapter->bar2); 5178 out_free_adapter: 5179 if (adapter->workq) 5180 destroy_workqueue(adapter->workq); 5181 5182 kfree(adapter->mbox_log); 5183 kfree(adapter); 5184 out_unmap_bar0: 5185 iounmap(regs); 5186 out_disable_device: 5187 pci_disable_pcie_error_reporting(pdev); 5188 pci_disable_device(pdev); 5189 out_release_regions: 5190 pci_release_regions(pdev); 5191 return err; 5192 } 5193 5194 static void remove_one(struct pci_dev *pdev) 5195 { 5196 struct adapter *adapter = pci_get_drvdata(pdev); 5197 5198 #ifdef CONFIG_PCI_IOV 5199 pci_disable_sriov(pdev); 5200 5201 #endif 5202 5203 if (adapter) { 5204 int i; 5205 5206 /* Tear down per-adapter Work Queue first since it can contain 5207 * references to our adapter data structure. 5208 */ 5209 destroy_workqueue(adapter->workq); 5210 5211 if (is_offload(adapter)) 5212 detach_ulds(adapter); 5213 5214 disable_interrupts(adapter); 5215 5216 for_each_port(adapter, i) 5217 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 5218 unregister_netdev(adapter->port[i]); 5219 5220 debugfs_remove_recursive(adapter->debugfs_root); 5221 5222 /* If we allocated filters, free up state associated with any 5223 * valid filters ... 5224 */ 5225 if (adapter->tids.ftid_tab) { 5226 struct filter_entry *f = &adapter->tids.ftid_tab[0]; 5227 for (i = 0; i < (adapter->tids.nftids + 5228 adapter->tids.nsftids); i++, f++) 5229 if (f->valid) 5230 clear_filter(adapter, f); 5231 } 5232 5233 if (adapter->flags & FULL_INIT_DONE) 5234 cxgb_down(adapter); 5235 5236 free_some_resources(adapter); 5237 #if IS_ENABLED(CONFIG_IPV6) 5238 t4_cleanup_clip_tbl(adapter); 5239 #endif 5240 iounmap(adapter->regs); 5241 if (!is_t4(adapter->params.chip)) 5242 iounmap(adapter->bar2); 5243 pci_disable_pcie_error_reporting(pdev); 5244 if ((adapter->flags & DEV_ENABLED)) { 5245 pci_disable_device(pdev); 5246 adapter->flags &= ~DEV_ENABLED; 5247 } 5248 pci_release_regions(pdev); 5249 kfree(adapter->mbox_log); 5250 synchronize_rcu(); 5251 kfree(adapter); 5252 } else 5253 pci_release_regions(pdev); 5254 } 5255 5256 static struct pci_driver cxgb4_driver = { 5257 .name = KBUILD_MODNAME, 5258 .id_table = cxgb4_pci_tbl, 5259 .probe = init_one, 5260 .remove = remove_one, 5261 .shutdown = remove_one, 5262 .err_handler = &cxgb4_eeh, 5263 }; 5264 5265 static int __init cxgb4_init_module(void) 5266 { 5267 int ret; 5268 5269 /* Debugfs support is optional, just warn if this fails */ 5270 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); 5271 if (!cxgb4_debugfs_root) 5272 pr_warn("could not create debugfs entry, continuing\n"); 5273 5274 ret = pci_register_driver(&cxgb4_driver); 5275 if (ret < 0) 5276 debugfs_remove(cxgb4_debugfs_root); 5277 5278 #if IS_ENABLED(CONFIG_IPV6) 5279 if (!inet6addr_registered) { 5280 register_inet6addr_notifier(&cxgb4_inet6addr_notifier); 5281 inet6addr_registered = true; 5282 } 5283 #endif 5284 5285 return ret; 5286 } 5287 5288 static void __exit cxgb4_cleanup_module(void) 5289 { 5290 #if IS_ENABLED(CONFIG_IPV6) 5291 if (inet6addr_registered) { 5292 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier); 5293 inet6addr_registered = false; 5294 } 5295 #endif 5296 pci_unregister_driver(&cxgb4_driver); 5297 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */ 5298 } 5299 5300 module_init(cxgb4_init_module); 5301 module_exit(cxgb4_cleanup_module); 5302