1 /************************************************************************* 2 * myri10ge.c: Myricom Myri-10G Ethernet driver. 3 * 4 * Copyright (C) 2005 - 2011 Myricom, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of Myricom, Inc. nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * 32 * If the eeprom on your board is not recent enough, you will need to get a 33 * newer firmware image at: 34 * http://www.myri.com/scs/download-Myri10GE.html 35 * 36 * Contact Information: 37 * <help@myri.com> 38 * Myricom, Inc., 325N Santa Anita Avenue, Arcadia, CA 91006 39 *************************************************************************/ 40 41 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 42 43 #include <linux/tcp.h> 44 #include <linux/netdevice.h> 45 #include <linux/skbuff.h> 46 #include <linux/string.h> 47 #include <linux/module.h> 48 #include <linux/pci.h> 49 #include <linux/dma-mapping.h> 50 #include <linux/etherdevice.h> 51 #include <linux/if_ether.h> 52 #include <linux/if_vlan.h> 53 #include <linux/dca.h> 54 #include <linux/ip.h> 55 #include <linux/inet.h> 56 #include <linux/in.h> 57 #include <linux/ethtool.h> 58 #include <linux/firmware.h> 59 #include <linux/delay.h> 60 #include <linux/timer.h> 61 #include <linux/vmalloc.h> 62 #include <linux/crc32.h> 63 #include <linux/moduleparam.h> 64 #include <linux/io.h> 65 #include <linux/log2.h> 66 #include <linux/slab.h> 67 #include <linux/prefetch.h> 68 #include <net/checksum.h> 69 #include <net/ip.h> 70 #include <net/tcp.h> 71 #include <asm/byteorder.h> 72 #include <asm/processor.h> 73 74 #include "myri10ge_mcp.h" 75 #include "myri10ge_mcp_gen_header.h" 76 77 #define MYRI10GE_VERSION_STR "1.5.3-1.534" 78 79 MODULE_DESCRIPTION("Myricom 10G driver (10GbE)"); 80 MODULE_AUTHOR("Maintainer: help@myri.com"); 81 MODULE_VERSION(MYRI10GE_VERSION_STR); 82 MODULE_LICENSE("Dual BSD/GPL"); 83 84 #define MYRI10GE_MAX_ETHER_MTU 9014 85 86 #define MYRI10GE_ETH_STOPPED 0 87 #define MYRI10GE_ETH_STOPPING 1 88 #define MYRI10GE_ETH_STARTING 2 89 #define MYRI10GE_ETH_RUNNING 3 90 #define MYRI10GE_ETH_OPEN_FAILED 4 91 92 #define MYRI10GE_EEPROM_STRINGS_SIZE 256 93 #define MYRI10GE_MAX_SEND_DESC_TSO ((65536 / 2048) * 2) 94 95 #define MYRI10GE_NO_CONFIRM_DATA htonl(0xffffffff) 96 #define MYRI10GE_NO_RESPONSE_RESULT 0xffffffff 97 98 #define MYRI10GE_ALLOC_ORDER 0 99 #define MYRI10GE_ALLOC_SIZE ((1 << MYRI10GE_ALLOC_ORDER) * PAGE_SIZE) 100 #define MYRI10GE_MAX_FRAGS_PER_FRAME (MYRI10GE_MAX_ETHER_MTU/MYRI10GE_ALLOC_SIZE + 1) 101 102 #define MYRI10GE_MAX_SLICES 32 103 104 struct myri10ge_rx_buffer_state { 105 struct page *page; 106 int page_offset; 107 DEFINE_DMA_UNMAP_ADDR(bus); 108 DEFINE_DMA_UNMAP_LEN(len); 109 }; 110 111 struct myri10ge_tx_buffer_state { 112 struct sk_buff *skb; 113 int last; 114 DEFINE_DMA_UNMAP_ADDR(bus); 115 DEFINE_DMA_UNMAP_LEN(len); 116 }; 117 118 struct myri10ge_cmd { 119 u32 data0; 120 u32 data1; 121 u32 data2; 122 }; 123 124 struct myri10ge_rx_buf { 125 struct mcp_kreq_ether_recv __iomem *lanai; /* lanai ptr for recv ring */ 126 struct mcp_kreq_ether_recv *shadow; /* host shadow of recv ring */ 127 struct myri10ge_rx_buffer_state *info; 128 struct page *page; 129 dma_addr_t bus; 130 int page_offset; 131 int cnt; 132 int fill_cnt; 133 int alloc_fail; 134 int mask; /* number of rx slots -1 */ 135 int watchdog_needed; 136 }; 137 138 struct myri10ge_tx_buf { 139 struct mcp_kreq_ether_send __iomem *lanai; /* lanai ptr for sendq */ 140 __be32 __iomem *send_go; /* "go" doorbell ptr */ 141 __be32 __iomem *send_stop; /* "stop" doorbell ptr */ 142 struct mcp_kreq_ether_send *req_list; /* host shadow of sendq */ 143 char *req_bytes; 144 struct myri10ge_tx_buffer_state *info; 145 int mask; /* number of transmit slots -1 */ 146 int req ____cacheline_aligned; /* transmit slots submitted */ 147 int pkt_start; /* packets started */ 148 int stop_queue; 149 int linearized; 150 int done ____cacheline_aligned; /* transmit slots completed */ 151 int pkt_done; /* packets completed */ 152 int wake_queue; 153 int queue_active; 154 }; 155 156 struct myri10ge_rx_done { 157 struct mcp_slot *entry; 158 dma_addr_t bus; 159 int cnt; 160 int idx; 161 }; 162 163 struct myri10ge_slice_netstats { 164 unsigned long rx_packets; 165 unsigned long tx_packets; 166 unsigned long rx_bytes; 167 unsigned long tx_bytes; 168 unsigned long rx_dropped; 169 unsigned long tx_dropped; 170 }; 171 172 struct myri10ge_slice_state { 173 struct myri10ge_tx_buf tx; /* transmit ring */ 174 struct myri10ge_rx_buf rx_small; 175 struct myri10ge_rx_buf rx_big; 176 struct myri10ge_rx_done rx_done; 177 struct net_device *dev; 178 struct napi_struct napi; 179 struct myri10ge_priv *mgp; 180 struct myri10ge_slice_netstats stats; 181 __be32 __iomem *irq_claim; 182 struct mcp_irq_data *fw_stats; 183 dma_addr_t fw_stats_bus; 184 int watchdog_tx_done; 185 int watchdog_tx_req; 186 int watchdog_rx_done; 187 int stuck; 188 #ifdef CONFIG_MYRI10GE_DCA 189 int cached_dca_tag; 190 int cpu; 191 __be32 __iomem *dca_tag; 192 #endif 193 char irq_desc[32]; 194 }; 195 196 struct myri10ge_priv { 197 struct myri10ge_slice_state *ss; 198 int tx_boundary; /* boundary transmits cannot cross */ 199 int num_slices; 200 int running; /* running? */ 201 int small_bytes; 202 int big_bytes; 203 int max_intr_slots; 204 struct net_device *dev; 205 u8 __iomem *sram; 206 int sram_size; 207 unsigned long board_span; 208 unsigned long iomem_base; 209 __be32 __iomem *irq_deassert; 210 char *mac_addr_string; 211 struct mcp_cmd_response *cmd; 212 dma_addr_t cmd_bus; 213 struct pci_dev *pdev; 214 int msi_enabled; 215 int msix_enabled; 216 struct msix_entry *msix_vectors; 217 #ifdef CONFIG_MYRI10GE_DCA 218 int dca_enabled; 219 int relaxed_order; 220 #endif 221 u32 link_state; 222 unsigned int rdma_tags_available; 223 int intr_coal_delay; 224 __be32 __iomem *intr_coal_delay_ptr; 225 int wc_cookie; 226 int down_cnt; 227 wait_queue_head_t down_wq; 228 struct work_struct watchdog_work; 229 struct timer_list watchdog_timer; 230 int watchdog_resets; 231 int watchdog_pause; 232 int pause; 233 bool fw_name_allocated; 234 char *fw_name; 235 char eeprom_strings[MYRI10GE_EEPROM_STRINGS_SIZE]; 236 char *product_code_string; 237 char fw_version[128]; 238 int fw_ver_major; 239 int fw_ver_minor; 240 int fw_ver_tiny; 241 int adopted_rx_filter_bug; 242 u8 mac_addr[ETH_ALEN]; /* eeprom mac address */ 243 unsigned long serial_number; 244 int vendor_specific_offset; 245 int fw_multicast_support; 246 u32 features; 247 u32 max_tso6; 248 u32 read_dma; 249 u32 write_dma; 250 u32 read_write_dma; 251 u32 link_changes; 252 u32 msg_enable; 253 unsigned int board_number; 254 int rebooted; 255 }; 256 257 static char *myri10ge_fw_unaligned = "myri10ge_ethp_z8e.dat"; 258 static char *myri10ge_fw_aligned = "myri10ge_eth_z8e.dat"; 259 static char *myri10ge_fw_rss_unaligned = "myri10ge_rss_ethp_z8e.dat"; 260 static char *myri10ge_fw_rss_aligned = "myri10ge_rss_eth_z8e.dat"; 261 MODULE_FIRMWARE("myri10ge_ethp_z8e.dat"); 262 MODULE_FIRMWARE("myri10ge_eth_z8e.dat"); 263 MODULE_FIRMWARE("myri10ge_rss_ethp_z8e.dat"); 264 MODULE_FIRMWARE("myri10ge_rss_eth_z8e.dat"); 265 266 /* Careful: must be accessed under kernel_param_lock() */ 267 static char *myri10ge_fw_name = NULL; 268 module_param(myri10ge_fw_name, charp, 0644); 269 MODULE_PARM_DESC(myri10ge_fw_name, "Firmware image name"); 270 271 #define MYRI10GE_MAX_BOARDS 8 272 static char *myri10ge_fw_names[MYRI10GE_MAX_BOARDS] = 273 {[0 ... (MYRI10GE_MAX_BOARDS - 1)] = NULL }; 274 module_param_array_named(myri10ge_fw_names, myri10ge_fw_names, charp, NULL, 275 0444); 276 MODULE_PARM_DESC(myri10ge_fw_names, "Firmware image names per board"); 277 278 static int myri10ge_ecrc_enable = 1; 279 module_param(myri10ge_ecrc_enable, int, 0444); 280 MODULE_PARM_DESC(myri10ge_ecrc_enable, "Enable Extended CRC on PCI-E"); 281 282 static int myri10ge_small_bytes = -1; /* -1 == auto */ 283 module_param(myri10ge_small_bytes, int, 0644); 284 MODULE_PARM_DESC(myri10ge_small_bytes, "Threshold of small packets"); 285 286 static int myri10ge_msi = 1; /* enable msi by default */ 287 module_param(myri10ge_msi, int, 0644); 288 MODULE_PARM_DESC(myri10ge_msi, "Enable Message Signalled Interrupts"); 289 290 static int myri10ge_intr_coal_delay = 75; 291 module_param(myri10ge_intr_coal_delay, int, 0444); 292 MODULE_PARM_DESC(myri10ge_intr_coal_delay, "Interrupt coalescing delay"); 293 294 static int myri10ge_flow_control = 1; 295 module_param(myri10ge_flow_control, int, 0444); 296 MODULE_PARM_DESC(myri10ge_flow_control, "Pause parameter"); 297 298 static int myri10ge_deassert_wait = 1; 299 module_param(myri10ge_deassert_wait, int, 0644); 300 MODULE_PARM_DESC(myri10ge_deassert_wait, 301 "Wait when deasserting legacy interrupts"); 302 303 static int myri10ge_force_firmware = 0; 304 module_param(myri10ge_force_firmware, int, 0444); 305 MODULE_PARM_DESC(myri10ge_force_firmware, 306 "Force firmware to assume aligned completions"); 307 308 static int myri10ge_initial_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 309 module_param(myri10ge_initial_mtu, int, 0444); 310 MODULE_PARM_DESC(myri10ge_initial_mtu, "Initial MTU"); 311 312 static int myri10ge_napi_weight = 64; 313 module_param(myri10ge_napi_weight, int, 0444); 314 MODULE_PARM_DESC(myri10ge_napi_weight, "Set NAPI weight"); 315 316 static int myri10ge_watchdog_timeout = 1; 317 module_param(myri10ge_watchdog_timeout, int, 0444); 318 MODULE_PARM_DESC(myri10ge_watchdog_timeout, "Set watchdog timeout"); 319 320 static int myri10ge_max_irq_loops = 1048576; 321 module_param(myri10ge_max_irq_loops, int, 0444); 322 MODULE_PARM_DESC(myri10ge_max_irq_loops, 323 "Set stuck legacy IRQ detection threshold"); 324 325 #define MYRI10GE_MSG_DEFAULT NETIF_MSG_LINK 326 327 static int myri10ge_debug = -1; /* defaults above */ 328 module_param(myri10ge_debug, int, 0); 329 MODULE_PARM_DESC(myri10ge_debug, "Debug level (0=none,...,16=all)"); 330 331 static int myri10ge_fill_thresh = 256; 332 module_param(myri10ge_fill_thresh, int, 0644); 333 MODULE_PARM_DESC(myri10ge_fill_thresh, "Number of empty rx slots allowed"); 334 335 static int myri10ge_reset_recover = 1; 336 337 static int myri10ge_max_slices = 1; 338 module_param(myri10ge_max_slices, int, 0444); 339 MODULE_PARM_DESC(myri10ge_max_slices, "Max tx/rx queues"); 340 341 static int myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_DST_PORT; 342 module_param(myri10ge_rss_hash, int, 0444); 343 MODULE_PARM_DESC(myri10ge_rss_hash, "Type of RSS hashing to do"); 344 345 static int myri10ge_dca = 1; 346 module_param(myri10ge_dca, int, 0444); 347 MODULE_PARM_DESC(myri10ge_dca, "Enable DCA if possible"); 348 349 #define MYRI10GE_FW_OFFSET 1024*1024 350 #define MYRI10GE_HIGHPART_TO_U32(X) \ 351 (sizeof (X) == 8) ? ((u32)((u64)(X) >> 32)) : (0) 352 #define MYRI10GE_LOWPART_TO_U32(X) ((u32)(X)) 353 354 #define myri10ge_pio_copy(to,from,size) __iowrite64_copy(to,from,size/8) 355 356 static void myri10ge_set_multicast_list(struct net_device *dev); 357 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 358 struct net_device *dev); 359 360 static inline void put_be32(__be32 val, __be32 __iomem * p) 361 { 362 __raw_writel((__force __u32) val, (__force void __iomem *)p); 363 } 364 365 static void myri10ge_get_stats(struct net_device *dev, 366 struct rtnl_link_stats64 *stats); 367 368 static void set_fw_name(struct myri10ge_priv *mgp, char *name, bool allocated) 369 { 370 if (mgp->fw_name_allocated) 371 kfree(mgp->fw_name); 372 mgp->fw_name = name; 373 mgp->fw_name_allocated = allocated; 374 } 375 376 static int 377 myri10ge_send_cmd(struct myri10ge_priv *mgp, u32 cmd, 378 struct myri10ge_cmd *data, int atomic) 379 { 380 struct mcp_cmd *buf; 381 char buf_bytes[sizeof(*buf) + 8]; 382 struct mcp_cmd_response *response = mgp->cmd; 383 char __iomem *cmd_addr = mgp->sram + MXGEFW_ETH_CMD; 384 u32 dma_low, dma_high, result, value; 385 int sleep_total = 0; 386 387 /* ensure buf is aligned to 8 bytes */ 388 buf = (struct mcp_cmd *)ALIGN((unsigned long)buf_bytes, 8); 389 390 buf->data0 = htonl(data->data0); 391 buf->data1 = htonl(data->data1); 392 buf->data2 = htonl(data->data2); 393 buf->cmd = htonl(cmd); 394 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 395 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 396 397 buf->response_addr.low = htonl(dma_low); 398 buf->response_addr.high = htonl(dma_high); 399 response->result = htonl(MYRI10GE_NO_RESPONSE_RESULT); 400 mb(); 401 myri10ge_pio_copy(cmd_addr, buf, sizeof(*buf)); 402 403 /* wait up to 15ms. Longest command is the DMA benchmark, 404 * which is capped at 5ms, but runs from a timeout handler 405 * that runs every 7.8ms. So a 15ms timeout leaves us with 406 * a 2.2ms margin 407 */ 408 if (atomic) { 409 /* if atomic is set, do not sleep, 410 * and try to get the completion quickly 411 * (1ms will be enough for those commands) */ 412 for (sleep_total = 0; 413 sleep_total < 1000 && 414 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 415 sleep_total += 10) { 416 udelay(10); 417 mb(); 418 } 419 } else { 420 /* use msleep for most command */ 421 for (sleep_total = 0; 422 sleep_total < 15 && 423 response->result == htonl(MYRI10GE_NO_RESPONSE_RESULT); 424 sleep_total++) 425 msleep(1); 426 } 427 428 result = ntohl(response->result); 429 value = ntohl(response->data); 430 if (result != MYRI10GE_NO_RESPONSE_RESULT) { 431 if (result == 0) { 432 data->data0 = value; 433 return 0; 434 } else if (result == MXGEFW_CMD_UNKNOWN) { 435 return -ENOSYS; 436 } else if (result == MXGEFW_CMD_ERROR_UNALIGNED) { 437 return -E2BIG; 438 } else if (result == MXGEFW_CMD_ERROR_RANGE && 439 cmd == MXGEFW_CMD_ENABLE_RSS_QUEUES && 440 (data-> 441 data1 & MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES) != 442 0) { 443 return -ERANGE; 444 } else { 445 dev_err(&mgp->pdev->dev, 446 "command %d failed, result = %d\n", 447 cmd, result); 448 return -ENXIO; 449 } 450 } 451 452 dev_err(&mgp->pdev->dev, "command %d timed out, result = %d\n", 453 cmd, result); 454 return -EAGAIN; 455 } 456 457 /* 458 * The eeprom strings on the lanaiX have the format 459 * SN=x\0 460 * MAC=x:x:x:x:x:x\0 461 * PT:ddd mmm xx xx:xx:xx xx\0 462 * PV:ddd mmm xx xx:xx:xx xx\0 463 */ 464 static int myri10ge_read_mac_addr(struct myri10ge_priv *mgp) 465 { 466 char *ptr, *limit; 467 int i; 468 469 ptr = mgp->eeprom_strings; 470 limit = mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE; 471 472 while (*ptr != '\0' && ptr < limit) { 473 if (memcmp(ptr, "MAC=", 4) == 0) { 474 ptr += 4; 475 mgp->mac_addr_string = ptr; 476 for (i = 0; i < 6; i++) { 477 if ((ptr + 2) > limit) 478 goto abort; 479 mgp->mac_addr[i] = 480 simple_strtoul(ptr, &ptr, 16); 481 ptr += 1; 482 } 483 } 484 if (memcmp(ptr, "PC=", 3) == 0) { 485 ptr += 3; 486 mgp->product_code_string = ptr; 487 } 488 if (memcmp((const void *)ptr, "SN=", 3) == 0) { 489 ptr += 3; 490 mgp->serial_number = simple_strtoul(ptr, &ptr, 10); 491 } 492 while (ptr < limit && *ptr++) ; 493 } 494 495 return 0; 496 497 abort: 498 dev_err(&mgp->pdev->dev, "failed to parse eeprom_strings\n"); 499 return -ENXIO; 500 } 501 502 /* 503 * Enable or disable periodic RDMAs from the host to make certain 504 * chipsets resend dropped PCIe messages 505 */ 506 507 static void myri10ge_dummy_rdma(struct myri10ge_priv *mgp, int enable) 508 { 509 char __iomem *submit; 510 __be32 buf[16] __attribute__ ((__aligned__(8))); 511 u32 dma_low, dma_high; 512 int i; 513 514 /* clear confirmation addr */ 515 mgp->cmd->data = 0; 516 mb(); 517 518 /* send a rdma command to the PCIe engine, and wait for the 519 * response in the confirmation address. The firmware should 520 * write a -1 there to indicate it is alive and well 521 */ 522 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 523 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 524 525 buf[0] = htonl(dma_high); /* confirm addr MSW */ 526 buf[1] = htonl(dma_low); /* confirm addr LSW */ 527 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 528 buf[3] = htonl(dma_high); /* dummy addr MSW */ 529 buf[4] = htonl(dma_low); /* dummy addr LSW */ 530 buf[5] = htonl(enable); /* enable? */ 531 532 submit = mgp->sram + MXGEFW_BOOT_DUMMY_RDMA; 533 534 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 535 for (i = 0; mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 20; i++) 536 msleep(1); 537 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) 538 dev_err(&mgp->pdev->dev, "dummy rdma %s failed\n", 539 (enable ? "enable" : "disable")); 540 } 541 542 static int 543 myri10ge_validate_firmware(struct myri10ge_priv *mgp, 544 struct mcp_gen_header *hdr) 545 { 546 struct device *dev = &mgp->pdev->dev; 547 548 /* check firmware type */ 549 if (ntohl(hdr->mcp_type) != MCP_TYPE_ETH) { 550 dev_err(dev, "Bad firmware type: 0x%x\n", ntohl(hdr->mcp_type)); 551 return -EINVAL; 552 } 553 554 /* save firmware version for ethtool */ 555 strncpy(mgp->fw_version, hdr->version, sizeof(mgp->fw_version)); 556 mgp->fw_version[sizeof(mgp->fw_version) - 1] = '\0'; 557 558 sscanf(mgp->fw_version, "%d.%d.%d", &mgp->fw_ver_major, 559 &mgp->fw_ver_minor, &mgp->fw_ver_tiny); 560 561 if (!(mgp->fw_ver_major == MXGEFW_VERSION_MAJOR && 562 mgp->fw_ver_minor == MXGEFW_VERSION_MINOR)) { 563 dev_err(dev, "Found firmware version %s\n", mgp->fw_version); 564 dev_err(dev, "Driver needs %d.%d\n", MXGEFW_VERSION_MAJOR, 565 MXGEFW_VERSION_MINOR); 566 return -EINVAL; 567 } 568 return 0; 569 } 570 571 static int myri10ge_load_hotplug_firmware(struct myri10ge_priv *mgp, u32 * size) 572 { 573 unsigned crc, reread_crc; 574 const struct firmware *fw; 575 struct device *dev = &mgp->pdev->dev; 576 unsigned char *fw_readback; 577 struct mcp_gen_header *hdr; 578 size_t hdr_offset; 579 int status; 580 unsigned i; 581 582 if ((status = request_firmware(&fw, mgp->fw_name, dev)) < 0) { 583 dev_err(dev, "Unable to load %s firmware image via hotplug\n", 584 mgp->fw_name); 585 status = -EINVAL; 586 goto abort_with_nothing; 587 } 588 589 /* check size */ 590 591 if (fw->size >= mgp->sram_size - MYRI10GE_FW_OFFSET || 592 fw->size < MCP_HEADER_PTR_OFFSET + 4) { 593 dev_err(dev, "Firmware size invalid:%d\n", (int)fw->size); 594 status = -EINVAL; 595 goto abort_with_fw; 596 } 597 598 /* check id */ 599 hdr_offset = ntohl(*(__be32 *) (fw->data + MCP_HEADER_PTR_OFFSET)); 600 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > fw->size) { 601 dev_err(dev, "Bad firmware file\n"); 602 status = -EINVAL; 603 goto abort_with_fw; 604 } 605 hdr = (void *)(fw->data + hdr_offset); 606 607 status = myri10ge_validate_firmware(mgp, hdr); 608 if (status != 0) 609 goto abort_with_fw; 610 611 crc = crc32(~0, fw->data, fw->size); 612 for (i = 0; i < fw->size; i += 256) { 613 myri10ge_pio_copy(mgp->sram + MYRI10GE_FW_OFFSET + i, 614 fw->data + i, 615 min(256U, (unsigned)(fw->size - i))); 616 mb(); 617 readb(mgp->sram); 618 } 619 fw_readback = vmalloc(fw->size); 620 if (!fw_readback) { 621 status = -ENOMEM; 622 goto abort_with_fw; 623 } 624 /* corruption checking is good for parity recovery and buggy chipset */ 625 memcpy_fromio(fw_readback, mgp->sram + MYRI10GE_FW_OFFSET, fw->size); 626 reread_crc = crc32(~0, fw_readback, fw->size); 627 vfree(fw_readback); 628 if (crc != reread_crc) { 629 dev_err(dev, "CRC failed(fw-len=%u), got 0x%x (expect 0x%x)\n", 630 (unsigned)fw->size, reread_crc, crc); 631 status = -EIO; 632 goto abort_with_fw; 633 } 634 *size = (u32) fw->size; 635 636 abort_with_fw: 637 release_firmware(fw); 638 639 abort_with_nothing: 640 return status; 641 } 642 643 static int myri10ge_adopt_running_firmware(struct myri10ge_priv *mgp) 644 { 645 struct mcp_gen_header *hdr; 646 struct device *dev = &mgp->pdev->dev; 647 const size_t bytes = sizeof(struct mcp_gen_header); 648 size_t hdr_offset; 649 int status; 650 651 /* find running firmware header */ 652 hdr_offset = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 653 654 if ((hdr_offset & 3) || hdr_offset + sizeof(*hdr) > mgp->sram_size) { 655 dev_err(dev, "Running firmware has bad header offset (%d)\n", 656 (int)hdr_offset); 657 return -EIO; 658 } 659 660 /* copy header of running firmware from SRAM to host memory to 661 * validate firmware */ 662 hdr = kmalloc(bytes, GFP_KERNEL); 663 if (hdr == NULL) 664 return -ENOMEM; 665 666 memcpy_fromio(hdr, mgp->sram + hdr_offset, bytes); 667 status = myri10ge_validate_firmware(mgp, hdr); 668 kfree(hdr); 669 670 /* check to see if adopted firmware has bug where adopting 671 * it will cause broadcasts to be filtered unless the NIC 672 * is kept in ALLMULTI mode */ 673 if (mgp->fw_ver_major == 1 && mgp->fw_ver_minor == 4 && 674 mgp->fw_ver_tiny >= 4 && mgp->fw_ver_tiny <= 11) { 675 mgp->adopted_rx_filter_bug = 1; 676 dev_warn(dev, "Adopting fw %d.%d.%d: " 677 "working around rx filter bug\n", 678 mgp->fw_ver_major, mgp->fw_ver_minor, 679 mgp->fw_ver_tiny); 680 } 681 return status; 682 } 683 684 static int myri10ge_get_firmware_capabilities(struct myri10ge_priv *mgp) 685 { 686 struct myri10ge_cmd cmd; 687 int status; 688 689 /* probe for IPv6 TSO support */ 690 mgp->features = NETIF_F_SG | NETIF_F_HW_CSUM | NETIF_F_TSO; 691 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_TSO6_HDR_SIZE, 692 &cmd, 0); 693 if (status == 0) { 694 mgp->max_tso6 = cmd.data0; 695 mgp->features |= NETIF_F_TSO6; 696 } 697 698 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 699 if (status != 0) { 700 dev_err(&mgp->pdev->dev, 701 "failed MXGEFW_CMD_GET_RX_RING_SIZE\n"); 702 return -ENXIO; 703 } 704 705 mgp->max_intr_slots = 2 * (cmd.data0 / sizeof(struct mcp_dma_addr)); 706 707 return 0; 708 } 709 710 static int myri10ge_load_firmware(struct myri10ge_priv *mgp, int adopt) 711 { 712 char __iomem *submit; 713 __be32 buf[16] __attribute__ ((__aligned__(8))); 714 u32 dma_low, dma_high, size; 715 int status, i; 716 717 size = 0; 718 status = myri10ge_load_hotplug_firmware(mgp, &size); 719 if (status) { 720 if (!adopt) 721 return status; 722 dev_warn(&mgp->pdev->dev, "hotplug firmware loading failed\n"); 723 724 /* Do not attempt to adopt firmware if there 725 * was a bad crc */ 726 if (status == -EIO) 727 return status; 728 729 status = myri10ge_adopt_running_firmware(mgp); 730 if (status != 0) { 731 dev_err(&mgp->pdev->dev, 732 "failed to adopt running firmware\n"); 733 return status; 734 } 735 dev_info(&mgp->pdev->dev, 736 "Successfully adopted running firmware\n"); 737 if (mgp->tx_boundary == 4096) { 738 dev_warn(&mgp->pdev->dev, 739 "Using firmware currently running on NIC" 740 ". For optimal\n"); 741 dev_warn(&mgp->pdev->dev, 742 "performance consider loading optimized " 743 "firmware\n"); 744 dev_warn(&mgp->pdev->dev, "via hotplug\n"); 745 } 746 747 set_fw_name(mgp, "adopted", false); 748 mgp->tx_boundary = 2048; 749 myri10ge_dummy_rdma(mgp, 1); 750 status = myri10ge_get_firmware_capabilities(mgp); 751 return status; 752 } 753 754 /* clear confirmation addr */ 755 mgp->cmd->data = 0; 756 mb(); 757 758 /* send a reload command to the bootstrap MCP, and wait for the 759 * response in the confirmation address. The firmware should 760 * write a -1 there to indicate it is alive and well 761 */ 762 dma_low = MYRI10GE_LOWPART_TO_U32(mgp->cmd_bus); 763 dma_high = MYRI10GE_HIGHPART_TO_U32(mgp->cmd_bus); 764 765 buf[0] = htonl(dma_high); /* confirm addr MSW */ 766 buf[1] = htonl(dma_low); /* confirm addr LSW */ 767 buf[2] = MYRI10GE_NO_CONFIRM_DATA; /* confirm data */ 768 769 /* FIX: All newest firmware should un-protect the bottom of 770 * the sram before handoff. However, the very first interfaces 771 * do not. Therefore the handoff copy must skip the first 8 bytes 772 */ 773 buf[3] = htonl(MYRI10GE_FW_OFFSET + 8); /* where the code starts */ 774 buf[4] = htonl(size - 8); /* length of code */ 775 buf[5] = htonl(8); /* where to copy to */ 776 buf[6] = htonl(0); /* where to jump to */ 777 778 submit = mgp->sram + MXGEFW_BOOT_HANDOFF; 779 780 myri10ge_pio_copy(submit, &buf, sizeof(buf)); 781 mb(); 782 msleep(1); 783 mb(); 784 i = 0; 785 while (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA && i < 9) { 786 msleep(1 << i); 787 i++; 788 } 789 if (mgp->cmd->data != MYRI10GE_NO_CONFIRM_DATA) { 790 dev_err(&mgp->pdev->dev, "handoff failed\n"); 791 return -ENXIO; 792 } 793 myri10ge_dummy_rdma(mgp, 1); 794 status = myri10ge_get_firmware_capabilities(mgp); 795 796 return status; 797 } 798 799 static int myri10ge_update_mac_address(struct myri10ge_priv *mgp, u8 * addr) 800 { 801 struct myri10ge_cmd cmd; 802 int status; 803 804 cmd.data0 = ((addr[0] << 24) | (addr[1] << 16) 805 | (addr[2] << 8) | addr[3]); 806 807 cmd.data1 = ((addr[4] << 8) | (addr[5])); 808 809 status = myri10ge_send_cmd(mgp, MXGEFW_SET_MAC_ADDRESS, &cmd, 0); 810 return status; 811 } 812 813 static int myri10ge_change_pause(struct myri10ge_priv *mgp, int pause) 814 { 815 struct myri10ge_cmd cmd; 816 int status, ctl; 817 818 ctl = pause ? MXGEFW_ENABLE_FLOW_CONTROL : MXGEFW_DISABLE_FLOW_CONTROL; 819 status = myri10ge_send_cmd(mgp, ctl, &cmd, 0); 820 821 if (status) { 822 netdev_err(mgp->dev, "Failed to set flow control mode\n"); 823 return status; 824 } 825 mgp->pause = pause; 826 return 0; 827 } 828 829 static void 830 myri10ge_change_promisc(struct myri10ge_priv *mgp, int promisc, int atomic) 831 { 832 struct myri10ge_cmd cmd; 833 int status, ctl; 834 835 ctl = promisc ? MXGEFW_ENABLE_PROMISC : MXGEFW_DISABLE_PROMISC; 836 status = myri10ge_send_cmd(mgp, ctl, &cmd, atomic); 837 if (status) 838 netdev_err(mgp->dev, "Failed to set promisc mode\n"); 839 } 840 841 static int myri10ge_dma_test(struct myri10ge_priv *mgp, int test_type) 842 { 843 struct myri10ge_cmd cmd; 844 int status; 845 u32 len; 846 struct page *dmatest_page; 847 dma_addr_t dmatest_bus; 848 char *test = " "; 849 850 dmatest_page = alloc_page(GFP_KERNEL); 851 if (!dmatest_page) 852 return -ENOMEM; 853 dmatest_bus = pci_map_page(mgp->pdev, dmatest_page, 0, PAGE_SIZE, 854 DMA_BIDIRECTIONAL); 855 if (unlikely(pci_dma_mapping_error(mgp->pdev, dmatest_bus))) { 856 __free_page(dmatest_page); 857 return -ENOMEM; 858 } 859 860 /* Run a small DMA test. 861 * The magic multipliers to the length tell the firmware 862 * to do DMA read, write, or read+write tests. The 863 * results are returned in cmd.data0. The upper 16 864 * bits or the return is the number of transfers completed. 865 * The lower 16 bits is the time in 0.5us ticks that the 866 * transfers took to complete. 867 */ 868 869 len = mgp->tx_boundary; 870 871 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 872 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 873 cmd.data2 = len * 0x10000; 874 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 875 if (status != 0) { 876 test = "read"; 877 goto abort; 878 } 879 mgp->read_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 880 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 881 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 882 cmd.data2 = len * 0x1; 883 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 884 if (status != 0) { 885 test = "write"; 886 goto abort; 887 } 888 mgp->write_dma = ((cmd.data0 >> 16) * len * 2) / (cmd.data0 & 0xffff); 889 890 cmd.data0 = MYRI10GE_LOWPART_TO_U32(dmatest_bus); 891 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(dmatest_bus); 892 cmd.data2 = len * 0x10001; 893 status = myri10ge_send_cmd(mgp, test_type, &cmd, 0); 894 if (status != 0) { 895 test = "read/write"; 896 goto abort; 897 } 898 mgp->read_write_dma = ((cmd.data0 >> 16) * len * 2 * 2) / 899 (cmd.data0 & 0xffff); 900 901 abort: 902 pci_unmap_page(mgp->pdev, dmatest_bus, PAGE_SIZE, DMA_BIDIRECTIONAL); 903 put_page(dmatest_page); 904 905 if (status != 0 && test_type != MXGEFW_CMD_UNALIGNED_TEST) 906 dev_warn(&mgp->pdev->dev, "DMA %s benchmark failed: %d\n", 907 test, status); 908 909 return status; 910 } 911 912 static int myri10ge_reset(struct myri10ge_priv *mgp) 913 { 914 struct myri10ge_cmd cmd; 915 struct myri10ge_slice_state *ss; 916 int i, status; 917 size_t bytes; 918 #ifdef CONFIG_MYRI10GE_DCA 919 unsigned long dca_tag_off; 920 #endif 921 922 /* try to send a reset command to the card to see if it 923 * is alive */ 924 memset(&cmd, 0, sizeof(cmd)); 925 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 926 if (status != 0) { 927 dev_err(&mgp->pdev->dev, "failed reset\n"); 928 return -ENXIO; 929 } 930 931 (void)myri10ge_dma_test(mgp, MXGEFW_DMA_TEST); 932 /* 933 * Use non-ndis mcp_slot (eg, 4 bytes total, 934 * no toeplitz hash value returned. Older firmware will 935 * not understand this command, but will use the correct 936 * sized mcp_slot, so we ignore error returns 937 */ 938 cmd.data0 = MXGEFW_RSS_MCP_SLOT_TYPE_MIN; 939 (void)myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_MCP_SLOT_TYPE, &cmd, 0); 940 941 /* Now exchange information about interrupts */ 942 943 bytes = mgp->max_intr_slots * sizeof(*mgp->ss[0].rx_done.entry); 944 cmd.data0 = (u32) bytes; 945 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 946 947 /* 948 * Even though we already know how many slices are supported 949 * via myri10ge_probe_slices() MXGEFW_CMD_GET_MAX_RSS_QUEUES 950 * has magic side effects, and must be called after a reset. 951 * It must be called prior to calling any RSS related cmds, 952 * including assigning an interrupt queue for anything but 953 * slice 0. It must also be called *after* 954 * MXGEFW_CMD_SET_INTRQ_SIZE, since the intrq size is used by 955 * the firmware to compute offsets. 956 */ 957 958 if (mgp->num_slices > 1) { 959 960 /* ask the maximum number of slices it supports */ 961 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, 962 &cmd, 0); 963 if (status != 0) { 964 dev_err(&mgp->pdev->dev, 965 "failed to get number of slices\n"); 966 } 967 968 /* 969 * MXGEFW_CMD_ENABLE_RSS_QUEUES must be called prior 970 * to setting up the interrupt queue DMA 971 */ 972 973 cmd.data0 = mgp->num_slices; 974 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 975 if (mgp->dev->real_num_tx_queues > 1) 976 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 977 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 978 &cmd, 0); 979 980 /* Firmware older than 1.4.32 only supports multiple 981 * RX queues, so if we get an error, first retry using a 982 * single TX queue before giving up */ 983 if (status != 0 && mgp->dev->real_num_tx_queues > 1) { 984 netif_set_real_num_tx_queues(mgp->dev, 1); 985 cmd.data0 = mgp->num_slices; 986 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 987 status = myri10ge_send_cmd(mgp, 988 MXGEFW_CMD_ENABLE_RSS_QUEUES, 989 &cmd, 0); 990 } 991 992 if (status != 0) { 993 dev_err(&mgp->pdev->dev, 994 "failed to set number of slices\n"); 995 996 return status; 997 } 998 } 999 for (i = 0; i < mgp->num_slices; i++) { 1000 ss = &mgp->ss[i]; 1001 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->rx_done.bus); 1002 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->rx_done.bus); 1003 cmd.data2 = i; 1004 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_DMA, 1005 &cmd, 0); 1006 } 1007 1008 status |= 1009 myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_ACK_OFFSET, &cmd, 0); 1010 for (i = 0; i < mgp->num_slices; i++) { 1011 ss = &mgp->ss[i]; 1012 ss->irq_claim = 1013 (__iomem __be32 *) (mgp->sram + cmd.data0 + 8 * i); 1014 } 1015 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_IRQ_DEASSERT_OFFSET, 1016 &cmd, 0); 1017 mgp->irq_deassert = (__iomem __be32 *) (mgp->sram + cmd.data0); 1018 1019 status |= myri10ge_send_cmd 1020 (mgp, MXGEFW_CMD_GET_INTR_COAL_DELAY_OFFSET, &cmd, 0); 1021 mgp->intr_coal_delay_ptr = (__iomem __be32 *) (mgp->sram + cmd.data0); 1022 if (status != 0) { 1023 dev_err(&mgp->pdev->dev, "failed set interrupt parameters\n"); 1024 return status; 1025 } 1026 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1027 1028 #ifdef CONFIG_MYRI10GE_DCA 1029 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_DCA_OFFSET, &cmd, 0); 1030 dca_tag_off = cmd.data0; 1031 for (i = 0; i < mgp->num_slices; i++) { 1032 ss = &mgp->ss[i]; 1033 if (status == 0) { 1034 ss->dca_tag = (__iomem __be32 *) 1035 (mgp->sram + dca_tag_off + 4 * i); 1036 } else { 1037 ss->dca_tag = NULL; 1038 } 1039 } 1040 #endif /* CONFIG_MYRI10GE_DCA */ 1041 1042 /* reset mcp/driver shared state back to 0 */ 1043 1044 mgp->link_changes = 0; 1045 for (i = 0; i < mgp->num_slices; i++) { 1046 ss = &mgp->ss[i]; 1047 1048 memset(ss->rx_done.entry, 0, bytes); 1049 ss->tx.req = 0; 1050 ss->tx.done = 0; 1051 ss->tx.pkt_start = 0; 1052 ss->tx.pkt_done = 0; 1053 ss->rx_big.cnt = 0; 1054 ss->rx_small.cnt = 0; 1055 ss->rx_done.idx = 0; 1056 ss->rx_done.cnt = 0; 1057 ss->tx.wake_queue = 0; 1058 ss->tx.stop_queue = 0; 1059 } 1060 1061 status = myri10ge_update_mac_address(mgp, mgp->dev->dev_addr); 1062 myri10ge_change_pause(mgp, mgp->pause); 1063 myri10ge_set_multicast_list(mgp->dev); 1064 return status; 1065 } 1066 1067 #ifdef CONFIG_MYRI10GE_DCA 1068 static int myri10ge_toggle_relaxed(struct pci_dev *pdev, int on) 1069 { 1070 int ret; 1071 u16 ctl; 1072 1073 pcie_capability_read_word(pdev, PCI_EXP_DEVCTL, &ctl); 1074 1075 ret = (ctl & PCI_EXP_DEVCTL_RELAX_EN) >> 4; 1076 if (ret != on) { 1077 ctl &= ~PCI_EXP_DEVCTL_RELAX_EN; 1078 ctl |= (on << 4); 1079 pcie_capability_write_word(pdev, PCI_EXP_DEVCTL, ctl); 1080 } 1081 return ret; 1082 } 1083 1084 static void 1085 myri10ge_write_dca(struct myri10ge_slice_state *ss, int cpu, int tag) 1086 { 1087 ss->cached_dca_tag = tag; 1088 put_be32(htonl(tag), ss->dca_tag); 1089 } 1090 1091 static inline void myri10ge_update_dca(struct myri10ge_slice_state *ss) 1092 { 1093 int cpu = get_cpu(); 1094 int tag; 1095 1096 if (cpu != ss->cpu) { 1097 tag = dca3_get_tag(&ss->mgp->pdev->dev, cpu); 1098 if (ss->cached_dca_tag != tag) 1099 myri10ge_write_dca(ss, cpu, tag); 1100 ss->cpu = cpu; 1101 } 1102 put_cpu(); 1103 } 1104 1105 static void myri10ge_setup_dca(struct myri10ge_priv *mgp) 1106 { 1107 int err, i; 1108 struct pci_dev *pdev = mgp->pdev; 1109 1110 if (mgp->ss[0].dca_tag == NULL || mgp->dca_enabled) 1111 return; 1112 if (!myri10ge_dca) { 1113 dev_err(&pdev->dev, "dca disabled by administrator\n"); 1114 return; 1115 } 1116 err = dca_add_requester(&pdev->dev); 1117 if (err) { 1118 if (err != -ENODEV) 1119 dev_err(&pdev->dev, 1120 "dca_add_requester() failed, err=%d\n", err); 1121 return; 1122 } 1123 mgp->relaxed_order = myri10ge_toggle_relaxed(pdev, 0); 1124 mgp->dca_enabled = 1; 1125 for (i = 0; i < mgp->num_slices; i++) { 1126 mgp->ss[i].cpu = -1; 1127 mgp->ss[i].cached_dca_tag = -1; 1128 myri10ge_update_dca(&mgp->ss[i]); 1129 } 1130 } 1131 1132 static void myri10ge_teardown_dca(struct myri10ge_priv *mgp) 1133 { 1134 struct pci_dev *pdev = mgp->pdev; 1135 1136 if (!mgp->dca_enabled) 1137 return; 1138 mgp->dca_enabled = 0; 1139 if (mgp->relaxed_order) 1140 myri10ge_toggle_relaxed(pdev, 1); 1141 dca_remove_requester(&pdev->dev); 1142 } 1143 1144 static int myri10ge_notify_dca_device(struct device *dev, void *data) 1145 { 1146 struct myri10ge_priv *mgp; 1147 unsigned long event; 1148 1149 mgp = dev_get_drvdata(dev); 1150 event = *(unsigned long *)data; 1151 1152 if (event == DCA_PROVIDER_ADD) 1153 myri10ge_setup_dca(mgp); 1154 else if (event == DCA_PROVIDER_REMOVE) 1155 myri10ge_teardown_dca(mgp); 1156 return 0; 1157 } 1158 #endif /* CONFIG_MYRI10GE_DCA */ 1159 1160 static inline void 1161 myri10ge_submit_8rx(struct mcp_kreq_ether_recv __iomem * dst, 1162 struct mcp_kreq_ether_recv *src) 1163 { 1164 __be32 low; 1165 1166 low = src->addr_low; 1167 src->addr_low = htonl(DMA_BIT_MASK(32)); 1168 myri10ge_pio_copy(dst, src, 4 * sizeof(*src)); 1169 mb(); 1170 myri10ge_pio_copy(dst + 4, src + 4, 4 * sizeof(*src)); 1171 mb(); 1172 src->addr_low = low; 1173 put_be32(low, &dst->addr_low); 1174 mb(); 1175 } 1176 1177 static void 1178 myri10ge_alloc_rx_pages(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx, 1179 int bytes, int watchdog) 1180 { 1181 struct page *page; 1182 dma_addr_t bus; 1183 int idx; 1184 #if MYRI10GE_ALLOC_SIZE > 4096 1185 int end_offset; 1186 #endif 1187 1188 if (unlikely(rx->watchdog_needed && !watchdog)) 1189 return; 1190 1191 /* try to refill entire ring */ 1192 while (rx->fill_cnt != (rx->cnt + rx->mask + 1)) { 1193 idx = rx->fill_cnt & rx->mask; 1194 if (rx->page_offset + bytes <= MYRI10GE_ALLOC_SIZE) { 1195 /* we can use part of previous page */ 1196 get_page(rx->page); 1197 } else { 1198 /* we need a new page */ 1199 page = 1200 alloc_pages(GFP_ATOMIC | __GFP_COMP, 1201 MYRI10GE_ALLOC_ORDER); 1202 if (unlikely(page == NULL)) { 1203 if (rx->fill_cnt - rx->cnt < 16) 1204 rx->watchdog_needed = 1; 1205 return; 1206 } 1207 1208 bus = pci_map_page(mgp->pdev, page, 0, 1209 MYRI10GE_ALLOC_SIZE, 1210 PCI_DMA_FROMDEVICE); 1211 if (unlikely(pci_dma_mapping_error(mgp->pdev, bus))) { 1212 __free_pages(page, MYRI10GE_ALLOC_ORDER); 1213 if (rx->fill_cnt - rx->cnt < 16) 1214 rx->watchdog_needed = 1; 1215 return; 1216 } 1217 1218 rx->page = page; 1219 rx->page_offset = 0; 1220 rx->bus = bus; 1221 1222 } 1223 rx->info[idx].page = rx->page; 1224 rx->info[idx].page_offset = rx->page_offset; 1225 /* note that this is the address of the start of the 1226 * page */ 1227 dma_unmap_addr_set(&rx->info[idx], bus, rx->bus); 1228 rx->shadow[idx].addr_low = 1229 htonl(MYRI10GE_LOWPART_TO_U32(rx->bus) + rx->page_offset); 1230 rx->shadow[idx].addr_high = 1231 htonl(MYRI10GE_HIGHPART_TO_U32(rx->bus)); 1232 1233 /* start next packet on a cacheline boundary */ 1234 rx->page_offset += SKB_DATA_ALIGN(bytes); 1235 1236 #if MYRI10GE_ALLOC_SIZE > 4096 1237 /* don't cross a 4KB boundary */ 1238 end_offset = rx->page_offset + bytes - 1; 1239 if ((unsigned)(rx->page_offset ^ end_offset) > 4095) 1240 rx->page_offset = end_offset & ~4095; 1241 #endif 1242 rx->fill_cnt++; 1243 1244 /* copy 8 descriptors to the firmware at a time */ 1245 if ((idx & 7) == 7) { 1246 myri10ge_submit_8rx(&rx->lanai[idx - 7], 1247 &rx->shadow[idx - 7]); 1248 } 1249 } 1250 } 1251 1252 static inline void 1253 myri10ge_unmap_rx_page(struct pci_dev *pdev, 1254 struct myri10ge_rx_buffer_state *info, int bytes) 1255 { 1256 /* unmap the recvd page if we're the only or last user of it */ 1257 if (bytes >= MYRI10GE_ALLOC_SIZE / 2 || 1258 (info->page_offset + 2 * bytes) > MYRI10GE_ALLOC_SIZE) { 1259 pci_unmap_page(pdev, (dma_unmap_addr(info, bus) 1260 & ~(MYRI10GE_ALLOC_SIZE - 1)), 1261 MYRI10GE_ALLOC_SIZE, PCI_DMA_FROMDEVICE); 1262 } 1263 } 1264 1265 /* 1266 * GRO does not support acceleration of tagged vlan frames, and 1267 * this NIC does not support vlan tag offload, so we must pop 1268 * the tag ourselves to be able to achieve GRO performance that 1269 * is comparable to LRO. 1270 */ 1271 1272 static inline void 1273 myri10ge_vlan_rx(struct net_device *dev, void *addr, struct sk_buff *skb) 1274 { 1275 u8 *va; 1276 struct vlan_ethhdr *veh; 1277 skb_frag_t *frag; 1278 __wsum vsum; 1279 1280 va = addr; 1281 va += MXGEFW_PAD; 1282 veh = (struct vlan_ethhdr *)va; 1283 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) == 1284 NETIF_F_HW_VLAN_CTAG_RX && 1285 veh->h_vlan_proto == htons(ETH_P_8021Q)) { 1286 /* fixup csum if needed */ 1287 if (skb->ip_summed == CHECKSUM_COMPLETE) { 1288 vsum = csum_partial(va + ETH_HLEN, VLAN_HLEN, 0); 1289 skb->csum = csum_sub(skb->csum, vsum); 1290 } 1291 /* pop tag */ 1292 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(veh->h_vlan_TCI)); 1293 memmove(va + VLAN_HLEN, va, 2 * ETH_ALEN); 1294 skb->len -= VLAN_HLEN; 1295 skb->data_len -= VLAN_HLEN; 1296 frag = skb_shinfo(skb)->frags; 1297 skb_frag_off_add(frag, VLAN_HLEN); 1298 skb_frag_size_sub(frag, VLAN_HLEN); 1299 } 1300 } 1301 1302 #define MYRI10GE_HLEN 64 /* Bytes to copy from page to skb linear memory */ 1303 1304 static inline int 1305 myri10ge_rx_done(struct myri10ge_slice_state *ss, int len, __wsum csum) 1306 { 1307 struct myri10ge_priv *mgp = ss->mgp; 1308 struct sk_buff *skb; 1309 skb_frag_t *rx_frags; 1310 struct myri10ge_rx_buf *rx; 1311 int i, idx, remainder, bytes; 1312 struct pci_dev *pdev = mgp->pdev; 1313 struct net_device *dev = mgp->dev; 1314 u8 *va; 1315 1316 if (len <= mgp->small_bytes) { 1317 rx = &ss->rx_small; 1318 bytes = mgp->small_bytes; 1319 } else { 1320 rx = &ss->rx_big; 1321 bytes = mgp->big_bytes; 1322 } 1323 1324 len += MXGEFW_PAD; 1325 idx = rx->cnt & rx->mask; 1326 va = page_address(rx->info[idx].page) + rx->info[idx].page_offset; 1327 prefetch(va); 1328 1329 skb = napi_get_frags(&ss->napi); 1330 if (unlikely(skb == NULL)) { 1331 ss->stats.rx_dropped++; 1332 for (i = 0, remainder = len; remainder > 0; i++) { 1333 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1334 put_page(rx->info[idx].page); 1335 rx->cnt++; 1336 idx = rx->cnt & rx->mask; 1337 remainder -= MYRI10GE_ALLOC_SIZE; 1338 } 1339 return 0; 1340 } 1341 rx_frags = skb_shinfo(skb)->frags; 1342 /* Fill skb_frag_t(s) with data from our receive */ 1343 for (i = 0, remainder = len; remainder > 0; i++) { 1344 myri10ge_unmap_rx_page(pdev, &rx->info[idx], bytes); 1345 skb_fill_page_desc(skb, i, rx->info[idx].page, 1346 rx->info[idx].page_offset, 1347 remainder < MYRI10GE_ALLOC_SIZE ? 1348 remainder : MYRI10GE_ALLOC_SIZE); 1349 rx->cnt++; 1350 idx = rx->cnt & rx->mask; 1351 remainder -= MYRI10GE_ALLOC_SIZE; 1352 } 1353 1354 /* remove padding */ 1355 skb_frag_off_add(&rx_frags[0], MXGEFW_PAD); 1356 skb_frag_size_sub(&rx_frags[0], MXGEFW_PAD); 1357 len -= MXGEFW_PAD; 1358 1359 skb->len = len; 1360 skb->data_len = len; 1361 skb->truesize += len; 1362 if (dev->features & NETIF_F_RXCSUM) { 1363 skb->ip_summed = CHECKSUM_COMPLETE; 1364 skb->csum = csum; 1365 } 1366 myri10ge_vlan_rx(mgp->dev, va, skb); 1367 skb_record_rx_queue(skb, ss - &mgp->ss[0]); 1368 1369 napi_gro_frags(&ss->napi); 1370 1371 return 1; 1372 } 1373 1374 static inline void 1375 myri10ge_tx_done(struct myri10ge_slice_state *ss, int mcp_index) 1376 { 1377 struct pci_dev *pdev = ss->mgp->pdev; 1378 struct myri10ge_tx_buf *tx = &ss->tx; 1379 struct netdev_queue *dev_queue; 1380 struct sk_buff *skb; 1381 int idx, len; 1382 1383 while (tx->pkt_done != mcp_index) { 1384 idx = tx->done & tx->mask; 1385 skb = tx->info[idx].skb; 1386 1387 /* Mark as free */ 1388 tx->info[idx].skb = NULL; 1389 if (tx->info[idx].last) { 1390 tx->pkt_done++; 1391 tx->info[idx].last = 0; 1392 } 1393 tx->done++; 1394 len = dma_unmap_len(&tx->info[idx], len); 1395 dma_unmap_len_set(&tx->info[idx], len, 0); 1396 if (skb) { 1397 ss->stats.tx_bytes += skb->len; 1398 ss->stats.tx_packets++; 1399 dev_consume_skb_irq(skb); 1400 if (len) 1401 pci_unmap_single(pdev, 1402 dma_unmap_addr(&tx->info[idx], 1403 bus), len, 1404 PCI_DMA_TODEVICE); 1405 } else { 1406 if (len) 1407 pci_unmap_page(pdev, 1408 dma_unmap_addr(&tx->info[idx], 1409 bus), len, 1410 PCI_DMA_TODEVICE); 1411 } 1412 } 1413 1414 dev_queue = netdev_get_tx_queue(ss->dev, ss - ss->mgp->ss); 1415 /* 1416 * Make a minimal effort to prevent the NIC from polling an 1417 * idle tx queue. If we can't get the lock we leave the queue 1418 * active. In this case, either a thread was about to start 1419 * using the queue anyway, or we lost a race and the NIC will 1420 * waste some of its resources polling an inactive queue for a 1421 * while. 1422 */ 1423 1424 if ((ss->mgp->dev->real_num_tx_queues > 1) && 1425 __netif_tx_trylock(dev_queue)) { 1426 if (tx->req == tx->done) { 1427 tx->queue_active = 0; 1428 put_be32(htonl(1), tx->send_stop); 1429 mb(); 1430 } 1431 __netif_tx_unlock(dev_queue); 1432 } 1433 1434 /* start the queue if we've stopped it */ 1435 if (netif_tx_queue_stopped(dev_queue) && 1436 tx->req - tx->done < (tx->mask >> 1) && 1437 ss->mgp->running == MYRI10GE_ETH_RUNNING) { 1438 tx->wake_queue++; 1439 netif_tx_wake_queue(dev_queue); 1440 } 1441 } 1442 1443 static inline int 1444 myri10ge_clean_rx_done(struct myri10ge_slice_state *ss, int budget) 1445 { 1446 struct myri10ge_rx_done *rx_done = &ss->rx_done; 1447 struct myri10ge_priv *mgp = ss->mgp; 1448 unsigned long rx_bytes = 0; 1449 unsigned long rx_packets = 0; 1450 unsigned long rx_ok; 1451 int idx = rx_done->idx; 1452 int cnt = rx_done->cnt; 1453 int work_done = 0; 1454 u16 length; 1455 __wsum checksum; 1456 1457 while (rx_done->entry[idx].length != 0 && work_done < budget) { 1458 length = ntohs(rx_done->entry[idx].length); 1459 rx_done->entry[idx].length = 0; 1460 checksum = csum_unfold(rx_done->entry[idx].checksum); 1461 rx_ok = myri10ge_rx_done(ss, length, checksum); 1462 rx_packets += rx_ok; 1463 rx_bytes += rx_ok * (unsigned long)length; 1464 cnt++; 1465 idx = cnt & (mgp->max_intr_slots - 1); 1466 work_done++; 1467 } 1468 rx_done->idx = idx; 1469 rx_done->cnt = cnt; 1470 ss->stats.rx_packets += rx_packets; 1471 ss->stats.rx_bytes += rx_bytes; 1472 1473 /* restock receive rings if needed */ 1474 if (ss->rx_small.fill_cnt - ss->rx_small.cnt < myri10ge_fill_thresh) 1475 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 1476 mgp->small_bytes + MXGEFW_PAD, 0); 1477 if (ss->rx_big.fill_cnt - ss->rx_big.cnt < myri10ge_fill_thresh) 1478 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 1479 1480 return work_done; 1481 } 1482 1483 static inline void myri10ge_check_statblock(struct myri10ge_priv *mgp) 1484 { 1485 struct mcp_irq_data *stats = mgp->ss[0].fw_stats; 1486 1487 if (unlikely(stats->stats_updated)) { 1488 unsigned link_up = ntohl(stats->link_up); 1489 if (mgp->link_state != link_up) { 1490 mgp->link_state = link_up; 1491 1492 if (mgp->link_state == MXGEFW_LINK_UP) { 1493 netif_info(mgp, link, mgp->dev, "link up\n"); 1494 netif_carrier_on(mgp->dev); 1495 mgp->link_changes++; 1496 } else { 1497 netif_info(mgp, link, mgp->dev, "link %s\n", 1498 (link_up == MXGEFW_LINK_MYRINET ? 1499 "mismatch (Myrinet detected)" : 1500 "down")); 1501 netif_carrier_off(mgp->dev); 1502 mgp->link_changes++; 1503 } 1504 } 1505 if (mgp->rdma_tags_available != 1506 ntohl(stats->rdma_tags_available)) { 1507 mgp->rdma_tags_available = 1508 ntohl(stats->rdma_tags_available); 1509 netdev_warn(mgp->dev, "RDMA timed out! %d tags left\n", 1510 mgp->rdma_tags_available); 1511 } 1512 mgp->down_cnt += stats->link_down; 1513 if (stats->link_down) 1514 wake_up(&mgp->down_wq); 1515 } 1516 } 1517 1518 static int myri10ge_poll(struct napi_struct *napi, int budget) 1519 { 1520 struct myri10ge_slice_state *ss = 1521 container_of(napi, struct myri10ge_slice_state, napi); 1522 int work_done; 1523 1524 #ifdef CONFIG_MYRI10GE_DCA 1525 if (ss->mgp->dca_enabled) 1526 myri10ge_update_dca(ss); 1527 #endif 1528 /* process as many rx events as NAPI will allow */ 1529 work_done = myri10ge_clean_rx_done(ss, budget); 1530 1531 if (work_done < budget) { 1532 napi_complete_done(napi, work_done); 1533 put_be32(htonl(3), ss->irq_claim); 1534 } 1535 return work_done; 1536 } 1537 1538 static irqreturn_t myri10ge_intr(int irq, void *arg) 1539 { 1540 struct myri10ge_slice_state *ss = arg; 1541 struct myri10ge_priv *mgp = ss->mgp; 1542 struct mcp_irq_data *stats = ss->fw_stats; 1543 struct myri10ge_tx_buf *tx = &ss->tx; 1544 u32 send_done_count; 1545 int i; 1546 1547 /* an interrupt on a non-zero receive-only slice is implicitly 1548 * valid since MSI-X irqs are not shared */ 1549 if ((mgp->dev->real_num_tx_queues == 1) && (ss != mgp->ss)) { 1550 napi_schedule(&ss->napi); 1551 return IRQ_HANDLED; 1552 } 1553 1554 /* make sure it is our IRQ, and that the DMA has finished */ 1555 if (unlikely(!stats->valid)) 1556 return IRQ_NONE; 1557 1558 /* low bit indicates receives are present, so schedule 1559 * napi poll handler */ 1560 if (stats->valid & 1) 1561 napi_schedule(&ss->napi); 1562 1563 if (!mgp->msi_enabled && !mgp->msix_enabled) { 1564 put_be32(0, mgp->irq_deassert); 1565 if (!myri10ge_deassert_wait) 1566 stats->valid = 0; 1567 mb(); 1568 } else 1569 stats->valid = 0; 1570 1571 /* Wait for IRQ line to go low, if using INTx */ 1572 i = 0; 1573 while (1) { 1574 i++; 1575 /* check for transmit completes and receives */ 1576 send_done_count = ntohl(stats->send_done_count); 1577 if (send_done_count != tx->pkt_done) 1578 myri10ge_tx_done(ss, (int)send_done_count); 1579 if (unlikely(i > myri10ge_max_irq_loops)) { 1580 netdev_warn(mgp->dev, "irq stuck?\n"); 1581 stats->valid = 0; 1582 schedule_work(&mgp->watchdog_work); 1583 } 1584 if (likely(stats->valid == 0)) 1585 break; 1586 cpu_relax(); 1587 barrier(); 1588 } 1589 1590 /* Only slice 0 updates stats */ 1591 if (ss == mgp->ss) 1592 myri10ge_check_statblock(mgp); 1593 1594 put_be32(htonl(3), ss->irq_claim + 1); 1595 return IRQ_HANDLED; 1596 } 1597 1598 static int 1599 myri10ge_get_link_ksettings(struct net_device *netdev, 1600 struct ethtool_link_ksettings *cmd) 1601 { 1602 struct myri10ge_priv *mgp = netdev_priv(netdev); 1603 char *ptr; 1604 int i; 1605 1606 cmd->base.autoneg = AUTONEG_DISABLE; 1607 cmd->base.speed = SPEED_10000; 1608 cmd->base.duplex = DUPLEX_FULL; 1609 1610 /* 1611 * parse the product code to deterimine the interface type 1612 * (CX4, XFP, Quad Ribbon Fiber) by looking at the character 1613 * after the 3rd dash in the driver's cached copy of the 1614 * EEPROM's product code string. 1615 */ 1616 ptr = mgp->product_code_string; 1617 if (ptr == NULL) { 1618 netdev_err(netdev, "Missing product code\n"); 1619 return 0; 1620 } 1621 for (i = 0; i < 3; i++, ptr++) { 1622 ptr = strchr(ptr, '-'); 1623 if (ptr == NULL) { 1624 netdev_err(netdev, "Invalid product code %s\n", 1625 mgp->product_code_string); 1626 return 0; 1627 } 1628 } 1629 if (*ptr == '2') 1630 ptr++; 1631 if (*ptr == 'R' || *ptr == 'Q' || *ptr == 'S') { 1632 /* We've found either an XFP, quad ribbon fiber, or SFP+ */ 1633 cmd->base.port = PORT_FIBRE; 1634 ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE); 1635 ethtool_link_ksettings_add_link_mode(cmd, advertising, FIBRE); 1636 } else { 1637 cmd->base.port = PORT_OTHER; 1638 } 1639 1640 return 0; 1641 } 1642 1643 static void 1644 myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info) 1645 { 1646 struct myri10ge_priv *mgp = netdev_priv(netdev); 1647 1648 strlcpy(info->driver, "myri10ge", sizeof(info->driver)); 1649 strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version)); 1650 strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version)); 1651 strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info)); 1652 } 1653 1654 static int 1655 myri10ge_get_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal) 1656 { 1657 struct myri10ge_priv *mgp = netdev_priv(netdev); 1658 1659 coal->rx_coalesce_usecs = mgp->intr_coal_delay; 1660 return 0; 1661 } 1662 1663 static int 1664 myri10ge_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *coal) 1665 { 1666 struct myri10ge_priv *mgp = netdev_priv(netdev); 1667 1668 mgp->intr_coal_delay = coal->rx_coalesce_usecs; 1669 put_be32(htonl(mgp->intr_coal_delay), mgp->intr_coal_delay_ptr); 1670 return 0; 1671 } 1672 1673 static void 1674 myri10ge_get_pauseparam(struct net_device *netdev, 1675 struct ethtool_pauseparam *pause) 1676 { 1677 struct myri10ge_priv *mgp = netdev_priv(netdev); 1678 1679 pause->autoneg = 0; 1680 pause->rx_pause = mgp->pause; 1681 pause->tx_pause = mgp->pause; 1682 } 1683 1684 static int 1685 myri10ge_set_pauseparam(struct net_device *netdev, 1686 struct ethtool_pauseparam *pause) 1687 { 1688 struct myri10ge_priv *mgp = netdev_priv(netdev); 1689 1690 if (pause->tx_pause != mgp->pause) 1691 return myri10ge_change_pause(mgp, pause->tx_pause); 1692 if (pause->rx_pause != mgp->pause) 1693 return myri10ge_change_pause(mgp, pause->rx_pause); 1694 if (pause->autoneg != 0) 1695 return -EINVAL; 1696 return 0; 1697 } 1698 1699 static void 1700 myri10ge_get_ringparam(struct net_device *netdev, 1701 struct ethtool_ringparam *ring) 1702 { 1703 struct myri10ge_priv *mgp = netdev_priv(netdev); 1704 1705 ring->rx_mini_max_pending = mgp->ss[0].rx_small.mask + 1; 1706 ring->rx_max_pending = mgp->ss[0].rx_big.mask + 1; 1707 ring->rx_jumbo_max_pending = 0; 1708 ring->tx_max_pending = mgp->ss[0].tx.mask + 1; 1709 ring->rx_mini_pending = ring->rx_mini_max_pending; 1710 ring->rx_pending = ring->rx_max_pending; 1711 ring->rx_jumbo_pending = ring->rx_jumbo_max_pending; 1712 ring->tx_pending = ring->tx_max_pending; 1713 } 1714 1715 static const char myri10ge_gstrings_main_stats[][ETH_GSTRING_LEN] = { 1716 "rx_packets", "tx_packets", "rx_bytes", "tx_bytes", "rx_errors", 1717 "tx_errors", "rx_dropped", "tx_dropped", "multicast", "collisions", 1718 "rx_length_errors", "rx_over_errors", "rx_crc_errors", 1719 "rx_frame_errors", "rx_fifo_errors", "rx_missed_errors", 1720 "tx_aborted_errors", "tx_carrier_errors", "tx_fifo_errors", 1721 "tx_heartbeat_errors", "tx_window_errors", 1722 /* device-specific stats */ 1723 "tx_boundary", "irq", "MSI", "MSIX", 1724 "read_dma_bw_MBs", "write_dma_bw_MBs", "read_write_dma_bw_MBs", 1725 "serial_number", "watchdog_resets", 1726 #ifdef CONFIG_MYRI10GE_DCA 1727 "dca_capable_firmware", "dca_device_present", 1728 #endif 1729 "link_changes", "link_up", "dropped_link_overflow", 1730 "dropped_link_error_or_filtered", 1731 "dropped_pause", "dropped_bad_phy", "dropped_bad_crc32", 1732 "dropped_unicast_filtered", "dropped_multicast_filtered", 1733 "dropped_runt", "dropped_overrun", "dropped_no_small_buffer", 1734 "dropped_no_big_buffer" 1735 }; 1736 1737 static const char myri10ge_gstrings_slice_stats[][ETH_GSTRING_LEN] = { 1738 "----------- slice ---------", 1739 "tx_pkt_start", "tx_pkt_done", "tx_req", "tx_done", 1740 "rx_small_cnt", "rx_big_cnt", 1741 "wake_queue", "stop_queue", "tx_linearized", 1742 }; 1743 1744 #define MYRI10GE_NET_STATS_LEN 21 1745 #define MYRI10GE_MAIN_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_main_stats) 1746 #define MYRI10GE_SLICE_STATS_LEN ARRAY_SIZE(myri10ge_gstrings_slice_stats) 1747 1748 static void 1749 myri10ge_get_strings(struct net_device *netdev, u32 stringset, u8 * data) 1750 { 1751 struct myri10ge_priv *mgp = netdev_priv(netdev); 1752 int i; 1753 1754 switch (stringset) { 1755 case ETH_SS_STATS: 1756 memcpy(data, *myri10ge_gstrings_main_stats, 1757 sizeof(myri10ge_gstrings_main_stats)); 1758 data += sizeof(myri10ge_gstrings_main_stats); 1759 for (i = 0; i < mgp->num_slices; i++) { 1760 memcpy(data, *myri10ge_gstrings_slice_stats, 1761 sizeof(myri10ge_gstrings_slice_stats)); 1762 data += sizeof(myri10ge_gstrings_slice_stats); 1763 } 1764 break; 1765 } 1766 } 1767 1768 static int myri10ge_get_sset_count(struct net_device *netdev, int sset) 1769 { 1770 struct myri10ge_priv *mgp = netdev_priv(netdev); 1771 1772 switch (sset) { 1773 case ETH_SS_STATS: 1774 return MYRI10GE_MAIN_STATS_LEN + 1775 mgp->num_slices * MYRI10GE_SLICE_STATS_LEN; 1776 default: 1777 return -EOPNOTSUPP; 1778 } 1779 } 1780 1781 static void 1782 myri10ge_get_ethtool_stats(struct net_device *netdev, 1783 struct ethtool_stats *stats, u64 * data) 1784 { 1785 struct myri10ge_priv *mgp = netdev_priv(netdev); 1786 struct myri10ge_slice_state *ss; 1787 struct rtnl_link_stats64 link_stats; 1788 int slice; 1789 int i; 1790 1791 /* force stats update */ 1792 memset(&link_stats, 0, sizeof(link_stats)); 1793 (void)myri10ge_get_stats(netdev, &link_stats); 1794 for (i = 0; i < MYRI10GE_NET_STATS_LEN; i++) 1795 data[i] = ((u64 *)&link_stats)[i]; 1796 1797 data[i++] = (unsigned int)mgp->tx_boundary; 1798 data[i++] = (unsigned int)mgp->pdev->irq; 1799 data[i++] = (unsigned int)mgp->msi_enabled; 1800 data[i++] = (unsigned int)mgp->msix_enabled; 1801 data[i++] = (unsigned int)mgp->read_dma; 1802 data[i++] = (unsigned int)mgp->write_dma; 1803 data[i++] = (unsigned int)mgp->read_write_dma; 1804 data[i++] = (unsigned int)mgp->serial_number; 1805 data[i++] = (unsigned int)mgp->watchdog_resets; 1806 #ifdef CONFIG_MYRI10GE_DCA 1807 data[i++] = (unsigned int)(mgp->ss[0].dca_tag != NULL); 1808 data[i++] = (unsigned int)(mgp->dca_enabled); 1809 #endif 1810 data[i++] = (unsigned int)mgp->link_changes; 1811 1812 /* firmware stats are useful only in the first slice */ 1813 ss = &mgp->ss[0]; 1814 data[i++] = (unsigned int)ntohl(ss->fw_stats->link_up); 1815 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_link_overflow); 1816 data[i++] = 1817 (unsigned int)ntohl(ss->fw_stats->dropped_link_error_or_filtered); 1818 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_pause); 1819 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_phy); 1820 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_bad_crc32); 1821 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_unicast_filtered); 1822 data[i++] = 1823 (unsigned int)ntohl(ss->fw_stats->dropped_multicast_filtered); 1824 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_runt); 1825 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_overrun); 1826 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_small_buffer); 1827 data[i++] = (unsigned int)ntohl(ss->fw_stats->dropped_no_big_buffer); 1828 1829 for (slice = 0; slice < mgp->num_slices; slice++) { 1830 ss = &mgp->ss[slice]; 1831 data[i++] = slice; 1832 data[i++] = (unsigned int)ss->tx.pkt_start; 1833 data[i++] = (unsigned int)ss->tx.pkt_done; 1834 data[i++] = (unsigned int)ss->tx.req; 1835 data[i++] = (unsigned int)ss->tx.done; 1836 data[i++] = (unsigned int)ss->rx_small.cnt; 1837 data[i++] = (unsigned int)ss->rx_big.cnt; 1838 data[i++] = (unsigned int)ss->tx.wake_queue; 1839 data[i++] = (unsigned int)ss->tx.stop_queue; 1840 data[i++] = (unsigned int)ss->tx.linearized; 1841 } 1842 } 1843 1844 static void myri10ge_set_msglevel(struct net_device *netdev, u32 value) 1845 { 1846 struct myri10ge_priv *mgp = netdev_priv(netdev); 1847 mgp->msg_enable = value; 1848 } 1849 1850 static u32 myri10ge_get_msglevel(struct net_device *netdev) 1851 { 1852 struct myri10ge_priv *mgp = netdev_priv(netdev); 1853 return mgp->msg_enable; 1854 } 1855 1856 /* 1857 * Use a low-level command to change the LED behavior. Rather than 1858 * blinking (which is the normal case), when identify is used, the 1859 * yellow LED turns solid. 1860 */ 1861 static int myri10ge_led(struct myri10ge_priv *mgp, int on) 1862 { 1863 struct mcp_gen_header *hdr; 1864 struct device *dev = &mgp->pdev->dev; 1865 size_t hdr_off, pattern_off, hdr_len; 1866 u32 pattern = 0xfffffffe; 1867 1868 /* find running firmware header */ 1869 hdr_off = swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)); 1870 if ((hdr_off & 3) || hdr_off + sizeof(*hdr) > mgp->sram_size) { 1871 dev_err(dev, "Running firmware has bad header offset (%d)\n", 1872 (int)hdr_off); 1873 return -EIO; 1874 } 1875 hdr_len = swab32(readl(mgp->sram + hdr_off + 1876 offsetof(struct mcp_gen_header, header_length))); 1877 pattern_off = hdr_off + offsetof(struct mcp_gen_header, led_pattern); 1878 if (pattern_off >= (hdr_len + hdr_off)) { 1879 dev_info(dev, "Firmware does not support LED identification\n"); 1880 return -EINVAL; 1881 } 1882 if (!on) 1883 pattern = swab32(readl(mgp->sram + pattern_off + 4)); 1884 writel(swab32(pattern), mgp->sram + pattern_off); 1885 return 0; 1886 } 1887 1888 static int 1889 myri10ge_phys_id(struct net_device *netdev, enum ethtool_phys_id_state state) 1890 { 1891 struct myri10ge_priv *mgp = netdev_priv(netdev); 1892 int rc; 1893 1894 switch (state) { 1895 case ETHTOOL_ID_ACTIVE: 1896 rc = myri10ge_led(mgp, 1); 1897 break; 1898 1899 case ETHTOOL_ID_INACTIVE: 1900 rc = myri10ge_led(mgp, 0); 1901 break; 1902 1903 default: 1904 rc = -EINVAL; 1905 } 1906 1907 return rc; 1908 } 1909 1910 static const struct ethtool_ops myri10ge_ethtool_ops = { 1911 .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS, 1912 .get_drvinfo = myri10ge_get_drvinfo, 1913 .get_coalesce = myri10ge_get_coalesce, 1914 .set_coalesce = myri10ge_set_coalesce, 1915 .get_pauseparam = myri10ge_get_pauseparam, 1916 .set_pauseparam = myri10ge_set_pauseparam, 1917 .get_ringparam = myri10ge_get_ringparam, 1918 .get_link = ethtool_op_get_link, 1919 .get_strings = myri10ge_get_strings, 1920 .get_sset_count = myri10ge_get_sset_count, 1921 .get_ethtool_stats = myri10ge_get_ethtool_stats, 1922 .set_msglevel = myri10ge_set_msglevel, 1923 .get_msglevel = myri10ge_get_msglevel, 1924 .set_phys_id = myri10ge_phys_id, 1925 .get_link_ksettings = myri10ge_get_link_ksettings, 1926 }; 1927 1928 static int myri10ge_allocate_rings(struct myri10ge_slice_state *ss) 1929 { 1930 struct myri10ge_priv *mgp = ss->mgp; 1931 struct myri10ge_cmd cmd; 1932 struct net_device *dev = mgp->dev; 1933 int tx_ring_size, rx_ring_size; 1934 int tx_ring_entries, rx_ring_entries; 1935 int i, slice, status; 1936 size_t bytes; 1937 1938 /* get ring sizes */ 1939 slice = ss - mgp->ss; 1940 cmd.data0 = slice; 1941 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_RING_SIZE, &cmd, 0); 1942 tx_ring_size = cmd.data0; 1943 cmd.data0 = slice; 1944 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_RX_RING_SIZE, &cmd, 0); 1945 if (status != 0) 1946 return status; 1947 rx_ring_size = cmd.data0; 1948 1949 tx_ring_entries = tx_ring_size / sizeof(struct mcp_kreq_ether_send); 1950 rx_ring_entries = rx_ring_size / sizeof(struct mcp_dma_addr); 1951 ss->tx.mask = tx_ring_entries - 1; 1952 ss->rx_small.mask = ss->rx_big.mask = rx_ring_entries - 1; 1953 1954 status = -ENOMEM; 1955 1956 /* allocate the host shadow rings */ 1957 1958 bytes = 8 + (MYRI10GE_MAX_SEND_DESC_TSO + 4) 1959 * sizeof(*ss->tx.req_list); 1960 ss->tx.req_bytes = kzalloc(bytes, GFP_KERNEL); 1961 if (ss->tx.req_bytes == NULL) 1962 goto abort_with_nothing; 1963 1964 /* ensure req_list entries are aligned to 8 bytes */ 1965 ss->tx.req_list = (struct mcp_kreq_ether_send *) 1966 ALIGN((unsigned long)ss->tx.req_bytes, 8); 1967 ss->tx.queue_active = 0; 1968 1969 bytes = rx_ring_entries * sizeof(*ss->rx_small.shadow); 1970 ss->rx_small.shadow = kzalloc(bytes, GFP_KERNEL); 1971 if (ss->rx_small.shadow == NULL) 1972 goto abort_with_tx_req_bytes; 1973 1974 bytes = rx_ring_entries * sizeof(*ss->rx_big.shadow); 1975 ss->rx_big.shadow = kzalloc(bytes, GFP_KERNEL); 1976 if (ss->rx_big.shadow == NULL) 1977 goto abort_with_rx_small_shadow; 1978 1979 /* allocate the host info rings */ 1980 1981 bytes = tx_ring_entries * sizeof(*ss->tx.info); 1982 ss->tx.info = kzalloc(bytes, GFP_KERNEL); 1983 if (ss->tx.info == NULL) 1984 goto abort_with_rx_big_shadow; 1985 1986 bytes = rx_ring_entries * sizeof(*ss->rx_small.info); 1987 ss->rx_small.info = kzalloc(bytes, GFP_KERNEL); 1988 if (ss->rx_small.info == NULL) 1989 goto abort_with_tx_info; 1990 1991 bytes = rx_ring_entries * sizeof(*ss->rx_big.info); 1992 ss->rx_big.info = kzalloc(bytes, GFP_KERNEL); 1993 if (ss->rx_big.info == NULL) 1994 goto abort_with_rx_small_info; 1995 1996 /* Fill the receive rings */ 1997 ss->rx_big.cnt = 0; 1998 ss->rx_small.cnt = 0; 1999 ss->rx_big.fill_cnt = 0; 2000 ss->rx_small.fill_cnt = 0; 2001 ss->rx_small.page_offset = MYRI10GE_ALLOC_SIZE; 2002 ss->rx_big.page_offset = MYRI10GE_ALLOC_SIZE; 2003 ss->rx_small.watchdog_needed = 0; 2004 ss->rx_big.watchdog_needed = 0; 2005 if (mgp->small_bytes == 0) { 2006 ss->rx_small.fill_cnt = ss->rx_small.mask + 1; 2007 } else { 2008 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 2009 mgp->small_bytes + MXGEFW_PAD, 0); 2010 } 2011 2012 if (ss->rx_small.fill_cnt < ss->rx_small.mask + 1) { 2013 netdev_err(dev, "slice-%d: alloced only %d small bufs\n", 2014 slice, ss->rx_small.fill_cnt); 2015 goto abort_with_rx_small_ring; 2016 } 2017 2018 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, mgp->big_bytes, 0); 2019 if (ss->rx_big.fill_cnt < ss->rx_big.mask + 1) { 2020 netdev_err(dev, "slice-%d: alloced only %d big bufs\n", 2021 slice, ss->rx_big.fill_cnt); 2022 goto abort_with_rx_big_ring; 2023 } 2024 2025 return 0; 2026 2027 abort_with_rx_big_ring: 2028 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2029 int idx = i & ss->rx_big.mask; 2030 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2031 mgp->big_bytes); 2032 put_page(ss->rx_big.info[idx].page); 2033 } 2034 2035 abort_with_rx_small_ring: 2036 if (mgp->small_bytes == 0) 2037 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2038 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2039 int idx = i & ss->rx_small.mask; 2040 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2041 mgp->small_bytes + MXGEFW_PAD); 2042 put_page(ss->rx_small.info[idx].page); 2043 } 2044 2045 kfree(ss->rx_big.info); 2046 2047 abort_with_rx_small_info: 2048 kfree(ss->rx_small.info); 2049 2050 abort_with_tx_info: 2051 kfree(ss->tx.info); 2052 2053 abort_with_rx_big_shadow: 2054 kfree(ss->rx_big.shadow); 2055 2056 abort_with_rx_small_shadow: 2057 kfree(ss->rx_small.shadow); 2058 2059 abort_with_tx_req_bytes: 2060 kfree(ss->tx.req_bytes); 2061 ss->tx.req_bytes = NULL; 2062 ss->tx.req_list = NULL; 2063 2064 abort_with_nothing: 2065 return status; 2066 } 2067 2068 static void myri10ge_free_rings(struct myri10ge_slice_state *ss) 2069 { 2070 struct myri10ge_priv *mgp = ss->mgp; 2071 struct sk_buff *skb; 2072 struct myri10ge_tx_buf *tx; 2073 int i, len, idx; 2074 2075 /* If not allocated, skip it */ 2076 if (ss->tx.req_list == NULL) 2077 return; 2078 2079 for (i = ss->rx_big.cnt; i < ss->rx_big.fill_cnt; i++) { 2080 idx = i & ss->rx_big.mask; 2081 if (i == ss->rx_big.fill_cnt - 1) 2082 ss->rx_big.info[idx].page_offset = MYRI10GE_ALLOC_SIZE; 2083 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_big.info[idx], 2084 mgp->big_bytes); 2085 put_page(ss->rx_big.info[idx].page); 2086 } 2087 2088 if (mgp->small_bytes == 0) 2089 ss->rx_small.fill_cnt = ss->rx_small.cnt; 2090 for (i = ss->rx_small.cnt; i < ss->rx_small.fill_cnt; i++) { 2091 idx = i & ss->rx_small.mask; 2092 if (i == ss->rx_small.fill_cnt - 1) 2093 ss->rx_small.info[idx].page_offset = 2094 MYRI10GE_ALLOC_SIZE; 2095 myri10ge_unmap_rx_page(mgp->pdev, &ss->rx_small.info[idx], 2096 mgp->small_bytes + MXGEFW_PAD); 2097 put_page(ss->rx_small.info[idx].page); 2098 } 2099 tx = &ss->tx; 2100 while (tx->done != tx->req) { 2101 idx = tx->done & tx->mask; 2102 skb = tx->info[idx].skb; 2103 2104 /* Mark as free */ 2105 tx->info[idx].skb = NULL; 2106 tx->done++; 2107 len = dma_unmap_len(&tx->info[idx], len); 2108 dma_unmap_len_set(&tx->info[idx], len, 0); 2109 if (skb) { 2110 ss->stats.tx_dropped++; 2111 dev_kfree_skb_any(skb); 2112 if (len) 2113 pci_unmap_single(mgp->pdev, 2114 dma_unmap_addr(&tx->info[idx], 2115 bus), len, 2116 PCI_DMA_TODEVICE); 2117 } else { 2118 if (len) 2119 pci_unmap_page(mgp->pdev, 2120 dma_unmap_addr(&tx->info[idx], 2121 bus), len, 2122 PCI_DMA_TODEVICE); 2123 } 2124 } 2125 kfree(ss->rx_big.info); 2126 2127 kfree(ss->rx_small.info); 2128 2129 kfree(ss->tx.info); 2130 2131 kfree(ss->rx_big.shadow); 2132 2133 kfree(ss->rx_small.shadow); 2134 2135 kfree(ss->tx.req_bytes); 2136 ss->tx.req_bytes = NULL; 2137 ss->tx.req_list = NULL; 2138 } 2139 2140 static int myri10ge_request_irq(struct myri10ge_priv *mgp) 2141 { 2142 struct pci_dev *pdev = mgp->pdev; 2143 struct myri10ge_slice_state *ss; 2144 struct net_device *netdev = mgp->dev; 2145 int i; 2146 int status; 2147 2148 mgp->msi_enabled = 0; 2149 mgp->msix_enabled = 0; 2150 status = 0; 2151 if (myri10ge_msi) { 2152 if (mgp->num_slices > 1) { 2153 status = pci_enable_msix_range(pdev, mgp->msix_vectors, 2154 mgp->num_slices, mgp->num_slices); 2155 if (status < 0) { 2156 dev_err(&pdev->dev, 2157 "Error %d setting up MSI-X\n", status); 2158 return status; 2159 } 2160 mgp->msix_enabled = 1; 2161 } 2162 if (mgp->msix_enabled == 0) { 2163 status = pci_enable_msi(pdev); 2164 if (status != 0) { 2165 dev_err(&pdev->dev, 2166 "Error %d setting up MSI; falling back to xPIC\n", 2167 status); 2168 } else { 2169 mgp->msi_enabled = 1; 2170 } 2171 } 2172 } 2173 if (mgp->msix_enabled) { 2174 for (i = 0; i < mgp->num_slices; i++) { 2175 ss = &mgp->ss[i]; 2176 snprintf(ss->irq_desc, sizeof(ss->irq_desc), 2177 "%s:slice-%d", netdev->name, i); 2178 status = request_irq(mgp->msix_vectors[i].vector, 2179 myri10ge_intr, 0, ss->irq_desc, 2180 ss); 2181 if (status != 0) { 2182 dev_err(&pdev->dev, 2183 "slice %d failed to allocate IRQ\n", i); 2184 i--; 2185 while (i >= 0) { 2186 free_irq(mgp->msix_vectors[i].vector, 2187 &mgp->ss[i]); 2188 i--; 2189 } 2190 pci_disable_msix(pdev); 2191 return status; 2192 } 2193 } 2194 } else { 2195 status = request_irq(pdev->irq, myri10ge_intr, IRQF_SHARED, 2196 mgp->dev->name, &mgp->ss[0]); 2197 if (status != 0) { 2198 dev_err(&pdev->dev, "failed to allocate IRQ\n"); 2199 if (mgp->msi_enabled) 2200 pci_disable_msi(pdev); 2201 } 2202 } 2203 return status; 2204 } 2205 2206 static void myri10ge_free_irq(struct myri10ge_priv *mgp) 2207 { 2208 struct pci_dev *pdev = mgp->pdev; 2209 int i; 2210 2211 if (mgp->msix_enabled) { 2212 for (i = 0; i < mgp->num_slices; i++) 2213 free_irq(mgp->msix_vectors[i].vector, &mgp->ss[i]); 2214 } else { 2215 free_irq(pdev->irq, &mgp->ss[0]); 2216 } 2217 if (mgp->msi_enabled) 2218 pci_disable_msi(pdev); 2219 if (mgp->msix_enabled) 2220 pci_disable_msix(pdev); 2221 } 2222 2223 static int myri10ge_get_txrx(struct myri10ge_priv *mgp, int slice) 2224 { 2225 struct myri10ge_cmd cmd; 2226 struct myri10ge_slice_state *ss; 2227 int status; 2228 2229 ss = &mgp->ss[slice]; 2230 status = 0; 2231 if (slice == 0 || (mgp->dev->real_num_tx_queues > 1)) { 2232 cmd.data0 = slice; 2233 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SEND_OFFSET, 2234 &cmd, 0); 2235 ss->tx.lanai = (struct mcp_kreq_ether_send __iomem *) 2236 (mgp->sram + cmd.data0); 2237 } 2238 cmd.data0 = slice; 2239 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_SMALL_RX_OFFSET, 2240 &cmd, 0); 2241 ss->rx_small.lanai = (struct mcp_kreq_ether_recv __iomem *) 2242 (mgp->sram + cmd.data0); 2243 2244 cmd.data0 = slice; 2245 status |= myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_BIG_RX_OFFSET, &cmd, 0); 2246 ss->rx_big.lanai = (struct mcp_kreq_ether_recv __iomem *) 2247 (mgp->sram + cmd.data0); 2248 2249 ss->tx.send_go = (__iomem __be32 *) 2250 (mgp->sram + MXGEFW_ETH_SEND_GO + 64 * slice); 2251 ss->tx.send_stop = (__iomem __be32 *) 2252 (mgp->sram + MXGEFW_ETH_SEND_STOP + 64 * slice); 2253 return status; 2254 2255 } 2256 2257 static int myri10ge_set_stats(struct myri10ge_priv *mgp, int slice) 2258 { 2259 struct myri10ge_cmd cmd; 2260 struct myri10ge_slice_state *ss; 2261 int status; 2262 2263 ss = &mgp->ss[slice]; 2264 cmd.data0 = MYRI10GE_LOWPART_TO_U32(ss->fw_stats_bus); 2265 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(ss->fw_stats_bus); 2266 cmd.data2 = sizeof(struct mcp_irq_data) | (slice << 16); 2267 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_STATS_DMA_V2, &cmd, 0); 2268 if (status == -ENOSYS) { 2269 dma_addr_t bus = ss->fw_stats_bus; 2270 if (slice != 0) 2271 return -EINVAL; 2272 bus += offsetof(struct mcp_irq_data, send_done_count); 2273 cmd.data0 = MYRI10GE_LOWPART_TO_U32(bus); 2274 cmd.data1 = MYRI10GE_HIGHPART_TO_U32(bus); 2275 status = myri10ge_send_cmd(mgp, 2276 MXGEFW_CMD_SET_STATS_DMA_OBSOLETE, 2277 &cmd, 0); 2278 /* Firmware cannot support multicast without STATS_DMA_V2 */ 2279 mgp->fw_multicast_support = 0; 2280 } else { 2281 mgp->fw_multicast_support = 1; 2282 } 2283 return 0; 2284 } 2285 2286 static int myri10ge_open(struct net_device *dev) 2287 { 2288 struct myri10ge_slice_state *ss; 2289 struct myri10ge_priv *mgp = netdev_priv(dev); 2290 struct myri10ge_cmd cmd; 2291 int i, status, big_pow2, slice; 2292 u8 __iomem *itable; 2293 2294 if (mgp->running != MYRI10GE_ETH_STOPPED) 2295 return -EBUSY; 2296 2297 mgp->running = MYRI10GE_ETH_STARTING; 2298 status = myri10ge_reset(mgp); 2299 if (status != 0) { 2300 netdev_err(dev, "failed reset\n"); 2301 goto abort_with_nothing; 2302 } 2303 2304 if (mgp->num_slices > 1) { 2305 cmd.data0 = mgp->num_slices; 2306 cmd.data1 = MXGEFW_SLICE_INTR_MODE_ONE_PER_SLICE; 2307 if (mgp->dev->real_num_tx_queues > 1) 2308 cmd.data1 |= MXGEFW_SLICE_ENABLE_MULTIPLE_TX_QUEUES; 2309 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ENABLE_RSS_QUEUES, 2310 &cmd, 0); 2311 if (status != 0) { 2312 netdev_err(dev, "failed to set number of slices\n"); 2313 goto abort_with_nothing; 2314 } 2315 /* setup the indirection table */ 2316 cmd.data0 = mgp->num_slices; 2317 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_TABLE_SIZE, 2318 &cmd, 0); 2319 2320 status |= myri10ge_send_cmd(mgp, 2321 MXGEFW_CMD_GET_RSS_TABLE_OFFSET, 2322 &cmd, 0); 2323 if (status != 0) { 2324 netdev_err(dev, "failed to setup rss tables\n"); 2325 goto abort_with_nothing; 2326 } 2327 2328 /* just enable an identity mapping */ 2329 itable = mgp->sram + cmd.data0; 2330 for (i = 0; i < mgp->num_slices; i++) 2331 __raw_writeb(i, &itable[i]); 2332 2333 cmd.data0 = 1; 2334 cmd.data1 = myri10ge_rss_hash; 2335 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_RSS_ENABLE, 2336 &cmd, 0); 2337 if (status != 0) { 2338 netdev_err(dev, "failed to enable slices\n"); 2339 goto abort_with_nothing; 2340 } 2341 } 2342 2343 status = myri10ge_request_irq(mgp); 2344 if (status != 0) 2345 goto abort_with_nothing; 2346 2347 /* decide what small buffer size to use. For good TCP rx 2348 * performance, it is important to not receive 1514 byte 2349 * frames into jumbo buffers, as it confuses the socket buffer 2350 * accounting code, leading to drops and erratic performance. 2351 */ 2352 2353 if (dev->mtu <= ETH_DATA_LEN) 2354 /* enough for a TCP header */ 2355 mgp->small_bytes = (128 > SMP_CACHE_BYTES) 2356 ? (128 - MXGEFW_PAD) 2357 : (SMP_CACHE_BYTES - MXGEFW_PAD); 2358 else 2359 /* enough for a vlan encapsulated ETH_DATA_LEN frame */ 2360 mgp->small_bytes = VLAN_ETH_FRAME_LEN; 2361 2362 /* Override the small buffer size? */ 2363 if (myri10ge_small_bytes >= 0) 2364 mgp->small_bytes = myri10ge_small_bytes; 2365 2366 /* Firmware needs the big buff size as a power of 2. Lie and 2367 * tell him the buffer is larger, because we only use 1 2368 * buffer/pkt, and the mtu will prevent overruns. 2369 */ 2370 big_pow2 = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2371 if (big_pow2 < MYRI10GE_ALLOC_SIZE / 2) { 2372 while (!is_power_of_2(big_pow2)) 2373 big_pow2++; 2374 mgp->big_bytes = dev->mtu + ETH_HLEN + VLAN_HLEN + MXGEFW_PAD; 2375 } else { 2376 big_pow2 = MYRI10GE_ALLOC_SIZE; 2377 mgp->big_bytes = big_pow2; 2378 } 2379 2380 /* setup the per-slice data structures */ 2381 for (slice = 0; slice < mgp->num_slices; slice++) { 2382 ss = &mgp->ss[slice]; 2383 2384 status = myri10ge_get_txrx(mgp, slice); 2385 if (status != 0) { 2386 netdev_err(dev, "failed to get ring sizes or locations\n"); 2387 goto abort_with_rings; 2388 } 2389 status = myri10ge_allocate_rings(ss); 2390 if (status != 0) 2391 goto abort_with_rings; 2392 2393 /* only firmware which supports multiple TX queues 2394 * supports setting up the tx stats on non-zero 2395 * slices */ 2396 if (slice == 0 || mgp->dev->real_num_tx_queues > 1) 2397 status = myri10ge_set_stats(mgp, slice); 2398 if (status) { 2399 netdev_err(dev, "Couldn't set stats DMA\n"); 2400 goto abort_with_rings; 2401 } 2402 2403 /* must happen prior to any irq */ 2404 napi_enable(&(ss)->napi); 2405 } 2406 2407 /* now give firmware buffers sizes, and MTU */ 2408 cmd.data0 = dev->mtu + ETH_HLEN + VLAN_HLEN; 2409 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_MTU, &cmd, 0); 2410 cmd.data0 = mgp->small_bytes; 2411 status |= 2412 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_SMALL_BUFFER_SIZE, &cmd, 0); 2413 cmd.data0 = big_pow2; 2414 status |= 2415 myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_BIG_BUFFER_SIZE, &cmd, 0); 2416 if (status) { 2417 netdev_err(dev, "Couldn't set buffer sizes\n"); 2418 goto abort_with_rings; 2419 } 2420 2421 /* 2422 * Set Linux style TSO mode; this is needed only on newer 2423 * firmware versions. Older versions default to Linux 2424 * style TSO 2425 */ 2426 cmd.data0 = 0; 2427 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_TSO_MODE, &cmd, 0); 2428 if (status && status != -ENOSYS) { 2429 netdev_err(dev, "Couldn't set TSO mode\n"); 2430 goto abort_with_rings; 2431 } 2432 2433 mgp->link_state = ~0U; 2434 mgp->rdma_tags_available = 15; 2435 2436 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_UP, &cmd, 0); 2437 if (status) { 2438 netdev_err(dev, "Couldn't bring up link\n"); 2439 goto abort_with_rings; 2440 } 2441 2442 mgp->running = MYRI10GE_ETH_RUNNING; 2443 mgp->watchdog_timer.expires = jiffies + myri10ge_watchdog_timeout * HZ; 2444 add_timer(&mgp->watchdog_timer); 2445 netif_tx_wake_all_queues(dev); 2446 2447 return 0; 2448 2449 abort_with_rings: 2450 while (slice) { 2451 slice--; 2452 napi_disable(&mgp->ss[slice].napi); 2453 } 2454 for (i = 0; i < mgp->num_slices; i++) 2455 myri10ge_free_rings(&mgp->ss[i]); 2456 2457 myri10ge_free_irq(mgp); 2458 2459 abort_with_nothing: 2460 mgp->running = MYRI10GE_ETH_STOPPED; 2461 return -ENOMEM; 2462 } 2463 2464 static int myri10ge_close(struct net_device *dev) 2465 { 2466 struct myri10ge_priv *mgp = netdev_priv(dev); 2467 struct myri10ge_cmd cmd; 2468 int status, old_down_cnt; 2469 int i; 2470 2471 if (mgp->running != MYRI10GE_ETH_RUNNING) 2472 return 0; 2473 2474 if (mgp->ss[0].tx.req_bytes == NULL) 2475 return 0; 2476 2477 del_timer_sync(&mgp->watchdog_timer); 2478 mgp->running = MYRI10GE_ETH_STOPPING; 2479 for (i = 0; i < mgp->num_slices; i++) 2480 napi_disable(&mgp->ss[i].napi); 2481 2482 netif_carrier_off(dev); 2483 2484 netif_tx_stop_all_queues(dev); 2485 if (mgp->rebooted == 0) { 2486 old_down_cnt = mgp->down_cnt; 2487 mb(); 2488 status = 2489 myri10ge_send_cmd(mgp, MXGEFW_CMD_ETHERNET_DOWN, &cmd, 0); 2490 if (status) 2491 netdev_err(dev, "Couldn't bring down link\n"); 2492 2493 wait_event_timeout(mgp->down_wq, old_down_cnt != mgp->down_cnt, 2494 HZ); 2495 if (old_down_cnt == mgp->down_cnt) 2496 netdev_err(dev, "never got down irq\n"); 2497 } 2498 netif_tx_disable(dev); 2499 myri10ge_free_irq(mgp); 2500 for (i = 0; i < mgp->num_slices; i++) 2501 myri10ge_free_rings(&mgp->ss[i]); 2502 2503 mgp->running = MYRI10GE_ETH_STOPPED; 2504 return 0; 2505 } 2506 2507 /* copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2508 * backwards one at a time and handle ring wraps */ 2509 2510 static inline void 2511 myri10ge_submit_req_backwards(struct myri10ge_tx_buf *tx, 2512 struct mcp_kreq_ether_send *src, int cnt) 2513 { 2514 int idx, starting_slot; 2515 starting_slot = tx->req; 2516 while (cnt > 1) { 2517 cnt--; 2518 idx = (starting_slot + cnt) & tx->mask; 2519 myri10ge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); 2520 mb(); 2521 } 2522 } 2523 2524 /* 2525 * copy an array of struct mcp_kreq_ether_send's to the mcp. Copy 2526 * at most 32 bytes at a time, so as to avoid involving the software 2527 * pio handler in the nic. We re-write the first segment's flags 2528 * to mark them valid only after writing the entire chain. 2529 */ 2530 2531 static inline void 2532 myri10ge_submit_req(struct myri10ge_tx_buf *tx, struct mcp_kreq_ether_send *src, 2533 int cnt) 2534 { 2535 int idx, i; 2536 struct mcp_kreq_ether_send __iomem *dstp, *dst; 2537 struct mcp_kreq_ether_send *srcp; 2538 u8 last_flags; 2539 2540 idx = tx->req & tx->mask; 2541 2542 last_flags = src->flags; 2543 src->flags = 0; 2544 mb(); 2545 dst = dstp = &tx->lanai[idx]; 2546 srcp = src; 2547 2548 if ((idx + cnt) < tx->mask) { 2549 for (i = 0; i < (cnt - 1); i += 2) { 2550 myri10ge_pio_copy(dstp, srcp, 2 * sizeof(*src)); 2551 mb(); /* force write every 32 bytes */ 2552 srcp += 2; 2553 dstp += 2; 2554 } 2555 } else { 2556 /* submit all but the first request, and ensure 2557 * that it is submitted below */ 2558 myri10ge_submit_req_backwards(tx, src, cnt); 2559 i = 0; 2560 } 2561 if (i < cnt) { 2562 /* submit the first request */ 2563 myri10ge_pio_copy(dstp, srcp, sizeof(*src)); 2564 mb(); /* barrier before setting valid flag */ 2565 } 2566 2567 /* re-write the last 32-bits with the valid flags */ 2568 src->flags = last_flags; 2569 put_be32(*((__be32 *) src + 3), (__be32 __iomem *) dst + 3); 2570 tx->req += cnt; 2571 mb(); 2572 } 2573 2574 static void myri10ge_unmap_tx_dma(struct myri10ge_priv *mgp, 2575 struct myri10ge_tx_buf *tx, int idx) 2576 { 2577 unsigned int len; 2578 int last_idx; 2579 2580 /* Free any DMA resources we've alloced and clear out the skb slot */ 2581 last_idx = (idx + 1) & tx->mask; 2582 idx = tx->req & tx->mask; 2583 do { 2584 len = dma_unmap_len(&tx->info[idx], len); 2585 if (len) { 2586 if (tx->info[idx].skb != NULL) 2587 pci_unmap_single(mgp->pdev, 2588 dma_unmap_addr(&tx->info[idx], 2589 bus), len, 2590 PCI_DMA_TODEVICE); 2591 else 2592 pci_unmap_page(mgp->pdev, 2593 dma_unmap_addr(&tx->info[idx], 2594 bus), len, 2595 PCI_DMA_TODEVICE); 2596 dma_unmap_len_set(&tx->info[idx], len, 0); 2597 tx->info[idx].skb = NULL; 2598 } 2599 idx = (idx + 1) & tx->mask; 2600 } while (idx != last_idx); 2601 } 2602 2603 /* 2604 * Transmit a packet. We need to split the packet so that a single 2605 * segment does not cross myri10ge->tx_boundary, so this makes segment 2606 * counting tricky. So rather than try to count segments up front, we 2607 * just give up if there are too few segments to hold a reasonably 2608 * fragmented packet currently available. If we run 2609 * out of segments while preparing a packet for DMA, we just linearize 2610 * it and try again. 2611 */ 2612 2613 static netdev_tx_t myri10ge_xmit(struct sk_buff *skb, 2614 struct net_device *dev) 2615 { 2616 struct myri10ge_priv *mgp = netdev_priv(dev); 2617 struct myri10ge_slice_state *ss; 2618 struct mcp_kreq_ether_send *req; 2619 struct myri10ge_tx_buf *tx; 2620 skb_frag_t *frag; 2621 struct netdev_queue *netdev_queue; 2622 dma_addr_t bus; 2623 u32 low; 2624 __be32 high_swapped; 2625 unsigned int len; 2626 int idx, avail, frag_cnt, frag_idx, count, mss, max_segments; 2627 u16 pseudo_hdr_offset, cksum_offset, queue; 2628 int cum_len, seglen, boundary, rdma_count; 2629 u8 flags, odd_flag; 2630 2631 queue = skb_get_queue_mapping(skb); 2632 ss = &mgp->ss[queue]; 2633 netdev_queue = netdev_get_tx_queue(mgp->dev, queue); 2634 tx = &ss->tx; 2635 2636 again: 2637 req = tx->req_list; 2638 avail = tx->mask - 1 - (tx->req - tx->done); 2639 2640 mss = 0; 2641 max_segments = MXGEFW_MAX_SEND_DESC; 2642 2643 if (skb_is_gso(skb)) { 2644 mss = skb_shinfo(skb)->gso_size; 2645 max_segments = MYRI10GE_MAX_SEND_DESC_TSO; 2646 } 2647 2648 if ((unlikely(avail < max_segments))) { 2649 /* we are out of transmit resources */ 2650 tx->stop_queue++; 2651 netif_tx_stop_queue(netdev_queue); 2652 return NETDEV_TX_BUSY; 2653 } 2654 2655 /* Setup checksum offloading, if needed */ 2656 cksum_offset = 0; 2657 pseudo_hdr_offset = 0; 2658 odd_flag = 0; 2659 flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST); 2660 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 2661 cksum_offset = skb_checksum_start_offset(skb); 2662 pseudo_hdr_offset = cksum_offset + skb->csum_offset; 2663 /* If the headers are excessively large, then we must 2664 * fall back to a software checksum */ 2665 if (unlikely(!mss && (cksum_offset > 255 || 2666 pseudo_hdr_offset > 127))) { 2667 if (skb_checksum_help(skb)) 2668 goto drop; 2669 cksum_offset = 0; 2670 pseudo_hdr_offset = 0; 2671 } else { 2672 odd_flag = MXGEFW_FLAGS_ALIGN_ODD; 2673 flags |= MXGEFW_FLAGS_CKSUM; 2674 } 2675 } 2676 2677 cum_len = 0; 2678 2679 if (mss) { /* TSO */ 2680 /* this removes any CKSUM flag from before */ 2681 flags = (MXGEFW_FLAGS_TSO_HDR | MXGEFW_FLAGS_FIRST); 2682 2683 /* negative cum_len signifies to the 2684 * send loop that we are still in the 2685 * header portion of the TSO packet. 2686 * TSO header can be at most 1KB long */ 2687 cum_len = -(skb_transport_offset(skb) + tcp_hdrlen(skb)); 2688 2689 /* for IPv6 TSO, the checksum offset stores the 2690 * TCP header length, to save the firmware from 2691 * the need to parse the headers */ 2692 if (skb_is_gso_v6(skb)) { 2693 cksum_offset = tcp_hdrlen(skb); 2694 /* Can only handle headers <= max_tso6 long */ 2695 if (unlikely(-cum_len > mgp->max_tso6)) 2696 return myri10ge_sw_tso(skb, dev); 2697 } 2698 /* for TSO, pseudo_hdr_offset holds mss. 2699 * The firmware figures out where to put 2700 * the checksum by parsing the header. */ 2701 pseudo_hdr_offset = mss; 2702 } else 2703 /* Mark small packets, and pad out tiny packets */ 2704 if (skb->len <= MXGEFW_SEND_SMALL_SIZE) { 2705 flags |= MXGEFW_FLAGS_SMALL; 2706 2707 /* pad frames to at least ETH_ZLEN bytes */ 2708 if (eth_skb_pad(skb)) { 2709 /* The packet is gone, so we must 2710 * return 0 */ 2711 ss->stats.tx_dropped += 1; 2712 return NETDEV_TX_OK; 2713 } 2714 } 2715 2716 /* map the skb for DMA */ 2717 len = skb_headlen(skb); 2718 bus = pci_map_single(mgp->pdev, skb->data, len, PCI_DMA_TODEVICE); 2719 if (unlikely(pci_dma_mapping_error(mgp->pdev, bus))) 2720 goto drop; 2721 2722 idx = tx->req & tx->mask; 2723 tx->info[idx].skb = skb; 2724 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2725 dma_unmap_len_set(&tx->info[idx], len, len); 2726 2727 frag_cnt = skb_shinfo(skb)->nr_frags; 2728 frag_idx = 0; 2729 count = 0; 2730 rdma_count = 0; 2731 2732 /* "rdma_count" is the number of RDMAs belonging to the 2733 * current packet BEFORE the current send request. For 2734 * non-TSO packets, this is equal to "count". 2735 * For TSO packets, rdma_count needs to be reset 2736 * to 0 after a segment cut. 2737 * 2738 * The rdma_count field of the send request is 2739 * the number of RDMAs of the packet starting at 2740 * that request. For TSO send requests with one ore more cuts 2741 * in the middle, this is the number of RDMAs starting 2742 * after the last cut in the request. All previous 2743 * segments before the last cut implicitly have 1 RDMA. 2744 * 2745 * Since the number of RDMAs is not known beforehand, 2746 * it must be filled-in retroactively - after each 2747 * segmentation cut or at the end of the entire packet. 2748 */ 2749 2750 while (1) { 2751 /* Break the SKB or Fragment up into pieces which 2752 * do not cross mgp->tx_boundary */ 2753 low = MYRI10GE_LOWPART_TO_U32(bus); 2754 high_swapped = htonl(MYRI10GE_HIGHPART_TO_U32(bus)); 2755 while (len) { 2756 u8 flags_next; 2757 int cum_len_next; 2758 2759 if (unlikely(count == max_segments)) 2760 goto abort_linearize; 2761 2762 boundary = 2763 (low + mgp->tx_boundary) & ~(mgp->tx_boundary - 1); 2764 seglen = boundary - low; 2765 if (seglen > len) 2766 seglen = len; 2767 flags_next = flags & ~MXGEFW_FLAGS_FIRST; 2768 cum_len_next = cum_len + seglen; 2769 if (mss) { /* TSO */ 2770 (req - rdma_count)->rdma_count = rdma_count + 1; 2771 2772 if (likely(cum_len >= 0)) { /* payload */ 2773 int next_is_first, chop; 2774 2775 chop = (cum_len_next > mss); 2776 cum_len_next = cum_len_next % mss; 2777 next_is_first = (cum_len_next == 0); 2778 flags |= chop * MXGEFW_FLAGS_TSO_CHOP; 2779 flags_next |= next_is_first * 2780 MXGEFW_FLAGS_FIRST; 2781 rdma_count |= -(chop | next_is_first); 2782 rdma_count += chop & ~next_is_first; 2783 } else if (likely(cum_len_next >= 0)) { /* header ends */ 2784 int small; 2785 2786 rdma_count = -1; 2787 cum_len_next = 0; 2788 seglen = -cum_len; 2789 small = (mss <= MXGEFW_SEND_SMALL_SIZE); 2790 flags_next = MXGEFW_FLAGS_TSO_PLD | 2791 MXGEFW_FLAGS_FIRST | 2792 (small * MXGEFW_FLAGS_SMALL); 2793 } 2794 } 2795 req->addr_high = high_swapped; 2796 req->addr_low = htonl(low); 2797 req->pseudo_hdr_offset = htons(pseudo_hdr_offset); 2798 req->pad = 0; /* complete solid 16-byte block; does this matter? */ 2799 req->rdma_count = 1; 2800 req->length = htons(seglen); 2801 req->cksum_offset = cksum_offset; 2802 req->flags = flags | ((cum_len & 1) * odd_flag); 2803 2804 low += seglen; 2805 len -= seglen; 2806 cum_len = cum_len_next; 2807 flags = flags_next; 2808 req++; 2809 count++; 2810 rdma_count++; 2811 if (cksum_offset != 0 && !(mss && skb_is_gso_v6(skb))) { 2812 if (unlikely(cksum_offset > seglen)) 2813 cksum_offset -= seglen; 2814 else 2815 cksum_offset = 0; 2816 } 2817 } 2818 if (frag_idx == frag_cnt) 2819 break; 2820 2821 /* map next fragment for DMA */ 2822 frag = &skb_shinfo(skb)->frags[frag_idx]; 2823 frag_idx++; 2824 len = skb_frag_size(frag); 2825 bus = skb_frag_dma_map(&mgp->pdev->dev, frag, 0, len, 2826 DMA_TO_DEVICE); 2827 if (unlikely(pci_dma_mapping_error(mgp->pdev, bus))) { 2828 myri10ge_unmap_tx_dma(mgp, tx, idx); 2829 goto drop; 2830 } 2831 idx = (count + tx->req) & tx->mask; 2832 dma_unmap_addr_set(&tx->info[idx], bus, bus); 2833 dma_unmap_len_set(&tx->info[idx], len, len); 2834 } 2835 2836 (req - rdma_count)->rdma_count = rdma_count; 2837 if (mss) 2838 do { 2839 req--; 2840 req->flags |= MXGEFW_FLAGS_TSO_LAST; 2841 } while (!(req->flags & (MXGEFW_FLAGS_TSO_CHOP | 2842 MXGEFW_FLAGS_FIRST))); 2843 idx = ((count - 1) + tx->req) & tx->mask; 2844 tx->info[idx].last = 1; 2845 myri10ge_submit_req(tx, tx->req_list, count); 2846 /* if using multiple tx queues, make sure NIC polls the 2847 * current slice */ 2848 if ((mgp->dev->real_num_tx_queues > 1) && tx->queue_active == 0) { 2849 tx->queue_active = 1; 2850 put_be32(htonl(1), tx->send_go); 2851 mb(); 2852 } 2853 tx->pkt_start++; 2854 if ((avail - count) < MXGEFW_MAX_SEND_DESC) { 2855 tx->stop_queue++; 2856 netif_tx_stop_queue(netdev_queue); 2857 } 2858 return NETDEV_TX_OK; 2859 2860 abort_linearize: 2861 myri10ge_unmap_tx_dma(mgp, tx, idx); 2862 2863 if (skb_is_gso(skb)) { 2864 netdev_err(mgp->dev, "TSO but wanted to linearize?!?!?\n"); 2865 goto drop; 2866 } 2867 2868 if (skb_linearize(skb)) 2869 goto drop; 2870 2871 tx->linearized++; 2872 goto again; 2873 2874 drop: 2875 dev_kfree_skb_any(skb); 2876 ss->stats.tx_dropped += 1; 2877 return NETDEV_TX_OK; 2878 2879 } 2880 2881 static netdev_tx_t myri10ge_sw_tso(struct sk_buff *skb, 2882 struct net_device *dev) 2883 { 2884 struct sk_buff *segs, *curr, *next; 2885 struct myri10ge_priv *mgp = netdev_priv(dev); 2886 struct myri10ge_slice_state *ss; 2887 netdev_tx_t status; 2888 2889 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO6); 2890 if (IS_ERR(segs)) 2891 goto drop; 2892 2893 skb_list_walk_safe(segs, curr, next) { 2894 skb_mark_not_on_list(curr); 2895 status = myri10ge_xmit(curr, dev); 2896 if (status != 0) { 2897 dev_kfree_skb_any(curr); 2898 if (segs != NULL) { 2899 curr = segs; 2900 segs = segs->next; 2901 curr->next = NULL; 2902 dev_kfree_skb_any(segs); 2903 } 2904 goto drop; 2905 } 2906 } 2907 dev_kfree_skb_any(skb); 2908 return NETDEV_TX_OK; 2909 2910 drop: 2911 ss = &mgp->ss[skb_get_queue_mapping(skb)]; 2912 dev_kfree_skb_any(skb); 2913 ss->stats.tx_dropped += 1; 2914 return NETDEV_TX_OK; 2915 } 2916 2917 static void myri10ge_get_stats(struct net_device *dev, 2918 struct rtnl_link_stats64 *stats) 2919 { 2920 const struct myri10ge_priv *mgp = netdev_priv(dev); 2921 const struct myri10ge_slice_netstats *slice_stats; 2922 int i; 2923 2924 for (i = 0; i < mgp->num_slices; i++) { 2925 slice_stats = &mgp->ss[i].stats; 2926 stats->rx_packets += slice_stats->rx_packets; 2927 stats->tx_packets += slice_stats->tx_packets; 2928 stats->rx_bytes += slice_stats->rx_bytes; 2929 stats->tx_bytes += slice_stats->tx_bytes; 2930 stats->rx_dropped += slice_stats->rx_dropped; 2931 stats->tx_dropped += slice_stats->tx_dropped; 2932 } 2933 } 2934 2935 static void myri10ge_set_multicast_list(struct net_device *dev) 2936 { 2937 struct myri10ge_priv *mgp = netdev_priv(dev); 2938 struct myri10ge_cmd cmd; 2939 struct netdev_hw_addr *ha; 2940 __be32 data[2] = { 0, 0 }; 2941 int err; 2942 2943 /* can be called from atomic contexts, 2944 * pass 1 to force atomicity in myri10ge_send_cmd() */ 2945 myri10ge_change_promisc(mgp, dev->flags & IFF_PROMISC, 1); 2946 2947 /* This firmware is known to not support multicast */ 2948 if (!mgp->fw_multicast_support) 2949 return; 2950 2951 /* Disable multicast filtering */ 2952 2953 err = myri10ge_send_cmd(mgp, MXGEFW_ENABLE_ALLMULTI, &cmd, 1); 2954 if (err != 0) { 2955 netdev_err(dev, "Failed MXGEFW_ENABLE_ALLMULTI, error status: %d\n", 2956 err); 2957 goto abort; 2958 } 2959 2960 if ((dev->flags & IFF_ALLMULTI) || mgp->adopted_rx_filter_bug) { 2961 /* request to disable multicast filtering, so quit here */ 2962 return; 2963 } 2964 2965 /* Flush the filters */ 2966 2967 err = myri10ge_send_cmd(mgp, MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, 2968 &cmd, 1); 2969 if (err != 0) { 2970 netdev_err(dev, "Failed MXGEFW_LEAVE_ALL_MULTICAST_GROUPS, error status: %d\n", 2971 err); 2972 goto abort; 2973 } 2974 2975 /* Walk the multicast list, and add each address */ 2976 netdev_for_each_mc_addr(ha, dev) { 2977 memcpy(data, &ha->addr, ETH_ALEN); 2978 cmd.data0 = ntohl(data[0]); 2979 cmd.data1 = ntohl(data[1]); 2980 err = myri10ge_send_cmd(mgp, MXGEFW_JOIN_MULTICAST_GROUP, 2981 &cmd, 1); 2982 2983 if (err != 0) { 2984 netdev_err(dev, "Failed MXGEFW_JOIN_MULTICAST_GROUP, error status:%d %pM\n", 2985 err, ha->addr); 2986 goto abort; 2987 } 2988 } 2989 /* Enable multicast filtering */ 2990 err = myri10ge_send_cmd(mgp, MXGEFW_DISABLE_ALLMULTI, &cmd, 1); 2991 if (err != 0) { 2992 netdev_err(dev, "Failed MXGEFW_DISABLE_ALLMULTI, error status: %d\n", 2993 err); 2994 goto abort; 2995 } 2996 2997 return; 2998 2999 abort: 3000 return; 3001 } 3002 3003 static int myri10ge_set_mac_address(struct net_device *dev, void *addr) 3004 { 3005 struct sockaddr *sa = addr; 3006 struct myri10ge_priv *mgp = netdev_priv(dev); 3007 int status; 3008 3009 if (!is_valid_ether_addr(sa->sa_data)) 3010 return -EADDRNOTAVAIL; 3011 3012 status = myri10ge_update_mac_address(mgp, sa->sa_data); 3013 if (status != 0) { 3014 netdev_err(dev, "changing mac address failed with %d\n", 3015 status); 3016 return status; 3017 } 3018 3019 /* change the dev structure */ 3020 memcpy(dev->dev_addr, sa->sa_data, ETH_ALEN); 3021 return 0; 3022 } 3023 3024 static int myri10ge_change_mtu(struct net_device *dev, int new_mtu) 3025 { 3026 struct myri10ge_priv *mgp = netdev_priv(dev); 3027 3028 netdev_info(dev, "changing mtu from %d to %d\n", dev->mtu, new_mtu); 3029 if (mgp->running) { 3030 /* if we change the mtu on an active device, we must 3031 * reset the device so the firmware sees the change */ 3032 myri10ge_close(dev); 3033 dev->mtu = new_mtu; 3034 myri10ge_open(dev); 3035 } else 3036 dev->mtu = new_mtu; 3037 3038 return 0; 3039 } 3040 3041 /* 3042 * Enable ECRC to align PCI-E Completion packets on an 8-byte boundary. 3043 * Only do it if the bridge is a root port since we don't want to disturb 3044 * any other device, except if forced with myri10ge_ecrc_enable > 1. 3045 */ 3046 3047 static void myri10ge_enable_ecrc(struct myri10ge_priv *mgp) 3048 { 3049 struct pci_dev *bridge = mgp->pdev->bus->self; 3050 struct device *dev = &mgp->pdev->dev; 3051 int cap; 3052 unsigned err_cap; 3053 int ret; 3054 3055 if (!myri10ge_ecrc_enable || !bridge) 3056 return; 3057 3058 /* check that the bridge is a root port */ 3059 if (pci_pcie_type(bridge) != PCI_EXP_TYPE_ROOT_PORT) { 3060 if (myri10ge_ecrc_enable > 1) { 3061 struct pci_dev *prev_bridge, *old_bridge = bridge; 3062 3063 /* Walk the hierarchy up to the root port 3064 * where ECRC has to be enabled */ 3065 do { 3066 prev_bridge = bridge; 3067 bridge = bridge->bus->self; 3068 if (!bridge || prev_bridge == bridge) { 3069 dev_err(dev, 3070 "Failed to find root port" 3071 " to force ECRC\n"); 3072 return; 3073 } 3074 } while (pci_pcie_type(bridge) != 3075 PCI_EXP_TYPE_ROOT_PORT); 3076 3077 dev_info(dev, 3078 "Forcing ECRC on non-root port %s" 3079 " (enabling on root port %s)\n", 3080 pci_name(old_bridge), pci_name(bridge)); 3081 } else { 3082 dev_err(dev, 3083 "Not enabling ECRC on non-root port %s\n", 3084 pci_name(bridge)); 3085 return; 3086 } 3087 } 3088 3089 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3090 if (!cap) 3091 return; 3092 3093 ret = pci_read_config_dword(bridge, cap + PCI_ERR_CAP, &err_cap); 3094 if (ret) { 3095 dev_err(dev, "failed reading ext-conf-space of %s\n", 3096 pci_name(bridge)); 3097 dev_err(dev, "\t pci=nommconf in use? " 3098 "or buggy/incomplete/absent ACPI MCFG attr?\n"); 3099 return; 3100 } 3101 if (!(err_cap & PCI_ERR_CAP_ECRC_GENC)) 3102 return; 3103 3104 err_cap |= PCI_ERR_CAP_ECRC_GENE; 3105 pci_write_config_dword(bridge, cap + PCI_ERR_CAP, err_cap); 3106 dev_info(dev, "Enabled ECRC on upstream bridge %s\n", pci_name(bridge)); 3107 } 3108 3109 /* 3110 * The Lanai Z8E PCI-E interface achieves higher Read-DMA throughput 3111 * when the PCI-E Completion packets are aligned on an 8-byte 3112 * boundary. Some PCI-E chip sets always align Completion packets; on 3113 * the ones that do not, the alignment can be enforced by enabling 3114 * ECRC generation (if supported). 3115 * 3116 * When PCI-E Completion packets are not aligned, it is actually more 3117 * efficient to limit Read-DMA transactions to 2KB, rather than 4KB. 3118 * 3119 * If the driver can neither enable ECRC nor verify that it has 3120 * already been enabled, then it must use a firmware image which works 3121 * around unaligned completion packets (myri10ge_rss_ethp_z8e.dat), and it 3122 * should also ensure that it never gives the device a Read-DMA which is 3123 * larger than 2KB by setting the tx_boundary to 2KB. If ECRC is 3124 * enabled, then the driver should use the aligned (myri10ge_rss_eth_z8e.dat) 3125 * firmware image, and set tx_boundary to 4KB. 3126 */ 3127 3128 static void myri10ge_firmware_probe(struct myri10ge_priv *mgp) 3129 { 3130 struct pci_dev *pdev = mgp->pdev; 3131 struct device *dev = &pdev->dev; 3132 int status; 3133 3134 mgp->tx_boundary = 4096; 3135 /* 3136 * Verify the max read request size was set to 4KB 3137 * before trying the test with 4KB. 3138 */ 3139 status = pcie_get_readrq(pdev); 3140 if (status < 0) { 3141 dev_err(dev, "Couldn't read max read req size: %d\n", status); 3142 goto abort; 3143 } 3144 if (status != 4096) { 3145 dev_warn(dev, "Max Read Request size != 4096 (%d)\n", status); 3146 mgp->tx_boundary = 2048; 3147 } 3148 /* 3149 * load the optimized firmware (which assumes aligned PCIe 3150 * completions) in order to see if it works on this host. 3151 */ 3152 set_fw_name(mgp, myri10ge_fw_aligned, false); 3153 status = myri10ge_load_firmware(mgp, 1); 3154 if (status != 0) { 3155 goto abort; 3156 } 3157 3158 /* 3159 * Enable ECRC if possible 3160 */ 3161 myri10ge_enable_ecrc(mgp); 3162 3163 /* 3164 * Run a DMA test which watches for unaligned completions and 3165 * aborts on the first one seen. 3166 */ 3167 3168 status = myri10ge_dma_test(mgp, MXGEFW_CMD_UNALIGNED_TEST); 3169 if (status == 0) 3170 return; /* keep the aligned firmware */ 3171 3172 if (status != -E2BIG) 3173 dev_warn(dev, "DMA test failed: %d\n", status); 3174 if (status == -ENOSYS) 3175 dev_warn(dev, "Falling back to ethp! " 3176 "Please install up to date fw\n"); 3177 abort: 3178 /* fall back to using the unaligned firmware */ 3179 mgp->tx_boundary = 2048; 3180 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3181 } 3182 3183 static void myri10ge_select_firmware(struct myri10ge_priv *mgp) 3184 { 3185 int overridden = 0; 3186 3187 if (myri10ge_force_firmware == 0) { 3188 int link_width; 3189 u16 lnk; 3190 3191 pcie_capability_read_word(mgp->pdev, PCI_EXP_LNKSTA, &lnk); 3192 link_width = (lnk >> 4) & 0x3f; 3193 3194 /* Check to see if Link is less than 8 or if the 3195 * upstream bridge is known to provide aligned 3196 * completions */ 3197 if (link_width < 8) { 3198 dev_info(&mgp->pdev->dev, "PCIE x%d Link\n", 3199 link_width); 3200 mgp->tx_boundary = 4096; 3201 set_fw_name(mgp, myri10ge_fw_aligned, false); 3202 } else { 3203 myri10ge_firmware_probe(mgp); 3204 } 3205 } else { 3206 if (myri10ge_force_firmware == 1) { 3207 dev_info(&mgp->pdev->dev, 3208 "Assuming aligned completions (forced)\n"); 3209 mgp->tx_boundary = 4096; 3210 set_fw_name(mgp, myri10ge_fw_aligned, false); 3211 } else { 3212 dev_info(&mgp->pdev->dev, 3213 "Assuming unaligned completions (forced)\n"); 3214 mgp->tx_boundary = 2048; 3215 set_fw_name(mgp, myri10ge_fw_unaligned, false); 3216 } 3217 } 3218 3219 kernel_param_lock(THIS_MODULE); 3220 if (myri10ge_fw_name != NULL) { 3221 char *fw_name = kstrdup(myri10ge_fw_name, GFP_KERNEL); 3222 if (fw_name) { 3223 overridden = 1; 3224 set_fw_name(mgp, fw_name, true); 3225 } 3226 } 3227 kernel_param_unlock(THIS_MODULE); 3228 3229 if (mgp->board_number < MYRI10GE_MAX_BOARDS && 3230 myri10ge_fw_names[mgp->board_number] != NULL && 3231 strlen(myri10ge_fw_names[mgp->board_number])) { 3232 set_fw_name(mgp, myri10ge_fw_names[mgp->board_number], false); 3233 overridden = 1; 3234 } 3235 if (overridden) 3236 dev_info(&mgp->pdev->dev, "overriding firmware to %s\n", 3237 mgp->fw_name); 3238 } 3239 3240 static void myri10ge_mask_surprise_down(struct pci_dev *pdev) 3241 { 3242 struct pci_dev *bridge = pdev->bus->self; 3243 int cap; 3244 u32 mask; 3245 3246 if (bridge == NULL) 3247 return; 3248 3249 cap = pci_find_ext_capability(bridge, PCI_EXT_CAP_ID_ERR); 3250 if (cap) { 3251 /* a sram parity error can cause a surprise link 3252 * down; since we expect and can recover from sram 3253 * parity errors, mask surprise link down events */ 3254 pci_read_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, &mask); 3255 mask |= 0x20; 3256 pci_write_config_dword(bridge, cap + PCI_ERR_UNCOR_MASK, mask); 3257 } 3258 } 3259 3260 static int __maybe_unused myri10ge_suspend(struct device *dev) 3261 { 3262 struct myri10ge_priv *mgp; 3263 struct net_device *netdev; 3264 3265 mgp = dev_get_drvdata(dev); 3266 if (mgp == NULL) 3267 return -EINVAL; 3268 netdev = mgp->dev; 3269 3270 netif_device_detach(netdev); 3271 if (netif_running(netdev)) { 3272 netdev_info(netdev, "closing\n"); 3273 rtnl_lock(); 3274 myri10ge_close(netdev); 3275 rtnl_unlock(); 3276 } 3277 myri10ge_dummy_rdma(mgp, 0); 3278 3279 return 0; 3280 } 3281 3282 static int __maybe_unused myri10ge_resume(struct device *dev) 3283 { 3284 struct pci_dev *pdev = to_pci_dev(dev); 3285 struct myri10ge_priv *mgp; 3286 struct net_device *netdev; 3287 int status; 3288 u16 vendor; 3289 3290 mgp = pci_get_drvdata(pdev); 3291 if (mgp == NULL) 3292 return -EINVAL; 3293 netdev = mgp->dev; 3294 msleep(5); /* give card time to respond */ 3295 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3296 if (vendor == 0xffff) { 3297 netdev_err(mgp->dev, "device disappeared!\n"); 3298 return -EIO; 3299 } 3300 3301 myri10ge_reset(mgp); 3302 myri10ge_dummy_rdma(mgp, 1); 3303 3304 if (netif_running(netdev)) { 3305 rtnl_lock(); 3306 status = myri10ge_open(netdev); 3307 rtnl_unlock(); 3308 if (status != 0) 3309 goto abort_with_enabled; 3310 3311 } 3312 netif_device_attach(netdev); 3313 3314 return 0; 3315 3316 abort_with_enabled: 3317 return -EIO; 3318 } 3319 3320 static u32 myri10ge_read_reboot(struct myri10ge_priv *mgp) 3321 { 3322 struct pci_dev *pdev = mgp->pdev; 3323 int vs = mgp->vendor_specific_offset; 3324 u32 reboot; 3325 3326 /*enter read32 mode */ 3327 pci_write_config_byte(pdev, vs + 0x10, 0x3); 3328 3329 /*read REBOOT_STATUS (0xfffffff0) */ 3330 pci_write_config_dword(pdev, vs + 0x18, 0xfffffff0); 3331 pci_read_config_dword(pdev, vs + 0x14, &reboot); 3332 return reboot; 3333 } 3334 3335 static void 3336 myri10ge_check_slice(struct myri10ge_slice_state *ss, int *reset_needed, 3337 int *busy_slice_cnt, u32 rx_pause_cnt) 3338 { 3339 struct myri10ge_priv *mgp = ss->mgp; 3340 int slice = ss - mgp->ss; 3341 3342 if (ss->tx.req != ss->tx.done && 3343 ss->tx.done == ss->watchdog_tx_done && 3344 ss->watchdog_tx_req != ss->watchdog_tx_done) { 3345 /* nic seems like it might be stuck.. */ 3346 if (rx_pause_cnt != mgp->watchdog_pause) { 3347 if (net_ratelimit()) 3348 netdev_warn(mgp->dev, "slice %d: TX paused, " 3349 "check link partner\n", slice); 3350 } else { 3351 netdev_warn(mgp->dev, 3352 "slice %d: TX stuck %d %d %d %d %d %d\n", 3353 slice, ss->tx.queue_active, ss->tx.req, 3354 ss->tx.done, ss->tx.pkt_start, 3355 ss->tx.pkt_done, 3356 (int)ntohl(mgp->ss[slice].fw_stats-> 3357 send_done_count)); 3358 *reset_needed = 1; 3359 ss->stuck = 1; 3360 } 3361 } 3362 if (ss->watchdog_tx_done != ss->tx.done || 3363 ss->watchdog_rx_done != ss->rx_done.cnt) { 3364 *busy_slice_cnt += 1; 3365 } 3366 ss->watchdog_tx_done = ss->tx.done; 3367 ss->watchdog_tx_req = ss->tx.req; 3368 ss->watchdog_rx_done = ss->rx_done.cnt; 3369 } 3370 3371 /* 3372 * This watchdog is used to check whether the board has suffered 3373 * from a parity error and needs to be recovered. 3374 */ 3375 static void myri10ge_watchdog(struct work_struct *work) 3376 { 3377 struct myri10ge_priv *mgp = 3378 container_of(work, struct myri10ge_priv, watchdog_work); 3379 struct myri10ge_slice_state *ss; 3380 u32 reboot, rx_pause_cnt; 3381 int status, rebooted; 3382 int i; 3383 int reset_needed = 0; 3384 int busy_slice_cnt = 0; 3385 u16 cmd, vendor; 3386 3387 mgp->watchdog_resets++; 3388 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3389 rebooted = 0; 3390 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3391 /* Bus master DMA disabled? Check to see 3392 * if the card rebooted due to a parity error 3393 * For now, just report it */ 3394 reboot = myri10ge_read_reboot(mgp); 3395 netdev_err(mgp->dev, "NIC rebooted (0x%x),%s resetting\n", 3396 reboot, myri10ge_reset_recover ? "" : " not"); 3397 if (myri10ge_reset_recover == 0) 3398 return; 3399 rtnl_lock(); 3400 mgp->rebooted = 1; 3401 rebooted = 1; 3402 myri10ge_close(mgp->dev); 3403 myri10ge_reset_recover--; 3404 mgp->rebooted = 0; 3405 /* 3406 * A rebooted nic will come back with config space as 3407 * it was after power was applied to PCIe bus. 3408 * Attempt to restore config space which was saved 3409 * when the driver was loaded, or the last time the 3410 * nic was resumed from power saving mode. 3411 */ 3412 pci_restore_state(mgp->pdev); 3413 3414 /* save state again for accounting reasons */ 3415 pci_save_state(mgp->pdev); 3416 3417 } else { 3418 /* if we get back -1's from our slot, perhaps somebody 3419 * powered off our card. Don't try to reset it in 3420 * this case */ 3421 if (cmd == 0xffff) { 3422 pci_read_config_word(mgp->pdev, PCI_VENDOR_ID, &vendor); 3423 if (vendor == 0xffff) { 3424 netdev_err(mgp->dev, "device disappeared!\n"); 3425 return; 3426 } 3427 } 3428 /* Perhaps it is a software error. See if stuck slice 3429 * has recovered, reset if not */ 3430 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3431 for (i = 0; i < mgp->num_slices; i++) { 3432 ss = mgp->ss; 3433 if (ss->stuck) { 3434 myri10ge_check_slice(ss, &reset_needed, 3435 &busy_slice_cnt, 3436 rx_pause_cnt); 3437 ss->stuck = 0; 3438 } 3439 } 3440 if (!reset_needed) { 3441 netdev_dbg(mgp->dev, "not resetting\n"); 3442 return; 3443 } 3444 3445 netdev_err(mgp->dev, "device timeout, resetting\n"); 3446 } 3447 3448 if (!rebooted) { 3449 rtnl_lock(); 3450 myri10ge_close(mgp->dev); 3451 } 3452 status = myri10ge_load_firmware(mgp, 1); 3453 if (status != 0) 3454 netdev_err(mgp->dev, "failed to load firmware\n"); 3455 else 3456 myri10ge_open(mgp->dev); 3457 rtnl_unlock(); 3458 } 3459 3460 /* 3461 * We use our own timer routine rather than relying upon 3462 * netdev->tx_timeout because we have a very large hardware transmit 3463 * queue. Due to the large queue, the netdev->tx_timeout function 3464 * cannot detect a NIC with a parity error in a timely fashion if the 3465 * NIC is lightly loaded. 3466 */ 3467 static void myri10ge_watchdog_timer(struct timer_list *t) 3468 { 3469 struct myri10ge_priv *mgp; 3470 struct myri10ge_slice_state *ss; 3471 int i, reset_needed, busy_slice_cnt; 3472 u32 rx_pause_cnt; 3473 u16 cmd; 3474 3475 mgp = from_timer(mgp, t, watchdog_timer); 3476 3477 rx_pause_cnt = ntohl(mgp->ss[0].fw_stats->dropped_pause); 3478 busy_slice_cnt = 0; 3479 for (i = 0, reset_needed = 0; 3480 i < mgp->num_slices && reset_needed == 0; ++i) { 3481 3482 ss = &mgp->ss[i]; 3483 if (ss->rx_small.watchdog_needed) { 3484 myri10ge_alloc_rx_pages(mgp, &ss->rx_small, 3485 mgp->small_bytes + MXGEFW_PAD, 3486 1); 3487 if (ss->rx_small.fill_cnt - ss->rx_small.cnt >= 3488 myri10ge_fill_thresh) 3489 ss->rx_small.watchdog_needed = 0; 3490 } 3491 if (ss->rx_big.watchdog_needed) { 3492 myri10ge_alloc_rx_pages(mgp, &ss->rx_big, 3493 mgp->big_bytes, 1); 3494 if (ss->rx_big.fill_cnt - ss->rx_big.cnt >= 3495 myri10ge_fill_thresh) 3496 ss->rx_big.watchdog_needed = 0; 3497 } 3498 myri10ge_check_slice(ss, &reset_needed, &busy_slice_cnt, 3499 rx_pause_cnt); 3500 } 3501 /* if we've sent or received no traffic, poll the NIC to 3502 * ensure it is still there. Otherwise, we risk not noticing 3503 * an error in a timely fashion */ 3504 if (busy_slice_cnt == 0) { 3505 pci_read_config_word(mgp->pdev, PCI_COMMAND, &cmd); 3506 if ((cmd & PCI_COMMAND_MASTER) == 0) { 3507 reset_needed = 1; 3508 } 3509 } 3510 mgp->watchdog_pause = rx_pause_cnt; 3511 3512 if (reset_needed) { 3513 schedule_work(&mgp->watchdog_work); 3514 } else { 3515 /* rearm timer */ 3516 mod_timer(&mgp->watchdog_timer, 3517 jiffies + myri10ge_watchdog_timeout * HZ); 3518 } 3519 } 3520 3521 static void myri10ge_free_slices(struct myri10ge_priv *mgp) 3522 { 3523 struct myri10ge_slice_state *ss; 3524 struct pci_dev *pdev = mgp->pdev; 3525 size_t bytes; 3526 int i; 3527 3528 if (mgp->ss == NULL) 3529 return; 3530 3531 for (i = 0; i < mgp->num_slices; i++) { 3532 ss = &mgp->ss[i]; 3533 if (ss->rx_done.entry != NULL) { 3534 bytes = mgp->max_intr_slots * 3535 sizeof(*ss->rx_done.entry); 3536 dma_free_coherent(&pdev->dev, bytes, 3537 ss->rx_done.entry, ss->rx_done.bus); 3538 ss->rx_done.entry = NULL; 3539 } 3540 if (ss->fw_stats != NULL) { 3541 bytes = sizeof(*ss->fw_stats); 3542 dma_free_coherent(&pdev->dev, bytes, 3543 ss->fw_stats, ss->fw_stats_bus); 3544 ss->fw_stats = NULL; 3545 } 3546 napi_hash_del(&ss->napi); 3547 netif_napi_del(&ss->napi); 3548 } 3549 /* Wait till napi structs are no longer used, and then free ss. */ 3550 synchronize_rcu(); 3551 kfree(mgp->ss); 3552 mgp->ss = NULL; 3553 } 3554 3555 static int myri10ge_alloc_slices(struct myri10ge_priv *mgp) 3556 { 3557 struct myri10ge_slice_state *ss; 3558 struct pci_dev *pdev = mgp->pdev; 3559 size_t bytes; 3560 int i; 3561 3562 bytes = sizeof(*mgp->ss) * mgp->num_slices; 3563 mgp->ss = kzalloc(bytes, GFP_KERNEL); 3564 if (mgp->ss == NULL) { 3565 return -ENOMEM; 3566 } 3567 3568 for (i = 0; i < mgp->num_slices; i++) { 3569 ss = &mgp->ss[i]; 3570 bytes = mgp->max_intr_slots * sizeof(*ss->rx_done.entry); 3571 ss->rx_done.entry = dma_alloc_coherent(&pdev->dev, bytes, 3572 &ss->rx_done.bus, 3573 GFP_KERNEL); 3574 if (ss->rx_done.entry == NULL) 3575 goto abort; 3576 bytes = sizeof(*ss->fw_stats); 3577 ss->fw_stats = dma_alloc_coherent(&pdev->dev, bytes, 3578 &ss->fw_stats_bus, 3579 GFP_KERNEL); 3580 if (ss->fw_stats == NULL) 3581 goto abort; 3582 ss->mgp = mgp; 3583 ss->dev = mgp->dev; 3584 netif_napi_add(ss->dev, &ss->napi, myri10ge_poll, 3585 myri10ge_napi_weight); 3586 } 3587 return 0; 3588 abort: 3589 myri10ge_free_slices(mgp); 3590 return -ENOMEM; 3591 } 3592 3593 /* 3594 * This function determines the number of slices supported. 3595 * The number slices is the minimum of the number of CPUS, 3596 * the number of MSI-X irqs supported, the number of slices 3597 * supported by the firmware 3598 */ 3599 static void myri10ge_probe_slices(struct myri10ge_priv *mgp) 3600 { 3601 struct myri10ge_cmd cmd; 3602 struct pci_dev *pdev = mgp->pdev; 3603 char *old_fw; 3604 bool old_allocated; 3605 int i, status, ncpus; 3606 3607 mgp->num_slices = 1; 3608 ncpus = netif_get_num_default_rss_queues(); 3609 3610 if (myri10ge_max_slices == 1 || !pdev->msix_cap || 3611 (myri10ge_max_slices == -1 && ncpus < 2)) 3612 return; 3613 3614 /* try to load the slice aware rss firmware */ 3615 old_fw = mgp->fw_name; 3616 old_allocated = mgp->fw_name_allocated; 3617 /* don't free old_fw if we override it. */ 3618 mgp->fw_name_allocated = false; 3619 3620 if (myri10ge_fw_name != NULL) { 3621 dev_info(&mgp->pdev->dev, "overriding rss firmware to %s\n", 3622 myri10ge_fw_name); 3623 set_fw_name(mgp, myri10ge_fw_name, false); 3624 } else if (old_fw == myri10ge_fw_aligned) 3625 set_fw_name(mgp, myri10ge_fw_rss_aligned, false); 3626 else 3627 set_fw_name(mgp, myri10ge_fw_rss_unaligned, false); 3628 status = myri10ge_load_firmware(mgp, 0); 3629 if (status != 0) { 3630 dev_info(&pdev->dev, "Rss firmware not found\n"); 3631 if (old_allocated) 3632 kfree(old_fw); 3633 return; 3634 } 3635 3636 /* hit the board with a reset to ensure it is alive */ 3637 memset(&cmd, 0, sizeof(cmd)); 3638 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_RESET, &cmd, 0); 3639 if (status != 0) { 3640 dev_err(&mgp->pdev->dev, "failed reset\n"); 3641 goto abort_with_fw; 3642 } 3643 3644 mgp->max_intr_slots = cmd.data0 / sizeof(struct mcp_slot); 3645 3646 /* tell it the size of the interrupt queues */ 3647 cmd.data0 = mgp->max_intr_slots * sizeof(struct mcp_slot); 3648 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_SET_INTRQ_SIZE, &cmd, 0); 3649 if (status != 0) { 3650 dev_err(&mgp->pdev->dev, "failed MXGEFW_CMD_SET_INTRQ_SIZE\n"); 3651 goto abort_with_fw; 3652 } 3653 3654 /* ask the maximum number of slices it supports */ 3655 status = myri10ge_send_cmd(mgp, MXGEFW_CMD_GET_MAX_RSS_QUEUES, &cmd, 0); 3656 if (status != 0) 3657 goto abort_with_fw; 3658 else 3659 mgp->num_slices = cmd.data0; 3660 3661 /* Only allow multiple slices if MSI-X is usable */ 3662 if (!myri10ge_msi) { 3663 goto abort_with_fw; 3664 } 3665 3666 /* if the admin did not specify a limit to how many 3667 * slices we should use, cap it automatically to the 3668 * number of CPUs currently online */ 3669 if (myri10ge_max_slices == -1) 3670 myri10ge_max_slices = ncpus; 3671 3672 if (mgp->num_slices > myri10ge_max_slices) 3673 mgp->num_slices = myri10ge_max_slices; 3674 3675 /* Now try to allocate as many MSI-X vectors as we have 3676 * slices. We give up on MSI-X if we can only get a single 3677 * vector. */ 3678 3679 mgp->msix_vectors = kcalloc(mgp->num_slices, sizeof(*mgp->msix_vectors), 3680 GFP_KERNEL); 3681 if (mgp->msix_vectors == NULL) 3682 goto no_msix; 3683 for (i = 0; i < mgp->num_slices; i++) { 3684 mgp->msix_vectors[i].entry = i; 3685 } 3686 3687 while (mgp->num_slices > 1) { 3688 mgp->num_slices = rounddown_pow_of_two(mgp->num_slices); 3689 if (mgp->num_slices == 1) 3690 goto no_msix; 3691 status = pci_enable_msix_range(pdev, 3692 mgp->msix_vectors, 3693 mgp->num_slices, 3694 mgp->num_slices); 3695 if (status < 0) 3696 goto no_msix; 3697 3698 pci_disable_msix(pdev); 3699 3700 if (status == mgp->num_slices) { 3701 if (old_allocated) 3702 kfree(old_fw); 3703 return; 3704 } else { 3705 mgp->num_slices = status; 3706 } 3707 } 3708 3709 no_msix: 3710 if (mgp->msix_vectors != NULL) { 3711 kfree(mgp->msix_vectors); 3712 mgp->msix_vectors = NULL; 3713 } 3714 3715 abort_with_fw: 3716 mgp->num_slices = 1; 3717 set_fw_name(mgp, old_fw, old_allocated); 3718 myri10ge_load_firmware(mgp, 0); 3719 } 3720 3721 static const struct net_device_ops myri10ge_netdev_ops = { 3722 .ndo_open = myri10ge_open, 3723 .ndo_stop = myri10ge_close, 3724 .ndo_start_xmit = myri10ge_xmit, 3725 .ndo_get_stats64 = myri10ge_get_stats, 3726 .ndo_validate_addr = eth_validate_addr, 3727 .ndo_change_mtu = myri10ge_change_mtu, 3728 .ndo_set_rx_mode = myri10ge_set_multicast_list, 3729 .ndo_set_mac_address = myri10ge_set_mac_address, 3730 }; 3731 3732 static int myri10ge_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3733 { 3734 struct net_device *netdev; 3735 struct myri10ge_priv *mgp; 3736 struct device *dev = &pdev->dev; 3737 int i; 3738 int status = -ENXIO; 3739 int dac_enabled; 3740 unsigned hdr_offset, ss_offset; 3741 static int board_number; 3742 3743 netdev = alloc_etherdev_mq(sizeof(*mgp), MYRI10GE_MAX_SLICES); 3744 if (netdev == NULL) 3745 return -ENOMEM; 3746 3747 SET_NETDEV_DEV(netdev, &pdev->dev); 3748 3749 mgp = netdev_priv(netdev); 3750 mgp->dev = netdev; 3751 mgp->pdev = pdev; 3752 mgp->pause = myri10ge_flow_control; 3753 mgp->intr_coal_delay = myri10ge_intr_coal_delay; 3754 mgp->msg_enable = netif_msg_init(myri10ge_debug, MYRI10GE_MSG_DEFAULT); 3755 mgp->board_number = board_number; 3756 init_waitqueue_head(&mgp->down_wq); 3757 3758 if (pci_enable_device(pdev)) { 3759 dev_err(&pdev->dev, "pci_enable_device call failed\n"); 3760 status = -ENODEV; 3761 goto abort_with_netdev; 3762 } 3763 3764 /* Find the vendor-specific cap so we can check 3765 * the reboot register later on */ 3766 mgp->vendor_specific_offset 3767 = pci_find_capability(pdev, PCI_CAP_ID_VNDR); 3768 3769 /* Set our max read request to 4KB */ 3770 status = pcie_set_readrq(pdev, 4096); 3771 if (status != 0) { 3772 dev_err(&pdev->dev, "Error %d writing PCI_EXP_DEVCTL\n", 3773 status); 3774 goto abort_with_enabled; 3775 } 3776 3777 myri10ge_mask_surprise_down(pdev); 3778 pci_set_master(pdev); 3779 dac_enabled = 1; 3780 status = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 3781 if (status != 0) { 3782 dac_enabled = 0; 3783 dev_err(&pdev->dev, 3784 "64-bit pci address mask was refused, " 3785 "trying 32-bit\n"); 3786 status = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 3787 } 3788 if (status != 0) { 3789 dev_err(&pdev->dev, "Error %d setting DMA mask\n", status); 3790 goto abort_with_enabled; 3791 } 3792 (void)pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 3793 mgp->cmd = dma_alloc_coherent(&pdev->dev, sizeof(*mgp->cmd), 3794 &mgp->cmd_bus, GFP_KERNEL); 3795 if (!mgp->cmd) { 3796 status = -ENOMEM; 3797 goto abort_with_enabled; 3798 } 3799 3800 mgp->board_span = pci_resource_len(pdev, 0); 3801 mgp->iomem_base = pci_resource_start(pdev, 0); 3802 mgp->wc_cookie = arch_phys_wc_add(mgp->iomem_base, mgp->board_span); 3803 mgp->sram = ioremap_wc(mgp->iomem_base, mgp->board_span); 3804 if (mgp->sram == NULL) { 3805 dev_err(&pdev->dev, "ioremap failed for %ld bytes at 0x%lx\n", 3806 mgp->board_span, mgp->iomem_base); 3807 status = -ENXIO; 3808 goto abort_with_mtrr; 3809 } 3810 hdr_offset = 3811 swab32(readl(mgp->sram + MCP_HEADER_PTR_OFFSET)) & 0xffffc; 3812 ss_offset = hdr_offset + offsetof(struct mcp_gen_header, string_specs); 3813 mgp->sram_size = swab32(readl(mgp->sram + ss_offset)); 3814 if (mgp->sram_size > mgp->board_span || 3815 mgp->sram_size <= MYRI10GE_FW_OFFSET) { 3816 dev_err(&pdev->dev, 3817 "invalid sram_size %dB or board span %ldB\n", 3818 mgp->sram_size, mgp->board_span); 3819 goto abort_with_ioremap; 3820 } 3821 memcpy_fromio(mgp->eeprom_strings, 3822 mgp->sram + mgp->sram_size, MYRI10GE_EEPROM_STRINGS_SIZE); 3823 memset(mgp->eeprom_strings + MYRI10GE_EEPROM_STRINGS_SIZE - 2, 0, 2); 3824 status = myri10ge_read_mac_addr(mgp); 3825 if (status) 3826 goto abort_with_ioremap; 3827 3828 for (i = 0; i < ETH_ALEN; i++) 3829 netdev->dev_addr[i] = mgp->mac_addr[i]; 3830 3831 myri10ge_select_firmware(mgp); 3832 3833 status = myri10ge_load_firmware(mgp, 1); 3834 if (status != 0) { 3835 dev_err(&pdev->dev, "failed to load firmware\n"); 3836 goto abort_with_ioremap; 3837 } 3838 myri10ge_probe_slices(mgp); 3839 status = myri10ge_alloc_slices(mgp); 3840 if (status != 0) { 3841 dev_err(&pdev->dev, "failed to alloc slice state\n"); 3842 goto abort_with_firmware; 3843 } 3844 netif_set_real_num_tx_queues(netdev, mgp->num_slices); 3845 netif_set_real_num_rx_queues(netdev, mgp->num_slices); 3846 status = myri10ge_reset(mgp); 3847 if (status != 0) { 3848 dev_err(&pdev->dev, "failed reset\n"); 3849 goto abort_with_slices; 3850 } 3851 #ifdef CONFIG_MYRI10GE_DCA 3852 myri10ge_setup_dca(mgp); 3853 #endif 3854 pci_set_drvdata(pdev, mgp); 3855 3856 /* MTU range: 68 - 9000 */ 3857 netdev->min_mtu = ETH_MIN_MTU; 3858 netdev->max_mtu = MYRI10GE_MAX_ETHER_MTU - ETH_HLEN; 3859 3860 if (myri10ge_initial_mtu > netdev->max_mtu) 3861 myri10ge_initial_mtu = netdev->max_mtu; 3862 if (myri10ge_initial_mtu < netdev->min_mtu) 3863 myri10ge_initial_mtu = netdev->min_mtu; 3864 3865 netdev->mtu = myri10ge_initial_mtu; 3866 3867 netdev->netdev_ops = &myri10ge_netdev_ops; 3868 netdev->hw_features = mgp->features | NETIF_F_RXCSUM; 3869 3870 /* fake NETIF_F_HW_VLAN_CTAG_RX for good GRO performance */ 3871 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 3872 3873 netdev->features = netdev->hw_features; 3874 3875 if (dac_enabled) 3876 netdev->features |= NETIF_F_HIGHDMA; 3877 3878 netdev->vlan_features |= mgp->features; 3879 if (mgp->fw_ver_tiny < 37) 3880 netdev->vlan_features &= ~NETIF_F_TSO6; 3881 if (mgp->fw_ver_tiny < 32) 3882 netdev->vlan_features &= ~NETIF_F_TSO; 3883 3884 /* make sure we can get an irq, and that MSI can be 3885 * setup (if available). */ 3886 status = myri10ge_request_irq(mgp); 3887 if (status != 0) 3888 goto abort_with_slices; 3889 myri10ge_free_irq(mgp); 3890 3891 /* Save configuration space to be restored if the 3892 * nic resets due to a parity error */ 3893 pci_save_state(pdev); 3894 3895 /* Setup the watchdog timer */ 3896 timer_setup(&mgp->watchdog_timer, myri10ge_watchdog_timer, 0); 3897 3898 netdev->ethtool_ops = &myri10ge_ethtool_ops; 3899 INIT_WORK(&mgp->watchdog_work, myri10ge_watchdog); 3900 status = register_netdev(netdev); 3901 if (status != 0) { 3902 dev_err(&pdev->dev, "register_netdev failed: %d\n", status); 3903 goto abort_with_state; 3904 } 3905 if (mgp->msix_enabled) 3906 dev_info(dev, "%d MSI-X IRQs, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3907 mgp->num_slices, mgp->tx_boundary, mgp->fw_name, 3908 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3909 else 3910 dev_info(dev, "%s IRQ %d, tx bndry %d, fw %s, MTRR %s, WC Enabled\n", 3911 mgp->msi_enabled ? "MSI" : "xPIC", 3912 pdev->irq, mgp->tx_boundary, mgp->fw_name, 3913 (mgp->wc_cookie > 0 ? "Enabled" : "Disabled")); 3914 3915 board_number++; 3916 return 0; 3917 3918 abort_with_state: 3919 pci_restore_state(pdev); 3920 3921 abort_with_slices: 3922 myri10ge_free_slices(mgp); 3923 3924 abort_with_firmware: 3925 myri10ge_dummy_rdma(mgp, 0); 3926 3927 abort_with_ioremap: 3928 if (mgp->mac_addr_string != NULL) 3929 dev_err(&pdev->dev, 3930 "myri10ge_probe() failed: MAC=%s, SN=%ld\n", 3931 mgp->mac_addr_string, mgp->serial_number); 3932 iounmap(mgp->sram); 3933 3934 abort_with_mtrr: 3935 arch_phys_wc_del(mgp->wc_cookie); 3936 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3937 mgp->cmd, mgp->cmd_bus); 3938 3939 abort_with_enabled: 3940 pci_disable_device(pdev); 3941 3942 abort_with_netdev: 3943 set_fw_name(mgp, NULL, false); 3944 free_netdev(netdev); 3945 return status; 3946 } 3947 3948 /* 3949 * myri10ge_remove 3950 * 3951 * Does what is necessary to shutdown one Myrinet device. Called 3952 * once for each Myrinet card by the kernel when a module is 3953 * unloaded. 3954 */ 3955 static void myri10ge_remove(struct pci_dev *pdev) 3956 { 3957 struct myri10ge_priv *mgp; 3958 struct net_device *netdev; 3959 3960 mgp = pci_get_drvdata(pdev); 3961 if (mgp == NULL) 3962 return; 3963 3964 cancel_work_sync(&mgp->watchdog_work); 3965 netdev = mgp->dev; 3966 unregister_netdev(netdev); 3967 3968 #ifdef CONFIG_MYRI10GE_DCA 3969 myri10ge_teardown_dca(mgp); 3970 #endif 3971 myri10ge_dummy_rdma(mgp, 0); 3972 3973 /* avoid a memory leak */ 3974 pci_restore_state(pdev); 3975 3976 iounmap(mgp->sram); 3977 arch_phys_wc_del(mgp->wc_cookie); 3978 myri10ge_free_slices(mgp); 3979 kfree(mgp->msix_vectors); 3980 dma_free_coherent(&pdev->dev, sizeof(*mgp->cmd), 3981 mgp->cmd, mgp->cmd_bus); 3982 3983 set_fw_name(mgp, NULL, false); 3984 free_netdev(netdev); 3985 pci_disable_device(pdev); 3986 } 3987 3988 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E 0x0008 3989 #define PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9 0x0009 3990 3991 static const struct pci_device_id myri10ge_pci_tbl[] = { 3992 {PCI_DEVICE(PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E)}, 3993 {PCI_DEVICE 3994 (PCI_VENDOR_ID_MYRICOM, PCI_DEVICE_ID_MYRICOM_MYRI10GE_Z8E_9)}, 3995 {0}, 3996 }; 3997 3998 MODULE_DEVICE_TABLE(pci, myri10ge_pci_tbl); 3999 4000 static SIMPLE_DEV_PM_OPS(myri10ge_pm_ops, myri10ge_suspend, myri10ge_resume); 4001 4002 static struct pci_driver myri10ge_driver = { 4003 .name = "myri10ge", 4004 .probe = myri10ge_probe, 4005 .remove = myri10ge_remove, 4006 .id_table = myri10ge_pci_tbl, 4007 .driver.pm = &myri10ge_pm_ops, 4008 }; 4009 4010 #ifdef CONFIG_MYRI10GE_DCA 4011 static int 4012 myri10ge_notify_dca(struct notifier_block *nb, unsigned long event, void *p) 4013 { 4014 int err = driver_for_each_device(&myri10ge_driver.driver, 4015 NULL, &event, 4016 myri10ge_notify_dca_device); 4017 4018 if (err) 4019 return NOTIFY_BAD; 4020 return NOTIFY_DONE; 4021 } 4022 4023 static struct notifier_block myri10ge_dca_notifier = { 4024 .notifier_call = myri10ge_notify_dca, 4025 .next = NULL, 4026 .priority = 0, 4027 }; 4028 #endif /* CONFIG_MYRI10GE_DCA */ 4029 4030 static __init int myri10ge_init_module(void) 4031 { 4032 pr_info("Version %s\n", MYRI10GE_VERSION_STR); 4033 4034 if (myri10ge_rss_hash > MXGEFW_RSS_HASH_TYPE_MAX) { 4035 pr_err("Illegal rssh hash type %d, defaulting to source port\n", 4036 myri10ge_rss_hash); 4037 myri10ge_rss_hash = MXGEFW_RSS_HASH_TYPE_SRC_PORT; 4038 } 4039 #ifdef CONFIG_MYRI10GE_DCA 4040 dca_register_notify(&myri10ge_dca_notifier); 4041 #endif 4042 if (myri10ge_max_slices > MYRI10GE_MAX_SLICES) 4043 myri10ge_max_slices = MYRI10GE_MAX_SLICES; 4044 4045 return pci_register_driver(&myri10ge_driver); 4046 } 4047 4048 module_init(myri10ge_init_module); 4049 4050 static __exit void myri10ge_cleanup_module(void) 4051 { 4052 #ifdef CONFIG_MYRI10GE_DCA 4053 dca_unregister_notify(&myri10ge_dca_notifier); 4054 #endif 4055 pci_unregister_driver(&myri10ge_driver); 4056 } 4057 4058 module_exit(myri10ge_cleanup_module); 4059