1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2016 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more details. 17 ***********************************************************************/ 18 #include <linux/module.h> 19 #include <linux/interrupt.h> 20 #include <linux/pci.h> 21 #include <net/vxlan.h> 22 #include "liquidio_common.h" 23 #include "octeon_droq.h" 24 #include "octeon_iq.h" 25 #include "response_manager.h" 26 #include "octeon_device.h" 27 #include "octeon_nic.h" 28 #include "octeon_main.h" 29 #include "octeon_network.h" 30 #include "cn23xx_vf_device.h" 31 32 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>"); 33 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Virtual Function Driver"); 34 MODULE_LICENSE("GPL"); 35 36 static int debug = -1; 37 module_param(debug, int, 0644); 38 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits"); 39 40 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK) 41 42 struct oct_timestamp_resp { 43 u64 rh; 44 u64 timestamp; 45 u64 status; 46 }; 47 48 union tx_info { 49 u64 u64; 50 struct { 51 #ifdef __BIG_ENDIAN_BITFIELD 52 u16 gso_size; 53 u16 gso_segs; 54 u32 reserved; 55 #else 56 u32 reserved; 57 u16 gso_segs; 58 u16 gso_size; 59 #endif 60 } s; 61 }; 62 63 #define OCTNIC_GSO_MAX_HEADER_SIZE 128 64 #define OCTNIC_GSO_MAX_SIZE \ 65 (CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE) 66 67 static int 68 liquidio_vf_probe(struct pci_dev *pdev, const struct pci_device_id *ent); 69 static void liquidio_vf_remove(struct pci_dev *pdev); 70 static int octeon_device_init(struct octeon_device *oct); 71 static int liquidio_stop(struct net_device *netdev); 72 73 static int lio_wait_for_oq_pkts(struct octeon_device *oct) 74 { 75 struct octeon_device_priv *oct_priv = 76 (struct octeon_device_priv *)oct->priv; 77 int retry = MAX_IO_PENDING_PKT_COUNT; 78 int pkt_cnt = 0, pending_pkts; 79 int i; 80 81 do { 82 pending_pkts = 0; 83 84 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) { 85 if (!(oct->io_qmask.oq & BIT_ULL(i))) 86 continue; 87 pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]); 88 } 89 if (pkt_cnt > 0) { 90 pending_pkts += pkt_cnt; 91 tasklet_schedule(&oct_priv->droq_tasklet); 92 } 93 pkt_cnt = 0; 94 schedule_timeout_uninterruptible(1); 95 96 } while (retry-- && pending_pkts); 97 98 return pkt_cnt; 99 } 100 101 /** 102 * pcierror_quiesce_device - Cause device to go quiet so it can be safely removed/reset/etc 103 * @oct: Pointer to Octeon device 104 */ 105 static void pcierror_quiesce_device(struct octeon_device *oct) 106 { 107 int i; 108 109 /* Disable the input and output queues now. No more packets will 110 * arrive from Octeon, but we should wait for all packet processing 111 * to finish. 112 */ 113 114 /* To allow for in-flight requests */ 115 schedule_timeout_uninterruptible(100); 116 117 if (wait_for_pending_requests(oct)) 118 dev_err(&oct->pci_dev->dev, "There were pending requests\n"); 119 120 /* Force all requests waiting to be fetched by OCTEON to complete. */ 121 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) { 122 struct octeon_instr_queue *iq; 123 124 if (!(oct->io_qmask.iq & BIT_ULL(i))) 125 continue; 126 iq = oct->instr_queue[i]; 127 128 if (atomic_read(&iq->instr_pending)) { 129 spin_lock_bh(&iq->lock); 130 iq->fill_cnt = 0; 131 iq->octeon_read_index = iq->host_write_index; 132 iq->stats.instr_processed += 133 atomic_read(&iq->instr_pending); 134 lio_process_iq_request_list(oct, iq, 0); 135 spin_unlock_bh(&iq->lock); 136 } 137 } 138 139 /* Force all pending ordered list requests to time out. */ 140 lio_process_ordered_list(oct, 1); 141 142 /* We do not need to wait for output queue packets to be processed. */ 143 } 144 145 /** 146 * cleanup_aer_uncorrect_error_status - Cleanup PCI AER uncorrectable error status 147 * @dev: Pointer to PCI device 148 */ 149 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev) 150 { 151 u32 status, mask; 152 int pos = 0x100; 153 154 pr_info("%s :\n", __func__); 155 156 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status); 157 pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask); 158 if (dev->error_state == pci_channel_io_normal) 159 status &= ~mask; /* Clear corresponding nonfatal bits */ 160 else 161 status &= mask; /* Clear corresponding fatal bits */ 162 pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status); 163 } 164 165 /** 166 * stop_pci_io - Stop all PCI IO to a given device 167 * @oct: Pointer to Octeon device 168 */ 169 static void stop_pci_io(struct octeon_device *oct) 170 { 171 struct msix_entry *msix_entries; 172 int i; 173 174 /* No more instructions will be forwarded. */ 175 atomic_set(&oct->status, OCT_DEV_IN_RESET); 176 177 for (i = 0; i < oct->ifcount; i++) 178 netif_device_detach(oct->props[i].netdev); 179 180 /* Disable interrupts */ 181 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR); 182 183 pcierror_quiesce_device(oct); 184 if (oct->msix_on) { 185 msix_entries = (struct msix_entry *)oct->msix_entries; 186 for (i = 0; i < oct->num_msix_irqs; i++) { 187 /* clear the affinity_cpumask */ 188 irq_set_affinity_hint(msix_entries[i].vector, 189 NULL); 190 free_irq(msix_entries[i].vector, 191 &oct->ioq_vector[i]); 192 } 193 pci_disable_msix(oct->pci_dev); 194 kfree(oct->msix_entries); 195 oct->msix_entries = NULL; 196 octeon_free_ioq_vector(oct); 197 } 198 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n", 199 lio_get_state_string(&oct->status)); 200 201 /* making it a common function for all OCTEON models */ 202 cleanup_aer_uncorrect_error_status(oct->pci_dev); 203 204 pci_disable_device(oct->pci_dev); 205 } 206 207 /** 208 * liquidio_pcie_error_detected - called when PCI error is detected 209 * @pdev: Pointer to PCI device 210 * @state: The current pci connection state 211 * 212 * This function is called after a PCI bus error affecting 213 * this device has been detected. 214 */ 215 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev, 216 pci_channel_state_t state) 217 { 218 struct octeon_device *oct = pci_get_drvdata(pdev); 219 220 /* Non-correctable Non-fatal errors */ 221 if (state == pci_channel_io_normal) { 222 dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n"); 223 cleanup_aer_uncorrect_error_status(oct->pci_dev); 224 return PCI_ERS_RESULT_CAN_RECOVER; 225 } 226 227 /* Non-correctable Fatal errors */ 228 dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n"); 229 stop_pci_io(oct); 230 231 return PCI_ERS_RESULT_DISCONNECT; 232 } 233 234 /* For PCI-E Advanced Error Recovery (AER) Interface */ 235 static const struct pci_error_handlers liquidio_vf_err_handler = { 236 .error_detected = liquidio_pcie_error_detected, 237 }; 238 239 static const struct pci_device_id liquidio_vf_pci_tbl[] = { 240 { 241 PCI_VENDOR_ID_CAVIUM, OCTEON_CN23XX_VF_VID, 242 PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 243 }, 244 { 245 0, 0, 0, 0, 0, 0, 0 246 } 247 }; 248 MODULE_DEVICE_TABLE(pci, liquidio_vf_pci_tbl); 249 250 static struct pci_driver liquidio_vf_pci_driver = { 251 .name = "LiquidIO_VF", 252 .id_table = liquidio_vf_pci_tbl, 253 .probe = liquidio_vf_probe, 254 .remove = liquidio_vf_remove, 255 .err_handler = &liquidio_vf_err_handler, /* For AER */ 256 }; 257 258 /** 259 * print_link_info - Print link information 260 * @netdev: network device 261 */ 262 static void print_link_info(struct net_device *netdev) 263 { 264 struct lio *lio = GET_LIO(netdev); 265 266 if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) && 267 ifstate_check(lio, LIO_IFSTATE_REGISTERED)) { 268 struct oct_link_info *linfo = &lio->linfo; 269 270 if (linfo->link.s.link_up) { 271 netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n", 272 linfo->link.s.speed, 273 (linfo->link.s.duplex) ? "Full" : "Half"); 274 } else { 275 netif_info(lio, link, lio->netdev, "Link Down\n"); 276 } 277 } 278 } 279 280 /** 281 * octnet_link_status_change - Routine to notify MTU change 282 * @work: work_struct data structure 283 */ 284 static void octnet_link_status_change(struct work_struct *work) 285 { 286 struct cavium_wk *wk = (struct cavium_wk *)work; 287 struct lio *lio = (struct lio *)wk->ctxptr; 288 289 /* lio->linfo.link.s.mtu always contains max MTU of the lio interface. 290 * this API is invoked only when new max-MTU of the interface is 291 * less than current MTU. 292 */ 293 rtnl_lock(); 294 dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu); 295 rtnl_unlock(); 296 } 297 298 /** 299 * setup_link_status_change_wq - Sets up the mtu status change work 300 * @netdev: network device 301 */ 302 static int setup_link_status_change_wq(struct net_device *netdev) 303 { 304 struct lio *lio = GET_LIO(netdev); 305 struct octeon_device *oct = lio->oct_dev; 306 307 lio->link_status_wq.wq = alloc_workqueue("link-status", 308 WQ_MEM_RECLAIM, 0); 309 if (!lio->link_status_wq.wq) { 310 dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n"); 311 return -1; 312 } 313 INIT_DELAYED_WORK(&lio->link_status_wq.wk.work, 314 octnet_link_status_change); 315 lio->link_status_wq.wk.ctxptr = lio; 316 317 return 0; 318 } 319 320 static void cleanup_link_status_change_wq(struct net_device *netdev) 321 { 322 struct lio *lio = GET_LIO(netdev); 323 324 if (lio->link_status_wq.wq) { 325 cancel_delayed_work_sync(&lio->link_status_wq.wk.work); 326 destroy_workqueue(lio->link_status_wq.wq); 327 } 328 } 329 330 /** 331 * update_link_status - Update link status 332 * @netdev: network device 333 * @ls: link status structure 334 * 335 * Called on receipt of a link status response from the core application to 336 * update each interface's link status. 337 */ 338 static void update_link_status(struct net_device *netdev, 339 union oct_link_status *ls) 340 { 341 struct lio *lio = GET_LIO(netdev); 342 int current_max_mtu = lio->linfo.link.s.mtu; 343 struct octeon_device *oct = lio->oct_dev; 344 345 if ((lio->intf_open) && (lio->linfo.link.u64 != ls->u64)) { 346 lio->linfo.link.u64 = ls->u64; 347 348 print_link_info(netdev); 349 lio->link_changes++; 350 351 if (lio->linfo.link.s.link_up) { 352 netif_carrier_on(netdev); 353 wake_txqs(netdev); 354 } else { 355 netif_carrier_off(netdev); 356 stop_txqs(netdev); 357 } 358 359 if (lio->linfo.link.s.mtu != current_max_mtu) { 360 dev_info(&oct->pci_dev->dev, 361 "Max MTU Changed from %d to %d\n", 362 current_max_mtu, lio->linfo.link.s.mtu); 363 netdev->max_mtu = lio->linfo.link.s.mtu; 364 } 365 366 if (lio->linfo.link.s.mtu < netdev->mtu) { 367 dev_warn(&oct->pci_dev->dev, 368 "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n", 369 netdev->mtu, lio->linfo.link.s.mtu); 370 queue_delayed_work(lio->link_status_wq.wq, 371 &lio->link_status_wq.wk.work, 0); 372 } 373 } 374 } 375 376 /** 377 * liquidio_vf_probe - PCI probe handler 378 * @pdev: PCI device structure 379 * @ent: unused 380 */ 381 static int 382 liquidio_vf_probe(struct pci_dev *pdev, 383 const struct pci_device_id __maybe_unused *ent) 384 { 385 struct octeon_device *oct_dev = NULL; 386 387 oct_dev = octeon_allocate_device(pdev->device, 388 sizeof(struct octeon_device_priv)); 389 390 if (!oct_dev) { 391 dev_err(&pdev->dev, "Unable to allocate device\n"); 392 return -ENOMEM; 393 } 394 oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED; 395 396 dev_info(&pdev->dev, "Initializing device %x:%x.\n", 397 (u32)pdev->vendor, (u32)pdev->device); 398 399 /* Assign octeon_device for this device to the private data area. */ 400 pci_set_drvdata(pdev, oct_dev); 401 402 /* set linux specific device pointer */ 403 oct_dev->pci_dev = pdev; 404 405 oct_dev->subsystem_id = pdev->subsystem_vendor | 406 (pdev->subsystem_device << 16); 407 408 if (octeon_device_init(oct_dev)) { 409 liquidio_vf_remove(pdev); 410 return -ENOMEM; 411 } 412 413 dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n"); 414 415 return 0; 416 } 417 418 /** 419 * octeon_pci_flr - PCI FLR for each Octeon device. 420 * @oct: octeon device 421 */ 422 static void octeon_pci_flr(struct octeon_device *oct) 423 { 424 pci_save_state(oct->pci_dev); 425 426 pci_cfg_access_lock(oct->pci_dev); 427 428 /* Quiesce the device completely */ 429 pci_write_config_word(oct->pci_dev, PCI_COMMAND, 430 PCI_COMMAND_INTX_DISABLE); 431 432 pcie_flr(oct->pci_dev); 433 434 pci_cfg_access_unlock(oct->pci_dev); 435 436 pci_restore_state(oct->pci_dev); 437 } 438 439 /** 440 * octeon_destroy_resources - Destroy resources associated with octeon device 441 * @oct: octeon device 442 */ 443 static void octeon_destroy_resources(struct octeon_device *oct) 444 { 445 struct octeon_device_priv *oct_priv = 446 (struct octeon_device_priv *)oct->priv; 447 struct msix_entry *msix_entries; 448 int i; 449 450 switch (atomic_read(&oct->status)) { 451 case OCT_DEV_RUNNING: 452 case OCT_DEV_CORE_OK: 453 /* No more instructions will be forwarded. */ 454 atomic_set(&oct->status, OCT_DEV_IN_RESET); 455 456 oct->app_mode = CVM_DRV_INVALID_APP; 457 dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n", 458 lio_get_state_string(&oct->status)); 459 460 schedule_timeout_uninterruptible(HZ / 10); 461 462 fallthrough; 463 case OCT_DEV_HOST_OK: 464 case OCT_DEV_IO_QUEUES_DONE: 465 if (lio_wait_for_instr_fetch(oct)) 466 dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n"); 467 468 if (wait_for_pending_requests(oct)) 469 dev_err(&oct->pci_dev->dev, "There were pending requests\n"); 470 471 /* Disable the input and output queues now. No more packets will 472 * arrive from Octeon, but we should wait for all packet 473 * processing to finish. 474 */ 475 oct->fn_list.disable_io_queues(oct); 476 477 if (lio_wait_for_oq_pkts(oct)) 478 dev_err(&oct->pci_dev->dev, "OQ had pending packets\n"); 479 480 /* Force all requests waiting to be fetched by OCTEON to 481 * complete. 482 */ 483 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) { 484 struct octeon_instr_queue *iq; 485 486 if (!(oct->io_qmask.iq & BIT_ULL(i))) 487 continue; 488 iq = oct->instr_queue[i]; 489 490 if (atomic_read(&iq->instr_pending)) { 491 spin_lock_bh(&iq->lock); 492 iq->fill_cnt = 0; 493 iq->octeon_read_index = iq->host_write_index; 494 iq->stats.instr_processed += 495 atomic_read(&iq->instr_pending); 496 lio_process_iq_request_list(oct, iq, 0); 497 spin_unlock_bh(&iq->lock); 498 } 499 } 500 501 lio_process_ordered_list(oct, 1); 502 octeon_free_sc_done_list(oct); 503 octeon_free_sc_zombie_list(oct); 504 505 fallthrough; 506 case OCT_DEV_INTR_SET_DONE: 507 /* Disable interrupts */ 508 oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR); 509 510 if (oct->msix_on) { 511 msix_entries = (struct msix_entry *)oct->msix_entries; 512 for (i = 0; i < oct->num_msix_irqs; i++) { 513 if (oct->ioq_vector[i].vector) { 514 irq_set_affinity_hint( 515 msix_entries[i].vector, 516 NULL); 517 free_irq(msix_entries[i].vector, 518 &oct->ioq_vector[i]); 519 oct->ioq_vector[i].vector = 0; 520 } 521 } 522 pci_disable_msix(oct->pci_dev); 523 kfree(oct->msix_entries); 524 oct->msix_entries = NULL; 525 kfree(oct->irq_name_storage); 526 oct->irq_name_storage = NULL; 527 } 528 /* Soft reset the octeon device before exiting */ 529 if (!pcie_reset_flr(oct->pci_dev, PCI_RESET_PROBE)) 530 octeon_pci_flr(oct); 531 else 532 cn23xx_vf_ask_pf_to_do_flr(oct); 533 534 fallthrough; 535 case OCT_DEV_MSIX_ALLOC_VECTOR_DONE: 536 octeon_free_ioq_vector(oct); 537 538 fallthrough; 539 case OCT_DEV_MBOX_SETUP_DONE: 540 oct->fn_list.free_mbox(oct); 541 542 fallthrough; 543 case OCT_DEV_IN_RESET: 544 case OCT_DEV_DROQ_INIT_DONE: 545 mdelay(100); 546 for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) { 547 if (!(oct->io_qmask.oq & BIT_ULL(i))) 548 continue; 549 octeon_delete_droq(oct, i); 550 } 551 552 fallthrough; 553 case OCT_DEV_RESP_LIST_INIT_DONE: 554 octeon_delete_response_list(oct); 555 556 fallthrough; 557 case OCT_DEV_INSTR_QUEUE_INIT_DONE: 558 for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) { 559 if (!(oct->io_qmask.iq & BIT_ULL(i))) 560 continue; 561 octeon_delete_instr_queue(oct, i); 562 } 563 564 fallthrough; 565 case OCT_DEV_SC_BUFF_POOL_INIT_DONE: 566 octeon_free_sc_buffer_pool(oct); 567 568 fallthrough; 569 case OCT_DEV_DISPATCH_INIT_DONE: 570 octeon_delete_dispatch_list(oct); 571 cancel_delayed_work_sync(&oct->nic_poll_work.work); 572 573 fallthrough; 574 case OCT_DEV_PCI_MAP_DONE: 575 octeon_unmap_pci_barx(oct, 0); 576 octeon_unmap_pci_barx(oct, 1); 577 578 fallthrough; 579 case OCT_DEV_PCI_ENABLE_DONE: 580 pci_clear_master(oct->pci_dev); 581 /* Disable the device, releasing the PCI INT */ 582 pci_disable_device(oct->pci_dev); 583 584 fallthrough; 585 case OCT_DEV_BEGIN_STATE: 586 /* Nothing to be done here either */ 587 break; 588 } 589 590 tasklet_kill(&oct_priv->droq_tasklet); 591 } 592 593 /** 594 * send_rx_ctrl_cmd - Send Rx control command 595 * @lio: per-network private data 596 * @start_stop: whether to start or stop 597 */ 598 static int send_rx_ctrl_cmd(struct lio *lio, int start_stop) 599 { 600 struct octeon_device *oct = (struct octeon_device *)lio->oct_dev; 601 struct octeon_soft_command *sc; 602 union octnet_cmd *ncmd; 603 int retval; 604 605 if (oct->props[lio->ifidx].rx_on == start_stop) 606 return 0; 607 608 sc = (struct octeon_soft_command *) 609 octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE, 610 16, 0); 611 if (!sc) { 612 netif_info(lio, rx_err, lio->netdev, 613 "Failed to allocate octeon_soft_command struct\n"); 614 return -ENOMEM; 615 } 616 617 ncmd = (union octnet_cmd *)sc->virtdptr; 618 619 ncmd->u64 = 0; 620 ncmd->s.cmd = OCTNET_CMD_RX_CTL; 621 ncmd->s.param1 = start_stop; 622 623 octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3)); 624 625 sc->iq_no = lio->linfo.txpciq[0].s.q_no; 626 627 octeon_prepare_soft_command(oct, sc, OPCODE_NIC, 628 OPCODE_NIC_CMD, 0, 0, 0); 629 630 init_completion(&sc->complete); 631 sc->sc_status = OCTEON_REQUEST_PENDING; 632 633 retval = octeon_send_soft_command(oct, sc); 634 if (retval == IQ_SEND_FAILED) { 635 netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n"); 636 octeon_free_soft_command(oct, sc); 637 } else { 638 /* Sleep on a wait queue till the cond flag indicates that the 639 * response arrived or timed-out. 640 */ 641 retval = wait_for_sc_completion_timeout(oct, sc, 0); 642 if (retval) 643 return retval; 644 645 oct->props[lio->ifidx].rx_on = start_stop; 646 WRITE_ONCE(sc->caller_is_done, true); 647 } 648 649 return retval; 650 } 651 652 /** 653 * liquidio_destroy_nic_device - Destroy NIC device interface 654 * @oct: octeon device 655 * @ifidx: which interface to destroy 656 * 657 * Cleanup associated with each interface for an Octeon device when NIC 658 * module is being unloaded or if initialization fails during load. 659 */ 660 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx) 661 { 662 struct net_device *netdev = oct->props[ifidx].netdev; 663 struct octeon_device_priv *oct_priv = 664 (struct octeon_device_priv *)oct->priv; 665 struct napi_struct *napi, *n; 666 struct lio *lio; 667 668 if (!netdev) { 669 dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n", 670 __func__, ifidx); 671 return; 672 } 673 674 lio = GET_LIO(netdev); 675 676 dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n"); 677 678 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) 679 liquidio_stop(netdev); 680 681 if (oct->props[lio->ifidx].napi_enabled == 1) { 682 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 683 napi_disable(napi); 684 685 oct->props[lio->ifidx].napi_enabled = 0; 686 687 oct->droq[0]->ops.poll_mode = 0; 688 } 689 690 /* Delete NAPI */ 691 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 692 netif_napi_del(napi); 693 694 tasklet_enable(&oct_priv->droq_tasklet); 695 696 if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED) 697 unregister_netdev(netdev); 698 699 cleanup_rx_oom_poll_fn(netdev); 700 701 cleanup_link_status_change_wq(netdev); 702 703 lio_delete_glists(lio); 704 705 free_netdev(netdev); 706 707 oct->props[ifidx].gmxport = -1; 708 709 oct->props[ifidx].netdev = NULL; 710 } 711 712 /** 713 * liquidio_stop_nic_module - Stop complete NIC functionality 714 * @oct: octeon device 715 */ 716 static int liquidio_stop_nic_module(struct octeon_device *oct) 717 { 718 struct lio *lio; 719 int i, j; 720 721 dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n"); 722 if (!oct->ifcount) { 723 dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n"); 724 return 1; 725 } 726 727 spin_lock_bh(&oct->cmd_resp_wqlock); 728 oct->cmd_resp_state = OCT_DRV_OFFLINE; 729 spin_unlock_bh(&oct->cmd_resp_wqlock); 730 731 for (i = 0; i < oct->ifcount; i++) { 732 lio = GET_LIO(oct->props[i].netdev); 733 for (j = 0; j < oct->num_oqs; j++) 734 octeon_unregister_droq_ops(oct, 735 lio->linfo.rxpciq[j].s.q_no); 736 } 737 738 for (i = 0; i < oct->ifcount; i++) 739 liquidio_destroy_nic_device(oct, i); 740 741 dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n"); 742 return 0; 743 } 744 745 /** 746 * liquidio_vf_remove - Cleans up resources at unload time 747 * @pdev: PCI device structure 748 */ 749 static void liquidio_vf_remove(struct pci_dev *pdev) 750 { 751 struct octeon_device *oct_dev = pci_get_drvdata(pdev); 752 753 dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n"); 754 755 if (oct_dev->app_mode == CVM_DRV_NIC_APP) 756 liquidio_stop_nic_module(oct_dev); 757 758 /* Reset the octeon device and cleanup all memory allocated for 759 * the octeon device by driver. 760 */ 761 octeon_destroy_resources(oct_dev); 762 763 dev_info(&oct_dev->pci_dev->dev, "Device removed\n"); 764 765 /* This octeon device has been removed. Update the global 766 * data structure to reflect this. Free the device structure. 767 */ 768 octeon_free_device_mem(oct_dev); 769 } 770 771 /** 772 * octeon_pci_os_setup - PCI initialization for each Octeon device. 773 * @oct: octeon device 774 */ 775 static int octeon_pci_os_setup(struct octeon_device *oct) 776 { 777 #ifdef CONFIG_PCI_IOV 778 /* setup PCI stuff first */ 779 if (!oct->pci_dev->physfn) 780 octeon_pci_flr(oct); 781 #endif 782 783 if (pci_enable_device(oct->pci_dev)) { 784 dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n"); 785 return 1; 786 } 787 788 if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) { 789 dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n"); 790 pci_disable_device(oct->pci_dev); 791 return 1; 792 } 793 794 /* Enable PCI DMA Master. */ 795 pci_set_master(oct->pci_dev); 796 797 return 0; 798 } 799 800 /** 801 * free_netbuf - Unmap and free network buffer 802 * @buf: buffer 803 */ 804 static void free_netbuf(void *buf) 805 { 806 struct octnet_buf_free_info *finfo; 807 struct sk_buff *skb; 808 struct lio *lio; 809 810 finfo = (struct octnet_buf_free_info *)buf; 811 skb = finfo->skb; 812 lio = finfo->lio; 813 814 dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len, 815 DMA_TO_DEVICE); 816 817 tx_buffer_free(skb); 818 } 819 820 /** 821 * free_netsgbuf - Unmap and free gather buffer 822 * @buf: buffer 823 */ 824 static void free_netsgbuf(void *buf) 825 { 826 struct octnet_buf_free_info *finfo; 827 struct octnic_gather *g; 828 struct sk_buff *skb; 829 int i, frags, iq; 830 struct lio *lio; 831 832 finfo = (struct octnet_buf_free_info *)buf; 833 skb = finfo->skb; 834 lio = finfo->lio; 835 g = finfo->g; 836 frags = skb_shinfo(skb)->nr_frags; 837 838 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 839 g->sg[0].ptr[0], (skb->len - skb->data_len), 840 DMA_TO_DEVICE); 841 842 i = 1; 843 while (frags--) { 844 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; 845 846 dma_unmap_page(&lio->oct_dev->pci_dev->dev, 847 g->sg[(i >> 2)].ptr[(i & 3)], 848 skb_frag_size(frag), DMA_TO_DEVICE); 849 i++; 850 } 851 852 iq = skb_iq(lio->oct_dev, skb); 853 854 spin_lock(&lio->glist_lock[iq]); 855 list_add_tail(&g->list, &lio->glist[iq]); 856 spin_unlock(&lio->glist_lock[iq]); 857 858 tx_buffer_free(skb); 859 } 860 861 /** 862 * free_netsgbuf_with_resp - Unmap and free gather buffer with response 863 * @buf: buffer 864 */ 865 static void free_netsgbuf_with_resp(void *buf) 866 { 867 struct octnet_buf_free_info *finfo; 868 struct octeon_soft_command *sc; 869 struct octnic_gather *g; 870 struct sk_buff *skb; 871 int i, frags, iq; 872 struct lio *lio; 873 874 sc = (struct octeon_soft_command *)buf; 875 skb = (struct sk_buff *)sc->callback_arg; 876 finfo = (struct octnet_buf_free_info *)&skb->cb; 877 878 lio = finfo->lio; 879 g = finfo->g; 880 frags = skb_shinfo(skb)->nr_frags; 881 882 dma_unmap_single(&lio->oct_dev->pci_dev->dev, 883 g->sg[0].ptr[0], (skb->len - skb->data_len), 884 DMA_TO_DEVICE); 885 886 i = 1; 887 while (frags--) { 888 skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1]; 889 890 dma_unmap_page(&lio->oct_dev->pci_dev->dev, 891 g->sg[(i >> 2)].ptr[(i & 3)], 892 skb_frag_size(frag), DMA_TO_DEVICE); 893 i++; 894 } 895 896 iq = skb_iq(lio->oct_dev, skb); 897 898 spin_lock(&lio->glist_lock[iq]); 899 list_add_tail(&g->list, &lio->glist[iq]); 900 spin_unlock(&lio->glist_lock[iq]); 901 902 /* Don't free the skb yet */ 903 } 904 905 /** 906 * liquidio_open - Net device open for LiquidIO 907 * @netdev: network device 908 */ 909 static int liquidio_open(struct net_device *netdev) 910 { 911 struct lio *lio = GET_LIO(netdev); 912 struct octeon_device *oct = lio->oct_dev; 913 struct octeon_device_priv *oct_priv = 914 (struct octeon_device_priv *)oct->priv; 915 struct napi_struct *napi, *n; 916 int ret = 0; 917 918 if (!oct->props[lio->ifidx].napi_enabled) { 919 tasklet_disable(&oct_priv->droq_tasklet); 920 921 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 922 napi_enable(napi); 923 924 oct->props[lio->ifidx].napi_enabled = 1; 925 926 oct->droq[0]->ops.poll_mode = 1; 927 } 928 929 ifstate_set(lio, LIO_IFSTATE_RUNNING); 930 931 /* Ready for link status updates */ 932 lio->intf_open = 1; 933 934 netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n"); 935 start_txqs(netdev); 936 937 INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats); 938 lio->stats_wk.ctxptr = lio; 939 schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies 940 (LIQUIDIO_NDEV_STATS_POLL_TIME_MS)); 941 942 /* tell Octeon to start forwarding packets to host */ 943 ret = send_rx_ctrl_cmd(lio, 1); 944 if (ret) 945 return ret; 946 947 dev_info(&oct->pci_dev->dev, "%s interface is opened\n", netdev->name); 948 949 return ret; 950 } 951 952 /** 953 * liquidio_stop - jNet device stop for LiquidIO 954 * @netdev: network device 955 */ 956 static int liquidio_stop(struct net_device *netdev) 957 { 958 struct lio *lio = GET_LIO(netdev); 959 struct octeon_device *oct = lio->oct_dev; 960 struct octeon_device_priv *oct_priv = 961 (struct octeon_device_priv *)oct->priv; 962 struct napi_struct *napi, *n; 963 int ret = 0; 964 965 /* tell Octeon to stop forwarding packets to host */ 966 ret = send_rx_ctrl_cmd(lio, 0); 967 if (ret) 968 return ret; 969 970 netif_info(lio, ifdown, lio->netdev, "Stopping interface!\n"); 971 /* Inform that netif carrier is down */ 972 lio->intf_open = 0; 973 lio->linfo.link.s.link_up = 0; 974 975 netif_carrier_off(netdev); 976 lio->link_changes++; 977 978 ifstate_reset(lio, LIO_IFSTATE_RUNNING); 979 980 stop_txqs(netdev); 981 982 /* Wait for any pending Rx descriptors */ 983 if (lio_wait_for_clean_oq(oct)) 984 netif_info(lio, rx_err, lio->netdev, 985 "Proceeding with stop interface after partial RX desc processing\n"); 986 987 if (oct->props[lio->ifidx].napi_enabled == 1) { 988 list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list) 989 napi_disable(napi); 990 991 oct->props[lio->ifidx].napi_enabled = 0; 992 993 oct->droq[0]->ops.poll_mode = 0; 994 995 tasklet_enable(&oct_priv->droq_tasklet); 996 } 997 998 cancel_delayed_work_sync(&lio->stats_wk.work); 999 1000 dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name); 1001 1002 return ret; 1003 } 1004 1005 /** 1006 * get_new_flags - Converts a mask based on net device flags 1007 * @netdev: network device 1008 * 1009 * This routine generates a octnet_ifflags mask from the net device flags 1010 * received from the OS. 1011 */ 1012 static enum octnet_ifflags get_new_flags(struct net_device *netdev) 1013 { 1014 enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST; 1015 1016 if (netdev->flags & IFF_PROMISC) 1017 f |= OCTNET_IFFLAG_PROMISC; 1018 1019 if (netdev->flags & IFF_ALLMULTI) 1020 f |= OCTNET_IFFLAG_ALLMULTI; 1021 1022 if (netdev->flags & IFF_MULTICAST) { 1023 f |= OCTNET_IFFLAG_MULTICAST; 1024 1025 /* Accept all multicast addresses if there are more than we 1026 * can handle 1027 */ 1028 if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR) 1029 f |= OCTNET_IFFLAG_ALLMULTI; 1030 } 1031 1032 if (netdev->flags & IFF_BROADCAST) 1033 f |= OCTNET_IFFLAG_BROADCAST; 1034 1035 return f; 1036 } 1037 1038 static void liquidio_set_uc_list(struct net_device *netdev) 1039 { 1040 struct lio *lio = GET_LIO(netdev); 1041 struct octeon_device *oct = lio->oct_dev; 1042 struct octnic_ctrl_pkt nctrl; 1043 struct netdev_hw_addr *ha; 1044 u64 *mac; 1045 1046 if (lio->netdev_uc_count == netdev_uc_count(netdev)) 1047 return; 1048 1049 if (netdev_uc_count(netdev) > MAX_NCTRL_UDD) { 1050 dev_err(&oct->pci_dev->dev, "too many MAC addresses in netdev uc list\n"); 1051 return; 1052 } 1053 1054 lio->netdev_uc_count = netdev_uc_count(netdev); 1055 1056 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1057 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_UC_LIST; 1058 nctrl.ncmd.s.more = lio->netdev_uc_count; 1059 nctrl.ncmd.s.param1 = oct->vf_num; 1060 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1061 nctrl.netpndev = (u64)netdev; 1062 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1063 1064 /* copy all the addresses into the udd */ 1065 mac = &nctrl.udd[0]; 1066 netdev_for_each_uc_addr(ha, netdev) { 1067 ether_addr_copy(((u8 *)mac) + 2, ha->addr); 1068 mac++; 1069 } 1070 1071 octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1072 } 1073 1074 /** 1075 * liquidio_set_mcast_list - Net device set_multicast_list 1076 * @netdev: network device 1077 */ 1078 static void liquidio_set_mcast_list(struct net_device *netdev) 1079 { 1080 int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR); 1081 struct lio *lio = GET_LIO(netdev); 1082 struct octeon_device *oct = lio->oct_dev; 1083 struct octnic_ctrl_pkt nctrl; 1084 struct netdev_hw_addr *ha; 1085 u64 *mc; 1086 int ret; 1087 1088 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1089 1090 /* Create a ctrl pkt command to be sent to core app. */ 1091 nctrl.ncmd.u64 = 0; 1092 nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST; 1093 nctrl.ncmd.s.param1 = get_new_flags(netdev); 1094 nctrl.ncmd.s.param2 = mc_count; 1095 nctrl.ncmd.s.more = mc_count; 1096 nctrl.netpndev = (u64)netdev; 1097 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1098 1099 /* copy all the addresses into the udd */ 1100 mc = &nctrl.udd[0]; 1101 netdev_for_each_mc_addr(ha, netdev) { 1102 *mc = 0; 1103 ether_addr_copy(((u8 *)mc) + 2, ha->addr); 1104 /* no need to swap bytes */ 1105 if (++mc > &nctrl.udd[mc_count]) 1106 break; 1107 } 1108 1109 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1110 1111 /* Apparently, any activity in this call from the kernel has to 1112 * be atomic. So we won't wait for response. 1113 */ 1114 1115 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1116 if (ret) { 1117 dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n", 1118 ret); 1119 } 1120 1121 liquidio_set_uc_list(netdev); 1122 } 1123 1124 /** 1125 * liquidio_set_mac - Net device set_mac_address 1126 * @netdev: network device 1127 * @p: opaque pointer to sockaddr 1128 */ 1129 static int liquidio_set_mac(struct net_device *netdev, void *p) 1130 { 1131 struct sockaddr *addr = (struct sockaddr *)p; 1132 struct lio *lio = GET_LIO(netdev); 1133 struct octeon_device *oct = lio->oct_dev; 1134 struct octnic_ctrl_pkt nctrl; 1135 int ret = 0; 1136 1137 if (!is_valid_ether_addr(addr->sa_data)) 1138 return -EADDRNOTAVAIL; 1139 1140 if (ether_addr_equal(addr->sa_data, netdev->dev_addr)) 1141 return 0; 1142 1143 if (lio->linfo.macaddr_is_admin_asgnd) 1144 return -EPERM; 1145 1146 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1147 1148 nctrl.ncmd.u64 = 0; 1149 nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR; 1150 nctrl.ncmd.s.param1 = 0; 1151 nctrl.ncmd.s.more = 1; 1152 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1153 nctrl.netpndev = (u64)netdev; 1154 1155 nctrl.udd[0] = 0; 1156 /* The MAC Address is presented in network byte order. */ 1157 ether_addr_copy((u8 *)&nctrl.udd[0] + 2, addr->sa_data); 1158 1159 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1160 if (ret < 0) { 1161 dev_err(&oct->pci_dev->dev, "MAC Address change failed\n"); 1162 return -ENOMEM; 1163 } 1164 1165 if (nctrl.sc_status == 1166 FIRMWARE_STATUS_CODE(OCTEON_REQUEST_NO_PERMISSION)) { 1167 dev_err(&oct->pci_dev->dev, "MAC Address change failed: no permission\n"); 1168 return -EPERM; 1169 } 1170 1171 eth_hw_addr_set(netdev, addr->sa_data); 1172 ether_addr_copy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data); 1173 1174 return 0; 1175 } 1176 1177 static void 1178 liquidio_get_stats64(struct net_device *netdev, 1179 struct rtnl_link_stats64 *lstats) 1180 { 1181 struct lio *lio = GET_LIO(netdev); 1182 struct octeon_device *oct; 1183 u64 pkts = 0, drop = 0, bytes = 0; 1184 struct oct_droq_stats *oq_stats; 1185 struct oct_iq_stats *iq_stats; 1186 int i, iq_no, oq_no; 1187 1188 oct = lio->oct_dev; 1189 1190 if (ifstate_check(lio, LIO_IFSTATE_RESETTING)) 1191 return; 1192 1193 for (i = 0; i < oct->num_iqs; i++) { 1194 iq_no = lio->linfo.txpciq[i].s.q_no; 1195 iq_stats = &oct->instr_queue[iq_no]->stats; 1196 pkts += iq_stats->tx_done; 1197 drop += iq_stats->tx_dropped; 1198 bytes += iq_stats->tx_tot_bytes; 1199 } 1200 1201 lstats->tx_packets = pkts; 1202 lstats->tx_bytes = bytes; 1203 lstats->tx_dropped = drop; 1204 1205 pkts = 0; 1206 drop = 0; 1207 bytes = 0; 1208 1209 for (i = 0; i < oct->num_oqs; i++) { 1210 oq_no = lio->linfo.rxpciq[i].s.q_no; 1211 oq_stats = &oct->droq[oq_no]->stats; 1212 pkts += oq_stats->rx_pkts_received; 1213 drop += (oq_stats->rx_dropped + 1214 oq_stats->dropped_nodispatch + 1215 oq_stats->dropped_toomany + 1216 oq_stats->dropped_nomem); 1217 bytes += oq_stats->rx_bytes_received; 1218 } 1219 1220 lstats->rx_bytes = bytes; 1221 lstats->rx_packets = pkts; 1222 lstats->rx_dropped = drop; 1223 1224 lstats->multicast = oct->link_stats.fromwire.fw_total_mcast; 1225 1226 /* detailed rx_errors: */ 1227 lstats->rx_length_errors = oct->link_stats.fromwire.l2_err; 1228 /* recved pkt with crc error */ 1229 lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err; 1230 /* recv'd frame alignment error */ 1231 lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err; 1232 1233 lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors + 1234 lstats->rx_frame_errors; 1235 1236 /* detailed tx_errors */ 1237 lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko; 1238 lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link; 1239 1240 lstats->tx_errors = lstats->tx_aborted_errors + 1241 lstats->tx_carrier_errors; 1242 } 1243 1244 /** 1245 * hwtstamp_ioctl - Handler for SIOCSHWTSTAMP ioctl 1246 * @netdev: network device 1247 * @ifr: interface request 1248 */ 1249 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr) 1250 { 1251 struct lio *lio = GET_LIO(netdev); 1252 struct hwtstamp_config conf; 1253 1254 if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf))) 1255 return -EFAULT; 1256 1257 switch (conf.tx_type) { 1258 case HWTSTAMP_TX_ON: 1259 case HWTSTAMP_TX_OFF: 1260 break; 1261 default: 1262 return -ERANGE; 1263 } 1264 1265 switch (conf.rx_filter) { 1266 case HWTSTAMP_FILTER_NONE: 1267 break; 1268 case HWTSTAMP_FILTER_ALL: 1269 case HWTSTAMP_FILTER_SOME: 1270 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 1271 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 1272 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 1273 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 1274 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 1275 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 1276 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 1277 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 1278 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 1279 case HWTSTAMP_FILTER_PTP_V2_EVENT: 1280 case HWTSTAMP_FILTER_PTP_V2_SYNC: 1281 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 1282 case HWTSTAMP_FILTER_NTP_ALL: 1283 conf.rx_filter = HWTSTAMP_FILTER_ALL; 1284 break; 1285 default: 1286 return -ERANGE; 1287 } 1288 1289 if (conf.rx_filter == HWTSTAMP_FILTER_ALL) 1290 ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED); 1291 1292 else 1293 ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED); 1294 1295 return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0; 1296 } 1297 1298 /** 1299 * liquidio_ioctl - ioctl handler 1300 * @netdev: network device 1301 * @ifr: interface request 1302 * @cmd: command 1303 */ 1304 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 1305 { 1306 switch (cmd) { 1307 case SIOCSHWTSTAMP: 1308 return hwtstamp_ioctl(netdev, ifr); 1309 default: 1310 return -EOPNOTSUPP; 1311 } 1312 } 1313 1314 static void handle_timestamp(struct octeon_device *oct, u32 status, void *buf) 1315 { 1316 struct sk_buff *skb = (struct sk_buff *)buf; 1317 struct octnet_buf_free_info *finfo; 1318 struct oct_timestamp_resp *resp; 1319 struct octeon_soft_command *sc; 1320 struct lio *lio; 1321 1322 finfo = (struct octnet_buf_free_info *)skb->cb; 1323 lio = finfo->lio; 1324 sc = finfo->sc; 1325 oct = lio->oct_dev; 1326 resp = (struct oct_timestamp_resp *)sc->virtrptr; 1327 1328 if (status != OCTEON_REQUEST_DONE) { 1329 dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n", 1330 CVM_CAST64(status)); 1331 resp->timestamp = 0; 1332 } 1333 1334 octeon_swap_8B_data(&resp->timestamp, 1); 1335 1336 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)) { 1337 struct skb_shared_hwtstamps ts; 1338 u64 ns = resp->timestamp; 1339 1340 netif_info(lio, tx_done, lio->netdev, 1341 "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n", 1342 skb, (unsigned long long)ns); 1343 ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust); 1344 skb_tstamp_tx(skb, &ts); 1345 } 1346 1347 octeon_free_soft_command(oct, sc); 1348 tx_buffer_free(skb); 1349 } 1350 1351 /* send_nic_timestamp_pkt - Send a data packet that will be timestamped 1352 * @oct: octeon device 1353 * @ndata: pointer to network data 1354 * @finfo: pointer to private network data 1355 */ 1356 static int send_nic_timestamp_pkt(struct octeon_device *oct, 1357 struct octnic_data_pkt *ndata, 1358 struct octnet_buf_free_info *finfo, 1359 int xmit_more) 1360 { 1361 struct octeon_soft_command *sc; 1362 int ring_doorbell; 1363 struct lio *lio; 1364 int retval; 1365 u32 len; 1366 1367 lio = finfo->lio; 1368 1369 sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd, 1370 sizeof(struct oct_timestamp_resp)); 1371 finfo->sc = sc; 1372 1373 if (!sc) { 1374 dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n"); 1375 return IQ_SEND_FAILED; 1376 } 1377 1378 if (ndata->reqtype == REQTYPE_NORESP_NET) 1379 ndata->reqtype = REQTYPE_RESP_NET; 1380 else if (ndata->reqtype == REQTYPE_NORESP_NET_SG) 1381 ndata->reqtype = REQTYPE_RESP_NET_SG; 1382 1383 sc->callback = handle_timestamp; 1384 sc->callback_arg = finfo->skb; 1385 sc->iq_no = ndata->q_no; 1386 1387 len = (u32)((struct octeon_instr_ih3 *)(&sc->cmd.cmd3.ih3))->dlengsz; 1388 1389 ring_doorbell = !xmit_more; 1390 1391 retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd, 1392 sc, len, ndata->reqtype); 1393 1394 if (retval == IQ_SEND_FAILED) { 1395 dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n", 1396 retval); 1397 octeon_free_soft_command(oct, sc); 1398 } else { 1399 netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n"); 1400 } 1401 1402 return retval; 1403 } 1404 1405 /** 1406 * liquidio_xmit - Transmit networks packets to the Octeon interface 1407 * @skb: skbuff struct to be passed to network layer. 1408 * @netdev: pointer to network device 1409 * @returns whether the packet was transmitted to the device okay or not 1410 * (NETDEV_TX_OK or NETDEV_TX_BUSY) 1411 */ 1412 static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev) 1413 { 1414 struct octnet_buf_free_info *finfo; 1415 union octnic_cmd_setup cmdsetup; 1416 struct octnic_data_pkt ndata; 1417 struct octeon_instr_irh *irh; 1418 struct oct_iq_stats *stats; 1419 struct octeon_device *oct; 1420 int q_idx = 0, iq_no = 0; 1421 union tx_info *tx_info; 1422 int xmit_more = 0; 1423 struct lio *lio; 1424 int status = 0; 1425 u64 dptr = 0; 1426 u32 tag = 0; 1427 int j; 1428 1429 lio = GET_LIO(netdev); 1430 oct = lio->oct_dev; 1431 1432 q_idx = skb_iq(lio->oct_dev, skb); 1433 tag = q_idx; 1434 iq_no = lio->linfo.txpciq[q_idx].s.q_no; 1435 1436 stats = &oct->instr_queue[iq_no]->stats; 1437 1438 /* Check for all conditions in which the current packet cannot be 1439 * transmitted. 1440 */ 1441 if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) || 1442 (!lio->linfo.link.s.link_up) || (skb->len <= 0)) { 1443 netif_info(lio, tx_err, lio->netdev, "Transmit failed link_status : %d\n", 1444 lio->linfo.link.s.link_up); 1445 goto lio_xmit_failed; 1446 } 1447 1448 /* Use space in skb->cb to store info used to unmap and 1449 * free the buffers. 1450 */ 1451 finfo = (struct octnet_buf_free_info *)skb->cb; 1452 finfo->lio = lio; 1453 finfo->skb = skb; 1454 finfo->sc = NULL; 1455 1456 /* Prepare the attributes for the data to be passed to OSI. */ 1457 memset(&ndata, 0, sizeof(struct octnic_data_pkt)); 1458 1459 ndata.buf = finfo; 1460 1461 ndata.q_no = iq_no; 1462 1463 if (octnet_iq_is_full(oct, ndata.q_no)) { 1464 /* defer sending if queue is full */ 1465 netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n", 1466 ndata.q_no); 1467 stats->tx_iq_busy++; 1468 return NETDEV_TX_BUSY; 1469 } 1470 1471 ndata.datasize = skb->len; 1472 1473 cmdsetup.u64 = 0; 1474 cmdsetup.s.iq_no = iq_no; 1475 1476 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1477 if (skb->encapsulation) { 1478 cmdsetup.s.tnl_csum = 1; 1479 stats->tx_vxlan++; 1480 } else { 1481 cmdsetup.s.transport_csum = 1; 1482 } 1483 } 1484 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { 1485 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 1486 cmdsetup.s.timestamp = 1; 1487 } 1488 1489 if (!skb_shinfo(skb)->nr_frags) { 1490 cmdsetup.s.u.datasize = skb->len; 1491 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag); 1492 /* Offload checksum calculation for TCP/UDP packets */ 1493 dptr = dma_map_single(&oct->pci_dev->dev, 1494 skb->data, 1495 skb->len, 1496 DMA_TO_DEVICE); 1497 if (dma_mapping_error(&oct->pci_dev->dev, dptr)) { 1498 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n", 1499 __func__); 1500 return NETDEV_TX_BUSY; 1501 } 1502 1503 ndata.cmd.cmd3.dptr = dptr; 1504 finfo->dptr = dptr; 1505 ndata.reqtype = REQTYPE_NORESP_NET; 1506 1507 } else { 1508 skb_frag_t *frag; 1509 struct octnic_gather *g; 1510 int i, frags; 1511 1512 spin_lock(&lio->glist_lock[q_idx]); 1513 g = (struct octnic_gather *) 1514 lio_list_delete_head(&lio->glist[q_idx]); 1515 spin_unlock(&lio->glist_lock[q_idx]); 1516 1517 if (!g) { 1518 netif_info(lio, tx_err, lio->netdev, 1519 "Transmit scatter gather: glist null!\n"); 1520 goto lio_xmit_failed; 1521 } 1522 1523 cmdsetup.s.gather = 1; 1524 cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1); 1525 octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag); 1526 1527 memset(g->sg, 0, g->sg_size); 1528 1529 g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev, 1530 skb->data, 1531 (skb->len - skb->data_len), 1532 DMA_TO_DEVICE); 1533 if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) { 1534 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n", 1535 __func__); 1536 return NETDEV_TX_BUSY; 1537 } 1538 add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0); 1539 1540 frags = skb_shinfo(skb)->nr_frags; 1541 i = 1; 1542 while (frags--) { 1543 frag = &skb_shinfo(skb)->frags[i - 1]; 1544 1545 g->sg[(i >> 2)].ptr[(i & 3)] = 1546 skb_frag_dma_map(&oct->pci_dev->dev, 1547 frag, 0, skb_frag_size(frag), 1548 DMA_TO_DEVICE); 1549 if (dma_mapping_error(&oct->pci_dev->dev, 1550 g->sg[i >> 2].ptr[i & 3])) { 1551 dma_unmap_single(&oct->pci_dev->dev, 1552 g->sg[0].ptr[0], 1553 skb->len - skb->data_len, 1554 DMA_TO_DEVICE); 1555 for (j = 1; j < i; j++) { 1556 frag = &skb_shinfo(skb)->frags[j - 1]; 1557 dma_unmap_page(&oct->pci_dev->dev, 1558 g->sg[j >> 2].ptr[j & 3], 1559 skb_frag_size(frag), 1560 DMA_TO_DEVICE); 1561 } 1562 dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n", 1563 __func__); 1564 return NETDEV_TX_BUSY; 1565 } 1566 1567 add_sg_size(&g->sg[(i >> 2)], skb_frag_size(frag), 1568 (i & 3)); 1569 i++; 1570 } 1571 1572 dptr = g->sg_dma_ptr; 1573 1574 ndata.cmd.cmd3.dptr = dptr; 1575 finfo->dptr = dptr; 1576 finfo->g = g; 1577 1578 ndata.reqtype = REQTYPE_NORESP_NET_SG; 1579 } 1580 1581 irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh; 1582 tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0]; 1583 1584 if (skb_shinfo(skb)->gso_size) { 1585 tx_info->s.gso_size = skb_shinfo(skb)->gso_size; 1586 tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs; 1587 } 1588 1589 /* HW insert VLAN tag */ 1590 if (skb_vlan_tag_present(skb)) { 1591 irh->priority = skb_vlan_tag_get(skb) >> VLAN_PRIO_SHIFT; 1592 irh->vlan = skb_vlan_tag_get(skb) & VLAN_VID_MASK; 1593 } 1594 1595 xmit_more = netdev_xmit_more(); 1596 1597 if (unlikely(cmdsetup.s.timestamp)) 1598 status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more); 1599 else 1600 status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more); 1601 if (status == IQ_SEND_FAILED) 1602 goto lio_xmit_failed; 1603 1604 netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n"); 1605 1606 if (status == IQ_SEND_STOP) { 1607 dev_err(&oct->pci_dev->dev, "Rcvd IQ_SEND_STOP signal; stopping IQ-%d\n", 1608 iq_no); 1609 netif_stop_subqueue(netdev, q_idx); 1610 } 1611 1612 netif_trans_update(netdev); 1613 1614 if (tx_info->s.gso_segs) 1615 stats->tx_done += tx_info->s.gso_segs; 1616 else 1617 stats->tx_done++; 1618 stats->tx_tot_bytes += ndata.datasize; 1619 1620 return NETDEV_TX_OK; 1621 1622 lio_xmit_failed: 1623 stats->tx_dropped++; 1624 netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n", 1625 iq_no, stats->tx_dropped); 1626 if (dptr) 1627 dma_unmap_single(&oct->pci_dev->dev, dptr, 1628 ndata.datasize, DMA_TO_DEVICE); 1629 1630 octeon_ring_doorbell_locked(oct, iq_no); 1631 1632 tx_buffer_free(skb); 1633 return NETDEV_TX_OK; 1634 } 1635 1636 /** 1637 * liquidio_tx_timeout - Network device Tx timeout 1638 * @netdev: pointer to network device 1639 * @txqueue: index of the hung transmit queue 1640 */ 1641 static void liquidio_tx_timeout(struct net_device *netdev, unsigned int txqueue) 1642 { 1643 struct lio *lio; 1644 1645 lio = GET_LIO(netdev); 1646 1647 netif_info(lio, tx_err, lio->netdev, 1648 "Transmit timeout tx_dropped:%ld, waking up queues now!!\n", 1649 netdev->stats.tx_dropped); 1650 netif_trans_update(netdev); 1651 wake_txqs(netdev); 1652 } 1653 1654 static int 1655 liquidio_vlan_rx_add_vid(struct net_device *netdev, 1656 __be16 proto __attribute__((unused)), u16 vid) 1657 { 1658 struct lio *lio = GET_LIO(netdev); 1659 struct octeon_device *oct = lio->oct_dev; 1660 struct octnic_ctrl_pkt nctrl; 1661 int ret = 0; 1662 1663 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1664 1665 nctrl.ncmd.u64 = 0; 1666 nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER; 1667 nctrl.ncmd.s.param1 = vid; 1668 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1669 nctrl.netpndev = (u64)netdev; 1670 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1671 1672 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1673 if (ret) { 1674 dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n", 1675 ret); 1676 return -EPERM; 1677 } 1678 1679 return 0; 1680 } 1681 1682 static int 1683 liquidio_vlan_rx_kill_vid(struct net_device *netdev, 1684 __be16 proto __attribute__((unused)), u16 vid) 1685 { 1686 struct lio *lio = GET_LIO(netdev); 1687 struct octeon_device *oct = lio->oct_dev; 1688 struct octnic_ctrl_pkt nctrl; 1689 int ret = 0; 1690 1691 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1692 1693 nctrl.ncmd.u64 = 0; 1694 nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER; 1695 nctrl.ncmd.s.param1 = vid; 1696 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1697 nctrl.netpndev = (u64)netdev; 1698 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1699 1700 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1701 if (ret) { 1702 dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n", 1703 ret); 1704 if (ret > 0) 1705 ret = -EIO; 1706 } 1707 return ret; 1708 } 1709 1710 /** Sending command to enable/disable RX checksum offload 1711 * @param netdev pointer to network device 1712 * @param command OCTNET_CMD_TNL_RX_CSUM_CTL 1713 * @param rx_cmd_bit OCTNET_CMD_RXCSUM_ENABLE/ 1714 * OCTNET_CMD_RXCSUM_DISABLE 1715 * @returns SUCCESS or FAILURE 1716 */ 1717 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command, 1718 u8 rx_cmd) 1719 { 1720 struct lio *lio = GET_LIO(netdev); 1721 struct octeon_device *oct = lio->oct_dev; 1722 struct octnic_ctrl_pkt nctrl; 1723 int ret = 0; 1724 1725 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1726 1727 nctrl.ncmd.u64 = 0; 1728 nctrl.ncmd.s.cmd = command; 1729 nctrl.ncmd.s.param1 = rx_cmd; 1730 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1731 nctrl.netpndev = (u64)netdev; 1732 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1733 1734 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1735 if (ret) { 1736 dev_err(&oct->pci_dev->dev, "DEVFLAGS RXCSUM change failed in core (ret:0x%x)\n", 1737 ret); 1738 if (ret > 0) 1739 ret = -EIO; 1740 } 1741 return ret; 1742 } 1743 1744 /** Sending command to add/delete VxLAN UDP port to firmware 1745 * @param netdev pointer to network device 1746 * @param command OCTNET_CMD_VXLAN_PORT_CONFIG 1747 * @param vxlan_port VxLAN port to be added or deleted 1748 * @param vxlan_cmd_bit OCTNET_CMD_VXLAN_PORT_ADD, 1749 * OCTNET_CMD_VXLAN_PORT_DEL 1750 * @returns SUCCESS or FAILURE 1751 */ 1752 static int liquidio_vxlan_port_command(struct net_device *netdev, int command, 1753 u16 vxlan_port, u8 vxlan_cmd_bit) 1754 { 1755 struct lio *lio = GET_LIO(netdev); 1756 struct octeon_device *oct = lio->oct_dev; 1757 struct octnic_ctrl_pkt nctrl; 1758 int ret = 0; 1759 1760 memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt)); 1761 1762 nctrl.ncmd.u64 = 0; 1763 nctrl.ncmd.s.cmd = command; 1764 nctrl.ncmd.s.more = vxlan_cmd_bit; 1765 nctrl.ncmd.s.param1 = vxlan_port; 1766 nctrl.iq_no = lio->linfo.txpciq[0].s.q_no; 1767 nctrl.netpndev = (u64)netdev; 1768 nctrl.cb_fn = liquidio_link_ctrl_cmd_completion; 1769 1770 ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl); 1771 if (ret) { 1772 dev_err(&oct->pci_dev->dev, 1773 "DEVFLAGS VxLAN port add/delete failed in core (ret : 0x%x)\n", 1774 ret); 1775 if (ret > 0) 1776 ret = -EIO; 1777 } 1778 return ret; 1779 } 1780 1781 static int liquidio_udp_tunnel_set_port(struct net_device *netdev, 1782 unsigned int table, unsigned int entry, 1783 struct udp_tunnel_info *ti) 1784 { 1785 return liquidio_vxlan_port_command(netdev, 1786 OCTNET_CMD_VXLAN_PORT_CONFIG, 1787 htons(ti->port), 1788 OCTNET_CMD_VXLAN_PORT_ADD); 1789 } 1790 1791 static int liquidio_udp_tunnel_unset_port(struct net_device *netdev, 1792 unsigned int table, 1793 unsigned int entry, 1794 struct udp_tunnel_info *ti) 1795 { 1796 return liquidio_vxlan_port_command(netdev, 1797 OCTNET_CMD_VXLAN_PORT_CONFIG, 1798 htons(ti->port), 1799 OCTNET_CMD_VXLAN_PORT_DEL); 1800 } 1801 1802 static const struct udp_tunnel_nic_info liquidio_udp_tunnels = { 1803 .set_port = liquidio_udp_tunnel_set_port, 1804 .unset_port = liquidio_udp_tunnel_unset_port, 1805 .tables = { 1806 { .n_entries = 1024, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 1807 }, 1808 }; 1809 1810 /** \brief Net device fix features 1811 * @param netdev pointer to network device 1812 * @param request features requested 1813 * @returns updated features list 1814 */ 1815 static netdev_features_t liquidio_fix_features(struct net_device *netdev, 1816 netdev_features_t request) 1817 { 1818 struct lio *lio = netdev_priv(netdev); 1819 1820 if ((request & NETIF_F_RXCSUM) && 1821 !(lio->dev_capability & NETIF_F_RXCSUM)) 1822 request &= ~NETIF_F_RXCSUM; 1823 1824 if ((request & NETIF_F_HW_CSUM) && 1825 !(lio->dev_capability & NETIF_F_HW_CSUM)) 1826 request &= ~NETIF_F_HW_CSUM; 1827 1828 if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO)) 1829 request &= ~NETIF_F_TSO; 1830 1831 if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6)) 1832 request &= ~NETIF_F_TSO6; 1833 1834 if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO)) 1835 request &= ~NETIF_F_LRO; 1836 1837 /* Disable LRO if RXCSUM is off */ 1838 if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) && 1839 (lio->dev_capability & NETIF_F_LRO)) 1840 request &= ~NETIF_F_LRO; 1841 1842 return request; 1843 } 1844 1845 /** \brief Net device set features 1846 * @param netdev pointer to network device 1847 * @param features features to enable/disable 1848 */ 1849 static int liquidio_set_features(struct net_device *netdev, 1850 netdev_features_t features) 1851 { 1852 struct lio *lio = netdev_priv(netdev); 1853 1854 if (!((netdev->features ^ features) & NETIF_F_LRO)) 1855 return 0; 1856 1857 if ((features & NETIF_F_LRO) && (lio->dev_capability & NETIF_F_LRO)) 1858 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE, 1859 OCTNIC_LROIPV4 | OCTNIC_LROIPV6); 1860 else if (!(features & NETIF_F_LRO) && 1861 (lio->dev_capability & NETIF_F_LRO)) 1862 liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE, 1863 OCTNIC_LROIPV4 | OCTNIC_LROIPV6); 1864 if (!(netdev->features & NETIF_F_RXCSUM) && 1865 (lio->enc_dev_capability & NETIF_F_RXCSUM) && 1866 (features & NETIF_F_RXCSUM)) 1867 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL, 1868 OCTNET_CMD_RXCSUM_ENABLE); 1869 else if ((netdev->features & NETIF_F_RXCSUM) && 1870 (lio->enc_dev_capability & NETIF_F_RXCSUM) && 1871 !(features & NETIF_F_RXCSUM)) 1872 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL, 1873 OCTNET_CMD_RXCSUM_DISABLE); 1874 1875 return 0; 1876 } 1877 1878 static const struct net_device_ops lionetdevops = { 1879 .ndo_open = liquidio_open, 1880 .ndo_stop = liquidio_stop, 1881 .ndo_start_xmit = liquidio_xmit, 1882 .ndo_get_stats64 = liquidio_get_stats64, 1883 .ndo_set_mac_address = liquidio_set_mac, 1884 .ndo_set_rx_mode = liquidio_set_mcast_list, 1885 .ndo_tx_timeout = liquidio_tx_timeout, 1886 .ndo_vlan_rx_add_vid = liquidio_vlan_rx_add_vid, 1887 .ndo_vlan_rx_kill_vid = liquidio_vlan_rx_kill_vid, 1888 .ndo_change_mtu = liquidio_change_mtu, 1889 .ndo_eth_ioctl = liquidio_ioctl, 1890 .ndo_fix_features = liquidio_fix_features, 1891 .ndo_set_features = liquidio_set_features, 1892 }; 1893 1894 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf) 1895 { 1896 struct octeon_device *oct = (struct octeon_device *)buf; 1897 struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt; 1898 union oct_link_status *ls; 1899 int gmxport = 0; 1900 int i; 1901 1902 if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) { 1903 dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n", 1904 recv_pkt->buffer_size[0], 1905 recv_pkt->rh.r_nic_info.gmxport); 1906 goto nic_info_err; 1907 } 1908 1909 gmxport = recv_pkt->rh.r_nic_info.gmxport; 1910 ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) + 1911 OCT_DROQ_INFO_SIZE); 1912 1913 octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3); 1914 1915 for (i = 0; i < oct->ifcount; i++) { 1916 if (oct->props[i].gmxport == gmxport) { 1917 update_link_status(oct->props[i].netdev, ls); 1918 break; 1919 } 1920 } 1921 1922 nic_info_err: 1923 for (i = 0; i < recv_pkt->buffer_count; i++) 1924 recv_buffer_free(recv_pkt->buffer_ptr[i]); 1925 octeon_free_recv_info(recv_info); 1926 return 0; 1927 } 1928 1929 /** 1930 * setup_nic_devices - Setup network interfaces 1931 * @octeon_dev: octeon device 1932 * 1933 * Called during init time for each device. It assumes the NIC 1934 * is already up and running. The link information for each 1935 * interface is passed in link_info. 1936 */ 1937 static int setup_nic_devices(struct octeon_device *octeon_dev) 1938 { 1939 int retval, num_iqueues, num_oqueues; 1940 u32 resp_size, data_size; 1941 struct liquidio_if_cfg_resp *resp; 1942 struct octeon_soft_command *sc; 1943 union oct_nic_if_cfg if_cfg; 1944 struct octdev_props *props; 1945 struct net_device *netdev; 1946 struct lio_version *vdata; 1947 struct lio *lio = NULL; 1948 u8 mac[ETH_ALEN], i, j; 1949 u32 ifidx_or_pfnum; 1950 1951 ifidx_or_pfnum = octeon_dev->pf_num; 1952 1953 /* This is to handle link status changes */ 1954 octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC, OPCODE_NIC_INFO, 1955 lio_nic_info, octeon_dev); 1956 1957 /* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions. 1958 * They are handled directly. 1959 */ 1960 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET, 1961 free_netbuf); 1962 1963 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG, 1964 free_netsgbuf); 1965 1966 octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG, 1967 free_netsgbuf_with_resp); 1968 1969 for (i = 0; i < octeon_dev->ifcount; i++) { 1970 resp_size = sizeof(struct liquidio_if_cfg_resp); 1971 data_size = sizeof(struct lio_version); 1972 sc = (struct octeon_soft_command *) 1973 octeon_alloc_soft_command(octeon_dev, data_size, 1974 resp_size, 0); 1975 resp = (struct liquidio_if_cfg_resp *)sc->virtrptr; 1976 vdata = (struct lio_version *)sc->virtdptr; 1977 1978 *((u64 *)vdata) = 0; 1979 vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION); 1980 vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION); 1981 vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION); 1982 1983 if_cfg.u64 = 0; 1984 1985 if_cfg.s.num_iqueues = octeon_dev->sriov_info.rings_per_vf; 1986 if_cfg.s.num_oqueues = octeon_dev->sriov_info.rings_per_vf; 1987 if_cfg.s.base_queue = 0; 1988 1989 sc->iq_no = 0; 1990 1991 octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC, 1992 OPCODE_NIC_IF_CFG, 0, if_cfg.u64, 1993 0); 1994 1995 init_completion(&sc->complete); 1996 sc->sc_status = OCTEON_REQUEST_PENDING; 1997 1998 retval = octeon_send_soft_command(octeon_dev, sc); 1999 if (retval == IQ_SEND_FAILED) { 2000 dev_err(&octeon_dev->pci_dev->dev, 2001 "iq/oq config failed status: %x\n", retval); 2002 /* Soft instr is freed by driver in case of failure. */ 2003 octeon_free_soft_command(octeon_dev, sc); 2004 return(-EIO); 2005 } 2006 2007 /* Sleep on a wait queue till the cond flag indicates that the 2008 * response arrived or timed-out. 2009 */ 2010 retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0); 2011 if (retval) 2012 return retval; 2013 2014 retval = resp->status; 2015 if (retval) { 2016 dev_err(&octeon_dev->pci_dev->dev, 2017 "iq/oq config failed, retval = %d\n", retval); 2018 WRITE_ONCE(sc->caller_is_done, true); 2019 return -EIO; 2020 } 2021 2022 snprintf(octeon_dev->fw_info.liquidio_firmware_version, 2023 32, "%s", 2024 resp->cfg_info.liquidio_firmware_version); 2025 2026 octeon_swap_8B_data((u64 *)(&resp->cfg_info), 2027 (sizeof(struct liquidio_if_cfg_info)) >> 3); 2028 2029 num_iqueues = hweight64(resp->cfg_info.iqmask); 2030 num_oqueues = hweight64(resp->cfg_info.oqmask); 2031 2032 if (!(num_iqueues) || !(num_oqueues)) { 2033 dev_err(&octeon_dev->pci_dev->dev, 2034 "Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n", 2035 resp->cfg_info.iqmask, resp->cfg_info.oqmask); 2036 WRITE_ONCE(sc->caller_is_done, true); 2037 goto setup_nic_dev_done; 2038 } 2039 dev_dbg(&octeon_dev->pci_dev->dev, 2040 "interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d\n", 2041 i, resp->cfg_info.iqmask, resp->cfg_info.oqmask, 2042 num_iqueues, num_oqueues); 2043 2044 netdev = alloc_etherdev_mq(LIO_SIZE, num_iqueues); 2045 2046 if (!netdev) { 2047 dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n"); 2048 WRITE_ONCE(sc->caller_is_done, true); 2049 goto setup_nic_dev_done; 2050 } 2051 2052 SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev); 2053 2054 /* Associate the routines that will handle different 2055 * netdev tasks. 2056 */ 2057 netdev->netdev_ops = &lionetdevops; 2058 2059 lio = GET_LIO(netdev); 2060 2061 memset(lio, 0, sizeof(struct lio)); 2062 2063 lio->ifidx = ifidx_or_pfnum; 2064 2065 props = &octeon_dev->props[i]; 2066 props->gmxport = resp->cfg_info.linfo.gmxport; 2067 props->netdev = netdev; 2068 2069 lio->linfo.num_rxpciq = num_oqueues; 2070 lio->linfo.num_txpciq = num_iqueues; 2071 2072 for (j = 0; j < num_oqueues; j++) { 2073 lio->linfo.rxpciq[j].u64 = 2074 resp->cfg_info.linfo.rxpciq[j].u64; 2075 } 2076 for (j = 0; j < num_iqueues; j++) { 2077 lio->linfo.txpciq[j].u64 = 2078 resp->cfg_info.linfo.txpciq[j].u64; 2079 } 2080 2081 lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr; 2082 lio->linfo.gmxport = resp->cfg_info.linfo.gmxport; 2083 lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64; 2084 lio->linfo.macaddr_is_admin_asgnd = 2085 resp->cfg_info.linfo.macaddr_is_admin_asgnd; 2086 lio->linfo.macaddr_spoofchk = 2087 resp->cfg_info.linfo.macaddr_spoofchk; 2088 2089 lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 2090 2091 lio->dev_capability = NETIF_F_HIGHDMA 2092 | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM 2093 | NETIF_F_SG | NETIF_F_RXCSUM 2094 | NETIF_F_TSO | NETIF_F_TSO6 2095 | NETIF_F_GRO 2096 | NETIF_F_LRO; 2097 netif_set_tso_max_size(netdev, OCTNIC_GSO_MAX_SIZE); 2098 2099 /* Copy of transmit encapsulation capabilities: 2100 * TSO, TSO6, Checksums for this device 2101 */ 2102 lio->enc_dev_capability = NETIF_F_IP_CSUM 2103 | NETIF_F_IPV6_CSUM 2104 | NETIF_F_GSO_UDP_TUNNEL 2105 | NETIF_F_HW_CSUM | NETIF_F_SG 2106 | NETIF_F_RXCSUM 2107 | NETIF_F_TSO | NETIF_F_TSO6 2108 | NETIF_F_LRO; 2109 2110 netdev->hw_enc_features = 2111 (lio->enc_dev_capability & ~NETIF_F_LRO); 2112 netdev->udp_tunnel_nic_info = &liquidio_udp_tunnels; 2113 2114 netdev->vlan_features = lio->dev_capability; 2115 /* Add any unchangeable hw features */ 2116 lio->dev_capability |= NETIF_F_HW_VLAN_CTAG_FILTER | 2117 NETIF_F_HW_VLAN_CTAG_RX | 2118 NETIF_F_HW_VLAN_CTAG_TX; 2119 2120 netdev->features = (lio->dev_capability & ~NETIF_F_LRO); 2121 2122 netdev->hw_features = lio->dev_capability; 2123 netdev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_RX; 2124 2125 /* MTU range: 68 - 16000 */ 2126 netdev->min_mtu = LIO_MIN_MTU_SIZE; 2127 netdev->max_mtu = LIO_MAX_MTU_SIZE; 2128 2129 WRITE_ONCE(sc->caller_is_done, true); 2130 2131 /* Point to the properties for octeon device to which this 2132 * interface belongs. 2133 */ 2134 lio->oct_dev = octeon_dev; 2135 lio->octprops = props; 2136 lio->netdev = netdev; 2137 2138 dev_dbg(&octeon_dev->pci_dev->dev, 2139 "if%d gmx: %d hw_addr: 0x%llx\n", i, 2140 lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr)); 2141 2142 /* 64-bit swap required on LE machines */ 2143 octeon_swap_8B_data(&lio->linfo.hw_addr, 1); 2144 for (j = 0; j < ETH_ALEN; j++) 2145 mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j)); 2146 2147 /* Copy MAC Address to OS network device structure */ 2148 eth_hw_addr_set(netdev, mac); 2149 2150 if (liquidio_setup_io_queues(octeon_dev, i, 2151 lio->linfo.num_txpciq, 2152 lio->linfo.num_rxpciq)) { 2153 dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n"); 2154 goto setup_nic_dev_free; 2155 } 2156 2157 ifstate_set(lio, LIO_IFSTATE_DROQ_OPS); 2158 2159 /* For VFs, enable Octeon device interrupts here, 2160 * as this is contingent upon IO queue setup 2161 */ 2162 octeon_dev->fn_list.enable_interrupt(octeon_dev, 2163 OCTEON_ALL_INTR); 2164 2165 /* By default all interfaces on a single Octeon uses the same 2166 * tx and rx queues 2167 */ 2168 lio->txq = lio->linfo.txpciq[0].s.q_no; 2169 lio->rxq = lio->linfo.rxpciq[0].s.q_no; 2170 2171 lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq); 2172 lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq); 2173 2174 if (lio_setup_glists(octeon_dev, lio, num_iqueues)) { 2175 dev_err(&octeon_dev->pci_dev->dev, 2176 "Gather list allocation failed\n"); 2177 goto setup_nic_dev_free; 2178 } 2179 2180 /* Register ethtool support */ 2181 liquidio_set_ethtool_ops(netdev); 2182 if (lio->oct_dev->chip_id == OCTEON_CN23XX_VF_VID) 2183 octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT; 2184 else 2185 octeon_dev->priv_flags = 0x0; 2186 2187 if (netdev->features & NETIF_F_LRO) 2188 liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE, 2189 OCTNIC_LROIPV4 | OCTNIC_LROIPV6); 2190 2191 if (setup_link_status_change_wq(netdev)) 2192 goto setup_nic_dev_free; 2193 2194 if (setup_rx_oom_poll_fn(netdev)) 2195 goto setup_nic_dev_free; 2196 2197 /* Register the network device with the OS */ 2198 if (register_netdev(netdev)) { 2199 dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n"); 2200 goto setup_nic_dev_free; 2201 } 2202 2203 dev_dbg(&octeon_dev->pci_dev->dev, 2204 "Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n", 2205 i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]); 2206 netif_carrier_off(netdev); 2207 lio->link_changes++; 2208 2209 ifstate_set(lio, LIO_IFSTATE_REGISTERED); 2210 2211 /* Sending command to firmware to enable Rx checksum offload 2212 * by default at the time of setup of Liquidio driver for 2213 * this device 2214 */ 2215 liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL, 2216 OCTNET_CMD_RXCSUM_ENABLE); 2217 liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL, 2218 OCTNET_CMD_TXCSUM_ENABLE); 2219 2220 dev_dbg(&octeon_dev->pci_dev->dev, 2221 "NIC ifidx:%d Setup successful\n", i); 2222 2223 octeon_dev->no_speed_setting = 1; 2224 } 2225 2226 return 0; 2227 2228 setup_nic_dev_free: 2229 2230 while (i--) { 2231 dev_err(&octeon_dev->pci_dev->dev, 2232 "NIC ifidx:%d Setup failed\n", i); 2233 liquidio_destroy_nic_device(octeon_dev, i); 2234 } 2235 2236 setup_nic_dev_done: 2237 2238 return -ENODEV; 2239 } 2240 2241 /** 2242 * liquidio_init_nic_module - initialize the NIC 2243 * @oct: octeon device 2244 * 2245 * This initialization routine is called once the Octeon device application is 2246 * up and running 2247 */ 2248 static int liquidio_init_nic_module(struct octeon_device *oct) 2249 { 2250 int num_nic_ports = 1; 2251 int i, retval = 0; 2252 2253 dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n"); 2254 2255 /* only default iq and oq were initialized 2256 * initialize the rest as well run port_config command for each port 2257 */ 2258 oct->ifcount = num_nic_ports; 2259 memset(oct->props, 0, 2260 sizeof(struct octdev_props) * num_nic_ports); 2261 2262 for (i = 0; i < MAX_OCTEON_LINKS; i++) 2263 oct->props[i].gmxport = -1; 2264 2265 retval = setup_nic_devices(oct); 2266 if (retval) { 2267 dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n"); 2268 goto octnet_init_failure; 2269 } 2270 2271 dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n"); 2272 2273 return retval; 2274 2275 octnet_init_failure: 2276 2277 oct->ifcount = 0; 2278 2279 return retval; 2280 } 2281 2282 /** 2283 * octeon_device_init - Device initialization for each Octeon device that is probed 2284 * @oct: octeon device 2285 */ 2286 static int octeon_device_init(struct octeon_device *oct) 2287 { 2288 u32 rev_id; 2289 int j; 2290 2291 atomic_set(&oct->status, OCT_DEV_BEGIN_STATE); 2292 2293 /* Enable access to the octeon device and make its DMA capability 2294 * known to the OS. 2295 */ 2296 if (octeon_pci_os_setup(oct)) 2297 return 1; 2298 atomic_set(&oct->status, OCT_DEV_PCI_ENABLE_DONE); 2299 2300 oct->chip_id = OCTEON_CN23XX_VF_VID; 2301 pci_read_config_dword(oct->pci_dev, 8, &rev_id); 2302 oct->rev_id = rev_id & 0xff; 2303 2304 if (cn23xx_setup_octeon_vf_device(oct)) 2305 return 1; 2306 2307 atomic_set(&oct->status, OCT_DEV_PCI_MAP_DONE); 2308 2309 oct->app_mode = CVM_DRV_NIC_APP; 2310 2311 /* Initialize the dispatch mechanism used to push packets arriving on 2312 * Octeon Output queues. 2313 */ 2314 if (octeon_init_dispatch_list(oct)) 2315 return 1; 2316 2317 atomic_set(&oct->status, OCT_DEV_DISPATCH_INIT_DONE); 2318 2319 if (octeon_set_io_queues_off(oct)) { 2320 dev_err(&oct->pci_dev->dev, "setting io queues off failed\n"); 2321 return 1; 2322 } 2323 2324 if (oct->fn_list.setup_device_regs(oct)) { 2325 dev_err(&oct->pci_dev->dev, "device registers configuration failed\n"); 2326 return 1; 2327 } 2328 2329 /* Initialize soft command buffer pool */ 2330 if (octeon_setup_sc_buffer_pool(oct)) { 2331 dev_err(&oct->pci_dev->dev, "sc buffer pool allocation failed\n"); 2332 return 1; 2333 } 2334 atomic_set(&oct->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE); 2335 2336 /* Setup the data structures that manage this Octeon's Input queues. */ 2337 if (octeon_setup_instr_queues(oct)) { 2338 dev_err(&oct->pci_dev->dev, "instruction queue initialization failed\n"); 2339 return 1; 2340 } 2341 atomic_set(&oct->status, OCT_DEV_INSTR_QUEUE_INIT_DONE); 2342 2343 /* Initialize lists to manage the requests of different types that 2344 * arrive from user & kernel applications for this octeon device. 2345 */ 2346 if (octeon_setup_response_list(oct)) { 2347 dev_err(&oct->pci_dev->dev, "Response list allocation failed\n"); 2348 return 1; 2349 } 2350 atomic_set(&oct->status, OCT_DEV_RESP_LIST_INIT_DONE); 2351 2352 if (octeon_setup_output_queues(oct)) { 2353 dev_err(&oct->pci_dev->dev, "Output queue initialization failed\n"); 2354 return 1; 2355 } 2356 atomic_set(&oct->status, OCT_DEV_DROQ_INIT_DONE); 2357 2358 if (oct->fn_list.setup_mbox(oct)) { 2359 dev_err(&oct->pci_dev->dev, "Mailbox setup failed\n"); 2360 return 1; 2361 } 2362 atomic_set(&oct->status, OCT_DEV_MBOX_SETUP_DONE); 2363 2364 if (octeon_allocate_ioq_vector(oct, oct->sriov_info.rings_per_vf)) { 2365 dev_err(&oct->pci_dev->dev, "ioq vector allocation failed\n"); 2366 return 1; 2367 } 2368 atomic_set(&oct->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE); 2369 2370 dev_info(&oct->pci_dev->dev, "OCTEON_CN23XX VF: %d ioqs\n", 2371 oct->sriov_info.rings_per_vf); 2372 2373 /* Setup the interrupt handler and record the INT SUM register address*/ 2374 if (octeon_setup_interrupt(oct, oct->sriov_info.rings_per_vf)) 2375 return 1; 2376 2377 atomic_set(&oct->status, OCT_DEV_INTR_SET_DONE); 2378 2379 /* *************************************************************** 2380 * The interrupts need to be enabled for the PF<-->VF handshake. 2381 * They are [re]-enabled after the PF<-->VF handshake so that the 2382 * correct OQ tick value is used (i.e. the value retrieved from 2383 * the PF as part of the handshake). 2384 */ 2385 2386 /* Enable Octeon device interrupts */ 2387 oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR); 2388 2389 if (cn23xx_octeon_pfvf_handshake(oct)) 2390 return 1; 2391 2392 /* Here we [re]-enable the interrupts so that the correct OQ tick value 2393 * is used (i.e. the value that was retrieved during the handshake) 2394 */ 2395 2396 /* Enable Octeon device interrupts */ 2397 oct->fn_list.enable_interrupt(oct, OCTEON_ALL_INTR); 2398 /* *************************************************************** */ 2399 2400 /* Enable the input and output queues for this Octeon device */ 2401 if (oct->fn_list.enable_io_queues(oct)) { 2402 dev_err(&oct->pci_dev->dev, "enabling io queues failed\n"); 2403 return 1; 2404 } 2405 2406 atomic_set(&oct->status, OCT_DEV_IO_QUEUES_DONE); 2407 2408 atomic_set(&oct->status, OCT_DEV_HOST_OK); 2409 2410 /* Send Credit for Octeon Output queues. Credits are always sent after 2411 * the output queue is enabled. 2412 */ 2413 for (j = 0; j < oct->num_oqs; j++) 2414 writel(oct->droq[j]->max_count, oct->droq[j]->pkts_credit_reg); 2415 2416 /* Packets can start arriving on the output queues from this point. */ 2417 2418 atomic_set(&oct->status, OCT_DEV_CORE_OK); 2419 2420 atomic_set(&oct->status, OCT_DEV_RUNNING); 2421 2422 if (liquidio_init_nic_module(oct)) 2423 return 1; 2424 2425 return 0; 2426 } 2427 2428 static int __init liquidio_vf_init(void) 2429 { 2430 octeon_init_device_list(0); 2431 return pci_register_driver(&liquidio_vf_pci_driver); 2432 } 2433 2434 static void __exit liquidio_vf_exit(void) 2435 { 2436 pci_unregister_driver(&liquidio_vf_pci_driver); 2437 2438 pr_info("LiquidIO_VF network module is now unloaded\n"); 2439 } 2440 2441 module_init(liquidio_vf_init); 2442 module_exit(liquidio_vf_exit); 2443