1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2016 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more 17 * details. 18 **********************************************************************/ 19 20 /*! \file octeon_network.h 21 * \brief Host NIC Driver: Structure and Macro definitions used by NIC Module. 22 */ 23 24 #ifndef __OCTEON_NETWORK_H__ 25 #define __OCTEON_NETWORK_H__ 26 #include <linux/ptp_clock_kernel.h> 27 28 #define LIO_MAX_MTU_SIZE (OCTNET_MAX_FRM_SIZE - OCTNET_FRM_HEADER_SIZE) 29 #define LIO_MIN_MTU_SIZE ETH_MIN_MTU 30 31 /* Bit mask values for lio->ifstate */ 32 #define LIO_IFSTATE_DROQ_OPS 0x01 33 #define LIO_IFSTATE_REGISTERED 0x02 34 #define LIO_IFSTATE_RUNNING 0x04 35 #define LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08 36 #define LIO_IFSTATE_RESETTING 0x10 37 38 struct liquidio_if_cfg_resp { 39 u64 rh; 40 struct liquidio_if_cfg_info cfg_info; 41 u64 status; 42 }; 43 44 #define LIO_IFCFG_WAIT_TIME 3000 /* In milli seconds */ 45 #define LIQUIDIO_NDEV_STATS_POLL_TIME_MS 200 46 47 /* Structure of a node in list of gather components maintained by 48 * NIC driver for each network device. 49 */ 50 struct octnic_gather { 51 /* List manipulation. Next and prev pointers. */ 52 struct list_head list; 53 54 /* Size of the gather component at sg in bytes. */ 55 int sg_size; 56 57 /* Number of bytes that sg was adjusted to make it 8B-aligned. */ 58 int adjust; 59 60 /* Gather component that can accommodate max sized fragment list 61 * received from the IP layer. 62 */ 63 struct octeon_sg_entry *sg; 64 65 dma_addr_t sg_dma_ptr; 66 }; 67 68 struct oct_nic_stats_resp { 69 u64 rh; 70 struct oct_link_stats stats; 71 u64 status; 72 }; 73 74 struct oct_nic_stats_ctrl { 75 struct completion complete; 76 struct net_device *netdev; 77 }; 78 79 struct oct_nic_seapi_resp { 80 u64 rh; 81 u32 speed; 82 u64 status; 83 }; 84 85 /** LiquidIO per-interface network private data */ 86 struct lio { 87 /** State of the interface. Rx/Tx happens only in the RUNNING state. */ 88 atomic_t ifstate; 89 90 /** Octeon Interface index number. This device will be represented as 91 * oct<ifidx> in the system. 92 */ 93 int ifidx; 94 95 /** Octeon Input queue to use to transmit for this network interface. */ 96 int txq; 97 98 /** Octeon Output queue from which pkts arrive 99 * for this network interface. 100 */ 101 int rxq; 102 103 /** Guards each glist */ 104 spinlock_t *glist_lock; 105 106 /** Array of gather component linked lists */ 107 struct list_head *glist; 108 void **glists_virt_base; 109 dma_addr_t *glists_dma_base; 110 u32 glist_entry_size; 111 112 /** Pointer to the NIC properties for the Octeon device this network 113 * interface is associated with. 114 */ 115 struct octdev_props *octprops; 116 117 /** Pointer to the octeon device structure. */ 118 struct octeon_device *oct_dev; 119 120 struct net_device *netdev; 121 122 /** Link information sent by the core application for this interface. */ 123 struct oct_link_info linfo; 124 125 /** counter of link changes */ 126 u64 link_changes; 127 128 /** Size of Tx queue for this octeon device. */ 129 u32 tx_qsize; 130 131 /** Size of Rx queue for this octeon device. */ 132 u32 rx_qsize; 133 134 /** Size of MTU this octeon device. */ 135 u32 mtu; 136 137 /** msg level flag per interface. */ 138 u32 msg_enable; 139 140 /** Copy of Interface capabilities: TSO, TSO6, LRO, Chescksums . */ 141 u64 dev_capability; 142 143 /* Copy of transmit encapsulation capabilities: 144 * TSO, TSO6, Checksums for this device for Kernel 145 * 3.10.0 onwards 146 */ 147 u64 enc_dev_capability; 148 149 /** Copy of beacaon reg in phy */ 150 u32 phy_beacon_val; 151 152 /** Copy of ctrl reg in phy */ 153 u32 led_ctrl_val; 154 155 /* PTP clock information */ 156 struct ptp_clock_info ptp_info; 157 struct ptp_clock *ptp_clock; 158 s64 ptp_adjust; 159 160 /* for atomic access to Octeon PTP reg and data struct */ 161 spinlock_t ptp_lock; 162 163 /* Interface info */ 164 u32 intf_open; 165 166 /* work queue for txq status */ 167 struct cavium_wq txq_status_wq; 168 169 /* work queue for rxq oom status */ 170 struct cavium_wq rxq_status_wq; 171 172 /* work queue for link status */ 173 struct cavium_wq link_status_wq; 174 175 /* work queue to regularly send local time to octeon firmware */ 176 struct cavium_wq sync_octeon_time_wq; 177 178 int netdev_uc_count; 179 struct cavium_wk stats_wk; 180 }; 181 182 #define LIO_SIZE (sizeof(struct lio)) 183 #define GET_LIO(netdev) ((struct lio *)netdev_priv(netdev)) 184 185 #define LIO_MAX_CORES 16 186 187 /** 188 * \brief Enable or disable feature 189 * @param netdev pointer to network device 190 * @param cmd Command that just requires acknowledgment 191 * @param param1 Parameter to command 192 */ 193 int liquidio_set_feature(struct net_device *netdev, int cmd, u16 param1); 194 195 int setup_rx_oom_poll_fn(struct net_device *netdev); 196 197 void cleanup_rx_oom_poll_fn(struct net_device *netdev); 198 199 /** 200 * \brief Link control command completion callback 201 * @param nctrl_ptr pointer to control packet structure 202 * 203 * This routine is called by the callback function when a ctrl pkt sent to 204 * core app completes. The nctrl_ptr contains a copy of the command type 205 * and data sent to the core app. This routine is only called if the ctrl 206 * pkt was sent successfully to the core app. 207 */ 208 void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr); 209 210 int liquidio_setup_io_queues(struct octeon_device *octeon_dev, int ifidx, 211 u32 num_iqs, u32 num_oqs); 212 213 irqreturn_t liquidio_msix_intr_handler(int irq __attribute__((unused)), 214 void *dev); 215 216 int octeon_setup_interrupt(struct octeon_device *oct, u32 num_ioqs); 217 218 void lio_fetch_stats(struct work_struct *work); 219 220 int lio_wait_for_clean_oq(struct octeon_device *oct); 221 /** 222 * \brief Register ethtool operations 223 * @param netdev pointer to network device 224 */ 225 void liquidio_set_ethtool_ops(struct net_device *netdev); 226 227 void lio_delete_glists(struct lio *lio); 228 229 int lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_qs); 230 231 int liquidio_get_speed(struct lio *lio); 232 int liquidio_set_speed(struct lio *lio, int speed); 233 234 /** 235 * \brief Net device change_mtu 236 * @param netdev network device 237 */ 238 int liquidio_change_mtu(struct net_device *netdev, int new_mtu); 239 #define LIO_CHANGE_MTU_SUCCESS 1 240 #define LIO_CHANGE_MTU_FAIL 2 241 242 #define SKB_ADJ_MASK 0x3F 243 #define SKB_ADJ (SKB_ADJ_MASK + 1) 244 245 #define MIN_SKB_SIZE 256 /* 8 bytes and more - 8 bytes for PTP */ 246 #define LIO_RXBUFFER_SZ 2048 247 248 static inline void 249 *recv_buffer_alloc(struct octeon_device *oct, 250 struct octeon_skb_page_info *pg_info) 251 { 252 struct page *page; 253 struct sk_buff *skb; 254 struct octeon_skb_page_info *skb_pg_info; 255 256 page = alloc_page(GFP_ATOMIC); 257 if (unlikely(!page)) 258 return NULL; 259 260 skb = dev_alloc_skb(MIN_SKB_SIZE + SKB_ADJ); 261 if (unlikely(!skb)) { 262 __free_page(page); 263 pg_info->page = NULL; 264 return NULL; 265 } 266 267 if ((unsigned long)skb->data & SKB_ADJ_MASK) { 268 u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK); 269 270 skb_reserve(skb, r); 271 } 272 273 skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 274 /* Get DMA info */ 275 pg_info->dma = dma_map_page(&oct->pci_dev->dev, page, 0, 276 PAGE_SIZE, DMA_FROM_DEVICE); 277 278 /* Mapping failed!! */ 279 if (dma_mapping_error(&oct->pci_dev->dev, pg_info->dma)) { 280 __free_page(page); 281 dev_kfree_skb_any((struct sk_buff *)skb); 282 pg_info->page = NULL; 283 return NULL; 284 } 285 286 pg_info->page = page; 287 pg_info->page_offset = 0; 288 skb_pg_info->page = page; 289 skb_pg_info->page_offset = 0; 290 skb_pg_info->dma = pg_info->dma; 291 292 return (void *)skb; 293 } 294 295 static inline void 296 *recv_buffer_fast_alloc(u32 size) 297 { 298 struct sk_buff *skb; 299 struct octeon_skb_page_info *skb_pg_info; 300 301 skb = dev_alloc_skb(size + SKB_ADJ); 302 if (unlikely(!skb)) 303 return NULL; 304 305 if ((unsigned long)skb->data & SKB_ADJ_MASK) { 306 u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK); 307 308 skb_reserve(skb, r); 309 } 310 311 skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 312 skb_pg_info->page = NULL; 313 skb_pg_info->page_offset = 0; 314 skb_pg_info->dma = 0; 315 316 return skb; 317 } 318 319 static inline int 320 recv_buffer_recycle(struct octeon_device *oct, void *buf) 321 { 322 struct octeon_skb_page_info *pg_info = buf; 323 324 if (!pg_info->page) { 325 dev_err(&oct->pci_dev->dev, "%s: pg_info->page NULL\n", 326 __func__); 327 return -ENOMEM; 328 } 329 330 if (unlikely(page_count(pg_info->page) != 1) || 331 unlikely(page_to_nid(pg_info->page) != numa_node_id())) { 332 dma_unmap_page(&oct->pci_dev->dev, 333 pg_info->dma, (PAGE_SIZE << 0), 334 DMA_FROM_DEVICE); 335 pg_info->dma = 0; 336 pg_info->page = NULL; 337 pg_info->page_offset = 0; 338 return -ENOMEM; 339 } 340 341 /* Flip to other half of the buffer */ 342 if (pg_info->page_offset == 0) 343 pg_info->page_offset = LIO_RXBUFFER_SZ; 344 else 345 pg_info->page_offset = 0; 346 page_ref_inc(pg_info->page); 347 348 return 0; 349 } 350 351 static inline void 352 *recv_buffer_reuse(struct octeon_device *oct, void *buf) 353 { 354 struct octeon_skb_page_info *pg_info = buf, *skb_pg_info; 355 struct sk_buff *skb; 356 357 skb = dev_alloc_skb(MIN_SKB_SIZE + SKB_ADJ); 358 if (unlikely(!skb)) { 359 dma_unmap_page(&oct->pci_dev->dev, 360 pg_info->dma, (PAGE_SIZE << 0), 361 DMA_FROM_DEVICE); 362 return NULL; 363 } 364 365 if ((unsigned long)skb->data & SKB_ADJ_MASK) { 366 u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK); 367 368 skb_reserve(skb, r); 369 } 370 371 skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 372 skb_pg_info->page = pg_info->page; 373 skb_pg_info->page_offset = pg_info->page_offset; 374 skb_pg_info->dma = pg_info->dma; 375 376 return skb; 377 } 378 379 static inline void 380 recv_buffer_destroy(void *buffer, struct octeon_skb_page_info *pg_info) 381 { 382 struct sk_buff *skb = (struct sk_buff *)buffer; 383 384 put_page(pg_info->page); 385 pg_info->dma = 0; 386 pg_info->page = NULL; 387 pg_info->page_offset = 0; 388 389 if (skb) 390 dev_kfree_skb_any(skb); 391 } 392 393 static inline void recv_buffer_free(void *buffer) 394 { 395 struct sk_buff *skb = (struct sk_buff *)buffer; 396 struct octeon_skb_page_info *pg_info; 397 398 pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 399 400 if (pg_info->page) { 401 put_page(pg_info->page); 402 pg_info->dma = 0; 403 pg_info->page = NULL; 404 pg_info->page_offset = 0; 405 } 406 407 dev_kfree_skb_any((struct sk_buff *)buffer); 408 } 409 410 static inline void 411 recv_buffer_fast_free(void *buffer) 412 { 413 dev_kfree_skb_any((struct sk_buff *)buffer); 414 } 415 416 static inline void tx_buffer_free(void *buffer) 417 { 418 dev_kfree_skb_any((struct sk_buff *)buffer); 419 } 420 421 #define lio_dma_alloc(oct, size, dma_addr) \ 422 dma_alloc_coherent(&(oct)->pci_dev->dev, size, dma_addr, GFP_KERNEL) 423 #define lio_dma_free(oct, size, virt_addr, dma_addr) \ 424 dma_free_coherent(&(oct)->pci_dev->dev, size, virt_addr, dma_addr) 425 426 static inline 427 void *get_rbd(struct sk_buff *skb) 428 { 429 struct octeon_skb_page_info *pg_info; 430 unsigned char *va; 431 432 pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 433 va = page_address(pg_info->page) + pg_info->page_offset; 434 435 return va; 436 } 437 438 static inline u64 439 lio_map_ring(void *buf) 440 { 441 dma_addr_t dma_addr; 442 443 struct sk_buff *skb = (struct sk_buff *)buf; 444 struct octeon_skb_page_info *pg_info; 445 446 pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 447 if (!pg_info->page) { 448 pr_err("%s: pg_info->page NULL\n", __func__); 449 WARN_ON(1); 450 } 451 452 /* Get DMA info */ 453 dma_addr = pg_info->dma; 454 if (!pg_info->dma) { 455 pr_err("%s: ERROR it should be already available\n", 456 __func__); 457 WARN_ON(1); 458 } 459 dma_addr += pg_info->page_offset; 460 461 return (u64)dma_addr; 462 } 463 464 static inline void 465 lio_unmap_ring(struct pci_dev *pci_dev, 466 u64 buf_ptr) 467 468 { 469 dma_unmap_page(&pci_dev->dev, 470 buf_ptr, (PAGE_SIZE << 0), 471 DMA_FROM_DEVICE); 472 } 473 474 static inline void *octeon_fast_packet_alloc(u32 size) 475 { 476 return recv_buffer_fast_alloc(size); 477 } 478 479 static inline void octeon_fast_packet_next(struct octeon_droq *droq, 480 struct sk_buff *nicbuf, 481 int copy_len, 482 int idx) 483 { 484 skb_put_data(nicbuf, get_rbd(droq->recv_buf_list[idx].buffer), 485 copy_len); 486 } 487 488 /** 489 * \brief check interface state 490 * @param lio per-network private data 491 * @param state_flag flag state to check 492 */ 493 static inline int ifstate_check(struct lio *lio, int state_flag) 494 { 495 return atomic_read(&lio->ifstate) & state_flag; 496 } 497 498 /** 499 * \brief set interface state 500 * @param lio per-network private data 501 * @param state_flag flag state to set 502 */ 503 static inline void ifstate_set(struct lio *lio, int state_flag) 504 { 505 atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag)); 506 } 507 508 /** 509 * \brief clear interface state 510 * @param lio per-network private data 511 * @param state_flag flag state to clear 512 */ 513 static inline void ifstate_reset(struct lio *lio, int state_flag) 514 { 515 atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag))); 516 } 517 518 /** 519 * \brief wait for all pending requests to complete 520 * @param oct Pointer to Octeon device 521 * 522 * Called during shutdown sequence 523 */ 524 static inline int wait_for_pending_requests(struct octeon_device *oct) 525 { 526 int i, pcount = 0; 527 528 for (i = 0; i < MAX_IO_PENDING_PKT_COUNT; i++) { 529 pcount = atomic_read( 530 &oct->response_list[OCTEON_ORDERED_SC_LIST] 531 .pending_req_count); 532 if (pcount) 533 schedule_timeout_uninterruptible(HZ / 10); 534 else 535 break; 536 } 537 538 if (pcount) 539 return 1; 540 541 return 0; 542 } 543 544 /** 545 * \brief Stop Tx queues 546 * @param netdev network device 547 */ 548 static inline void stop_txqs(struct net_device *netdev) 549 { 550 int i; 551 552 for (i = 0; i < netdev->real_num_tx_queues; i++) 553 netif_stop_subqueue(netdev, i); 554 } 555 556 /** 557 * \brief Wake Tx queues 558 * @param netdev network device 559 */ 560 static inline void wake_txqs(struct net_device *netdev) 561 { 562 struct lio *lio = GET_LIO(netdev); 563 int i, qno; 564 565 for (i = 0; i < netdev->real_num_tx_queues; i++) { 566 qno = lio->linfo.txpciq[i % lio->oct_dev->num_iqs].s.q_no; 567 568 if (__netif_subqueue_stopped(netdev, i)) { 569 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno, 570 tx_restart, 1); 571 netif_wake_subqueue(netdev, i); 572 } 573 } 574 } 575 576 /** 577 * \brief Start Tx queues 578 * @param netdev network device 579 */ 580 static inline void start_txqs(struct net_device *netdev) 581 { 582 struct lio *lio = GET_LIO(netdev); 583 int i; 584 585 if (lio->linfo.link.s.link_up) { 586 for (i = 0; i < netdev->real_num_tx_queues; i++) 587 netif_start_subqueue(netdev, i); 588 } 589 } 590 591 static inline int skb_iq(struct octeon_device *oct, struct sk_buff *skb) 592 { 593 return skb->queue_mapping % oct->num_iqs; 594 } 595 596 /** 597 * Remove the node at the head of the list. The list would be empty at 598 * the end of this call if there are no more nodes in the list. 599 */ 600 static inline struct list_head *lio_list_delete_head(struct list_head *root) 601 { 602 struct list_head *node; 603 604 if (root->prev == root && root->next == root) 605 node = NULL; 606 else 607 node = root->next; 608 609 if (node) 610 list_del(node); 611 612 return node; 613 } 614 615 #endif 616