1 /* 2 * Copyright (C) 2015 Cavium Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 * 28 */ 29 30 #ifndef NIC_H 31 #define NIC_H 32 33 /* PCI vendor ID */ 34 #define PCI_VENDOR_ID_CAVIUM 0x177D 35 /* PCI device IDs */ 36 #define PCI_DEVICE_ID_THUNDER_NIC_PF 0xA01E 37 #define PCI_DEVICE_ID_THUNDER_PASS1_NIC_VF 0x0011 38 #define PCI_DEVICE_ID_THUNDER_NIC_VF 0xA034 39 #define PCI_DEVICE_ID_THUNDER_BGX 0xA026 40 41 /* PCI BAR nos */ 42 #define PCI_CFG_REG_BAR_NUM 0 43 #define PCI_MSIX_REG_BAR_NUM 4 44 45 /* NIC SRIOV VF count */ 46 #define MAX_NUM_VFS_SUPPORTED 128 47 #define DEFAULT_NUM_VF_ENABLED 8 48 49 #define NIC_TNS_BYPASS_MODE 0 50 #define NIC_TNS_MODE 1 51 52 /* NIC priv flags */ 53 #define NIC_SRIOV_ENABLED (1 << 0) 54 #define NIC_TNS_ENABLED (1 << 1) 55 56 /* ARM64TODO */ 57 #if 0 58 /* VNIC HW optimiation features */ 59 #define VNIC_RSS_SUPPORT 60 #define VNIC_MULTI_QSET_SUPPORT 61 #endif 62 63 /* Min/Max packet size */ 64 #define NIC_HW_MIN_FRS 64 65 #define NIC_HW_MAX_FRS 9200 /* 9216 max packet including FCS */ 66 67 /* Max pkinds */ 68 #define NIC_MAX_PKIND 16 69 70 /* 71 * Rx Channels */ 72 /* Receive channel configuration in TNS bypass mode 73 * Below is configuration in TNS bypass mode 74 * BGX0-LMAC0-CHAN0 - VNIC CHAN0 75 * BGX0-LMAC1-CHAN0 - VNIC CHAN16 76 * ... 77 * BGX1-LMAC0-CHAN0 - VNIC CHAN128 78 * ... 79 * BGX1-LMAC3-CHAN0 - VNIC CHAN174 80 */ 81 #define NIC_INTF_COUNT 2 /* Interfaces btw VNIC and TNS/BGX */ 82 #define NIC_CHANS_PER_INF 128 83 #define NIC_MAX_CHANS (NIC_INTF_COUNT * NIC_CHANS_PER_INF) 84 #define NIC_CPI_COUNT 2048 /* No of channel parse indices */ 85 86 /* TNS bypass mode: 1-1 mapping between VNIC and BGX:LMAC */ 87 #define NIC_MAX_BGX MAX_BGX_PER_CN88XX 88 #define NIC_CPI_PER_BGX (NIC_CPI_COUNT / NIC_MAX_BGX) 89 #define NIC_MAX_CPI_PER_LMAC 64 /* Max when CPI_ALG is IP diffserv */ 90 #define NIC_RSSI_PER_BGX (NIC_RSSI_COUNT / NIC_MAX_BGX) 91 92 /* Tx scheduling */ 93 #define NIC_MAX_TL4 1024 94 #define NIC_MAX_TL4_SHAPERS 256 /* 1 shaper for 4 TL4s */ 95 #define NIC_MAX_TL3 256 96 #define NIC_MAX_TL3_SHAPERS 64 /* 1 shaper for 4 TL3s */ 97 #define NIC_MAX_TL2 64 98 #define NIC_MAX_TL2_SHAPERS 2 /* 1 shaper for 32 TL2s */ 99 #define NIC_MAX_TL1 2 100 101 /* TNS bypass mode */ 102 #define NIC_TL2_PER_BGX 32 103 #define NIC_TL4_PER_BGX (NIC_MAX_TL4 / NIC_MAX_BGX) 104 #define NIC_TL4_PER_LMAC (NIC_MAX_TL4 / NIC_CHANS_PER_INF) 105 106 /* NIC VF Interrupts */ 107 #define NICVF_INTR_CQ 0 108 #define NICVF_INTR_SQ 1 109 #define NICVF_INTR_RBDR 2 110 #define NICVF_INTR_PKT_DROP 3 111 #define NICVF_INTR_TCP_TIMER 4 112 #define NICVF_INTR_MBOX 5 113 #define NICVF_INTR_QS_ERR 6 114 115 #define NICVF_INTR_CQ_SHIFT 0 116 #define NICVF_INTR_SQ_SHIFT 8 117 #define NICVF_INTR_RBDR_SHIFT 16 118 #define NICVF_INTR_PKT_DROP_SHIFT 20 119 #define NICVF_INTR_TCP_TIMER_SHIFT 21 120 #define NICVF_INTR_MBOX_SHIFT 22 121 #define NICVF_INTR_QS_ERR_SHIFT 23 122 123 #define NICVF_INTR_CQ_MASK (0xFF << NICVF_INTR_CQ_SHIFT) 124 #define NICVF_INTR_SQ_MASK (0xFF << NICVF_INTR_SQ_SHIFT) 125 #define NICVF_INTR_RBDR_MASK (0x03 << NICVF_INTR_RBDR_SHIFT) 126 #define NICVF_INTR_PKT_DROP_MASK (1 << NICVF_INTR_PKT_DROP_SHIFT) 127 #define NICVF_INTR_TCP_TIMER_MASK (1 << NICVF_INTR_TCP_TIMER_SHIFT) 128 #define NICVF_INTR_MBOX_MASK (1 << NICVF_INTR_MBOX_SHIFT) 129 #define NICVF_INTR_QS_ERR_MASK (1 << NICVF_INTR_QS_ERR_SHIFT) 130 131 /* MSI-X interrupts */ 132 #define NIC_PF_MSIX_VECTORS 10 133 #define NIC_VF_MSIX_VECTORS 20 134 135 #define NIC_PF_INTR_ID_ECC0_SBE 0 136 #define NIC_PF_INTR_ID_ECC0_DBE 1 137 #define NIC_PF_INTR_ID_ECC1_SBE 2 138 #define NIC_PF_INTR_ID_ECC1_DBE 3 139 #define NIC_PF_INTR_ID_ECC2_SBE 4 140 #define NIC_PF_INTR_ID_ECC2_DBE 5 141 #define NIC_PF_INTR_ID_ECC3_SBE 6 142 #define NIC_PF_INTR_ID_ECC3_DBE 7 143 #define NIC_PF_INTR_ID_MBOX0 8 144 #define NIC_PF_INTR_ID_MBOX1 9 145 146 struct msix_entry { 147 struct resource * irq_res; 148 void * handle; 149 }; 150 151 /* 152 * Global timer for CQ timer thresh interrupts 153 * Calculated for SCLK of 700Mhz 154 * value written should be a 1/16th of what is expected 155 * 156 * 1 tick per 0.05usec = value of 2.2 157 * This 10% would be covered in CQ timer thresh value 158 */ 159 #define NICPF_CLK_PER_INT_TICK 2 160 161 /* 162 * Time to wait before we decide that a SQ is stuck. 163 * 164 * Since both pkt rx and tx notifications are done with same CQ, 165 * when packets are being received at very high rate (eg: L2 forwarding) 166 * then freeing transmitted skbs will be delayed and watchdog 167 * will kick in, resetting interface. Hence keeping this value high. 168 */ 169 #define NICVF_TX_TIMEOUT (50 * HZ) 170 171 #define NIC_RSSI_COUNT 4096 /* Total no of RSS indices */ 172 #define NIC_MAX_RSS_HASH_BITS 8 173 #define NIC_MAX_RSS_IDR_TBL_SIZE (1 << NIC_MAX_RSS_HASH_BITS) 174 #define RSS_HASH_KEY_SIZE 5 /* 320 bit key */ 175 176 enum rx_stats_reg_offset { 177 RX_OCTS = 0x0, 178 RX_UCAST = 0x1, 179 RX_BCAST = 0x2, 180 RX_MCAST = 0x3, 181 RX_RED = 0x4, 182 RX_RED_OCTS = 0x5, 183 RX_ORUN = 0x6, 184 RX_ORUN_OCTS = 0x7, 185 RX_FCS = 0x8, 186 RX_L2ERR = 0x9, 187 RX_DRP_BCAST = 0xa, 188 RX_DRP_MCAST = 0xb, 189 RX_DRP_L3BCAST = 0xc, 190 RX_DRP_L3MCAST = 0xd, 191 RX_STATS_ENUM_LAST, 192 }; 193 194 enum tx_stats_reg_offset { 195 TX_OCTS = 0x0, 196 TX_UCAST = 0x1, 197 TX_BCAST = 0x2, 198 TX_MCAST = 0x3, 199 TX_DROP = 0x4, 200 TX_STATS_ENUM_LAST, 201 }; 202 203 struct nicvf_hw_stats { 204 uint64_t rx_bytes; 205 uint64_t rx_ucast_frames; 206 uint64_t rx_bcast_frames; 207 uint64_t rx_mcast_frames; 208 uint64_t rx_fcs_errors; 209 uint64_t rx_l2_errors; 210 uint64_t rx_drop_red; 211 uint64_t rx_drop_red_bytes; 212 uint64_t rx_drop_overrun; 213 uint64_t rx_drop_overrun_bytes; 214 uint64_t rx_drop_bcast; 215 uint64_t rx_drop_mcast; 216 uint64_t rx_drop_l3_bcast; 217 uint64_t rx_drop_l3_mcast; 218 uint64_t rx_bgx_truncated_pkts; 219 uint64_t rx_jabber_errs; 220 uint64_t rx_fcs_errs; 221 uint64_t rx_bgx_errs; 222 uint64_t rx_prel2_errs; 223 uint64_t rx_l2_hdr_malformed; 224 uint64_t rx_oversize; 225 uint64_t rx_undersize; 226 uint64_t rx_l2_len_mismatch; 227 uint64_t rx_l2_pclp; 228 uint64_t rx_ip_ver_errs; 229 uint64_t rx_ip_csum_errs; 230 uint64_t rx_ip_hdr_malformed; 231 uint64_t rx_ip_payload_malformed; 232 uint64_t rx_ip_ttl_errs; 233 uint64_t rx_l3_pclp; 234 uint64_t rx_l4_malformed; 235 uint64_t rx_l4_csum_errs; 236 uint64_t rx_udp_len_errs; 237 uint64_t rx_l4_port_errs; 238 uint64_t rx_tcp_flag_errs; 239 uint64_t rx_tcp_offset_errs; 240 uint64_t rx_l4_pclp; 241 uint64_t rx_truncated_pkts; 242 243 uint64_t tx_bytes_ok; 244 uint64_t tx_ucast_frames_ok; 245 uint64_t tx_bcast_frames_ok; 246 uint64_t tx_mcast_frames_ok; 247 uint64_t tx_drops; 248 }; 249 250 struct nicvf_drv_stats { 251 /* Rx */ 252 uint64_t rx_frames_ok; 253 uint64_t rx_frames_64; 254 uint64_t rx_frames_127; 255 uint64_t rx_frames_255; 256 uint64_t rx_frames_511; 257 uint64_t rx_frames_1023; 258 uint64_t rx_frames_1518; 259 uint64_t rx_frames_jumbo; 260 uint64_t rx_drops; 261 262 /* Tx */ 263 uint64_t tx_frames_ok; 264 uint64_t tx_drops; 265 uint64_t tx_tso; 266 uint64_t txq_stop; 267 uint64_t txq_wake; 268 }; 269 270 struct nicvf { 271 struct nicvf *pnicvf; 272 device_t dev; 273 274 struct ifnet * ifp; 275 struct sx core_sx; 276 struct ifmedia if_media; 277 uint32_t if_flags; 278 279 uint8_t hwaddr[ETHER_ADDR_LEN]; 280 uint8_t vf_id; 281 uint8_t node; 282 boolean_t tns_mode:1; 283 boolean_t sqs_mode:1; 284 bool loopback_supported:1; 285 uint16_t mtu; 286 struct queue_set *qs; 287 uint8_t rx_queues; 288 uint8_t tx_queues; 289 uint8_t max_queues; 290 struct resource *reg_base; 291 boolean_t link_up; 292 uint8_t duplex; 293 uint32_t speed; 294 uint8_t cpi_alg; 295 /* Interrupt coalescing settings */ 296 uint32_t cq_coalesce_usecs; 297 298 uint32_t msg_enable; 299 struct nicvf_hw_stats hw_stats; 300 struct nicvf_drv_stats drv_stats; 301 struct bgx_stats bgx_stats; 302 303 /* Interface statistics */ 304 struct callout stats_callout; 305 struct mtx stats_mtx; 306 307 /* MSI-X */ 308 boolean_t msix_enabled; 309 uint8_t num_vec; 310 struct msix_entry msix_entries[NIC_VF_MSIX_VECTORS]; 311 struct resource * msix_table_res; 312 char irq_name[NIC_VF_MSIX_VECTORS][20]; 313 boolean_t irq_allocated[NIC_VF_MSIX_VECTORS]; 314 315 /* VF <-> PF mailbox communication */ 316 boolean_t pf_acked; 317 boolean_t pf_nacked; 318 } __aligned(CACHE_LINE_SIZE); 319 320 /* 321 * PF <--> VF Mailbox communication 322 * Eight 64bit registers are shared between PF and VF. 323 * Separate set for each VF. 324 * Writing '1' into last register mbx7 means end of message. 325 */ 326 327 /* PF <--> VF mailbox communication */ 328 #define NIC_PF_VF_MAILBOX_SIZE 2 329 #define NIC_MBOX_MSG_TIMEOUT 2000 /* ms */ 330 331 /* Mailbox message types */ 332 #define NIC_MBOX_MSG_READY 0x01 /* Is PF ready to rcv msgs */ 333 #define NIC_MBOX_MSG_ACK 0x02 /* ACK the message received */ 334 #define NIC_MBOX_MSG_NACK 0x03 /* NACK the message received */ 335 #define NIC_MBOX_MSG_QS_CFG 0x04 /* Configure Qset */ 336 #define NIC_MBOX_MSG_RQ_CFG 0x05 /* Configure receive queue */ 337 #define NIC_MBOX_MSG_SQ_CFG 0x06 /* Configure Send queue */ 338 #define NIC_MBOX_MSG_RQ_DROP_CFG 0x07 /* Configure receive queue */ 339 #define NIC_MBOX_MSG_SET_MAC 0x08 /* Add MAC ID to DMAC filter */ 340 #define NIC_MBOX_MSG_SET_MAX_FRS 0x09 /* Set max frame size */ 341 #define NIC_MBOX_MSG_CPI_CFG 0x0A /* Config CPI, RSSI */ 342 #define NIC_MBOX_MSG_RSS_SIZE 0x0B /* Get RSS indir_tbl size */ 343 #define NIC_MBOX_MSG_RSS_CFG 0x0C /* Config RSS table */ 344 #define NIC_MBOX_MSG_RSS_CFG_CONT 0x0D /* RSS config continuation */ 345 #define NIC_MBOX_MSG_RQ_BP_CFG 0x0E /* RQ backpressure config */ 346 #define NIC_MBOX_MSG_RQ_SW_SYNC 0x0F /* Flush inflight pkts to RQ */ 347 #define NIC_MBOX_MSG_BGX_STATS 0x10 /* Get stats from BGX */ 348 #define NIC_MBOX_MSG_BGX_LINK_CHANGE 0x11 /* BGX:LMAC link status */ 349 #define NIC_MBOX_MSG_ALLOC_SQS 0x12 /* Allocate secondary Qset */ 350 #define NIC_MBOX_MSG_NICVF_PTR 0x13 /* Send nicvf ptr to PF */ 351 #define NIC_MBOX_MSG_PNICVF_PTR 0x14 /* Get primary qset nicvf ptr */ 352 #define NIC_MBOX_MSG_SNICVF_PTR 0x15 /* Send sqet nicvf ptr to PVF */ 353 #define NIC_MBOX_MSG_LOOPBACK 0x16 /* Set interface in loopback */ 354 #define NIC_MBOX_MSG_CFG_DONE 0xF0 /* VF configuration done */ 355 #define NIC_MBOX_MSG_SHUTDOWN 0xF1 /* VF is being shutdown */ 356 357 struct nic_cfg_msg { 358 uint8_t msg; 359 uint8_t vf_id; 360 uint8_t node_id; 361 boolean_t tns_mode:1; 362 boolean_t sqs_mode:1; 363 boolean_t loopback_supported:1; 364 uint8_t mac_addr[ETHER_ADDR_LEN]; 365 }; 366 367 /* Qset configuration */ 368 struct qs_cfg_msg { 369 uint8_t msg; 370 uint8_t num; 371 uint8_t sqs_count; 372 uint64_t cfg; 373 }; 374 375 /* Receive queue configuration */ 376 struct rq_cfg_msg { 377 uint8_t msg; 378 uint8_t qs_num; 379 uint8_t rq_num; 380 uint64_t cfg; 381 }; 382 383 /* Send queue configuration */ 384 struct sq_cfg_msg { 385 uint8_t msg; 386 uint8_t qs_num; 387 uint8_t sq_num; 388 boolean_t sqs_mode; 389 uint64_t cfg; 390 }; 391 392 /* Set VF's MAC address */ 393 struct set_mac_msg { 394 uint8_t msg; 395 uint8_t vf_id; 396 uint8_t mac_addr[ETHER_ADDR_LEN]; 397 }; 398 399 /* Set Maximum frame size */ 400 struct set_frs_msg { 401 uint8_t msg; 402 uint8_t vf_id; 403 uint16_t max_frs; 404 }; 405 406 /* Set CPI algorithm type */ 407 struct cpi_cfg_msg { 408 uint8_t msg; 409 uint8_t vf_id; 410 uint8_t rq_cnt; 411 uint8_t cpi_alg; 412 }; 413 414 /* Get RSS table size */ 415 struct rss_sz_msg { 416 uint8_t msg; 417 uint8_t vf_id; 418 uint16_t ind_tbl_size; 419 }; 420 421 /* Set RSS configuration */ 422 struct rss_cfg_msg { 423 uint8_t msg; 424 uint8_t vf_id; 425 uint8_t hash_bits; 426 uint8_t tbl_len; 427 uint8_t tbl_offset; 428 #define RSS_IND_TBL_LEN_PER_MBX_MSG 8 429 uint8_t ind_tbl[RSS_IND_TBL_LEN_PER_MBX_MSG]; 430 }; 431 432 struct bgx_stats_msg { 433 uint8_t msg; 434 uint8_t vf_id; 435 uint8_t rx; 436 uint8_t idx; 437 uint64_t stats; 438 }; 439 440 /* Physical interface link status */ 441 struct bgx_link_status { 442 uint8_t msg; 443 uint8_t link_up; 444 uint8_t duplex; 445 uint32_t speed; 446 }; 447 448 /* Set interface in loopback mode */ 449 struct set_loopback { 450 uint8_t msg; 451 uint8_t vf_id; 452 boolean_t enable; 453 }; 454 455 /* 128 bit shared memory between PF and each VF */ 456 union nic_mbx { 457 struct { 458 uint8_t msg; 459 } msg; 460 struct nic_cfg_msg nic_cfg; 461 struct qs_cfg_msg qs; 462 struct rq_cfg_msg rq; 463 struct sq_cfg_msg sq; 464 struct set_mac_msg mac; 465 struct set_frs_msg frs; 466 struct cpi_cfg_msg cpi_cfg; 467 struct rss_sz_msg rss_size; 468 struct rss_cfg_msg rss_cfg; 469 struct bgx_stats_msg bgx_stats; 470 struct bgx_link_status link_status; 471 struct set_loopback lbk; 472 }; 473 474 #define NIC_NODE_ID_MASK 0x03 475 #define NIC_NODE_ID_SHIFT 44 476 477 static __inline int 478 nic_get_node_id(struct resource *res) 479 { 480 pci_addr_t addr; 481 482 addr = rman_get_start(res); 483 return ((addr >> NIC_NODE_ID_SHIFT) & NIC_NODE_ID_MASK); 484 } 485 486 int nicvf_send_msg_to_pf(struct nicvf *vf, union nic_mbx *mbx); 487 488 #endif /* NIC_H */ 489