1# Chelsio T4 Factory Default configuration file. 2# 3# Copyright (C) 2010 Chelsio Communications. All rights reserved. 4# 5 6# This file provides the default, power-on configuration for 4-port T4-based 7# adapters shipped from the factory. These defaults are designed to address 8# the needs of the vast majority of T4 customers. The basic idea is to have 9# a default configuration which allows a customer to plug a T4 adapter in and 10# have it work regardless of OS, driver or application except in the most 11# unusual and/or demanding customer applications. 12# 13# Many of the T4 resources which are described by this configuration are 14# finite. This requires balancing the configuration/operation needs of 15# device drivers across OSes and a large number of customer application. 16# 17# Some of the more important resources to allocate and their constaints are: 18# 1. Virtual Interfaces: 128. 19# 2. Ingress Queues with Free Lists: 1024. PCI-E SR-IOV Virtual Functions 20# must use a power of 2 Ingress Queues. 21# 3. Egress Queues: 128K. PCI-E SR-IOV Virtual Functions must use a 22# power of 2 Egress Queues. 23# 4. MSI-X Vectors: 1088. A complication here is that the PCI-E SR-IOV 24# Virtual Functions based off of a Physical Function all get the 25# same umber of MSI-X Vectors as the base Physical Function. 26# Additionally, regardless of whether Virtual Functions are enabled or 27# not, their MSI-X "needs" are counted by the PCI-E implementation. 28# And finally, all Physical Funcations capable of supporting Virtual 29# Functions (PF0-3) must have the same number of configured TotalVFs in 30# their SR-IOV Capabilities. 31# 5. Multi-Port Support (MPS) TCAM: 336 entries to support MAC destination 32# address matching on Ingress Packets. 33# 34# Some of the important OS/Driver resource needs are: 35# 6. Some OS Drivers will manage all resources through a single Physical 36# Function (currently PF0 but it could be any Physical Function). Thus, 37# this "Unified PF" will need to have enough resources allocated to it 38# to allow for this. And because of the MSI-X resource allocation 39# constraints mentioned above, this probably means we'll either have to 40# severely limit the TotalVFs if we continue to use PF0 as the Unified PF 41# or we'll need to move the Unified PF into the PF4-7 range since those 42# Physical Functions don't have any Virtual Functions associated with 43# them. 44# 7. Some OS Drivers will manage different ports and functions (NIC, 45# storage, etc.) on different Physical Functions. For example, NIC 46# functions for ports 0-3 on PF0-3, FCoE on PF4, iSCSI on PF5, etc. 47# 48# Some of the customer application needs which need to be accommodated: 49# 8. Some customers will want to support large CPU count systems with 50# good scaling. Thus, we'll need to accommodate a number of 51# Ingress Queues and MSI-X Vectors to allow up to some number of CPUs 52# to be involved per port and per application function. For example, 53# in the case where all ports and application functions will be 54# managed via a single Unified PF and we want to accommodate scaling up 55# to 8 CPUs, we would want: 56# 57# 4 ports * 58# 3 application functions (NIC, FCoE, iSCSI) per port * 59# 8 Ingress Queue/MSI-X Vectors per application function 60# 61# for a total of 96 Ingress Queues and MSI-X Vectors on the Unified PF. 62# (Plus a few for Firmware Event Queues, etc.) 63# 64# 9. Some customers will want to use T4's PCI-E SR-IOV Capability to allow 65# Virtual Machines to directly access T4 functionality via SR-IOV 66# Virtual Functions and "PCI Device Passthrough" -- this is especially 67# true for the NIC application functionality. (Note that there is 68# currently no ability to use the TOE, FCoE, iSCSI, etc. via Virtual 69# Functions so this is in fact solely limited to NIC.) 70# 71 72 73# Global configuration settings. 74# 75[global] 76 rss_glb_config_mode = basicvirtual 77 rss_glb_config_options = tnlmapen,hashtoeplitz,tnlalllkp 78 79 # The following Scatter Gather Engine (SGE) settings assume a 4KB Host 80 # Page Size and a 64B L1 Cache Line Size. It programs the 81 # EgrStatusPageSize and IngPadBoundary to 64B and the PktShift to 2. 82 # If a Master PF Driver finds itself on a machine with different 83 # parameters, then the Master PF Driver is responsible for initializing 84 # these parameters to appropriate values. 85 # 86 # Notes: 87 # 1. The Free List Buffer Sizes below are raw and the firmware will 88 # round them up to the Ingress Padding Boundary. 89 # 2. The SGE Timer Values below are expressed below in microseconds. 90 # The firmware will convert these values to Core Clock Ticks when 91 # it processes the configuration parameters. 92 # 93 reg[0x1008] = 0x40810/0x21c70 # SGE_CONTROL 94 reg[0x100c] = 0x22222222 # SGE_HOST_PAGE_SIZE 95 reg[0x10a0] = 0x01040810 # SGE_INGRESS_RX_THRESHOLD 96 reg[0x1044] = 4096 # SGE_FL_BUFFER_SIZE0 97 reg[0x1048] = 65536 # SGE_FL_BUFFER_SIZE1 98 reg[0x104c] = 1536 # SGE_FL_BUFFER_SIZE2 99 reg[0x1050] = 9024 # SGE_FL_BUFFER_SIZE3 100 reg[0x1054] = 9216 # SGE_FL_BUFFER_SIZE4 101 reg[0x1058] = 2048 # SGE_FL_BUFFER_SIZE5 102 reg[0x105c] = 128 # SGE_FL_BUFFER_SIZE6 103 reg[0x1060] = 8192 # SGE_FL_BUFFER_SIZE7 104 reg[0x1064] = 16384 # SGE_FL_BUFFER_SIZE8 105 reg[0x10a4] = 0xa000a000/0xf000f000 # SGE_DBFIFO_STATUS 106 reg[0x10a8] = 0x2000/0x2000 # SGE_DOORBELL_CONTROL 107 sge_timer_value = 5, 10, 20, 50, 100, 200 # SGE_TIMER_VALUE* in usecs 108 109 reg[0x7dc0] = 0x64f8849 # TP_SHIFT_CNT 110 111 # Selection of tuples for LE filter lookup, fields (and widths which 112 # must sum to <= 36): { IP Fragment (1), MPS Match Type (3), 113 # IP Protocol (8), [Inner] VLAN (17), Port (3), FCoE (1) } 114 # 115 filterMode = fragmentation, mpshittype, protocol, vnic_id, port, fcoe 116 117 # Percentage of dynamic memory (in either the EDRAM or external MEM) 118 # to use for TP RX payload 119 tp_pmrx = 30 120 121 # TP RX payload page size 122 tp_pmrx_pagesize = 64K 123 124 # Percentage of dynamic memory (in either the EDRAM or external MEM) 125 # to use for TP TX payload 126 tp_pmtx = 50 127 128 # TP TX payload page size 129 tp_pmtx_pagesize = 64K 130 131# Some "definitions" to make the rest of this a bit more readable. We support 132# 4 ports, 3 functions (NIC, FCoE and iSCSI), scaling up to 8 "CPU Queue Sets" 133# per function per port ... 134# 135# NMSIX = 1088 # available MSI-X Vectors 136# NVI = 128 # available Virtual Interfaces 137# NMPSTCAM = 336 # MPS TCAM entries 138# 139# NPORTS = 4 # ports 140# NCPUS = 8 # CPUs we want to support scalably 141# NFUNCS = 3 # functions per port (NIC, FCoE, iSCSI) 142 143# Breakdown of Virtual Interface/Queue/Interrupt resources for the "Unified 144# PF" which many OS Drivers will use to manage most or all functions. 145# 146# Each Ingress Queue can use one MSI-X interrupt but some Ingress Queues can 147# use Forwarded Interrupt Ingress Queues. For these latter, an Ingress Queue 148# would be created and the Queue ID of a Forwarded Interrupt Ingress Queue 149# will be specified as the "Ingress Queue Asynchronous Destination Index." 150# Thus, the number of MSI-X Vectors assigned to the Unified PF will be less 151# than or equal to the number of Ingress Queues ... 152# 153# NVI_NIC = 4 # NIC access to NPORTS 154# NFLIQ_NIC = 32 # NIC Ingress Queues with Free Lists 155# NETHCTRL_NIC = 32 # NIC Ethernet Control/TX Queues 156# NEQ_NIC = 64 # NIC Egress Queues (FL, ETHCTRL/TX) 157# NMPSTCAM_NIC = 16 # NIC MPS TCAM Entries (NPORTS*4) 158# NMSIX_NIC = 32 # NIC MSI-X Interrupt Vectors (FLIQ) 159# 160# NVI_OFLD = 0 # Offload uses NIC function to access ports 161# NFLIQ_OFLD = 16 # Offload Ingress Queues with Free Lists 162# NETHCTRL_OFLD = 0 # Offload Ethernet Control/TX Queues 163# NEQ_OFLD = 16 # Offload Egress Queues (FL) 164# NMPSTCAM_OFLD = 0 # Offload MPS TCAM Entries (uses NIC's) 165# NMSIX_OFLD = 16 # Offload MSI-X Interrupt Vectors (FLIQ) 166# 167# NVI_RDMA = 0 # RDMA uses NIC function to access ports 168# NFLIQ_RDMA = 4 # RDMA Ingress Queues with Free Lists 169# NETHCTRL_RDMA = 0 # RDMA Ethernet Control/TX Queues 170# NEQ_RDMA = 4 # RDMA Egress Queues (FL) 171# NMPSTCAM_RDMA = 0 # RDMA MPS TCAM Entries (uses NIC's) 172# NMSIX_RDMA = 4 # RDMA MSI-X Interrupt Vectors (FLIQ) 173# 174# NEQ_WD = 128 # Wire Direct TX Queues and FLs 175# NETHCTRL_WD = 64 # Wire Direct TX Queues 176# NFLIQ_WD = 64 ` # Wire Direct Ingress Queues with Free Lists 177# 178# NVI_ISCSI = 4 # ISCSI access to NPORTS 179# NFLIQ_ISCSI = 4 # ISCSI Ingress Queues with Free Lists 180# NETHCTRL_ISCSI = 0 # ISCSI Ethernet Control/TX Queues 181# NEQ_ISCSI = 4 # ISCSI Egress Queues (FL) 182# NMPSTCAM_ISCSI = 4 # ISCSI MPS TCAM Entries (NPORTS) 183# NMSIX_ISCSI = 4 # ISCSI MSI-X Interrupt Vectors (FLIQ) 184# 185# NVI_FCOE = 4 # FCOE access to NPORTS 186# NFLIQ_FCOE = 34 # FCOE Ingress Queues with Free Lists 187# NETHCTRL_FCOE = 32 # FCOE Ethernet Control/TX Queues 188# NEQ_FCOE = 66 # FCOE Egress Queues (FL) 189# NMPSTCAM_FCOE = 32 # FCOE MPS TCAM Entries (NPORTS) 190# NMSIX_FCOE = 34 # FCOE MSI-X Interrupt Vectors (FLIQ) 191 192# Two extra Ingress Queues per function for Firmware Events and Forwarded 193# Interrupts, and two extra interrupts per function for Firmware Events (or a 194# Forwarded Interrupt Queue) and General Interrupts per function. 195# 196# NFLIQ_EXTRA = 6 # "extra" Ingress Queues 2*NFUNCS (Firmware and 197# # Forwarded Interrupts 198# NMSIX_EXTRA = 6 # extra interrupts 2*NFUNCS (Firmware and 199# # General Interrupts 200 201# Microsoft HyperV resources. The HyperV Virtual Ingress Queues will have 202# their interrupts forwarded to another set of Forwarded Interrupt Queues. 203# 204# NVI_HYPERV = 16 # VMs we want to support 205# NVIIQ_HYPERV = 2 # Virtual Ingress Queues with Free Lists per VM 206# NFLIQ_HYPERV = 40 # VIQs + NCPUS Forwarded Interrupt Queues 207# NEQ_HYPERV = 32 # VIQs Free Lists 208# NMPSTCAM_HYPERV = 16 # MPS TCAM Entries (NVI_HYPERV) 209# NMSIX_HYPERV = 8 # NCPUS Forwarded Interrupt Queues 210 211# Adding all of the above Unified PF resource needs together: (NIC + OFLD + 212# RDMA + ISCSI + FCOE + EXTRA + HYPERV) 213# 214# NVI_UNIFIED = 28 215# NFLIQ_UNIFIED = 106 216# NETHCTRL_UNIFIED = 32 217# NEQ_UNIFIED = 124 218# NMPSTCAM_UNIFIED = 40 219# 220# The sum of all the MSI-X resources above is 74 MSI-X Vectors but we'll round 221# that up to 128 to make sure the Unified PF doesn't run out of resources. 222# 223# NMSIX_UNIFIED = 128 224# 225# The Storage PFs could need up to NPORTS*NCPUS + NMSIX_EXTRA MSI-X Vectors 226# which is 34 but they're probably safe with 32. 227# 228# NMSIX_STORAGE = 32 229 230# Note: The UnifiedPF is PF4 which doesn't have any Virtual Functions 231# associated with it. Thus, the MSI-X Vector allocations we give to the 232# UnifiedPF aren't inherited by any Virtual Functions. As a result we can 233# provision many more Virtual Functions than we can if the UnifiedPF were 234# one of PF0-3. 235# 236 237# All of the below PCI-E parameters are actually stored in various *_init.txt 238# files. We include them below essentially as comments. 239# 240# For PF0-3 we assign 8 vectors each for NIC Ingress Queues of the associated 241# ports 0-3. 242# 243# For PF4, the Unified PF, we give it an MSI-X Table Size as outlined above. 244# 245# For PF5-6 we assign enough MSI-X Vectors to support FCoE and iSCSI 246# storage applications across all four possible ports. 247# 248# Additionally, since the UnifiedPF isn't one of the per-port Physical 249# Functions, we give the UnifiedPF and the PF0-3 Physical Functions 250# different PCI Device IDs which will allow Unified and Per-Port Drivers 251# to directly select the type of Physical Function to which they wish to be 252# attached. 253# 254# Note that the actual values used for the PCI-E Intelectual Property will be 255# 1 less than those below since that's the way it "counts" things. For 256# readability, we use the number we actually mean ... 257# 258# PF0_INT = 8 # NCPUS 259# PF1_INT = 8 # NCPUS 260# PF2_INT = 8 # NCPUS 261# PF3_INT = 8 # NCPUS 262# PF0_3_INT = 32 # PF0_INT + PF1_INT + PF2_INT + PF3_INT 263# 264# PF4_INT = 128 # NMSIX_UNIFIED 265# PF5_INT = 32 # NMSIX_STORAGE 266# PF6_INT = 32 # NMSIX_STORAGE 267# PF7_INT = 0 # Nothing Assigned 268# PF4_7_INT = 192 # PF4_INT + PF5_INT + PF6_INT + PF7_INT 269# 270# PF0_7_INT = 224 # PF0_3_INT + PF4_7_INT 271# 272# With the above we can get 17 VFs/PF0-3 (limited by 336 MPS TCAM entries) 273# but we'll lower that to 16 to make our total 64 and a nice power of 2 ... 274# 275# NVF = 16 276 277# For those OSes which manage different ports on different PFs, we need 278# only enough resources to support a single port's NIC application functions 279# on PF0-3. The below assumes that we're only doing NIC with NCPUS "Queue 280# Sets" for ports 0-3. The FCoE and iSCSI functions for such OSes will be 281# managed on the "storage PFs" (see below). 282# 283[function "0"] 284 nvf = 16 # NVF on this function 285 wx_caps = all # write/execute permissions for all commands 286 r_caps = all # read permissions for all commands 287 nvi = 1 # 1 port 288 niqflint = 8 # NCPUS "Queue Sets" 289 nethctrl = 8 # NCPUS "Queue Sets" 290 neq = 16 # niqflint + nethctrl Egress Queues 291 nexactf = 8 # number of exact MPSTCAM MAC filters 292 cmask = all # access to all channels 293 pmask = 0x1 # access to only one port 294 295[function "1"] 296 nvf = 16 # NVF on this function 297 wx_caps = all # write/execute permissions for all commands 298 r_caps = all # read permissions for all commands 299 nvi = 1 # 1 port 300 niqflint = 8 # NCPUS "Queue Sets" 301 nethctrl = 8 # NCPUS "Queue Sets" 302 neq = 16 # niqflint + nethctrl Egress Queues 303 nexactf = 8 # number of exact MPSTCAM MAC filters 304 cmask = all # access to all channels 305 pmask = 0x2 # access to only one port 306 307[function "2"] 308 nvf = 16 # NVF on this function 309 wx_caps = all # write/execute permissions for all commands 310 r_caps = all # read permissions for all commands 311 nvi = 1 # 1 port 312 niqflint = 8 # NCPUS "Queue Sets" 313 nethctrl = 8 # NCPUS "Queue Sets" 314 neq = 16 # niqflint + nethctrl Egress Queues 315 nexactf = 8 # number of exact MPSTCAM MAC filters 316 cmask = all # access to all channels 317 pmask = 0x4 # access to only one port 318 319[function "3"] 320 nvf = 16 # NVF on this function 321 wx_caps = all # write/execute permissions for all commands 322 r_caps = all # read permissions for all commands 323 nvi = 1 # 1 port 324 niqflint = 8 # NCPUS "Queue Sets" 325 nethctrl = 8 # NCPUS "Queue Sets" 326 neq = 16 # niqflint + nethctrl Egress Queues 327 nexactf = 8 # number of exact MPSTCAM MAC filters 328 cmask = all # access to all channels 329 pmask = 0x8 # access to only one port 330 331# Some OS Drivers manage all application functions for all ports via PF4. 332# Thus we need to provide a large number of resources here. For Egress 333# Queues we need to account for both TX Queues as well as Free List Queues 334# (because the host is responsible for producing Free List Buffers for the 335# hardware to consume). 336# 337[function "4"] 338 wx_caps = all # write/execute permissions for all commands 339 r_caps = all # read permissions for all commands 340 nvi = 28 # NVI_UNIFIED 341 niqflint = 170 # NFLIQ_UNIFIED + NLFIQ_WD 342 nethctrl = 96 # NETHCTRL_UNIFIED + NETHCTRL_WD 343 neq = 252 # NEQ_UNIFIED + NEQ_WD 344 nexactf = 40 # NMPSTCAM_UNIFIED 345 cmask = all # access to all channels 346 pmask = all # access to all four ports ... 347 nroute = 32 # number of routing region entries 348 nclip = 32 # number of clip region entries 349 nfilter = 768 # number of filter region entries 350 nserver = 256 # number of server region entries 351 nhash = 0 # number of hash region entries 352 protocol = nic_vm, ofld, rddp, rdmac, iscsi_initiator_pdu, iscsi_target_pdu 353 tp_l2t = 100 354 tp_ddp = 2 355 tp_ddp_iscsi = 2 356 tp_stag = 2 357 tp_pbl = 5 358 tp_rq = 7 359 360# We have FCoE and iSCSI storage functions on PF5 and PF6 each of which may 361# need to have Virtual Interfaces on each of the four ports with up to NCPUS 362# "Queue Sets" each. 363# 364[function "5"] 365 wx_caps = all # write/execute permissions for all commands 366 r_caps = all # read permissions for all commands 367 nvi = 4 # NPORTS 368 niqflint = 34 # NPORTS*NCPUS + NMSIX_EXTRA 369 nethctrl = 32 # NPORTS*NCPUS 370 neq = 64 # NPORTS*NCPUS * 2 (FL, ETHCTRL/TX) 371 nexactf = 4 # NPORTS 372 cmask = all # access to all channels 373 pmask = all # access to all four ports ... 374 375[function "6"] 376 wx_caps = all # write/execute permissions for all commands 377 r_caps = all # read permissions for all commands 378 nvi = 4 # NPORTS 379 niqflint = 34 # NPORTS*NCPUS + NMSIX_EXTRA 380 nethctrl = 32 # NPORTS*NCPUS 381 neq = 66 # NPORTS*NCPUS * 2 (FL, ETHCTRL/TX) + 2 (EXTRA) 382 nexactf = 32 # NPORTS + adding 28 exact entries for FCoE 383 # which is OK since < MIN(SUM PF0..3, PF4) 384 # and we never load PF0..3 and PF4 concurrently 385 cmask = all # access to all channels 386 pmask = all # access to all four ports ... 387 nhash = 0 388 protocol = fcoe_initiator 389 tp_ddp = 2 390 fcoe_nfcf = 16 391 fcoe_nvnp = 32 392 fcoe_nssn = 1024 393 394# For Virtual functions, we only allow NIC functionality and we only allow 395# access to one port (1 << PF). Note that because of limitations in the 396# Scatter Gather Engine (SGE) hardware which checks writes to VF KDOORBELL 397# and GTS registers, the number of Ingress and Egress Queues must be a power 398# of 2. 399# 400[function "0/*"] # NVF 401 wx_caps = 0x82 # DMAQ | VF 402 r_caps = 0x86 # DMAQ | VF | PORT 403 nvi = 1 # 1 port 404 niqflint = 4 # 2 "Queue Sets" + NXIQ 405 nethctrl = 2 # 2 "Queue Sets" 406 neq = 4 # 2 "Queue Sets" * 2 407 nexactf = 4 408 cmask = all # access to all channels 409 pmask = 0x1 # access to only one port ... 410 411[function "1/*"] # NVF 412 wx_caps = 0x82 # DMAQ | VF 413 r_caps = 0x86 # DMAQ | VF | PORT 414 nvi = 1 # 1 port 415 niqflint = 4 # 2 "Queue Sets" + NXIQ 416 nethctrl = 2 # 2 "Queue Sets" 417 neq = 4 # 2 "Queue Sets" * 2 418 nexactf = 4 419 cmask = all # access to all channels 420 pmask = 0x2 # access to only one port ... 421 422[function "2/*"] # NVF 423 wx_caps = 0x82 # DMAQ | VF 424 r_caps = 0x86 # DMAQ | VF | PORT 425 nvi = 1 # 1 port 426 niqflint = 4 # 2 "Queue Sets" + NXIQ 427 nethctrl = 2 # 2 "Queue Sets" 428 neq = 4 # 2 "Queue Sets" * 2 429 nexactf = 4 430 cmask = all # access to all channels 431 pmask = 0x4 # access to only one port ... 432 433[function "3/*"] # NVF 434 wx_caps = 0x82 # DMAQ | VF 435 r_caps = 0x86 # DMAQ | VF | PORT 436 nvi = 1 # 1 port 437 niqflint = 4 # 2 "Queue Sets" + NXIQ 438 nethctrl = 2 # 2 "Queue Sets" 439 neq = 4 # 2 "Queue Sets" * 2 440 nexactf = 4 441 cmask = all # access to all channels 442 pmask = 0x8 # access to only one port ... 443 444# MPS features a 196608 bytes ingress buffer that is used for ingress buffering 445# for packets from the wire as well as the loopback path of the L2 switch. The 446# folling params control how the buffer memory is distributed and the L2 flow 447# control settings: 448# 449# bg_mem: %-age of mem to use for port/buffer group 450# lpbk_mem: %-age of port/bg mem to use for loopback 451# hwm: high watermark; bytes available when starting to send pause 452# frames (in units of 0.1 MTU) 453# lwm: low watermark; bytes remaining when sending 'unpause' frame 454# (in inuits of 0.1 MTU) 455# dwm: minimum delta between high and low watermark (in units of 100 456# Bytes) 457# 458[port "0"] 459 dcb = ppp, dcbx # configure for DCB PPP and enable DCBX offload 460 bg_mem = 25 461 lpbk_mem = 25 462 hwm = 30 463 lwm = 15 464 dwm = 30 465 466[port "1"] 467 dcb = ppp, dcbx 468 bg_mem = 25 469 lpbk_mem = 25 470 hwm = 30 471 lwm = 15 472 dwm = 30 473 474[port "2"] 475 dcb = ppp, dcbx 476 bg_mem = 25 477 lpbk_mem = 25 478 hwm = 30 479 lwm = 15 480 dwm = 30 481 482[port "3"] 483 dcb = ppp, dcbx 484 bg_mem = 25 485 lpbk_mem = 25 486 hwm = 30 487 lwm = 15 488 dwm = 30 489 490[fini] 491 version = 0x14250007 492 checksum = 0xfcbadefb 493 494# Total resources used by above allocations: 495# Virtual Interfaces: 104 496# Ingress Queues/w Free Lists and Interrupts: 526 497# Egress Queues: 702 498# MPS TCAM Entries: 336 499# MSI-X Vectors: 736 500# Virtual Functions: 64 501# 502# $FreeBSD$ 503# 504